Commit 3ca7b3c5b64d35fe02c35b5d44c2c58b49499fee

Authored by Hugh Dickins
Committed by Linus Torvalds
1 parent bb3ab59683

mm: define PAGE_MAPPING_FLAGS

At present we define PageAnon(page) by the low PAGE_MAPPING_ANON bit set
in page->mapping, with the higher bits a pointer to the anon_vma; and have
defined PageKsm(page) as that with NULL anon_vma.

But KSM swapping will need to store a pointer there: so in preparation for
that, now define PAGE_MAPPING_FLAGS as the low two bits, including
PAGE_MAPPING_KSM (always set along with PAGE_MAPPING_ANON, until some
other use for the bit emerges).

Declare page_rmapping(page) to return the pointer part of page->mapping,
and page_anon_vma(page) to return the anon_vma pointer when that's what it
is.  Use these in a few appropriate places: notably, unuse_vma() has been
testing page->mapping, but is better to be testing page_anon_vma() (cases
may be added in which flag bits are set without any pointer).

Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Wu Fengguang <fengguang.wu@intel.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 6 changed files with 35 additions and 15 deletions Side-by-side Diff

... ... @@ -38,7 +38,8 @@
38 38 */
39 39 static inline int PageKsm(struct page *page)
40 40 {
41   - return ((unsigned long)page->mapping == PAGE_MAPPING_ANON);
  41 + return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
  42 + (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
42 43 }
43 44  
44 45 /*
... ... @@ -47,7 +48,7 @@
47 48 static inline void page_add_ksm_rmap(struct page *page)
48 49 {
49 50 if (atomic_inc_and_test(&page->_mapcount)) {
50   - page->mapping = (void *) PAGE_MAPPING_ANON;
  51 + page->mapping = (void *) (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
51 52 __inc_zone_page_state(page, NR_ANON_PAGES);
52 53 }
53 54 }
... ... @@ -620,13 +620,22 @@
620 620 /*
621 621 * On an anonymous page mapped into a user virtual memory area,
622 622 * page->mapping points to its anon_vma, not to a struct address_space;
623   - * with the PAGE_MAPPING_ANON bit set to distinguish it.
  623 + * with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
624 624 *
  625 + * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
  626 + * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
  627 + * and then page->mapping points, not to an anon_vma, but to a private
  628 + * structure which KSM associates with that merged page. See ksm.h.
  629 + *
  630 + * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
  631 + *
625 632 * Please note that, confusingly, "page_mapping" refers to the inode
626 633 * address_space which maps the page from disk; whereas "page_mapped"
627 634 * refers to user virtual address space into which the page is mapped.
628 635 */
629 636 #define PAGE_MAPPING_ANON 1
  637 +#define PAGE_MAPPING_KSM 2
  638 +#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
630 639  
631 640 extern struct address_space swapper_space;
632 641 static inline struct address_space *page_mapping(struct page *page)
... ... @@ -642,6 +651,12 @@
642 651 if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
643 652 mapping = NULL;
644 653 return mapping;
  654 +}
  655 +
  656 +/* Neutral page->mapping pointer to address_space or anon_vma or other */
  657 +static inline void *page_rmapping(struct page *page)
  658 +{
  659 + return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);
645 660 }
646 661  
647 662 static inline int PageAnon(struct page *page)
include/linux/rmap.h
... ... @@ -39,6 +39,14 @@
39 39  
40 40 #ifdef CONFIG_MMU
41 41  
  42 +static inline struct anon_vma *page_anon_vma(struct page *page)
  43 +{
  44 + if (((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) !=
  45 + PAGE_MAPPING_ANON)
  46 + return NULL;
  47 + return page_rmapping(page);
  48 +}
  49 +
42 50 static inline void anon_vma_lock(struct vm_area_struct *vma)
43 51 {
44 52 struct anon_vma *anon_vma = vma->anon_vma;
... ... @@ -172,17 +172,14 @@
172 172 {
173 173 struct anon_vma *anon_vma;
174 174 struct vm_area_struct *vma;
175   - unsigned long mapping;
176 175  
177   - mapping = (unsigned long)new->mapping;
178   -
179   - if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
180   - return;
181   -
182 176 /*
183 177 * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
184 178 */
185   - anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
  179 + anon_vma = page_anon_vma(new);
  180 + if (!anon_vma)
  181 + return;
  182 +
186 183 spin_lock(&anon_vma->lock);
187 184  
188 185 list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
... ... @@ -203,7 +203,7 @@
203 203  
204 204 rcu_read_lock();
205 205 anon_mapping = (unsigned long) page->mapping;
206   - if (!(anon_mapping & PAGE_MAPPING_ANON))
  206 + if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
207 207 goto out;
208 208 if (!page_mapped(page))
209 209 goto out;
... ... @@ -248,8 +248,7 @@
248 248 unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
249 249 {
250 250 if (PageAnon(page)) {
251   - if ((void *)vma->anon_vma !=
252   - (void *)page->mapping - PAGE_MAPPING_ANON)
  251 + if (vma->anon_vma != page_anon_vma(page))
253 252 return -EFAULT;
254 253 } else if (page->mapping && !(vma->vm_flags & VM_NONLINEAR)) {
255 254 if (!vma->vm_file ||
... ... @@ -513,7 +512,7 @@
513 512 referenced++;
514 513  
515 514 *vm_flags = 0;
516   - if (page_mapped(page) && page->mapping) {
  515 + if (page_mapped(page) && page_rmapping(page)) {
517 516 if (PageAnon(page))
518 517 referenced += page_referenced_anon(page, mem_cont,
519 518 vm_flags);
... ... @@ -938,7 +938,7 @@
938 938 unsigned long addr, end, next;
939 939 int ret;
940 940  
941   - if (page->mapping) {
  941 + if (page_anon_vma(page)) {
942 942 addr = page_address_in_vma(page, vma);
943 943 if (addr == -EFAULT)
944 944 return 0;