memcg: clear pc->mem_cgroup if necessary.

This is a preparation before removing a flag PCG_ACCT_LRU in page_cgroup and reducing atomic ops/complexity in memcg LRU handling. In some cases, pages are added to lru before charge to memcg and pages are not classfied to memory cgroup at lru addtion. Now, the lru where the page should be added is determined a bit in page_cgroup->flags and pc->mem_cgroup. I'd like to remove the check of flag. To handle the case pc->mem_cgroup may contain stale pointers if pages are added to LRU before classification. This patch resets pc->mem_cgroup to root_mem_cgroup before lru additions. [akpm@linux-foundation.org: fix CONFIG_CGROUP_MEM_CONT=n build] [hughd@google.com: fix CONFIG_CGROUP_MEM_RES_CTLR=y CONFIG_CGROUP_MEM_RES_CTLR_SWAP=n build] [akpm@linux-foundation.org: ksm.c needs memcontrol.h, per Michal] [hughd@google.com: stop oops in mem_cgroup_reset_owner()] [hughd@google.com: fix page migration to reset_owner] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Miklos Szeredi <mszeredi@suse.cz> Acked-by: Michal Hocko <mhocko@suse.cz> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Ying Han <yinghan@google.com> Signed-off-by: Hugh Dickins <hughd@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

memcg: clear pc->mem_cgroup if necessary.
This is a preparation before removing a flag PCG_ACCT_LRU in page_cgroup and reducing atomic ops/complexity in memcg LRU handling. In some cases, pages are added to lru before charge to memcg and pages are not classfied to memory cgroup at lru addtion. Now, the lru where the page should be added is determined a bit in page_cgroup->flags and pc->mem_cgroup. I'd like to remove the check of flag. To handle the case pc->mem_cgroup may contain stale pointers if pages are added to LRU before classification. This patch resets pc->mem_cgroup to root_mem_cgroup before lru additions. [akpm@linux-foundation.org: fix CONFIG_CGROUP_MEM_CONT=n build] [hughd@google.com: fix CONFIG_CGROUP_MEM_RES_CTLR=y CONFIG_CGROUP_MEM_RES_CTLR_SWAP=n build] [akpm@linux-foundation.org: ksm.c needs memcontrol.h, per Michal] [hughd@google.com: stop oops in mem_cgroup_reset_owner()] [hughd@google.com: fix page migration to reset_owner] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Cc: Miklos Szeredi <mszeredi@suse.cz> Acked-by: Michal Hocko <mhocko@suse.cz> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Ying Han <yinghan@google.com> Signed-off-by: Hugh Dickins <hughd@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
KAMEZAWA Hiroyuki · Linus Torvalds
1 parent 36b62ad539
Showing 5 changed files with 45 additions and 0 deletions Side-by-side Diff
include/linux/memcontrol.h
mm/ksm.c
mm/memcontrol.c
mm/migrate.c
mm/swap_state.c
@@ -129,6 +129,7 @@
 extern void mem_cgroup_replace_page_cache(struct page *oldpage,
 					struct page *newpage);
  
+extern void mem_cgroup_reset_owner(struct page *page);
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
 extern int do_swap_account;
 #endif
@@ -389,6 +390,10 @@
 }
 static inline void mem_cgroup_replace_page_cache(struct page *oldpage,
 				struct page *newpage)
+{
+}
+
+static inline void mem_cgroup_reset_owner(struct page *page)
 {
 }
 #endif /* CONFIG_CGROUP_MEM_CONT */
@@ -28,6 +28,7 @@
 #include <linux/kthread.h>
 #include <linux/wait.h>
 #include <linux/slab.h>
+#include <linux/memcontrol.h>
 #include <linux/rbtree.h>
 #include <linux/memory.h>
 #include <linux/mmu_notifier.h>
@@ -1571,6 +1572,16 @@
  
 	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
 	if (new_page) {
+		/*
+		 * The memcg-specific accounting when moving
+		 * pages around the LRU lists relies on the
+		 * page's owner (memcg) to be valid.  Usually,
+		 * pages are assigned to a new owner before
+		 * being put on the LRU list, but since this
+		 * is not the case here, the stale owner from
+		 * a previous allocation cycle must be reset.
+		 */
+		mem_cgroup_reset_owner(new_page);
 		copy_user_highpage(new_page, page, address, vma);
  
 		SetPageDirty(new_page);
@@ -3050,6 +3050,23 @@
 	batch->memcg = NULL;
 }
  
+/*
+ * A function for resetting pc->mem_cgroup for newly allocated pages.
+ * This function should be called if the newpage will be added to LRU
+ * before start accounting.
+ */
+void mem_cgroup_reset_owner(struct page *newpage)
+{
+	struct page_cgroup *pc;
+
+	if (mem_cgroup_disabled())
+		return;
+
+	pc = lookup_page_cgroup(newpage);
+	VM_BUG_ON(PageCgroupUsed(pc));
+	pc->mem_cgroup = root_mem_cgroup;
+}
+
 #ifdef CONFIG_SWAP
 /*
  * called after __delete_from_swap_cache() and drop "page" account.
@@ -777,6 +777,8 @@
 	if (!newpage)
 		return -ENOMEM;
  
+	mem_cgroup_reset_owner(newpage);
+
 	if (page_count(page) == 1) {
 		/* page was freed from under us. So we are done. */
 		goto out;
@@ -300,6 +300,16 @@
 			new_page = alloc_page_vma(gfp_mask, vma, addr);
 			if (!new_page)
 				break;		/* Out of memory */
+			/*
+			 * The memcg-specific accounting when moving
+			 * pages around the LRU lists relies on the
+			 * page's owner (memcg) to be valid.  Usually,
+			 * pages are assigned to a new owner before
+			 * being put on the LRU list, but since this
+			 * is not the case here, the stale owner from
+			 * a previous allocation cycle must be reset.
+			 */
+			mem_cgroup_reset_owner(new_page);
 		}
  
 		/*
...	...	@@ -129,6 +129,7 @@
129	129	extern void mem_cgroup_replace_page_cache(struct page *oldpage,
130	130	struct page *newpage);
131	131
	132	+extern void mem_cgroup_reset_owner(struct page *page);
132	133	#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
133	134	extern int do_swap_account;
134	135	#endif
...	...	@@ -389,6 +390,10 @@
389	390	}
390	391	static inline void mem_cgroup_replace_page_cache(struct page *oldpage,
391	392	struct page *newpage)
	393	+{
	394	+}
	395	+
	396	+static inline void mem_cgroup_reset_owner(struct page *page)
392	397	{
393	398	}
394	399	#endif /* CONFIG_CGROUP_MEM_CONT */
...	...	@@ -28,6 +28,7 @@
28	28	#include <linux/kthread.h>
29	29	#include <linux/wait.h>
30	30	#include <linux/slab.h>
	31	+#include <linux/memcontrol.h>
31	32	#include <linux/rbtree.h>
32	33	#include <linux/memory.h>
33	34	#include <linux/mmu_notifier.h>
...	...	@@ -1571,6 +1572,16 @@
1571	1572
1572	1573	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
1573	1574	if (new_page) {
	1575	+ /*
	1576	+ * The memcg-specific accounting when moving
	1577	+ * pages around the LRU lists relies on the
	1578	+ * page's owner (memcg) to be valid. Usually,
	1579	+ * pages are assigned to a new owner before
	1580	+ * being put on the LRU list, but since this
	1581	+ * is not the case here, the stale owner from
	1582	+ * a previous allocation cycle must be reset.
	1583	+ */
	1584	+ mem_cgroup_reset_owner(new_page);
1574	1585	copy_user_highpage(new_page, page, address, vma);
1575	1586
1576	1587	SetPageDirty(new_page);
...	...	@@ -3050,6 +3050,23 @@
3050	3050	batch->memcg = NULL;
3051	3051	}
3052	3052
	3053	+/*
	3054	+ * A function for resetting pc->mem_cgroup for newly allocated pages.
	3055	+ * This function should be called if the newpage will be added to LRU
	3056	+ * before start accounting.
	3057	+ */
	3058	+void mem_cgroup_reset_owner(struct page *newpage)
	3059	+{
	3060	+ struct page_cgroup *pc;
	3061	+
	3062	+ if (mem_cgroup_disabled())
	3063	+ return;
	3064	+
	3065	+ pc = lookup_page_cgroup(newpage);
	3066	+ VM_BUG_ON(PageCgroupUsed(pc));
	3067	+ pc->mem_cgroup = root_mem_cgroup;
	3068	+}
	3069	+
3053	3070	#ifdef CONFIG_SWAP
3054	3071	/*
3055	3072	* called after __delete_from_swap_cache() and drop "page" account.
...	...	@@ -777,6 +777,8 @@
777	777	if (!newpage)
778	778	return -ENOMEM;
779	779
	780	+ mem_cgroup_reset_owner(newpage);
	781	+
780	782	if (page_count(page) == 1) {
781	783	/* page was freed from under us. So we are done. */
782	784	goto out;
...	...	@@ -300,6 +300,16 @@
300	300	new_page = alloc_page_vma(gfp_mask, vma, addr);
301	301	if (!new_page)
302	302	break; /* Out of memory */
	303	+ /*
	304	+ * The memcg-specific accounting when moving
	305	+ * pages around the LRU lists relies on the
	306	+ * page's owner (memcg) to be valid. Usually,
	307	+ * pages are assigned to a new owner before
	308	+ * being put on the LRU list, but since this
	309	+ * is not the case here, the stale owner from
	310	+ * a previous allocation cycle must be reset.
	311	+ */
	312	+ mem_cgroup_reset_owner(new_page);
303	313	}
304	314
305	315	/*