mm: numa: Migrate on reference policy

This is the simplest possible policy that still does something of note. When a pte_numa is faulted, it is moved immediately. Any replacement policy must at least do better than this and in all likelihood this policy regresses normal workloads. Signed-off-by: Mel Gorman <mgorman@suse.de> Acked-by: Rik van Riel <riel@redhat.com>

mm: numa: Migrate on reference policy
This is the simplest possible policy that still does something of note. When a pte_numa is faulted, it is moved immediately. Any replacement policy must at least do better than this and in all likelihood this policy regresses normal workloads. Signed-off-by: Mel Gorman <mgorman@suse.de> Acked-by: Rik van Riel <riel@redhat.com>
Mel Gorman
1 parent 03c5a6e163
Showing 2 changed files with 37 additions and 2 deletions Side-by-side Diff
include/uapi/linux/mempolicy.h
mm/mempolicy.c
@@ -67,6 +67,7 @@
 #define MPOL_F_LOCAL   (1 << 1)	/* preferred local allocation */
 #define MPOL_F_REBINDING (1 << 2)	/* identify policies in rebinding */
 #define MPOL_F_MOF	(1 << 3) /* this policy wants migrate on fault */
+#define MPOL_F_MORON	(1 << 4) /* Migrate On pte_numa Reference On Node */
  
  
 #endif /* _UAPI_LINUX_MEMPOLICY_H */
@@ -118,6 +118,26 @@
 	.flags = MPOL_F_LOCAL,
 };
  
+static struct mempolicy preferred_node_policy[MAX_NUMNODES];
+
+static struct mempolicy *get_task_policy(struct task_struct *p)
+{
+	struct mempolicy *pol = p->mempolicy;
+	int node;
+
+	if (!pol) {
+		node = numa_node_id();
+		if (node != -1)
+			pol = &preferred_node_policy[node];
+
+		/* preferred_node_policy is not initialised early in boot */
+		if (!pol->mode)
+			pol = NULL;
+	}
+
+	return pol;
+}
+
 static const struct mempolicy_operations {
 	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
 	/*
@@ -1598,7 +1618,7 @@
 struct mempolicy *get_vma_policy(struct task_struct *task,
 		struct vm_area_struct *vma, unsigned long addr)
 {
-	struct mempolicy *pol = task->mempolicy;
+	struct mempolicy *pol = get_task_policy(task);
  
 	if (vma) {
 		if (vma->vm_ops && vma->vm_ops->get_policy) {
@@ -2021,7 +2041,7 @@
  */
 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
-	struct mempolicy *pol = current->mempolicy;
+	struct mempolicy *pol = get_task_policy(current);
 	struct page *page;
 	unsigned int cpuset_mems_cookie;
  
@@ -2295,6 +2315,11 @@
 	default:
 		BUG();
 	}
+
+	/* Migrate the page towards the node whose CPU is referencing it */
+	if (pol->flags & MPOL_F_MORON)
+		polnid = numa_node_id();
+
 	if (curnid != polnid)
 		ret = polnid;
 out:
@@ -2482,6 +2507,15 @@
 	sn_cache = kmem_cache_create("shared_policy_node",
 				     sizeof(struct sp_node),
 				     0, SLAB_PANIC, NULL);
+
+	for_each_node(nid) {
+		preferred_node_policy[nid] = (struct mempolicy) {
+			.refcnt = ATOMIC_INIT(1),
+			.mode = MPOL_PREFERRED,
+			.flags = MPOL_F_MOF | MPOL_F_MORON,
+			.v = { .preferred_node = nid, },
+		};
+	}
  
 	/*
 	 * Set interleaving policy for system init. Interleaving is only
...	...	@@ -67,6 +67,7 @@
67	67	#define MPOL_F_LOCAL (1 << 1) /* preferred local allocation */
68	68	#define MPOL_F_REBINDING (1 << 2) /* identify policies in rebinding */
69	69	#define MPOL_F_MOF (1 << 3) /* this policy wants migrate on fault */
	70	+#define MPOL_F_MORON (1 << 4) /* Migrate On pte_numa Reference On Node */
70	71
71	72
72	73	#endif /* _UAPI_LINUX_MEMPOLICY_H */
...	...	@@ -118,6 +118,26 @@
118	118	.flags = MPOL_F_LOCAL,
119	119	};
120	120
	121	+static struct mempolicy preferred_node_policy[MAX_NUMNODES];
	122	+
	123	+static struct mempolicy get_task_policy(struct task_struct p)
	124	+{
	125	+ struct mempolicy *pol = p->mempolicy;
	126	+ int node;
	127	+
	128	+ if (!pol) {
	129	+ node = numa_node_id();
	130	+ if (node != -1)
	131	+ pol = &preferred_node_policy[node];
	132	+
	133	+ /* preferred_node_policy is not initialised early in boot */
	134	+ if (!pol->mode)
	135	+ pol = NULL;
	136	+ }
	137	+
	138	+ return pol;
	139	+}
	140	+
121	141	static const struct mempolicy_operations {
122	142	int (create)(struct mempolicy pol, const nodemask_t *nodes);
123	143	/*
...	...	@@ -1598,7 +1618,7 @@
1598	1618	struct mempolicy get_vma_policy(struct task_struct task,
1599	1619	struct vm_area_struct *vma, unsigned long addr)
1600	1620	{
1601		- struct mempolicy *pol = task->mempolicy;
	1621	+ struct mempolicy *pol = get_task_policy(task);
1602	1622
1603	1623	if (vma) {
1604	1624	if (vma->vm_ops && vma->vm_ops->get_policy) {
...	...	@@ -2021,7 +2041,7 @@
2021	2041	*/
2022	2042	struct page *alloc_pages_current(gfp_t gfp, unsigned order)
2023	2043	{
2024		- struct mempolicy *pol = current->mempolicy;
	2044	+ struct mempolicy *pol = get_task_policy(current);
2025	2045	struct page *page;
2026	2046	unsigned int cpuset_mems_cookie;
2027	2047
...	...	@@ -2295,6 +2315,11 @@
2295	2315	default:
2296	2316	BUG();
2297	2317	}
	2318	+
	2319	+ /* Migrate the page towards the node whose CPU is referencing it */
	2320	+ if (pol->flags & MPOL_F_MORON)
	2321	+ polnid = numa_node_id();
	2322	+
2298	2323	if (curnid != polnid)
2299	2324	ret = polnid;
2300	2325	out:
...	...	@@ -2482,6 +2507,15 @@
2482	2507	sn_cache = kmem_cache_create("shared_policy_node",
2483	2508	sizeof(struct sp_node),
2484	2509	0, SLAB_PANIC, NULL);
	2510	+
	2511	+ for_each_node(nid) {
	2512	+ preferred_node_policy[nid] = (struct mempolicy) {
	2513	+ .refcnt = ATOMIC_INIT(1),
	2514	+ .mode = MPOL_PREFERRED,
	2515	+ .flags = MPOL_F_MOF \| MPOL_F_MORON,
	2516	+ .v = { .preferred_node = nid, },
	2517	+ };
	2518	+ }
2485	2519
2486	2520	/*
2487	2521	* Set interleaving policy for system init. Interleaving is only