percpu: remove rbtree and use page->index instead

Impact: use page->index for addr to chunk mapping instead of dedicated rbtree The rbtree is used to determine the chunk from the virtual address. However, we can already determine the page struct from a virtual address and there are several unused fields in page struct used by vmalloc. Use the index field to store a pointer to the chunk. Then there is no need anymore for an rbtree. tj: * s/(set|get)_chunk/pcpu_\1_page_chunk/ * Drop inline from the above two functions and moved them upwards so that they are with other simple helpers. * Initial pages might not (actually most of the time don't) live in the vmalloc area. With the previous patch to manually reverse-map both first chunks, this is no longer an issue. Removed pcpu_set_chunk() call on initial pages. Signed-off-by: Christoph Lameter <cl@linux.com> Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: rusty@rustcorp.com.au Cc: Paul Mundt <lethal@linux-sh.org> Cc: rmk@arm.linux.org.uk Cc: starvik@axis.com Cc: ralf@linux-mips.org Cc: davem@davemloft.net Cc: cooloney@kernel.org Cc: kyle@mcmartin.ca Cc: matthew@wil.cx Cc: grundler@parisc-linux.org Cc: takata@linux-m32r.org Cc: benh@kernel.crashing.org Cc: rth@twiddle.net Cc: ink@jurassic.park.msu.ru Cc: heiko.carstens@de.ibm.com Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Nick Piggin <npiggin@suse.de> LKML-Reference: <49D43D58.4050102@kernel.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>

percpu: remove rbtree and use page->index instead
Impact: use page->index for addr to chunk mapping instead of dedicated rbtree The rbtree is used to determine the chunk from the virtual address. However, we can already determine the page struct from a virtual address and there are several unused fields in page struct used by vmalloc. Use the index field to store a pointer to the chunk. Then there is no need anymore for an rbtree. tj: * s/(set|get)_chunk/pcpu_\1_page_chunk/ * Drop inline from the above two functions and moved them upwards so that they are with other simple helpers. * Initial pages might not (actually most of the time don't) live in the vmalloc area. With the previous patch to manually reverse-map both first chunks, this is no longer an issue. Removed pcpu_set_chunk() call on initial pages. Signed-off-by: Christoph Lameter <cl@linux.com> Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: rusty@rustcorp.com.au Cc: Paul Mundt <lethal@linux-sh.org> Cc: rmk@arm.linux.org.uk Cc: starvik@axis.com Cc: ralf@linux-mips.org Cc: davem@davemloft.net Cc: cooloney@kernel.org Cc: kyle@mcmartin.ca Cc: matthew@wil.cx Cc: grundler@parisc-linux.org Cc: takata@linux-m32r.org Cc: benh@kernel.crashing.org Cc: rth@twiddle.net Cc: ink@jurassic.park.msu.ru Cc: heiko.carstens@de.ibm.com Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Nick Piggin <npiggin@suse.de> LKML-Reference: <49D43D58.4050102@kernel.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Christoph Lameter · Ingo Molnar
1 parent ae9e6bc9f7
Showing 1 changed file with 20 additions and 80 deletions Side-by-side Diff
mm/percpu.c
@@ -23,7 +23,7 @@
  * Allocation is done in offset-size areas of single unit space.  Ie,
  * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
  * c1:u1, c1:u2 and c1:u3.  Percpu access can be done by configuring
- * percpu base registers UNIT_SIZE apart.
+ * percpu base registers pcpu_unit_size apart.
  *
  * There are usually many small percpu allocations many of them as
  * small as 4 bytes.  The allocator organizes chunks into lists
@@ -38,8 +38,8 @@
  * region and negative allocated.  Allocation inside a chunk is done
  * by scanning this map sequentially and serving the first matching
  * entry.  This is mostly copied from the percpu_modalloc() allocator.
- * Chunks are also linked into a rb tree to ease address to chunk
- * mapping during free.
+ * Chunks can be determined from the address using the index field
+ * in the page struct. The index field contains a pointer to the chunk.
  *
  * To use this allocator, arch code should do the followings.
  *
@@ -61,7 +61,6 @@
 #include <linux/mutex.h>
 #include <linux/percpu.h>
 #include <linux/pfn.h>
-#include <linux/rbtree.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/vmalloc.h>
@@ -88,7 +87,6 @@
  
 struct pcpu_chunk {
 	struct list_head	list;		/* linked to pcpu_slot lists */
-	struct rb_node		rb_node;	/* key is chunk->vm->addr */
 	int			free_size;	/* free bytes in the chunk */
 	int			contig_hint;	/* max contiguous size hint */
 	struct vm_struct	*vm;		/* mapped vmalloc region */
@@ -133,7 +131,7 @@
  * There are two locks - pcpu_alloc_mutex and pcpu_lock.  The former
  * protects allocation/reclaim paths, chunks and chunk->page arrays.
  * The latter is a spinlock and protects the index data structures -
- * chunk slots, rbtree, chunks and area maps in chunks.
+ * chunk slots, chunks and area maps in chunks.
  *
  * During allocation, pcpu_alloc_mutex is kept locked all the time and
  * pcpu_lock is grabbed and released as necessary.  All actual memory
@@ -152,7 +150,6 @@
 static DEFINE_SPINLOCK(pcpu_lock);	/* protects index data structures */
  
 static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
-static struct rb_root pcpu_addr_root = RB_ROOT;	/* chunks by address */
  
 /* reclaim work to release fully free chunks, scheduled from free path */
 static void pcpu_reclaim(struct work_struct *work);
@@ -203,6 +200,18 @@
 	return *pcpu_chunk_pagep(chunk, 0, page_idx) != NULL;
 }
  
+/* set the pointer to a chunk in a page struct */
+static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
+{
+	page->index = (unsigned long)pcpu;
+}
+
+/* obtain pointer to a chunk from a page struct */
+static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
+{
+	return (struct pcpu_chunk *)page->index;
+}
+
 /**
  * pcpu_mem_alloc - allocate memory
  * @size: bytes to allocate
  
  
  
@@ -269,49 +278,16 @@
 	}
 }
  
-static struct rb_node **pcpu_chunk_rb_search(void *addr,
-					     struct rb_node **parentp)
-{
-	struct rb_node **p = &pcpu_addr_root.rb_node;
-	struct rb_node *parent = NULL;
-	struct pcpu_chunk *chunk;
-
-	while (*p) {
-		parent = *p;
-		chunk = rb_entry(parent, struct pcpu_chunk, rb_node);
-
-		if (addr < chunk->vm->addr)
-			p = &(*p)->rb_left;
-		else if (addr > chunk->vm->addr)
-			p = &(*p)->rb_right;
-		else
-			break;
-	}
-
-	if (parentp)
-		*parentp = parent;
-	return p;
-}
-
 /**
- * pcpu_chunk_addr_search - search for chunk containing specified address
- * @addr: address to search for
+ * pcpu_chunk_addr_search - determine chunk containing specified address
+ * @addr: address for which the chunk needs to be determined.
  *
- * Look for chunk which might contain @addr.  More specifically, it
- * searchs for the chunk with the highest start address which isn't
- * beyond @addr.
- *
- * CONTEXT:
- * pcpu_lock.
- *
  * RETURNS:
  * The address of the found chunk.
  */
 static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
 {
 	void *first_start = pcpu_first_chunk->vm->addr;
-	struct rb_node *n, *parent;
-	struct pcpu_chunk *chunk;
  
 	/* is it in the first chunk? */
 	if (addr >= first_start && addr < first_start + pcpu_chunk_size) {
  
@@ -321,45 +297,10 @@
 		return pcpu_first_chunk;
 	}
  
-	/* nah... search the regular ones */
-	n = *pcpu_chunk_rb_search(addr, &parent);
-	if (!n) {
-		/* no exactly matching chunk, the parent is the closest */
-		n = parent;
-		BUG_ON(!n);
-	}
-	chunk = rb_entry(n, struct pcpu_chunk, rb_node);
-
-	if (addr < chunk->vm->addr) {
-		/* the parent was the next one, look for the previous one */
-		n = rb_prev(n);
-		BUG_ON(!n);
-		chunk = rb_entry(n, struct pcpu_chunk, rb_node);
-	}
-
-	return chunk;
+	return pcpu_get_page_chunk(vmalloc_to_page(addr));
 }
  
 /**
- * pcpu_chunk_addr_insert - insert chunk into address rb tree
- * @new: chunk to insert
- *
- * Insert @new into address rb tree.
- *
- * CONTEXT:
- * pcpu_lock.
- */
-static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
-{
-	struct rb_node **p, *parent;
-
-	p = pcpu_chunk_rb_search(new->vm->addr, &parent);
-	BUG_ON(*p);
-	rb_link_node(&new->rb_node, parent, p);
-	rb_insert_color(&new->rb_node, &pcpu_addr_root);
-}
-
-/**
  * pcpu_extend_area_map - extend area map for allocation
  * @chunk: target chunk
  *
@@ -768,6 +709,7 @@
 						  alloc_mask, 0);
 			if (!*pagep)
 				goto err;
+			pcpu_set_page_chunk(*pagep, chunk);
 		}
 	}
  
@@ -892,7 +834,6 @@
  
 	spin_lock_irq(&pcpu_lock);
 	pcpu_chunk_relocate(chunk, -1);
-	pcpu_chunk_addr_insert(chunk);
 	goto restart;
  
 area_found:
@@ -981,7 +922,6 @@
 		if (chunk == list_first_entry(head, struct pcpu_chunk, list))
 			continue;
  
-		rb_erase(&chunk->rb_node, &pcpu_addr_root);
 		list_move(&chunk->list, &todo);
 	}
...	...	@@ -23,7 +23,7 @@
23	23	* Allocation is done in offset-size areas of single unit space. Ie,
24	24	* an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0,
25	25	* c1:u1, c1:u2 and c1:u3. Percpu access can be done by configuring
26		- * percpu base registers UNIT_SIZE apart.
	26	+ * percpu base registers pcpu_unit_size apart.
27	27	*
28	28	* There are usually many small percpu allocations many of them as
29	29	* small as 4 bytes. The allocator organizes chunks into lists
...	...	@@ -38,8 +38,8 @@
38	38	* region and negative allocated. Allocation inside a chunk is done
39	39	* by scanning this map sequentially and serving the first matching
40	40	* entry. This is mostly copied from the percpu_modalloc() allocator.
41		- * Chunks are also linked into a rb tree to ease address to chunk
42		- * mapping during free.
	41	+ * Chunks can be determined from the address using the index field
	42	+ * in the page struct. The index field contains a pointer to the chunk.
43	43	*
44	44	* To use this allocator, arch code should do the followings.
45	45	*
...	...	@@ -61,7 +61,6 @@
61	61	#include <linux/mutex.h>
62	62	#include <linux/percpu.h>
63	63	#include <linux/pfn.h>
64		-#include <linux/rbtree.h>
65	64	#include <linux/slab.h>
66	65	#include <linux/spinlock.h>
67	66	#include <linux/vmalloc.h>
...	...	@@ -88,7 +87,6 @@
88	87
89	88	struct pcpu_chunk {
90	89	struct list_head list; /* linked to pcpu_slot lists */
91		- struct rb_node rb_node; /* key is chunk->vm->addr */
92	90	int free_size; /* free bytes in the chunk */
93	91	int contig_hint; /* max contiguous size hint */
94	92	struct vm_struct vm; / mapped vmalloc region */
...	...	@@ -133,7 +131,7 @@
133	131	* There are two locks - pcpu_alloc_mutex and pcpu_lock. The former
134	132	* protects allocation/reclaim paths, chunks and chunk->page arrays.
135	133	* The latter is a spinlock and protects the index data structures -
136		- * chunk slots, rbtree, chunks and area maps in chunks.
	134	+ * chunk slots, chunks and area maps in chunks.
137	135	*
138	136	* During allocation, pcpu_alloc_mutex is kept locked all the time and
139	137	* pcpu_lock is grabbed and released as necessary. All actual memory
...	...	@@ -152,7 +150,6 @@
152	150	static DEFINE_SPINLOCK(pcpu_lock); /* protects index data structures */
153	151
154	152	static struct list_head pcpu_slot __read_mostly; / chunk list slots */
155		-static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */
156	153
157	154	/* reclaim work to release fully free chunks, scheduled from free path */
158	155	static void pcpu_reclaim(struct work_struct *work);
...	...	@@ -203,6 +200,18 @@
203	200	return *pcpu_chunk_pagep(chunk, 0, page_idx) != NULL;
204	201	}
205	202
	203	+/* set the pointer to a chunk in a page struct */
	204	+static void pcpu_set_page_chunk(struct page page, struct pcpu_chunk pcpu)
	205	+{
	206	+ page->index = (unsigned long)pcpu;
	207	+}
	208	+
	209	+/* obtain pointer to a chunk from a page struct */
	210	+static struct pcpu_chunk pcpu_get_page_chunk(struct page page)
	211	+{
	212	+ return (struct pcpu_chunk *)page->index;
	213	+}
	214	+
206	215	/**
207	216	* pcpu_mem_alloc - allocate memory
208	217	* @size: bytes to allocate
209	218
210	219
211	220
...	...	@@ -269,49 +278,16 @@
269	278	}
270	279	}
271	280
272		-static struct rb_node *pcpu_chunk_rb_search(void addr,
273		- struct rb_node **parentp)
274		-{
275		- struct rb_node **p = &pcpu_addr_root.rb_node;
276		- struct rb_node *parent = NULL;
277		- struct pcpu_chunk *chunk;
278		-
279		- while (*p) {
280		- parent = *p;
281		- chunk = rb_entry(parent, struct pcpu_chunk, rb_node);
282		-
283		- if (addr < chunk->vm->addr)
284		- p = &(*p)->rb_left;
285		- else if (addr > chunk->vm->addr)
286		- p = &(*p)->rb_right;
287		- else
288		- break;
289		- }
290		-
291		- if (parentp)
292		- *parentp = parent;
293		- return p;
294		-}
295		-
296	281	/**
297		- * pcpu_chunk_addr_search - search for chunk containing specified address
298		- * @addr: address to search for
	282	+ * pcpu_chunk_addr_search - determine chunk containing specified address
	283	+ * @addr: address for which the chunk needs to be determined.
299	284	*
300		- * Look for chunk which might contain @addr. More specifically, it
301		- * searchs for the chunk with the highest start address which isn't
302		- * beyond @addr.
303		- *
304		- * CONTEXT:
305		- * pcpu_lock.
306		- *
307	285	* RETURNS:
308	286	* The address of the found chunk.
309	287	*/
310	288	static struct pcpu_chunk pcpu_chunk_addr_search(void addr)
311	289	{
312	290	void *first_start = pcpu_first_chunk->vm->addr;
313		- struct rb_node n, parent;
314		- struct pcpu_chunk *chunk;
315	291
316	292	/* is it in the first chunk? */
317	293	if (addr >= first_start && addr < first_start + pcpu_chunk_size) {
318	294
...	...	@@ -321,45 +297,10 @@
321	297	return pcpu_first_chunk;
322	298	}
323	299
324		- /* nah... search the regular ones */
325		- n = *pcpu_chunk_rb_search(addr, &parent);
326		- if (!n) {
327		- /* no exactly matching chunk, the parent is the closest */
328		- n = parent;
329		- BUG_ON(!n);
330		- }
331		- chunk = rb_entry(n, struct pcpu_chunk, rb_node);
332		-
333		- if (addr < chunk->vm->addr) {
334		- /* the parent was the next one, look for the previous one */
335		- n = rb_prev(n);
336		- BUG_ON(!n);
337		- chunk = rb_entry(n, struct pcpu_chunk, rb_node);
338		- }
339		-
340		- return chunk;
	300	+ return pcpu_get_page_chunk(vmalloc_to_page(addr));
341	301	}
342	302
343	303	/**
344		- * pcpu_chunk_addr_insert - insert chunk into address rb tree
345		- * @new: chunk to insert
346		- *
347		- * Insert @new into address rb tree.
348		- *
349		- * CONTEXT:
350		- * pcpu_lock.
351		- */
352		-static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
353		-{
354		- struct rb_node *p, parent;
355		-
356		- p = pcpu_chunk_rb_search(new->vm->addr, &parent);
357		- BUG_ON(*p);
358		- rb_link_node(&new->rb_node, parent, p);
359		- rb_insert_color(&new->rb_node, &pcpu_addr_root);
360		-}
361		-
362		-/**
363	304	* pcpu_extend_area_map - extend area map for allocation
364	305	* @chunk: target chunk
365	306	*
...	...	@@ -768,6 +709,7 @@
768	709	alloc_mask, 0);
769	710	if (!*pagep)
770	711	goto err;
	712	+ pcpu_set_page_chunk(*pagep, chunk);
771	713	}
772	714	}
773	715
...	...	@@ -892,7 +834,6 @@
892	834
893	835	spin_lock_irq(&pcpu_lock);
894	836	pcpu_chunk_relocate(chunk, -1);
895		- pcpu_chunk_addr_insert(chunk);
896	837	goto restart;
897	838
898	839	area_found:
...	...	@@ -981,7 +922,6 @@
981	922	if (chunk == list_first_entry(head, struct pcpu_chunk, list))
982	923	continue;
983	924
984		- rb_erase(&chunk->rb_node, &pcpu_addr_root);
985	925	list_move(&chunk->list, &todo);
986	926	}
987	927