[PATCH] change gen_pool allocator to not touch managed memory

Modify the gen_pool allocator (lib/genalloc.c) to utilize a bitmap scheme instead of the buddy scheme. The purpose of this change is to eliminate the touching of the actual memory being allocated. Since the change modifies the interface, a change to the uncached allocator (arch/ia64/kernel/uncached.c) is also required. Both Andrey Volkov and Jes Sorenson have expressed a desire that the gen_pool allocator not write to the memory being managed. See the following: http://marc.theaimsgroup.com/?l=linux-kernel&m=113518602713125&w=2 http://marc.theaimsgroup.com/?l=linux-kernel&m=113533568827916&w=2 Signed-off-by: Dean Nelson <dcn@sgi.com> Cc: Andrey Volkov <avolkov@varma-el.com> Acked-by: Jes Sorensen <jes@trained-monkey.org> Cc: "Luck, Tony" <tony.luck@intel.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] change gen_pool allocator to not touch managed memory
Modify the gen_pool allocator (lib/genalloc.c) to utilize a bitmap scheme instead of the buddy scheme. The purpose of this change is to eliminate the touching of the actual memory being allocated. Since the change modifies the interface, a change to the uncached allocator (arch/ia64/kernel/uncached.c) is also required. Both Andrey Volkov and Jes Sorenson have expressed a desire that the gen_pool allocator not write to the memory being managed. See the following: http://marc.theaimsgroup.com/?l=linux-kernel&m=113518602713125&w=2 http://marc.theaimsgroup.com/?l=linux-kernel&m=113533568827916&w=2 Signed-off-by: Dean Nelson <dcn@sgi.com> Cc: Andrey Volkov <avolkov@varma-el.com> Acked-by: Jes Sorensen <jes@trained-monkey.org> Cc: "Luck, Tony" <tony.luck@intel.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Dean Nelson · Linus Torvalds
1 parent 833423143c
Showing 4 changed files with 252 additions and 261 deletions Side-by-side Diff
arch/ia64/kernel/uncached.c
arch/ia64/sn/kernel/sn2/cache.c
include/linux/genalloc.h
lib/genalloc.c
 /*
- * Copyright (C) 2001-2005 Silicon Graphics, Inc.  All rights reserved.
+ * Copyright (C) 2001-2006 Silicon Graphics, Inc.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of version 2 of the GNU General Public License
  
  
@@ -29,16 +29,9 @@
 #include <asm/tlbflush.h>
 #include <asm/sn/arch.h>
  
-#define DEBUG	0
  
-#if DEBUG
-#define dprintk			printk
-#else
-#define dprintk(x...)		do { } while (0)
-#endif
+extern void __init efi_memmap_walk_uc(efi_freemem_callback_t, void *);
  
-void __init efi_memmap_walk_uc (efi_freemem_callback_t callback);
-
 #define MAX_UNCACHED_GRANULES	5
 static int allocated_granules;
  
@@ -60,6 +53,7 @@
 static void uncached_ipi_mc_drain(void *data)
 {
 	int status;
+
 	status = ia64_pal_mc_drain();
 	if (status)
 		printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on "
  
  
  
  
  
  
  
@@ -67,30 +61,35 @@
 }
  
  
-static unsigned long
-uncached_get_new_chunk(struct gen_pool *poolp)
+/*
+ * Add a new chunk of uncached memory pages to the specified pool.
+ *
+ * @pool: pool to add new chunk of uncached memory to
+ * @nid: node id of node to allocate memory from, or -1
+ *
+ * This is accomplished by first allocating a granule of cached memory pages
+ * and then converting them to uncached memory pages.
+ */
+static int uncached_add_chunk(struct gen_pool *pool, int nid)
 {
 	struct page *page;
-	void *tmp;
 	int status, i;
-	unsigned long addr, node;
+	unsigned long c_addr, uc_addr;
  
 	if (allocated_granules >= MAX_UNCACHED_GRANULES)
-		return 0;
+		return -1;
  
-	node = poolp->private;
-	page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO,
+	/* attempt to allocate a granule's worth of cached memory pages */
+
+	page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO,
 				IA64_GRANULE_SHIFT-PAGE_SHIFT);
+	if (!page)
+		return -1;
  
-	dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n",
-		page, (unsigned long)(page-vmem_map) << PAGE_SHIFT);
+	/* convert the memory pages from cached to uncached */
  
-	/*
-	 * Do magic if no mem on local node! XXX
-	 */
-	if (!page)
-		return 0;
-	tmp = page_address(page);
+	c_addr = (unsigned long)page_address(page);
+	uc_addr = c_addr - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
  
 	/*
 	 * There's a small race here where it's possible for someone to
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -100,76 +99,90 @@
 	for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
 		SetPageUncached(&page[i]);
  
-	flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE);
+	flush_tlb_kernel_range(uc_addr, uc_adddr + IA64_GRANULE_SIZE);
  
 	status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
-
-	dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n",
-		status, raw_smp_processor_id());
-
 	if (!status) {
 		status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1);
 		if (status)
-			printk(KERN_WARNING "smp_call_function failed for "
-			       "uncached_ipi_visibility! (%i)\n", status);
+			goto failed;
 	}
  
+	preempt_disable();
+
 	if (ia64_platform_is("sn2"))
-		sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE);
+		sn_flush_all_caches(uc_addr, IA64_GRANULE_SIZE);
 	else
-		flush_icache_range((unsigned long)tmp,
-				   (unsigned long)tmp+IA64_GRANULE_SIZE);
+		flush_icache_range(uc_addr, uc_addr + IA64_GRANULE_SIZE);
  
+	/* flush the just introduced uncached translation from the TLB */
+	local_flush_tlb_all();
+
+	preempt_enable();
+
 	ia64_pal_mc_drain();
 	status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1);
 	if (status)
-		printk(KERN_WARNING "smp_call_function failed for "
-		       "uncached_ipi_mc_drain! (%i)\n", status);
+		goto failed;
  
-	addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
+	/*
+	 * The chunk of memory pages has been converted to uncached so now we
+	 * can add it to the pool.
+	 */
+	status = gen_pool_add(pool, uc_addr, IA64_GRANULE_SIZE, nid);
+	if (status)
+		goto failed;
  
 	allocated_granules++;
-	return addr;
+	return 0;
+
+	/* failed to convert or add the chunk so give it back to the kernel */
+failed:
+	for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
+		ClearPageUncached(&page[i]);
+
+	free_pages(c_addr, IA64_GRANULE_SHIFT-PAGE_SHIFT);
+	return -1;
 }
  
  
 /*
  * uncached_alloc_page
  *
+ * @starting_nid: node id of node to start with, or -1
+ *
  * Allocate 1 uncached page. Allocates on the requested node. If no
  * uncached pages are available on the requested node, roundrobin starting
- * with higher nodes.
+ * with the next higher node.
  */
-unsigned long
-uncached_alloc_page(int nid)
+unsigned long uncached_alloc_page(int starting_nid)
 {
-	unsigned long maddr;
+	unsigned long uc_addr;
+	struct gen_pool *pool;
+	int nid;
  
-	maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE);
+	if (unlikely(starting_nid >= MAX_NUMNODES))
+		return 0;
  
-	dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n",
-		maddr, nid);
+	if (starting_nid < 0)
+		starting_nid = numa_node_id();
+	nid = starting_nid;
  
-	/*
-	 * If no memory is availble on our local node, try the
-	 * remaining nodes in the system.
-	 */
-	if (!maddr) {
-		int i;
+	do {
+		if (!node_online(nid))
+			continue;
+		pool = uncached_pool[nid];
+		if (pool == NULL)
+			continue;
+		do {
+			uc_addr = gen_pool_alloc(pool, PAGE_SIZE);
+			if (uc_addr != 0)
+				return uc_addr;
+		} while (uncached_add_chunk(pool, nid) == 0);
  
-		for (i = MAX_NUMNODES - 1; i >= 0; i--) {
-			if (i == nid || !node_online(i))
-				continue;
-			maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE);
-			dprintk(KERN_DEBUG "uncached_alloc_page alternate search "
-				"returns %lx on node %i\n", maddr, i);
-			if (maddr) {
-				break;
-			}
-		}
-	}
+	} while ((nid = (nid + 1) % MAX_NUMNODES) != starting_nid);
  
-	return maddr;
+	return 0;
 }
 EXPORT_SYMBOL(uncached_alloc_page);
  
  
  
  
  
  
@@ -177,21 +190,22 @@
 /*
  * uncached_free_page
  *
+ * @uc_addr: uncached address of page to free
+ *
  * Free a single uncached page.
  */
-void
-uncached_free_page(unsigned long maddr)
+void uncached_free_page(unsigned long uc_addr)
 {
-	int node;
+	int nid = paddr_to_nid(uc_addr - __IA64_UNCACHED_OFFSET);
+	struct gen_pool *pool = uncached_pool[nid];
  
-	node = paddr_to_nid(maddr - __IA64_UNCACHED_OFFSET);
+	if (unlikely(pool == NULL))
+		return;
  
-	dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node);
+	if ((uc_addr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
+		panic("uncached_free_page invalid address %lx\n", uc_addr);
  
-	if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
-		panic("uncached_free_page invalid address %lx\n", maddr);
-
-	gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE);
+	gen_pool_free(pool, uc_addr, PAGE_SIZE);
 }
 EXPORT_SYMBOL(uncached_free_page);
  
  
  
  
  
  
  
  
  
  
@@ -199,43 +213,39 @@
 /*
  * uncached_build_memmap,
  *
+ * @uc_start: uncached starting address of a chunk of uncached memory
+ * @uc_end: uncached ending address of a chunk of uncached memory
+ * @arg: ignored, (NULL argument passed in on call to efi_memmap_walk_uc())
+ *
  * Called at boot time to build a map of pages that can be used for
  * memory special operations.
  */
-static int __init
-uncached_build_memmap(unsigned long start, unsigned long end, void *arg)
+static int __init uncached_build_memmap(unsigned long uc_start,
+					unsigned long uc_end, void *arg)
 {
-	long length = end - start;
-	int node;
+	int nid = paddr_to_nid(uc_start - __IA64_UNCACHED_OFFSET);
+	struct gen_pool *pool = uncached_pool[nid];
+	size_t size = uc_end - uc_start;
  
-	dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end);
-
 	touch_softlockup_watchdog();
-	memset((char *)start, 0, length);
  
-	node = paddr_to_nid(start - __IA64_UNCACHED_OFFSET);
-
-	for (; start < end ; start += PAGE_SIZE) {
-		dprintk(KERN_INFO "sticking %lx into the pool!\n", start);
-		gen_pool_free(uncached_pool[node], start, PAGE_SIZE);
+	if (pool != NULL) {
+		memset((char *)uc_start, 0, size);
+		(void) gen_pool_add(pool, uc_start, size, nid);
 	}
-
 	return 0;
 }
  
  
-static int __init uncached_init(void) {
-	int i;
+static int __init uncached_init(void)
+{
+	int nid;
  
-	for (i = 0; i < MAX_NUMNODES; i++) {
-		if (!node_online(i))
-			continue;
-		uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT,
-						   &uncached_get_new_chunk, i);
+	for_each_online_node(nid) {
+		uncached_pool[nid] = gen_pool_create(PAGE_SHIFT, nid);
 	}
  
-	efi_memmap_walk_uc(uncached_build_memmap);
-
+	efi_memmap_walk_uc(uncached_build_memmap, NULL);
 	return 0;
 }
  
@@ -3,11 +3,12 @@
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  * 
- * Copyright (C) 2001-2003 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 2001-2003, 2006 Silicon Graphics, Inc. All rights reserved.
  *
  */
 #include <linux/module.h>
 #include <asm/pgalloc.h>
+#include <asm/sn/arch.h>
  
 /**
  * sn_flush_all_caches - flush a range of address from all caches (incl. L4)
  
  
@@ -17,18 +18,24 @@
  * Flush a range of addresses from all caches including L4. 
  * All addresses fully or partially contained within 
  * @flush_addr to @flush_addr + @bytes are flushed
- * from the all caches.
+ * from all caches.
  */
 void
 sn_flush_all_caches(long flush_addr, long bytes)
 {
-	flush_icache_range(flush_addr, flush_addr+bytes);
+	unsigned long addr = flush_addr;
+
+	/* SHub1 requires a cached address */
+	if (is_shub1() && (addr & RGN_BITS) == RGN_BASE(RGN_UNCACHED))
+		addr = (addr - RGN_BASE(RGN_UNCACHED)) + RGN_BASE(RGN_KERNEL);
+
+	flush_icache_range(addr, addr + bytes);
 	/*
 	 * The last call may have returned before the caches
 	 * were actually flushed, so we call it again to make
 	 * sure.
 	 */
-	flush_icache_range(flush_addr, flush_addr+bytes);
+	flush_icache_range(addr, addr + bytes);
 	mb();
 }
 EXPORT_SYMBOL(sn_flush_all_caches);
@@ -4,38 +4,33 @@
  * Uses for this includes on-device special memory, uncached memory
  * etc.
  *
- * This code is based on the buddy allocator found in the sym53c8xx_2
- * driver, adapted for general purpose use.
- *
  * This source code is licensed under the GNU General Public License,
  * Version 2.  See the file COPYING for more details.
  */
  
-#include <linux/spinlock.h>
  
-#define ALLOC_MIN_SHIFT		5 /* 32 bytes minimum */
 /*
- *  Link between free memory chunks of a given size.
+ *  General purpose special memory pool descriptor.
  */
-struct gen_pool_link {
-	struct gen_pool_link *next;
+struct gen_pool {
+	rwlock_t lock;
+	struct list_head chunks;	/* list of chunks in this pool */
+	int min_alloc_order;		/* minimum allocation order */
 };
  
 /*
- *  Memory pool descriptor.
+ *  General purpose special memory pool chunk descriptor.
  */
-struct gen_pool {
+struct gen_pool_chunk {
 	spinlock_t lock;
-	unsigned long (*get_new_chunk)(struct gen_pool *);
-	struct gen_pool *next;
-	struct gen_pool_link *h;
-	unsigned long private;
-	int max_chunk_shift;
+	struct list_head next_chunk;	/* next chunk in pool */
+	unsigned long start_addr;	/* starting address of memory chunk */
+	unsigned long end_addr;		/* ending address of memory chunk */
+	unsigned long bits[0];		/* bitmap for allocating memory chunk */
 };
  
-unsigned long gen_pool_alloc(struct gen_pool *poolp, int size);
-void gen_pool_free(struct gen_pool *mp, unsigned long ptr, int size);
-struct gen_pool *gen_pool_create(int nr_chunks, int max_chunk_shift,
-				 unsigned long (*fp)(struct gen_pool *),
-				 unsigned long data);
+extern struct gen_pool *gen_pool_create(int, int);
+extern int gen_pool_add(struct gen_pool *, unsigned long, size_t, int);
+extern unsigned long gen_pool_alloc(struct gen_pool *, size_t);
+extern void gen_pool_free(struct gen_pool *, unsigned long, size_t);
@@ -4,10 +4,6 @@
  * Uses for this includes on-device special memory, uncached memory
  * etc.
  *
- * This code is based on the buddy allocator found in the sym53c8xx_2
- * driver Copyright (C) 1999-2001  Gerard Roudier <groudier@free.fr>,
- * and adapted for general purpose use.
- *
  * Copyright 2005 (C) Jes Sorensen <jes@trained-monkey.org>
  *
  * This source code is licensed under the GNU General Public License,
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -15,173 +11,156 @@
  */
  
 #include <linux/module.h>
-#include <linux/stddef.h>
-#include <linux/kernel.h>
-#include <linux/string.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/mm.h>
-#include <linux/spinlock.h>
 #include <linux/genalloc.h>
  
-#include <asm/page.h>
  
-
-struct gen_pool *gen_pool_create(int nr_chunks, int max_chunk_shift,
-				 unsigned long (*fp)(struct gen_pool *),
-				 unsigned long data)
+/*
+ * Create a new special memory pool.
+ *
+ * @min_alloc_order: log base 2 of number of bytes each bitmap bit represents
+ * @nid: node id of the node the pool structure should be allocated on, or -1
+ */
+struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
 {
-	struct gen_pool *poolp;
-	unsigned long tmp;
-	int i;
+	struct gen_pool *pool;
  
-	/*
-	 * This is really an arbitrary limit, +10 is enough for
-	 * IA64_GRANULE_SHIFT, aka 16MB. If anyone needs a large limit
-	 * this can be increased without problems.
-	 */
-	if ((max_chunk_shift > (PAGE_SHIFT + 10)) ||
-	    ((max_chunk_shift < ALLOC_MIN_SHIFT) && max_chunk_shift))
-		return NULL;
+	pool = kmalloc_node(sizeof(struct gen_pool), GFP_KERNEL, nid);
+	if (pool != NULL) {
+		rwlock_init(&pool->lock);
+		INIT_LIST_HEAD(&pool->chunks);
+		pool->min_alloc_order = min_alloc_order;
+	}
+	return pool;
+}
+EXPORT_SYMBOL(gen_pool_create);
  
-	if (!max_chunk_shift)
-		max_chunk_shift = PAGE_SHIFT;
  
-	poolp = kmalloc(sizeof(struct gen_pool), GFP_KERNEL);
-	if (!poolp)
-		return NULL;
-	memset(poolp, 0, sizeof(struct gen_pool));
-	poolp->h = kmalloc(sizeof(struct gen_pool_link) *
-			   (max_chunk_shift - ALLOC_MIN_SHIFT + 1),
-			   GFP_KERNEL);
-	if (!poolp->h) {
-		printk(KERN_WARNING "gen_pool_alloc() failed to allocate\n");
-		kfree(poolp);
-		return NULL;
-	}
-	memset(poolp->h, 0, sizeof(struct gen_pool_link) *
-	       (max_chunk_shift - ALLOC_MIN_SHIFT + 1));
+/*
+ * Add a new chunk of memory to the specified pool.
+ *
+ * @pool: pool to add new memory chunk to
+ * @addr: starting address of memory chunk to add to pool
+ * @size: size in bytes of the memory chunk to add to pool
+ * @nid: node id of the node the chunk structure and bitmap should be
+ *       allocated on, or -1
+ */
+int gen_pool_add(struct gen_pool *pool, unsigned long addr, size_t size,
+		 int nid)
+{
+	struct gen_pool_chunk *chunk;
+	int nbits = size >> pool->min_alloc_order;
+	int nbytes = sizeof(struct gen_pool_chunk) +
+				(nbits + BITS_PER_BYTE - 1) / BITS_PER_BYTE;
  
-	spin_lock_init(&poolp->lock);
-	poolp->get_new_chunk = fp;
-	poolp->max_chunk_shift = max_chunk_shift;
-	poolp->private = data;
+	chunk = kmalloc_node(nbytes, GFP_KERNEL, nid);
+	if (unlikely(chunk == NULL))
+		return -1;
  
-	for (i = 0; i < nr_chunks; i++) {
-		tmp = poolp->get_new_chunk(poolp);
-		printk(KERN_INFO "allocated %lx\n", tmp);
-		if (!tmp)
-			break;
-		gen_pool_free(poolp, tmp, (1 << poolp->max_chunk_shift));
-	}
+	memset(chunk, 0, nbytes);
+	spin_lock_init(&chunk->lock);
+	chunk->start_addr = addr;
+	chunk->end_addr = addr + size;
  
-	return poolp;
+	write_lock(&pool->lock);
+	list_add(&chunk->next_chunk, &pool->chunks);
+	write_unlock(&pool->lock);
+
+	return 0;
 }
-EXPORT_SYMBOL(gen_pool_create);
+EXPORT_SYMBOL(gen_pool_add);
  
  
 /*
- *  Simple power of two buddy-like generic allocator.
- *  Provides naturally aligned memory chunks.
+ * Allocate the requested number of bytes from the specified pool.
+ * Uses a first-fit algorithm.
+ *
+ * @pool: pool to allocate from
+ * @size: number of bytes to allocate from the pool
  */
-unsigned long gen_pool_alloc(struct gen_pool *poolp, int size)
+unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
 {
-	int j, i, s, max_chunk_size;
-	unsigned long a, flags;
-	struct gen_pool_link *h = poolp->h;
+	struct list_head *_chunk;
+	struct gen_pool_chunk *chunk;
+	unsigned long addr, flags;
+	int order = pool->min_alloc_order;
+	int nbits, bit, start_bit, end_bit;
  
-	max_chunk_size = 1 << poolp->max_chunk_shift;
-
-	if (size > max_chunk_size)
+	if (size == 0)
 		return 0;
  
-	size = max(size, 1 << ALLOC_MIN_SHIFT);
-	i = fls(size - 1);
-	s = 1 << i;
-	j = i -= ALLOC_MIN_SHIFT;
+	nbits = (size + (1UL << order) - 1) >> order;
  
-	spin_lock_irqsave(&poolp->lock, flags);
-	while (!h[j].next) {
-		if (s == max_chunk_size) {
-			struct gen_pool_link *ptr;
-			spin_unlock_irqrestore(&poolp->lock, flags);
-			ptr = (struct gen_pool_link *)poolp->get_new_chunk(poolp);
-			spin_lock_irqsave(&poolp->lock, flags);
-			h[j].next = ptr;
-			if (h[j].next)
-				h[j].next->next = NULL;
-			break;
+	read_lock(&pool->lock);
+	list_for_each(_chunk, &pool->chunks) {
+		chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
+
+		end_bit = (chunk->end_addr - chunk->start_addr) >> order;
+		end_bit -= nbits + 1;
+
+		spin_lock_irqsave(&chunk->lock, flags);
+		bit = -1;
+		while (bit + 1 < end_bit) {
+			bit = find_next_zero_bit(chunk->bits, end_bit, bit + 1);
+			if (bit >= end_bit)
+				break;
+
+			start_bit = bit;
+			if (nbits > 1) {
+				bit = find_next_bit(chunk->bits, bit + nbits,
+							bit + 1);
+				if (bit - start_bit < nbits)
+					continue;
+			}
+
+			addr = chunk->start_addr +
+					    ((unsigned long)start_bit << order);
+			while (nbits--)
+				__set_bit(start_bit++, &chunk->bits);
+			spin_unlock_irqrestore(&chunk->lock, flags);
+			read_unlock(&pool->lock);
+			return addr;
 		}
-		j++;
-		s <<= 1;
+		spin_unlock_irqrestore(&chunk->lock, flags);
 	}
-	a = (unsigned long) h[j].next;
-	if (a) {
-		h[j].next = h[j].next->next;
-		/*
-		 * This should be split into a seperate function doing
-		 * the chunk split in order to support custom
-		 * handling memory not physically accessible by host
-		 */
-		while (j > i) {
-			j -= 1;
-			s >>= 1;
-			h[j].next = (struct gen_pool_link *) (a + s);
-			h[j].next->next = NULL;
-		}
-	}
-	spin_unlock_irqrestore(&poolp->lock, flags);
-	return a;
+	read_unlock(&pool->lock);
+	return 0;
 }
 EXPORT_SYMBOL(gen_pool_alloc);
  
  
 /*
- *  Counter-part of the generic allocator.
+ * Free the specified memory back to the specified pool.
+ *
+ * @pool: pool to free to
+ * @addr: starting address of memory to free back to pool
+ * @size: size in bytes of memory to free
  */
-void gen_pool_free(struct gen_pool *poolp, unsigned long ptr, int size)
+void gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
 {
-	struct gen_pool_link *q;
-	struct gen_pool_link *h = poolp->h;
-	unsigned long a, b, flags;
-	int i, s, max_chunk_size;
+	struct list_head *_chunk;
+	struct gen_pool_chunk *chunk;
+	unsigned long flags;
+	int order = pool->min_alloc_order;
+	int bit, nbits;
  
-	max_chunk_size = 1 << poolp->max_chunk_shift;
+	nbits = (size + (1UL << order) - 1) >> order;
  
-	if (size > max_chunk_size)
-		return;
+	read_lock(&pool->lock);
+	list_for_each(_chunk, &pool->chunks) {
+		chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
  
-	size = max(size, 1 << ALLOC_MIN_SHIFT);
-	i = fls(size - 1);
-	s = 1 << i;
-	i -= ALLOC_MIN_SHIFT;
-
-	a = ptr;
-
-	spin_lock_irqsave(&poolp->lock, flags);
-	while (1) {
-		if (s == max_chunk_size) {
-			((struct gen_pool_link *)a)->next = h[i].next;
-			h[i].next = (struct gen_pool_link *)a;
+		if (addr >= chunk->start_addr && addr < chunk->end_addr) {
+			BUG_ON(addr + size > chunk->end_addr);
+			spin_lock_irqsave(&chunk->lock, flags);
+			bit = (addr - chunk->start_addr) >> order;
+			while (nbits--)
+				__clear_bit(bit++, &chunk->bits);
+			spin_unlock_irqrestore(&chunk->lock, flags);
 			break;
 		}
-		b = a ^ s;
-		q = &h[i];
-
-		while (q->next && q->next != (struct gen_pool_link *)b)
-			q = q->next;
-
-		if (!q->next) {
-			((struct gen_pool_link *)a)->next = h[i].next;
-			h[i].next = (struct gen_pool_link *)a;
-			break;
-		}
-		q->next = q->next->next;
-		a = a & b;
-		s <<= 1;
-		i++;
 	}
-	spin_unlock_irqrestore(&poolp->lock, flags);
+	BUG_ON(nbits > 0);
+	read_unlock(&pool->lock);
 }
 EXPORT_SYMBOL(gen_pool_free);
1	1	/*
2		- * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved.
	2	+ * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights reserved.
3	3	*
4	4	* This program is free software; you can redistribute it and/or modify it
5	5	* under the terms of version 2 of the GNU General Public License
6	6
7	7
...	...	@@ -29,16 +29,9 @@
29	29	#include <asm/tlbflush.h>
30	30	#include <asm/sn/arch.h>
31	31
32		-#define DEBUG 0
33	32
34		-#if DEBUG
35		-#define dprintk printk
36		-#else
37		-#define dprintk(x...) do { } while (0)
38		-#endif
	33	+extern void __init efi_memmap_walk_uc(efi_freemem_callback_t, void *);
39	34
40		-void __init efi_memmap_walk_uc (efi_freemem_callback_t callback);
41		-
42	35	#define MAX_UNCACHED_GRANULES 5
43	36	static int allocated_granules;
44	37
...	...	@@ -60,6 +53,7 @@
60	53	static void uncached_ipi_mc_drain(void *data)
61	54	{
62	55	int status;
	56	+
63	57	status = ia64_pal_mc_drain();
64	58	if (status)
65	59	printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on "
66	60
67	61
68	62
69	63
70	64
71	65
72	66
...	...	@@ -67,30 +61,35 @@
67	61	}
68	62
69	63
70		-static unsigned long
71		-uncached_get_new_chunk(struct gen_pool *poolp)
	64	+/*
	65	+ * Add a new chunk of uncached memory pages to the specified pool.
	66	+ *
	67	+ * @pool: pool to add new chunk of uncached memory to
	68	+ * @nid: node id of node to allocate memory from, or -1
	69	+ *
	70	+ * This is accomplished by first allocating a granule of cached memory pages
	71	+ * and then converting them to uncached memory pages.
	72	+ */
	73	+static int uncached_add_chunk(struct gen_pool *pool, int nid)
72	74	{
73	75	struct page *page;
74		- void *tmp;
75	76	int status, i;
76		- unsigned long addr, node;
	77	+ unsigned long c_addr, uc_addr;
77	78
78	79	if (allocated_granules >= MAX_UNCACHED_GRANULES)
79		- return 0;
	80	+ return -1;
80	81
81		- node = poolp->private;
82		- page = alloc_pages_node(node, GFP_KERNEL \| __GFP_ZERO,
	82	+ /* attempt to allocate a granule's worth of cached memory pages */
	83	+
	84	+ page = alloc_pages_node(nid, GFP_KERNEL \| __GFP_ZERO,
83	85	IA64_GRANULE_SHIFT-PAGE_SHIFT);
	86	+ if (!page)
	87	+ return -1;
84	88
85		- dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n",
86		- page, (unsigned long)(page-vmem_map) << PAGE_SHIFT);
	89	+ /* convert the memory pages from cached to uncached */
87	90
88		- /*
89		- * Do magic if no mem on local node! XXX
90		- */
91		- if (!page)
92		- return 0;
93		- tmp = page_address(page);
	91	+ c_addr = (unsigned long)page_address(page);
	92	+ uc_addr = c_addr - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
94	93
95	94	/*
96	95	* There's a small race here where it's possible for someone to
97	96
98	97
99	98
100	99
101	100
102	101
103	102
104	103
105	104
106	105
107	106
108	107
109	108
110	109
111	110
112	111
113	112
114	113
...	...	@@ -100,76 +99,90 @@
100	99	for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
101	100	SetPageUncached(&page[i]);
102	101
103		- flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE);
	102	+ flush_tlb_kernel_range(uc_addr, uc_adddr + IA64_GRANULE_SIZE);
104	103
105	104	status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
106		-
107		- dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n",
108		- status, raw_smp_processor_id());
109		-
110	105	if (!status) {
111	106	status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1);
112	107	if (status)
113		- printk(KERN_WARNING "smp_call_function failed for "
114		- "uncached_ipi_visibility! (%i)\n", status);
	108	+ goto failed;
115	109	}
116	110
	111	+ preempt_disable();
	112	+
117	113	if (ia64_platform_is("sn2"))
118		- sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE);
	114	+ sn_flush_all_caches(uc_addr, IA64_GRANULE_SIZE);
119	115	else
120		- flush_icache_range((unsigned long)tmp,
121		- (unsigned long)tmp+IA64_GRANULE_SIZE);
	116	+ flush_icache_range(uc_addr, uc_addr + IA64_GRANULE_SIZE);
122	117
	118	+ /* flush the just introduced uncached translation from the TLB */
	119	+ local_flush_tlb_all();
	120	+
	121	+ preempt_enable();
	122	+
123	123	ia64_pal_mc_drain();
124	124	status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1);
125	125	if (status)
126		- printk(KERN_WARNING "smp_call_function failed for "
127		- "uncached_ipi_mc_drain! (%i)\n", status);
	126	+ goto failed;
128	127
129		- addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
	128	+ /*
	129	+ * The chunk of memory pages has been converted to uncached so now we
	130	+ * can add it to the pool.
	131	+ */
	132	+ status = gen_pool_add(pool, uc_addr, IA64_GRANULE_SIZE, nid);
	133	+ if (status)
	134	+ goto failed;
130	135
131	136	allocated_granules++;
132		- return addr;
	137	+ return 0;
	138	+
	139	+ /* failed to convert or add the chunk so give it back to the kernel */
	140	+failed:
	141	+ for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
	142	+ ClearPageUncached(&page[i]);
	143	+
	144	+ free_pages(c_addr, IA64_GRANULE_SHIFT-PAGE_SHIFT);
	145	+ return -1;
133	146	}
134	147
135	148
136	149	/*
137	150	* uncached_alloc_page
138	151	*
	152	+ * @starting_nid: node id of node to start with, or -1
	153	+ *
139	154	* Allocate 1 uncached page. Allocates on the requested node. If no
140	155	* uncached pages are available on the requested node, roundrobin starting
141		- * with higher nodes.
	156	+ * with the next higher node.
142	157	*/
143		-unsigned long
144		-uncached_alloc_page(int nid)
	158	+unsigned long uncached_alloc_page(int starting_nid)
145	159	{
146		- unsigned long maddr;
	160	+ unsigned long uc_addr;
	161	+ struct gen_pool *pool;
	162	+ int nid;
147	163
148		- maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE);
	164	+ if (unlikely(starting_nid >= MAX_NUMNODES))
	165	+ return 0;
149	166
150		- dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n",
151		- maddr, nid);
	167	+ if (starting_nid < 0)
	168	+ starting_nid = numa_node_id();
	169	+ nid = starting_nid;
152	170
153		- /*
154		- * If no memory is availble on our local node, try the
155		- * remaining nodes in the system.
156		- */
157		- if (!maddr) {
158		- int i;
	171	+ do {
	172	+ if (!node_online(nid))
	173	+ continue;
	174	+ pool = uncached_pool[nid];
	175	+ if (pool == NULL)
	176	+ continue;
	177	+ do {
	178	+ uc_addr = gen_pool_alloc(pool, PAGE_SIZE);
	179	+ if (uc_addr != 0)
	180	+ return uc_addr;
	181	+ } while (uncached_add_chunk(pool, nid) == 0);
159	182
160		- for (i = MAX_NUMNODES - 1; i >= 0; i--) {
161		- if (i == nid \|\| !node_online(i))
162		- continue;
163		- maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE);
164		- dprintk(KERN_DEBUG "uncached_alloc_page alternate search "
165		- "returns %lx on node %i\n", maddr, i);
166		- if (maddr) {
167		- break;
168		- }
169		- }
170		- }
	183	+ } while ((nid = (nid + 1) % MAX_NUMNODES) != starting_nid);
171	184
172		- return maddr;
	185	+ return 0;
173	186	}
174	187	EXPORT_SYMBOL(uncached_alloc_page);
175	188
176	189
177	190
178	191
179	192
180	193
...	...	@@ -177,21 +190,22 @@
177	190	/*
178	191	* uncached_free_page
179	192	*
	193	+ * @uc_addr: uncached address of page to free
	194	+ *
180	195	* Free a single uncached page.
181	196	*/
182		-void
183		-uncached_free_page(unsigned long maddr)
	197	+void uncached_free_page(unsigned long uc_addr)
184	198	{
185		- int node;
	199	+ int nid = paddr_to_nid(uc_addr - __IA64_UNCACHED_OFFSET);
	200	+ struct gen_pool *pool = uncached_pool[nid];
186	201
187		- node = paddr_to_nid(maddr - __IA64_UNCACHED_OFFSET);
	202	+ if (unlikely(pool == NULL))
	203	+ return;
188	204
189		- dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node);
	205	+ if ((uc_addr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
	206	+ panic("uncached_free_page invalid address %lx\n", uc_addr);
190	207
191		- if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
192		- panic("uncached_free_page invalid address %lx\n", maddr);
193		-
194		- gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE);
	208	+ gen_pool_free(pool, uc_addr, PAGE_SIZE);
195	209	}
196	210	EXPORT_SYMBOL(uncached_free_page);
197	211
198	212
199	213
200	214
201	215
202	216
203	217
204	218
205	219
206	220
...	...	@@ -199,43 +213,39 @@
199	213	/*
200	214	* uncached_build_memmap,
201	215	*
	216	+ * @uc_start: uncached starting address of a chunk of uncached memory
	217	+ * @uc_end: uncached ending address of a chunk of uncached memory
	218	+ * @arg: ignored, (NULL argument passed in on call to efi_memmap_walk_uc())
	219	+ *
202	220	* Called at boot time to build a map of pages that can be used for
203	221	* memory special operations.
204	222	*/
205		-static int __init
206		-uncached_build_memmap(unsigned long start, unsigned long end, void *arg)
	223	+static int __init uncached_build_memmap(unsigned long uc_start,
	224	+ unsigned long uc_end, void *arg)
207	225	{
208		- long length = end - start;
209		- int node;
	226	+ int nid = paddr_to_nid(uc_start - __IA64_UNCACHED_OFFSET);
	227	+ struct gen_pool *pool = uncached_pool[nid];
	228	+ size_t size = uc_end - uc_start;
210	229
211		- dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end);
212		-
213	230	touch_softlockup_watchdog();
214		- memset((char *)start, 0, length);
215	231
216		- node = paddr_to_nid(start - __IA64_UNCACHED_OFFSET);
217		-
218		- for (; start < end ; start += PAGE_SIZE) {
219		- dprintk(KERN_INFO "sticking %lx into the pool!\n", start);
220		- gen_pool_free(uncached_pool[node], start, PAGE_SIZE);
	232	+ if (pool != NULL) {
	233	+ memset((char *)uc_start, 0, size);
	234	+ (void) gen_pool_add(pool, uc_start, size, nid);
221	235	}
222		-
223	236	return 0;
224	237	}
225	238
226	239
227		-static int __init uncached_init(void) {
228		- int i;
	240	+static int __init uncached_init(void)
	241	+{
	242	+ int nid;
229	243
230		- for (i = 0; i < MAX_NUMNODES; i++) {
231		- if (!node_online(i))
232		- continue;
233		- uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT,
234		- &uncached_get_new_chunk, i);
	244	+ for_each_online_node(nid) {
	245	+ uncached_pool[nid] = gen_pool_create(PAGE_SHIFT, nid);
235	246	}
236	247
237		- efi_memmap_walk_uc(uncached_build_memmap);
238		-
	248	+ efi_memmap_walk_uc(uncached_build_memmap, NULL);
239	249	return 0;
240	250	}
241	251
...	...	@@ -3,11 +3,12 @@
3	3	* License. See the file "COPYING" in the main directory of this archive
4	4	* for more details.
5	5	*
6		- * Copyright (C) 2001-2003 Silicon Graphics, Inc. All rights reserved.
	6	+ * Copyright (C) 2001-2003, 2006 Silicon Graphics, Inc. All rights reserved.
7	7	*
8	8	*/
9	9	#include <linux/module.h>
10	10	#include <asm/pgalloc.h>
	11	+#include <asm/sn/arch.h>
11	12
12	13	/**
13	14	* sn_flush_all_caches - flush a range of address from all caches (incl. L4)
14	15
15	16
...	...	@@ -17,18 +18,24 @@
17	18	* Flush a range of addresses from all caches including L4.
18	19	* All addresses fully or partially contained within
19	20	* @flush_addr to @flush_addr + @bytes are flushed
20		- * from the all caches.
	21	+ * from all caches.
21	22	*/
22	23	void
23	24	sn_flush_all_caches(long flush_addr, long bytes)
24	25	{
25		- flush_icache_range(flush_addr, flush_addr+bytes);
	26	+ unsigned long addr = flush_addr;
	27	+
	28	+ /* SHub1 requires a cached address */
	29	+ if (is_shub1() && (addr & RGN_BITS) == RGN_BASE(RGN_UNCACHED))
	30	+ addr = (addr - RGN_BASE(RGN_UNCACHED)) + RGN_BASE(RGN_KERNEL);
	31	+
	32	+ flush_icache_range(addr, addr + bytes);
26	33	/*
27	34	* The last call may have returned before the caches
28	35	* were actually flushed, so we call it again to make
29	36	* sure.
30	37	*/
31		- flush_icache_range(flush_addr, flush_addr+bytes);
	38	+ flush_icache_range(addr, addr + bytes);
32	39	mb();
33	40	}
34	41	EXPORT_SYMBOL(sn_flush_all_caches);
...	...	@@ -4,38 +4,33 @@
4	4	* Uses for this includes on-device special memory, uncached memory
5	5	* etc.
6	6	*
7		- * This code is based on the buddy allocator found in the sym53c8xx_2
8		- * driver, adapted for general purpose use.
9		- *
10	7	* This source code is licensed under the GNU General Public License,
11	8	* Version 2. See the file COPYING for more details.
12	9	*/
13	10
14		-#include <linux/spinlock.h>
15	11
16		-#define ALLOC_MIN_SHIFT 5 /* 32 bytes minimum */
17	12	/*
18		- * Link between free memory chunks of a given size.
	13	+ * General purpose special memory pool descriptor.
19	14	*/
20		-struct gen_pool_link {
21		- struct gen_pool_link *next;
	15	+struct gen_pool {
	16	+ rwlock_t lock;
	17	+ struct list_head chunks; /* list of chunks in this pool */
	18	+ int min_alloc_order; /* minimum allocation order */
22	19	};
23	20
24	21	/*
25		- * Memory pool descriptor.
	22	+ * General purpose special memory pool chunk descriptor.
26	23	*/
27		-struct gen_pool {
	24	+struct gen_pool_chunk {
28	25	spinlock_t lock;
29		- unsigned long (get_new_chunk)(struct gen_pool );
30		- struct gen_pool *next;
31		- struct gen_pool_link *h;
32		- unsigned long private;
33		- int max_chunk_shift;
	26	+ struct list_head next_chunk; /* next chunk in pool */
	27	+ unsigned long start_addr; /* starting address of memory chunk */
	28	+ unsigned long end_addr; /* ending address of memory chunk */
	29	+ unsigned long bits[0]; /* bitmap for allocating memory chunk */
34	30	};
35	31
36		-unsigned long gen_pool_alloc(struct gen_pool *poolp, int size);
37		-void gen_pool_free(struct gen_pool *mp, unsigned long ptr, int size);
38		-struct gen_pool *gen_pool_create(int nr_chunks, int max_chunk_shift,
39		- unsigned long (fp)(struct gen_pool ),
40		- unsigned long data);
	32	+extern struct gen_pool *gen_pool_create(int, int);
	33	+extern int gen_pool_add(struct gen_pool *, unsigned long, size_t, int);
	34	+extern unsigned long gen_pool_alloc(struct gen_pool *, size_t);
	35	+extern void gen_pool_free(struct gen_pool *, unsigned long, size_t);
...	...	@@ -4,10 +4,6 @@
4	4	* Uses for this includes on-device special memory, uncached memory
5	5	* etc.
6	6	*
7		- * This code is based on the buddy allocator found in the sym53c8xx_2
8		- * driver Copyright (C) 1999-2001 Gerard Roudier <groudier@free.fr>,
9		- * and adapted for general purpose use.
10		- *
11	7	* Copyright 2005 (C) Jes Sorensen <jes@trained-monkey.org>
12	8	*
13	9	* This source code is licensed under the GNU General Public License,
14	10
15	11
16	12
17	13
18	14
19	15
20	16
21	17
22	18
23	19
24	20
25	21
26	22
27	23
28	24
29	25
30	26
31	27
32	28
33	29
34	30
35	31
36	32
37	33
38	34
39	35
...	...	@@ -15,173 +11,156 @@
15	11	*/
16	12
17	13	#include <linux/module.h>
18		-#include <linux/stddef.h>
19		-#include <linux/kernel.h>
20		-#include <linux/string.h>
21		-#include <linux/slab.h>
22		-#include <linux/init.h>
23		-#include <linux/mm.h>
24		-#include <linux/spinlock.h>
25	14	#include <linux/genalloc.h>
26	15
27		-#include <asm/page.h>
28	16
29		-
30		-struct gen_pool *gen_pool_create(int nr_chunks, int max_chunk_shift,
31		- unsigned long (fp)(struct gen_pool ),
32		- unsigned long data)
	17	+/*
	18	+ * Create a new special memory pool.
	19	+ *
	20	+ * @min_alloc_order: log base 2 of number of bytes each bitmap bit represents
	21	+ * @nid: node id of the node the pool structure should be allocated on, or -1
	22	+ */
	23	+struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
33	24	{
34		- struct gen_pool *poolp;
35		- unsigned long tmp;
36		- int i;
	25	+ struct gen_pool *pool;
37	26
38		- /*
39		- * This is really an arbitrary limit, +10 is enough for
40		- * IA64_GRANULE_SHIFT, aka 16MB. If anyone needs a large limit
41		- * this can be increased without problems.
42		- */
43		- if ((max_chunk_shift > (PAGE_SHIFT + 10)) \|\|
44		- ((max_chunk_shift < ALLOC_MIN_SHIFT) && max_chunk_shift))
45		- return NULL;
	27	+ pool = kmalloc_node(sizeof(struct gen_pool), GFP_KERNEL, nid);
	28	+ if (pool != NULL) {
	29	+ rwlock_init(&pool->lock);
	30	+ INIT_LIST_HEAD(&pool->chunks);
	31	+ pool->min_alloc_order = min_alloc_order;
	32	+ }
	33	+ return pool;
	34	+}
	35	+EXPORT_SYMBOL(gen_pool_create);
46	36
47		- if (!max_chunk_shift)
48		- max_chunk_shift = PAGE_SHIFT;
49	37
50		- poolp = kmalloc(sizeof(struct gen_pool), GFP_KERNEL);
51		- if (!poolp)
52		- return NULL;
53		- memset(poolp, 0, sizeof(struct gen_pool));
54		- poolp->h = kmalloc(sizeof(struct gen_pool_link) *
55		- (max_chunk_shift - ALLOC_MIN_SHIFT + 1),
56		- GFP_KERNEL);
57		- if (!poolp->h) {
58		- printk(KERN_WARNING "gen_pool_alloc() failed to allocate\n");
59		- kfree(poolp);
60		- return NULL;
61		- }
62		- memset(poolp->h, 0, sizeof(struct gen_pool_link) *
63		- (max_chunk_shift - ALLOC_MIN_SHIFT + 1));
	38	+/*
	39	+ * Add a new chunk of memory to the specified pool.
	40	+ *
	41	+ * @pool: pool to add new memory chunk to
	42	+ * @addr: starting address of memory chunk to add to pool
	43	+ * @size: size in bytes of the memory chunk to add to pool
	44	+ * @nid: node id of the node the chunk structure and bitmap should be
	45	+ * allocated on, or -1
	46	+ */
	47	+int gen_pool_add(struct gen_pool *pool, unsigned long addr, size_t size,
	48	+ int nid)
	49	+{
	50	+ struct gen_pool_chunk *chunk;
	51	+ int nbits = size >> pool->min_alloc_order;
	52	+ int nbytes = sizeof(struct gen_pool_chunk) +
	53	+ (nbits + BITS_PER_BYTE - 1) / BITS_PER_BYTE;
64	54
65		- spin_lock_init(&poolp->lock);
66		- poolp->get_new_chunk = fp;
67		- poolp->max_chunk_shift = max_chunk_shift;
68		- poolp->private = data;
	55	+ chunk = kmalloc_node(nbytes, GFP_KERNEL, nid);
	56	+ if (unlikely(chunk == NULL))
	57	+ return -1;
69	58
70		- for (i = 0; i < nr_chunks; i++) {
71		- tmp = poolp->get_new_chunk(poolp);
72		- printk(KERN_INFO "allocated %lx\n", tmp);
73		- if (!tmp)
74		- break;
75		- gen_pool_free(poolp, tmp, (1 << poolp->max_chunk_shift));
76		- }
	59	+ memset(chunk, 0, nbytes);
	60	+ spin_lock_init(&chunk->lock);
	61	+ chunk->start_addr = addr;
	62	+ chunk->end_addr = addr + size;
77	63
78		- return poolp;
	64	+ write_lock(&pool->lock);
	65	+ list_add(&chunk->next_chunk, &pool->chunks);
	66	+ write_unlock(&pool->lock);
	67	+
	68	+ return 0;
79	69	}
80		-EXPORT_SYMBOL(gen_pool_create);
	70	+EXPORT_SYMBOL(gen_pool_add);
81	71
82	72
83	73	/*
84		- * Simple power of two buddy-like generic allocator.
85		- * Provides naturally aligned memory chunks.
	74	+ * Allocate the requested number of bytes from the specified pool.
	75	+ * Uses a first-fit algorithm.
	76	+ *
	77	+ * @pool: pool to allocate from
	78	+ * @size: number of bytes to allocate from the pool
86	79	*/
87		-unsigned long gen_pool_alloc(struct gen_pool *poolp, int size)
	80	+unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
88	81	{
89		- int j, i, s, max_chunk_size;
90		- unsigned long a, flags;
91		- struct gen_pool_link *h = poolp->h;
	82	+ struct list_head *_chunk;
	83	+ struct gen_pool_chunk *chunk;
	84	+ unsigned long addr, flags;
	85	+ int order = pool->min_alloc_order;
	86	+ int nbits, bit, start_bit, end_bit;
92	87
93		- max_chunk_size = 1 << poolp->max_chunk_shift;
94		-
95		- if (size > max_chunk_size)
	88	+ if (size == 0)
96	89	return 0;
97	90
98		- size = max(size, 1 << ALLOC_MIN_SHIFT);
99		- i = fls(size - 1);
100		- s = 1 << i;
101		- j = i -= ALLOC_MIN_SHIFT;
	91	+ nbits = (size + (1UL << order) - 1) >> order;
102	92
103		- spin_lock_irqsave(&poolp->lock, flags);
104		- while (!h[j].next) {
105		- if (s == max_chunk_size) {
106		- struct gen_pool_link *ptr;
107		- spin_unlock_irqrestore(&poolp->lock, flags);
108		- ptr = (struct gen_pool_link *)poolp->get_new_chunk(poolp);
109		- spin_lock_irqsave(&poolp->lock, flags);
110		- h[j].next = ptr;
111		- if (h[j].next)
112		- h[j].next->next = NULL;
113		- break;
	93	+ read_lock(&pool->lock);
	94	+ list_for_each(_chunk, &pool->chunks) {
	95	+ chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
	96	+
	97	+ end_bit = (chunk->end_addr - chunk->start_addr) >> order;
	98	+ end_bit -= nbits + 1;
	99	+
	100	+ spin_lock_irqsave(&chunk->lock, flags);
	101	+ bit = -1;
	102	+ while (bit + 1 < end_bit) {
	103	+ bit = find_next_zero_bit(chunk->bits, end_bit, bit + 1);
	104	+ if (bit >= end_bit)
	105	+ break;
	106	+
	107	+ start_bit = bit;
	108	+ if (nbits > 1) {
	109	+ bit = find_next_bit(chunk->bits, bit + nbits,
	110	+ bit + 1);
	111	+ if (bit - start_bit < nbits)
	112	+ continue;
	113	+ }
	114	+
	115	+ addr = chunk->start_addr +
	116	+ ((unsigned long)start_bit << order);
	117	+ while (nbits--)
	118	+ __set_bit(start_bit++, &chunk->bits);
	119	+ spin_unlock_irqrestore(&chunk->lock, flags);
	120	+ read_unlock(&pool->lock);
	121	+ return addr;
114	122	}
115		- j++;
116		- s <<= 1;
	123	+ spin_unlock_irqrestore(&chunk->lock, flags);
117	124	}
118		- a = (unsigned long) h[j].next;
119		- if (a) {
120		- h[j].next = h[j].next->next;
121		- /*
122		- * This should be split into a seperate function doing
123		- * the chunk split in order to support custom
124		- * handling memory not physically accessible by host
125		- */
126		- while (j > i) {
127		- j -= 1;
128		- s >>= 1;
129		- h[j].next = (struct gen_pool_link *) (a + s);
130		- h[j].next->next = NULL;
131		- }
132		- }
133		- spin_unlock_irqrestore(&poolp->lock, flags);
134		- return a;
	125	+ read_unlock(&pool->lock);
	126	+ return 0;
135	127	}
136	128	EXPORT_SYMBOL(gen_pool_alloc);
137	129
138	130
139	131	/*
140		- * Counter-part of the generic allocator.
	132	+ * Free the specified memory back to the specified pool.
	133	+ *
	134	+ * @pool: pool to free to
	135	+ * @addr: starting address of memory to free back to pool
	136	+ * @size: size in bytes of memory to free
141	137	*/
142		-void gen_pool_free(struct gen_pool *poolp, unsigned long ptr, int size)
	138	+void gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
143	139	{
144		- struct gen_pool_link *q;
145		- struct gen_pool_link *h = poolp->h;
146		- unsigned long a, b, flags;
147		- int i, s, max_chunk_size;
	140	+ struct list_head *_chunk;
	141	+ struct gen_pool_chunk *chunk;
	142	+ unsigned long flags;
	143	+ int order = pool->min_alloc_order;
	144	+ int bit, nbits;
148	145
149		- max_chunk_size = 1 << poolp->max_chunk_shift;
	146	+ nbits = (size + (1UL << order) - 1) >> order;
150	147
151		- if (size > max_chunk_size)
152		- return;
	148	+ read_lock(&pool->lock);
	149	+ list_for_each(_chunk, &pool->chunks) {
	150	+ chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
153	151
154		- size = max(size, 1 << ALLOC_MIN_SHIFT);
155		- i = fls(size - 1);
156		- s = 1 << i;
157		- i -= ALLOC_MIN_SHIFT;
158		-
159		- a = ptr;
160		-
161		- spin_lock_irqsave(&poolp->lock, flags);
162		- while (1) {
163		- if (s == max_chunk_size) {
164		- ((struct gen_pool_link *)a)->next = h[i].next;
165		- h[i].next = (struct gen_pool_link *)a;
	152	+ if (addr >= chunk->start_addr && addr < chunk->end_addr) {
	153	+ BUG_ON(addr + size > chunk->end_addr);
	154	+ spin_lock_irqsave(&chunk->lock, flags);
	155	+ bit = (addr - chunk->start_addr) >> order;
	156	+ while (nbits--)
	157	+ __clear_bit(bit++, &chunk->bits);
	158	+ spin_unlock_irqrestore(&chunk->lock, flags);
166	159	break;
167	160	}
168		- b = a ^ s;
169		- q = &h[i];
170		-
171		- while (q->next && q->next != (struct gen_pool_link *)b)
172		- q = q->next;
173		-
174		- if (!q->next) {
175		- ((struct gen_pool_link *)a)->next = h[i].next;
176		- h[i].next = (struct gen_pool_link *)a;
177		- break;
178		- }
179		- q->next = q->next->next;
180		- a = a & b;
181		- s <<= 1;
182		- i++;
183	161	}
184		- spin_unlock_irqrestore(&poolp->lock, flags);
	162	+ BUG_ON(nbits > 0);
	163	+ read_unlock(&pool->lock);
185	164	}
186	165	EXPORT_SYMBOL(gen_pool_free);