Blame view
mm/mempool.c
9 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
/* * linux/mm/mempool.c * * memory buffer pool support. Such pools are mostly used * for guaranteed, deadlock-free memory allocations during * extreme VM load. * * started by Ingo Molnar, Copyright (C) 2001 */ #include <linux/mm.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/mempool.h> #include <linux/blkdev.h> #include <linux/writeback.h> static void add_element(mempool_t *pool, void *element) { BUG_ON(pool->curr_nr >= pool->min_nr); pool->elements[pool->curr_nr++] = element; } static void *remove_element(mempool_t *pool) { BUG_ON(pool->curr_nr <= 0); return pool->elements[--pool->curr_nr]; } static void free_pool(mempool_t *pool) { while (pool->curr_nr) { void *element = remove_element(pool); pool->free(element, pool->pool_data); } kfree(pool->elements); kfree(pool); } /** * mempool_create - create a memory pool * @min_nr: the minimum number of elements guaranteed to be * allocated for this pool. * @alloc_fn: user-defined element-allocation function. * @free_fn: user-defined element-freeing function. * @pool_data: optional private data available to the user-defined functions. * * this function creates and allocates a guaranteed size, preallocated * memory pool. The pool can be used from the mempool_alloc and mempool_free * functions. This function might sleep. Both the alloc_fn() and the free_fn() * functions might sleep - as long as the mempool_alloc function is not called * from IRQ contexts. */ |
1946089a1 [PATCH] NUMA awar... |
54 |
mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, |
1da177e4c Linux-2.6.12-rc2 |
55 56 |
mempool_free_t *free_fn, void *pool_data) { |
1946089a1 [PATCH] NUMA awar... |
57 58 59 |
return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1); } EXPORT_SYMBOL(mempool_create); |
1da177e4c Linux-2.6.12-rc2 |
60 |
|
1946089a1 [PATCH] NUMA awar... |
61 62 63 64 65 |
mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, mempool_free_t *free_fn, void *pool_data, int node_id) { mempool_t *pool; pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id); |
1da177e4c Linux-2.6.12-rc2 |
66 67 68 |
if (!pool) return NULL; memset(pool, 0, sizeof(*pool)); |
1946089a1 [PATCH] NUMA awar... |
69 70 |
pool->elements = kmalloc_node(min_nr * sizeof(void *), GFP_KERNEL, node_id); |
1da177e4c Linux-2.6.12-rc2 |
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
if (!pool->elements) { kfree(pool); return NULL; } spin_lock_init(&pool->lock); pool->min_nr = min_nr; pool->pool_data = pool_data; init_waitqueue_head(&pool->wait); pool->alloc = alloc_fn; pool->free = free_fn; /* * First pre-allocate the guaranteed number of buffers. */ while (pool->curr_nr < pool->min_nr) { void *element; element = pool->alloc(GFP_KERNEL, pool->pool_data); if (unlikely(!element)) { free_pool(pool); return NULL; } add_element(pool, element); } return pool; } |
1946089a1 [PATCH] NUMA awar... |
97 |
EXPORT_SYMBOL(mempool_create_node); |
1da177e4c Linux-2.6.12-rc2 |
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
/** * mempool_resize - resize an existing memory pool * @pool: pointer to the memory pool which was allocated via * mempool_create(). * @new_min_nr: the new minimum number of elements guaranteed to be * allocated for this pool. * @gfp_mask: the usual allocation bitmask. * * This function shrinks/grows the pool. In the case of growing, * it cannot be guaranteed that the pool will be grown to the new * size immediately, but new mempool_free() calls will refill it. * * Note, the caller must guarantee that no mempool_destroy is called * while this function is running. mempool_alloc() & mempool_free() * might be called (eg. from IRQ contexts) while this function executes. */ |
dd0fc66fb [PATCH] gfp flags... |
115 |
int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) |
1da177e4c Linux-2.6.12-rc2 |
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 |
{ void *element; void **new_elements; unsigned long flags; BUG_ON(new_min_nr <= 0); spin_lock_irqsave(&pool->lock, flags); if (new_min_nr <= pool->min_nr) { while (new_min_nr < pool->curr_nr) { element = remove_element(pool); spin_unlock_irqrestore(&pool->lock, flags); pool->free(element, pool->pool_data); spin_lock_irqsave(&pool->lock, flags); } pool->min_nr = new_min_nr; goto out_unlock; } spin_unlock_irqrestore(&pool->lock, flags); /* Grow the pool */ new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); if (!new_elements) return -ENOMEM; spin_lock_irqsave(&pool->lock, flags); if (unlikely(new_min_nr <= pool->min_nr)) { /* Raced, other resize will do our work */ spin_unlock_irqrestore(&pool->lock, flags); kfree(new_elements); goto out; } memcpy(new_elements, pool->elements, pool->curr_nr * sizeof(*new_elements)); kfree(pool->elements); pool->elements = new_elements; pool->min_nr = new_min_nr; while (pool->curr_nr < pool->min_nr) { spin_unlock_irqrestore(&pool->lock, flags); element = pool->alloc(gfp_mask, pool->pool_data); if (!element) goto out; spin_lock_irqsave(&pool->lock, flags); if (pool->curr_nr < pool->min_nr) { add_element(pool, element); } else { spin_unlock_irqrestore(&pool->lock, flags); pool->free(element, pool->pool_data); /* Raced */ goto out; } } out_unlock: spin_unlock_irqrestore(&pool->lock, flags); out: return 0; } EXPORT_SYMBOL(mempool_resize); /** * mempool_destroy - deallocate a memory pool * @pool: pointer to the memory pool which was allocated via * mempool_create(). * * this function only sleeps if the free_fn() function sleeps. The caller * has to guarantee that all elements have been returned to the pool (ie: * freed) prior to calling mempool_destroy(). */ void mempool_destroy(mempool_t *pool) { |
f02e1fafb BUG_ON() Conversi... |
186 187 |
/* Check for outstanding elements */ BUG_ON(pool->curr_nr != pool->min_nr); |
1da177e4c Linux-2.6.12-rc2 |
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 |
free_pool(pool); } EXPORT_SYMBOL(mempool_destroy); /** * mempool_alloc - allocate an element from a specific memory pool * @pool: pointer to the memory pool which was allocated via * mempool_create(). * @gfp_mask: the usual allocation bitmask. * * this function only sleeps if the alloc_fn function sleeps or * returns NULL. Note that due to preallocation, this function * *never* fails when called from process contexts. (it might * fail if called from an IRQ context.) */ |
dd0fc66fb [PATCH] gfp flags... |
203 |
void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) |
1da177e4c Linux-2.6.12-rc2 |
204 205 206 |
{ void *element; unsigned long flags; |
01890a4c1 [PATCH] mempool -... |
207 |
wait_queue_t wait; |
6daa0e286 [PATCH] gfp_t: mm... |
208 |
gfp_t gfp_temp; |
20a77776c [PATCH] mempool: ... |
209 210 |
might_sleep_if(gfp_mask & __GFP_WAIT); |
b84a35be0 [PATCH] mempool: ... |
211 212 213 214 |
gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ gfp_mask |= __GFP_NOWARN; /* failures are OK */ |
1da177e4c Linux-2.6.12-rc2 |
215 |
|
20a77776c [PATCH] mempool: ... |
216 |
gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); |
1da177e4c Linux-2.6.12-rc2 |
217 |
repeat_alloc: |
20a77776c [PATCH] mempool: ... |
218 219 |
element = pool->alloc(gfp_temp, pool->pool_data); |
1da177e4c Linux-2.6.12-rc2 |
220 221 |
if (likely(element != NULL)) return element; |
1da177e4c Linux-2.6.12-rc2 |
222 223 224 225 226 227 228 229 230 231 232 |
spin_lock_irqsave(&pool->lock, flags); if (likely(pool->curr_nr)) { element = remove_element(pool); spin_unlock_irqrestore(&pool->lock, flags); return element; } spin_unlock_irqrestore(&pool->lock, flags); /* We must not sleep in the GFP_ATOMIC case */ if (!(gfp_mask & __GFP_WAIT)) return NULL; |
20a77776c [PATCH] mempool: ... |
233 234 |
/* Now start performing page reclaim */ gfp_temp = gfp_mask; |
01890a4c1 [PATCH] mempool -... |
235 |
init_wait(&wait); |
1da177e4c Linux-2.6.12-rc2 |
236 |
prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); |
d59dd4620 [PATCH] use smp_m... |
237 |
smp_mb(); |
0b1d647a0 [PATCH] dm: work ... |
238 239 240 241 242 243 244 |
if (!pool->curr_nr) { /* * FIXME: this should be io_schedule(). The timeout is there * as a workaround for some DM problems in 2.6.18. */ io_schedule_timeout(5*HZ); } |
1da177e4c Linux-2.6.12-rc2 |
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 |
finish_wait(&pool->wait, &wait); goto repeat_alloc; } EXPORT_SYMBOL(mempool_alloc); /** * mempool_free - return an element to the pool. * @element: pool element pointer. * @pool: pointer to the memory pool which was allocated via * mempool_create(). * * this function only sleeps if the free_fn() function sleeps. */ void mempool_free(void *element, mempool_t *pool) { unsigned long flags; |
d59dd4620 [PATCH] use smp_m... |
262 |
smp_mb(); |
1da177e4c Linux-2.6.12-rc2 |
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 |
if (pool->curr_nr < pool->min_nr) { spin_lock_irqsave(&pool->lock, flags); if (pool->curr_nr < pool->min_nr) { add_element(pool, element); spin_unlock_irqrestore(&pool->lock, flags); wake_up(&pool->wait); return; } spin_unlock_irqrestore(&pool->lock, flags); } pool->free(element, pool->pool_data); } EXPORT_SYMBOL(mempool_free); /* * A commonly used alloc and free fn. */ |
dd0fc66fb [PATCH] gfp flags... |
280 |
void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) |
1da177e4c Linux-2.6.12-rc2 |
281 |
{ |
fcc234f88 [PATCH] mm: kill ... |
282 |
struct kmem_cache *mem = pool_data; |
1da177e4c Linux-2.6.12-rc2 |
283 284 285 286 287 288 |
return kmem_cache_alloc(mem, gfp_mask); } EXPORT_SYMBOL(mempool_alloc_slab); void mempool_free_slab(void *element, void *pool_data) { |
fcc234f88 [PATCH] mm: kill ... |
289 |
struct kmem_cache *mem = pool_data; |
1da177e4c Linux-2.6.12-rc2 |
290 291 292 |
kmem_cache_free(mem, element); } EXPORT_SYMBOL(mempool_free_slab); |
6e0678f39 [PATCH] mempool: ... |
293 294 |
/* |
53184082b [PATCH] mempool: ... |
295 296 297 298 299 |
* A commonly used alloc and free fn that kmalloc/kfrees the amount of memory * specfied by pool_data */ void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) { |
f183323d3 [PATCH] mempool: ... |
300 |
size_t size = (size_t)(long)pool_data; |
53184082b [PATCH] mempool: ... |
301 302 303 |
return kmalloc(size, gfp_mask); } EXPORT_SYMBOL(mempool_kmalloc); |
f183323d3 [PATCH] mempool: ... |
304 305 306 307 308 309 |
void *mempool_kzalloc(gfp_t gfp_mask, void *pool_data) { size_t size = (size_t) pool_data; return kzalloc(size, gfp_mask); } EXPORT_SYMBOL(mempool_kzalloc); |
53184082b [PATCH] mempool: ... |
310 311 312 313 314 315 316 |
void mempool_kfree(void *element, void *pool_data) { kfree(element); } EXPORT_SYMBOL(mempool_kfree); /* |
6e0678f39 [PATCH] mempool: ... |
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 |
* A simple mempool-backed page allocator that allocates pages * of the order specified by pool_data. */ void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) { int order = (int)(long)pool_data; return alloc_pages(gfp_mask, order); } EXPORT_SYMBOL(mempool_alloc_pages); void mempool_free_pages(void *element, void *pool_data) { int order = (int)(long)pool_data; __free_pages(element, order); } EXPORT_SYMBOL(mempool_free_pages); |