Blame view

lib/genalloc.c 21.6 KB
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
1
  /*
7f184275a   Huang Ying   lib, Make gen_poo...
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
   * Basic general purpose allocator for managing special purpose
   * memory, for example, memory that is not managed by the regular
   * kmalloc/kfree interface.  Uses for this includes on-device special
   * memory, uncached memory etc.
   *
   * It is safe to use the allocator in NMI handlers and other special
   * unblockable contexts that could otherwise deadlock on locks.  This
   * is implemented by using atomic operations and retries on any
   * conflicts.  The disadvantage is that there may be livelocks in
   * extreme cases.  For better scalability, one allocator can be used
   * for each CPU.
   *
   * The lockless operation only works if there is enough memory
   * available.  If new memory is added to the pool a lock has to be
   * still taken.  So any user relying on locklessness has to ensure
   * that sufficient memory is preallocated.
   *
   * The basic atomic operation of this allocator is cmpxchg on long.
   * On architectures that don't have NMI-safe cmpxchg implementation,
   * the allocator can NOT be used in NMI handler.  So code uses the
   * allocator in NMI handler should depend on
   * CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG.
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
24
   *
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
25
26
27
28
29
   * Copyright 2005 (C) Jes Sorensen <jes@trained-monkey.org>
   *
   * This source code is licensed under the GNU General Public License,
   * Version 2.  See the file COPYING for more details.
   */
5a0e3ad6a   Tejun Heo   include cleanup: ...
30
  #include <linux/slab.h>
8bc3bcc93   Paul Gortmaker   lib: reduce the u...
31
  #include <linux/export.h>
243797f59   Akinobu Mita   genalloc: use bit...
32
  #include <linux/bitmap.h>
7f184275a   Huang Ying   lib, Make gen_poo...
33
34
  #include <linux/rculist.h>
  #include <linux/interrupt.h>
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
35
  #include <linux/genalloc.h>
9375db07a   Philipp Zabel   genalloc: add dev...
36
  #include <linux/of_device.h>
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
37

674470d97   Joonyoung Shim   lib/genalloc.c: f...
38
39
40
41
  static inline size_t chunk_size(const struct gen_pool_chunk *chunk)
  {
  	return chunk->end_addr - chunk->start_addr + 1;
  }
7f184275a   Huang Ying   lib, Make gen_poo...
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
  static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set)
  {
  	unsigned long val, nval;
  
  	nval = *addr;
  	do {
  		val = nval;
  		if (val & mask_to_set)
  			return -EBUSY;
  		cpu_relax();
  	} while ((nval = cmpxchg(addr, val, val | mask_to_set)) != val);
  
  	return 0;
  }
  
  static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
  {
  	unsigned long val, nval;
  
  	nval = *addr;
  	do {
  		val = nval;
  		if ((val & mask_to_clear) != mask_to_clear)
  			return -EBUSY;
  		cpu_relax();
  	} while ((nval = cmpxchg(addr, val, val & ~mask_to_clear)) != val);
  
  	return 0;
  }
  
  /*
   * bitmap_set_ll - set the specified number of bits at the specified position
   * @map: pointer to a bitmap
   * @start: a bit position in @map
   * @nr: number of bits to set
   *
   * Set @nr bits start from @start in @map lock-lessly. Several users
   * can set/clear the same bitmap simultaneously without lock. If two
   * users set the same bit, one user will return remain bits, otherwise
   * return 0.
   */
  static int bitmap_set_ll(unsigned long *map, int start, int nr)
  {
  	unsigned long *p = map + BIT_WORD(start);
  	const int size = start + nr;
  	int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
  	unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
  
  	while (nr - bits_to_set >= 0) {
  		if (set_bits_ll(p, mask_to_set))
  			return nr;
  		nr -= bits_to_set;
  		bits_to_set = BITS_PER_LONG;
  		mask_to_set = ~0UL;
  		p++;
  	}
  	if (nr) {
  		mask_to_set &= BITMAP_LAST_WORD_MASK(size);
  		if (set_bits_ll(p, mask_to_set))
  			return nr;
  	}
  
  	return 0;
  }
  
  /*
   * bitmap_clear_ll - clear the specified number of bits at the specified position
   * @map: pointer to a bitmap
   * @start: a bit position in @map
   * @nr: number of bits to set
   *
   * Clear @nr bits start from @start in @map lock-lessly. Several users
   * can set/clear the same bitmap simultaneously without lock. If two
   * users clear the same bit, one user will return remain bits,
   * otherwise return 0.
   */
  static int bitmap_clear_ll(unsigned long *map, int start, int nr)
  {
  	unsigned long *p = map + BIT_WORD(start);
  	const int size = start + nr;
  	int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
  	unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
  
  	while (nr - bits_to_clear >= 0) {
  		if (clear_bits_ll(p, mask_to_clear))
  			return nr;
  		nr -= bits_to_clear;
  		bits_to_clear = BITS_PER_LONG;
  		mask_to_clear = ~0UL;
  		p++;
  	}
  	if (nr) {
  		mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
  		if (clear_bits_ll(p, mask_to_clear))
  			return nr;
  	}
  
  	return 0;
  }
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
141

a58cbd7c2   Dean Nelson   [PATCH] make genp...
142
143
  /**
   * gen_pool_create - create a new special memory pool
929f97276   Dean Nelson   [PATCH] change ge...
144
145
   * @min_alloc_order: log base 2 of number of bytes each bitmap bit represents
   * @nid: node id of the node the pool structure should be allocated on, or -1
a58cbd7c2   Dean Nelson   [PATCH] make genp...
146
147
148
   *
   * Create a new special memory pool that can be used to manage special purpose
   * memory not managed by the regular kmalloc/kfree interface.
929f97276   Dean Nelson   [PATCH] change ge...
149
150
   */
  struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
151
  {
929f97276   Dean Nelson   [PATCH] change ge...
152
  	struct gen_pool *pool;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
153

929f97276   Dean Nelson   [PATCH] change ge...
154
155
  	pool = kmalloc_node(sizeof(struct gen_pool), GFP_KERNEL, nid);
  	if (pool != NULL) {
7f184275a   Huang Ying   lib, Make gen_poo...
156
  		spin_lock_init(&pool->lock);
929f97276   Dean Nelson   [PATCH] change ge...
157
158
  		INIT_LIST_HEAD(&pool->chunks);
  		pool->min_alloc_order = min_alloc_order;
ca279cf10   Benjamin Gaignard   genalloc: make it...
159
160
  		pool->algo = gen_pool_first_fit;
  		pool->data = NULL;
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
161
  		pool->name = NULL;
929f97276   Dean Nelson   [PATCH] change ge...
162
163
  	}
  	return pool;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
164
165
  }
  EXPORT_SYMBOL(gen_pool_create);
a58cbd7c2   Dean Nelson   [PATCH] make genp...
166
  /**
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
167
   * gen_pool_add_virt - add a new chunk of special memory to the pool
929f97276   Dean Nelson   [PATCH] change ge...
168
   * @pool: pool to add new memory chunk to
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
169
170
   * @virt: virtual starting address of memory chunk to add to pool
   * @phys: physical starting address of memory chunk to add to pool
929f97276   Dean Nelson   [PATCH] change ge...
171
172
173
   * @size: size in bytes of the memory chunk to add to pool
   * @nid: node id of the node the chunk structure and bitmap should be
   *       allocated on, or -1
a58cbd7c2   Dean Nelson   [PATCH] make genp...
174
175
   *
   * Add a new chunk of special memory to the specified pool.
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
176
177
   *
   * Returns 0 on success or a -ve errno on failure.
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
178
   */
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
179
180
  int gen_pool_add_virt(struct gen_pool *pool, unsigned long virt, phys_addr_t phys,
  		 size_t size, int nid)
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
181
  {
929f97276   Dean Nelson   [PATCH] change ge...
182
183
184
  	struct gen_pool_chunk *chunk;
  	int nbits = size >> pool->min_alloc_order;
  	int nbytes = sizeof(struct gen_pool_chunk) +
eedce141c   Thadeu Lima de Souza Cascardo   genalloc: stop cr...
185
  				BITS_TO_LONGS(nbits) * sizeof(long);
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
186

ade34a357   Joe Perches   lib/genalloc.c: c...
187
  	chunk = kzalloc_node(nbytes, GFP_KERNEL, nid);
929f97276   Dean Nelson   [PATCH] change ge...
188
  	if (unlikely(chunk == NULL))
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
189
  		return -ENOMEM;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
190

3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
191
192
  	chunk->phys_addr = phys;
  	chunk->start_addr = virt;
674470d97   Joonyoung Shim   lib/genalloc.c: f...
193
  	chunk->end_addr = virt + size - 1;
346008fe4   Stephen Bates   lib/genalloc.c: m...
194
  	atomic_long_set(&chunk->avail, size);
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
195

7f184275a   Huang Ying   lib, Make gen_poo...
196
197
198
  	spin_lock(&pool->lock);
  	list_add_rcu(&chunk->next_chunk, &pool->chunks);
  	spin_unlock(&pool->lock);
929f97276   Dean Nelson   [PATCH] change ge...
199
200
  
  	return 0;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
201
  }
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
202
203
204
205
206
207
208
209
210
211
212
  EXPORT_SYMBOL(gen_pool_add_virt);
  
  /**
   * gen_pool_virt_to_phys - return the physical address of memory
   * @pool: pool to allocate from
   * @addr: starting address of memory
   *
   * Returns the physical address on success, or -1 on error.
   */
  phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long addr)
  {
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
213
  	struct gen_pool_chunk *chunk;
7f184275a   Huang Ying   lib, Make gen_poo...
214
  	phys_addr_t paddr = -1;
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
215

7f184275a   Huang Ying   lib, Make gen_poo...
216
217
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
674470d97   Joonyoung Shim   lib/genalloc.c: f...
218
  		if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
7f184275a   Huang Ying   lib, Make gen_poo...
219
220
221
  			paddr = chunk->phys_addr + (addr - chunk->start_addr);
  			break;
  		}
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
222
  	}
7f184275a   Huang Ying   lib, Make gen_poo...
223
  	rcu_read_unlock();
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
224

7f184275a   Huang Ying   lib, Make gen_poo...
225
  	return paddr;
3c8f370de   Jean-Christophe PLAGNIOL-VILLARD   lib/genalloc.c: a...
226
227
  }
  EXPORT_SYMBOL(gen_pool_virt_to_phys);
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
228

a58cbd7c2   Dean Nelson   [PATCH] make genp...
229
230
  /**
   * gen_pool_destroy - destroy a special memory pool
322acc96d   Steve Wise   [PATCH] LIB: add ...
231
   * @pool: pool to destroy
a58cbd7c2   Dean Nelson   [PATCH] make genp...
232
233
234
   *
   * Destroy the specified special memory pool. Verifies that there are no
   * outstanding allocations.
322acc96d   Steve Wise   [PATCH] LIB: add ...
235
236
237
238
239
240
241
   */
  void gen_pool_destroy(struct gen_pool *pool)
  {
  	struct list_head *_chunk, *_next_chunk;
  	struct gen_pool_chunk *chunk;
  	int order = pool->min_alloc_order;
  	int bit, end_bit;
322acc96d   Steve Wise   [PATCH] LIB: add ...
242
243
244
  	list_for_each_safe(_chunk, _next_chunk, &pool->chunks) {
  		chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
  		list_del(&chunk->next_chunk);
674470d97   Joonyoung Shim   lib/genalloc.c: f...
245
  		end_bit = chunk_size(chunk) >> order;
322acc96d   Steve Wise   [PATCH] LIB: add ...
246
247
248
249
250
  		bit = find_next_bit(chunk->bits, end_bit, 0);
  		BUG_ON(bit < end_bit);
  
  		kfree(chunk);
  	}
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
251
  	kfree_const(pool->name);
322acc96d   Steve Wise   [PATCH] LIB: add ...
252
  	kfree(pool);
322acc96d   Steve Wise   [PATCH] LIB: add ...
253
254
  }
  EXPORT_SYMBOL(gen_pool_destroy);
a58cbd7c2   Dean Nelson   [PATCH] make genp...
255
256
  /**
   * gen_pool_alloc - allocate special memory from the pool
929f97276   Dean Nelson   [PATCH] change ge...
257
258
   * @pool: pool to allocate from
   * @size: number of bytes to allocate from the pool
a58cbd7c2   Dean Nelson   [PATCH] make genp...
259
260
   *
   * Allocate the requested number of bytes from the specified pool.
ca279cf10   Benjamin Gaignard   genalloc: make it...
261
262
263
   * Uses the pool allocation function (with first-fit algorithm by default).
   * Can not be used in NMI handler on architectures without
   * NMI-safe cmpxchg implementation.
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
264
   */
929f97276   Dean Nelson   [PATCH] change ge...
265
  unsigned long gen_pool_alloc(struct gen_pool *pool, size_t size)
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
266
  {
de2dd0eb3   Zhao Qiang   genalloc:support ...
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
  	return gen_pool_alloc_algo(pool, size, pool->algo, pool->data);
  }
  EXPORT_SYMBOL(gen_pool_alloc);
  
  /**
   * gen_pool_alloc_algo - allocate special memory from the pool
   * @pool: pool to allocate from
   * @size: number of bytes to allocate from the pool
   * @algo: algorithm passed from caller
   * @data: data passed to algorithm
   *
   * Allocate the requested number of bytes from the specified pool.
   * Uses the pool allocation function (with first-fit algorithm by default).
   * Can not be used in NMI handler on architectures without
   * NMI-safe cmpxchg implementation.
   */
  unsigned long gen_pool_alloc_algo(struct gen_pool *pool, size_t size,
  		genpool_algo_t algo, void *data)
  {
929f97276   Dean Nelson   [PATCH] change ge...
286
  	struct gen_pool_chunk *chunk;
7f184275a   Huang Ying   lib, Make gen_poo...
287
  	unsigned long addr = 0;
929f97276   Dean Nelson   [PATCH] change ge...
288
  	int order = pool->min_alloc_order;
62e931fac   Daniel Mentz   lib/genalloc.c: s...
289
  	int nbits, start_bit, end_bit, remain;
7f184275a   Huang Ying   lib, Make gen_poo...
290
291
292
293
  
  #ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
  	BUG_ON(in_nmi());
  #endif
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
294

929f97276   Dean Nelson   [PATCH] change ge...
295
296
  	if (size == 0)
  		return 0;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
297

929f97276   Dean Nelson   [PATCH] change ge...
298
  	nbits = (size + (1UL << order) - 1) >> order;
7f184275a   Huang Ying   lib, Make gen_poo...
299
300
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
346008fe4   Stephen Bates   lib/genalloc.c: m...
301
  		if (size > atomic_long_read(&chunk->avail))
7f184275a   Huang Ying   lib, Make gen_poo...
302
  			continue;
929f97276   Dean Nelson   [PATCH] change ge...
303

62e931fac   Daniel Mentz   lib/genalloc.c: s...
304
  		start_bit = 0;
674470d97   Joonyoung Shim   lib/genalloc.c: f...
305
  		end_bit = chunk_size(chunk) >> order;
7f184275a   Huang Ying   lib, Make gen_poo...
306
  retry:
de2dd0eb3   Zhao Qiang   genalloc:support ...
307
308
  		start_bit = algo(chunk->bits, end_bit, start_bit,
  				 nbits, data, pool);
7f184275a   Huang Ying   lib, Make gen_poo...
309
  		if (start_bit >= end_bit)
243797f59   Akinobu Mita   genalloc: use bit...
310
  			continue;
7f184275a   Huang Ying   lib, Make gen_poo...
311
312
313
314
315
316
  		remain = bitmap_set_ll(chunk->bits, start_bit, nbits);
  		if (remain) {
  			remain = bitmap_clear_ll(chunk->bits, start_bit,
  						 nbits - remain);
  			BUG_ON(remain);
  			goto retry;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
317
  		}
243797f59   Akinobu Mita   genalloc: use bit...
318
319
  
  		addr = chunk->start_addr + ((unsigned long)start_bit << order);
7f184275a   Huang Ying   lib, Make gen_poo...
320
  		size = nbits << order;
346008fe4   Stephen Bates   lib/genalloc.c: m...
321
  		atomic_long_sub(size, &chunk->avail);
7f184275a   Huang Ying   lib, Make gen_poo...
322
  		break;
929f97276   Dean Nelson   [PATCH] change ge...
323
  	}
7f184275a   Huang Ying   lib, Make gen_poo...
324
325
  	rcu_read_unlock();
  	return addr;
929f97276   Dean Nelson   [PATCH] change ge...
326
  }
de2dd0eb3   Zhao Qiang   genalloc:support ...
327
  EXPORT_SYMBOL(gen_pool_alloc_algo);
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
328

a58cbd7c2   Dean Nelson   [PATCH] make genp...
329
  /**
684f0d3d1   Nicolin Chen   lib/genalloc: add...
330
331
332
   * gen_pool_dma_alloc - allocate special memory from the pool for DMA usage
   * @pool: pool to allocate from
   * @size: number of bytes to allocate from the pool
0368dfd01   Lad, Prabhakar   lib/genalloc.c: a...
333
   * @dma: dma-view physical address return value.  Use NULL if unneeded.
684f0d3d1   Nicolin Chen   lib/genalloc: add...
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
   *
   * Allocate the requested number of bytes from the specified pool.
   * Uses the pool allocation function (with first-fit algorithm by default).
   * Can not be used in NMI handler on architectures without
   * NMI-safe cmpxchg implementation.
   */
  void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma)
  {
  	unsigned long vaddr;
  
  	if (!pool)
  		return NULL;
  
  	vaddr = gen_pool_alloc(pool, size);
  	if (!vaddr)
  		return NULL;
0368dfd01   Lad, Prabhakar   lib/genalloc.c: a...
350
351
  	if (dma)
  		*dma = gen_pool_virt_to_phys(pool, vaddr);
684f0d3d1   Nicolin Chen   lib/genalloc: add...
352
353
354
355
356
357
  
  	return (void *)vaddr;
  }
  EXPORT_SYMBOL(gen_pool_dma_alloc);
  
  /**
a58cbd7c2   Dean Nelson   [PATCH] make genp...
358
   * gen_pool_free - free allocated special memory back to the pool
929f97276   Dean Nelson   [PATCH] change ge...
359
360
361
   * @pool: pool to free to
   * @addr: starting address of memory to free back to pool
   * @size: size in bytes of memory to free
a58cbd7c2   Dean Nelson   [PATCH] make genp...
362
   *
7f184275a   Huang Ying   lib, Make gen_poo...
363
364
365
   * Free previously allocated special memory back to the specified
   * pool.  Can not be used in NMI handler on architectures without
   * NMI-safe cmpxchg implementation.
929f97276   Dean Nelson   [PATCH] change ge...
366
367
368
   */
  void gen_pool_free(struct gen_pool *pool, unsigned long addr, size_t size)
  {
929f97276   Dean Nelson   [PATCH] change ge...
369
  	struct gen_pool_chunk *chunk;
929f97276   Dean Nelson   [PATCH] change ge...
370
  	int order = pool->min_alloc_order;
7f184275a   Huang Ying   lib, Make gen_poo...
371
  	int start_bit, nbits, remain;
929f97276   Dean Nelson   [PATCH] change ge...
372

7f184275a   Huang Ying   lib, Make gen_poo...
373
374
375
  #ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
  	BUG_ON(in_nmi());
  #endif
929f97276   Dean Nelson   [PATCH] change ge...
376

7f184275a   Huang Ying   lib, Make gen_poo...
377
378
379
  	nbits = (size + (1UL << order) - 1) >> order;
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
674470d97   Joonyoung Shim   lib/genalloc.c: f...
380
381
  		if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
  			BUG_ON(addr + size - 1 > chunk->end_addr);
7f184275a   Huang Ying   lib, Make gen_poo...
382
383
384
385
  			start_bit = (addr - chunk->start_addr) >> order;
  			remain = bitmap_clear_ll(chunk->bits, start_bit, nbits);
  			BUG_ON(remain);
  			size = nbits << order;
346008fe4   Stephen Bates   lib/genalloc.c: m...
386
  			atomic_long_add(size, &chunk->avail);
7f184275a   Huang Ying   lib, Make gen_poo...
387
388
  			rcu_read_unlock();
  			return;
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
389
  		}
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
390
  	}
7f184275a   Huang Ying   lib, Make gen_poo...
391
392
  	rcu_read_unlock();
  	BUG();
f14f75b81   Jes Sorensen   [PATCH] ia64 unca...
393
394
  }
  EXPORT_SYMBOL(gen_pool_free);
7f184275a   Huang Ying   lib, Make gen_poo...
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
  
  /**
   * gen_pool_for_each_chunk - call func for every chunk of generic memory pool
   * @pool:	the generic memory pool
   * @func:	func to call
   * @data:	additional data used by @func
   *
   * Call @func for every chunk of generic memory pool.  The @func is
   * called with rcu_read_lock held.
   */
  void gen_pool_for_each_chunk(struct gen_pool *pool,
  	void (*func)(struct gen_pool *pool, struct gen_pool_chunk *chunk, void *data),
  	void *data)
  {
  	struct gen_pool_chunk *chunk;
  
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk)
  		func(pool, chunk, data);
  	rcu_read_unlock();
  }
  EXPORT_SYMBOL(gen_pool_for_each_chunk);
  
  /**
9efb3a421   Laura Abbott   lib/genalloc.c: a...
419
420
421
422
423
424
425
426
427
428
429
430
   * addr_in_gen_pool - checks if an address falls within the range of a pool
   * @pool:	the generic memory pool
   * @start:	start address
   * @size:	size of the region
   *
   * Check if the range of addresses falls within the specified pool. Returns
   * true if the entire range is contained in the pool and false otherwise.
   */
  bool addr_in_gen_pool(struct gen_pool *pool, unsigned long start,
  			size_t size)
  {
  	bool found = false;
ad3d5d2f7   Toshi Kikuchi   lib/genalloc.c: f...
431
  	unsigned long end = start + size - 1;
9efb3a421   Laura Abbott   lib/genalloc.c: a...
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
  	struct gen_pool_chunk *chunk;
  
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk) {
  		if (start >= chunk->start_addr && start <= chunk->end_addr) {
  			if (end <= chunk->end_addr) {
  				found = true;
  				break;
  			}
  		}
  	}
  	rcu_read_unlock();
  	return found;
  }
  
  /**
7f184275a   Huang Ying   lib, Make gen_poo...
448
449
450
451
452
453
454
455
456
457
458
459
   * gen_pool_avail - get available free space of the pool
   * @pool: pool to get available free space
   *
   * Return available free space of the specified pool.
   */
  size_t gen_pool_avail(struct gen_pool *pool)
  {
  	struct gen_pool_chunk *chunk;
  	size_t avail = 0;
  
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
346008fe4   Stephen Bates   lib/genalloc.c: m...
460
  		avail += atomic_long_read(&chunk->avail);
7f184275a   Huang Ying   lib, Make gen_poo...
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
  	rcu_read_unlock();
  	return avail;
  }
  EXPORT_SYMBOL_GPL(gen_pool_avail);
  
  /**
   * gen_pool_size - get size in bytes of memory managed by the pool
   * @pool: pool to get size
   *
   * Return size in bytes of memory managed by the pool.
   */
  size_t gen_pool_size(struct gen_pool *pool)
  {
  	struct gen_pool_chunk *chunk;
  	size_t size = 0;
  
  	rcu_read_lock();
  	list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
674470d97   Joonyoung Shim   lib/genalloc.c: f...
479
  		size += chunk_size(chunk);
7f184275a   Huang Ying   lib, Make gen_poo...
480
481
482
483
  	rcu_read_unlock();
  	return size;
  }
  EXPORT_SYMBOL_GPL(gen_pool_size);
ca279cf10   Benjamin Gaignard   genalloc: make it...
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
  
  /**
   * gen_pool_set_algo - set the allocation algorithm
   * @pool: pool to change allocation algorithm
   * @algo: custom algorithm function
   * @data: additional data used by @algo
   *
   * Call @algo for each memory allocation in the pool.
   * If @algo is NULL use gen_pool_first_fit as default
   * memory allocation function.
   */
  void gen_pool_set_algo(struct gen_pool *pool, genpool_algo_t algo, void *data)
  {
  	rcu_read_lock();
  
  	pool->algo = algo;
  	if (!pool->algo)
  		pool->algo = gen_pool_first_fit;
  
  	pool->data = data;
  
  	rcu_read_unlock();
  }
  EXPORT_SYMBOL(gen_pool_set_algo);
  
  /**
   * gen_pool_first_fit - find the first available region
   * of memory matching the size requirement (no alignment constraint)
   * @map: The address to base the search on
   * @size: The bitmap size in bits
   * @start: The bitnumber to start searching at
   * @nr: The number of zeroed bits we're looking for
   * @data: additional data - unused
de2dd0eb3   Zhao Qiang   genalloc:support ...
517
   * @pool: pool to find the fit region memory from
ca279cf10   Benjamin Gaignard   genalloc: make it...
518
519
   */
  unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size,
de2dd0eb3   Zhao Qiang   genalloc:support ...
520
521
  		unsigned long start, unsigned int nr, void *data,
  		struct gen_pool *pool)
ca279cf10   Benjamin Gaignard   genalloc: make it...
522
523
524
525
526
527
  {
  	return bitmap_find_next_zero_area(map, size, start, nr, 0);
  }
  EXPORT_SYMBOL(gen_pool_first_fit);
  
  /**
de2dd0eb3   Zhao Qiang   genalloc:support ...
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
   * gen_pool_first_fit_align - find the first available region
   * of memory matching the size requirement (alignment constraint)
   * @map: The address to base the search on
   * @size: The bitmap size in bits
   * @start: The bitnumber to start searching at
   * @nr: The number of zeroed bits we're looking for
   * @data: data for alignment
   * @pool: pool to get order from
   */
  unsigned long gen_pool_first_fit_align(unsigned long *map, unsigned long size,
  		unsigned long start, unsigned int nr, void *data,
  		struct gen_pool *pool)
  {
  	struct genpool_data_align *alignment;
  	unsigned long align_mask;
  	int order;
  
  	alignment = data;
  	order = pool->min_alloc_order;
  	align_mask = ((alignment->align + (1UL << order) - 1) >> order) - 1;
  	return bitmap_find_next_zero_area(map, size, start, nr, align_mask);
  }
  EXPORT_SYMBOL(gen_pool_first_fit_align);
  
  /**
b26981c8f   Zhao Qiang   genalloc:support ...
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
   * gen_pool_fixed_alloc - reserve a specific region
   * @map: The address to base the search on
   * @size: The bitmap size in bits
   * @start: The bitnumber to start searching at
   * @nr: The number of zeroed bits we're looking for
   * @data: data for alignment
   * @pool: pool to get order from
   */
  unsigned long gen_pool_fixed_alloc(unsigned long *map, unsigned long size,
  		unsigned long start, unsigned int nr, void *data,
  		struct gen_pool *pool)
  {
  	struct genpool_data_fixed *fixed_data;
  	int order;
  	unsigned long offset_bit;
  	unsigned long start_bit;
  
  	fixed_data = data;
  	order = pool->min_alloc_order;
  	offset_bit = fixed_data->offset >> order;
0e6e01ff6   Zhao Qiang   CPM/QE: use genal...
573
  	if (WARN_ON(fixed_data->offset & ((1UL << order) - 1)))
b26981c8f   Zhao Qiang   genalloc:support ...
574
575
576
577
578
579
580
581
582
583
584
  		return size;
  
  	start_bit = bitmap_find_next_zero_area(map, size,
  			start + offset_bit, nr, 0);
  	if (start_bit != offset_bit)
  		start_bit = size;
  	return start_bit;
  }
  EXPORT_SYMBOL(gen_pool_fixed_alloc);
  
  /**
505e3be6c   Laura Abbott   lib/genalloc.c: a...
585
586
587
588
589
590
591
592
   * gen_pool_first_fit_order_align - find the first available region
   * of memory matching the size requirement. The region will be aligned
   * to the order of the size specified.
   * @map: The address to base the search on
   * @size: The bitmap size in bits
   * @start: The bitnumber to start searching at
   * @nr: The number of zeroed bits we're looking for
   * @data: additional data - unused
de2dd0eb3   Zhao Qiang   genalloc:support ...
593
   * @pool: pool to find the fit region memory from
505e3be6c   Laura Abbott   lib/genalloc.c: a...
594
595
596
   */
  unsigned long gen_pool_first_fit_order_align(unsigned long *map,
  		unsigned long size, unsigned long start,
de2dd0eb3   Zhao Qiang   genalloc:support ...
597
  		unsigned int nr, void *data, struct gen_pool *pool)
505e3be6c   Laura Abbott   lib/genalloc.c: a...
598
599
600
601
602
603
604
605
  {
  	unsigned long align_mask = roundup_pow_of_two(nr) - 1;
  
  	return bitmap_find_next_zero_area(map, size, start, nr, align_mask);
  }
  EXPORT_SYMBOL(gen_pool_first_fit_order_align);
  
  /**
ca279cf10   Benjamin Gaignard   genalloc: make it...
606
607
608
609
610
611
612
   * gen_pool_best_fit - find the best fitting region of memory
   * macthing the size requirement (no alignment constraint)
   * @map: The address to base the search on
   * @size: The bitmap size in bits
   * @start: The bitnumber to start searching at
   * @nr: The number of zeroed bits we're looking for
   * @data: additional data - unused
de2dd0eb3   Zhao Qiang   genalloc:support ...
613
   * @pool: pool to find the fit region memory from
ca279cf10   Benjamin Gaignard   genalloc: make it...
614
615
616
617
618
   *
   * Iterate over the bitmap to find the smallest free region
   * which we can allocate the memory.
   */
  unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size,
de2dd0eb3   Zhao Qiang   genalloc:support ...
619
620
  		unsigned long start, unsigned int nr, void *data,
  		struct gen_pool *pool)
ca279cf10   Benjamin Gaignard   genalloc: make it...
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
  {
  	unsigned long start_bit = size;
  	unsigned long len = size + 1;
  	unsigned long index;
  
  	index = bitmap_find_next_zero_area(map, size, start, nr, 0);
  
  	while (index < size) {
  		int next_bit = find_next_bit(map, size, index + nr);
  		if ((next_bit - index) < len) {
  			len = next_bit - index;
  			start_bit = index;
  			if (len == nr)
  				return start_bit;
  		}
  		index = bitmap_find_next_zero_area(map, size,
  						   next_bit + 1, nr, 0);
  	}
  
  	return start_bit;
  }
  EXPORT_SYMBOL(gen_pool_best_fit);
9375db07a   Philipp Zabel   genalloc: add dev...
643
644
645
646
647
  
  static void devm_gen_pool_release(struct device *dev, void *res)
  {
  	gen_pool_destroy(*(struct gen_pool **)res);
  }
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
648
649
650
651
652
653
654
655
656
657
658
659
660
  static int devm_gen_pool_match(struct device *dev, void *res, void *data)
  {
  	struct gen_pool **p = res;
  
  	/* NULL data matches only a pool without an assigned name */
  	if (!data && !(*p)->name)
  		return 1;
  
  	if (!data || !(*p)->name)
  		return 0;
  
  	return !strcmp((*p)->name, data);
  }
9375db07a   Philipp Zabel   genalloc: add dev...
661
  /**
738581735   Vladimir Zapolskiy   genalloc: add nam...
662
663
664
665
666
667
668
669
670
   * gen_pool_get - Obtain the gen_pool (if any) for a device
   * @dev: device to retrieve the gen_pool from
   * @name: name of a gen_pool or NULL, identifies a particular gen_pool on device
   *
   * Returns the gen_pool for the device if one is present, or NULL.
   */
  struct gen_pool *gen_pool_get(struct device *dev, const char *name)
  {
  	struct gen_pool **p;
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
671
672
  	p = devres_find(dev, devm_gen_pool_release, devm_gen_pool_match,
  			(void *)name);
738581735   Vladimir Zapolskiy   genalloc: add nam...
673
674
675
676
677
678
679
  	if (!p)
  		return NULL;
  	return *p;
  }
  EXPORT_SYMBOL_GPL(gen_pool_get);
  
  /**
9375db07a   Philipp Zabel   genalloc: add dev...
680
681
682
   * devm_gen_pool_create - managed gen_pool_create
   * @dev: device that provides the gen_pool
   * @min_alloc_order: log base 2 of number of bytes each bitmap bit represents
738581735   Vladimir Zapolskiy   genalloc: add nam...
683
684
   * @nid: node selector for allocated gen_pool, %NUMA_NO_NODE for all nodes
   * @name: name of a gen_pool or NULL, identifies a particular gen_pool on device
9375db07a   Philipp Zabel   genalloc: add dev...
685
686
687
688
689
690
   *
   * Create a new special memory pool that can be used to manage special purpose
   * memory not managed by the regular kmalloc/kfree interface. The pool will be
   * automatically destroyed by the device management code.
   */
  struct gen_pool *devm_gen_pool_create(struct device *dev, int min_alloc_order,
738581735   Vladimir Zapolskiy   genalloc: add nam...
691
  				      int nid, const char *name)
9375db07a   Philipp Zabel   genalloc: add dev...
692
693
  {
  	struct gen_pool **ptr, *pool;
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
694
  	const char *pool_name = NULL;
9375db07a   Philipp Zabel   genalloc: add dev...
695

738581735   Vladimir Zapolskiy   genalloc: add nam...
696
697
698
  	/* Check that genpool to be created is uniquely addressed on device */
  	if (gen_pool_get(dev, name))
  		return ERR_PTR(-EINVAL);
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
699
700
701
702
703
  	if (name) {
  		pool_name = kstrdup_const(name, GFP_KERNEL);
  		if (!pool_name)
  			return ERR_PTR(-ENOMEM);
  	}
9375db07a   Philipp Zabel   genalloc: add dev...
704
  	ptr = devres_alloc(devm_gen_pool_release, sizeof(*ptr), GFP_KERNEL);
310ee9e8f   Jan Kara   lib/genalloc.c: c...
705
  	if (!ptr)
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
706
  		goto free_pool_name;
9375db07a   Philipp Zabel   genalloc: add dev...
707
708
  
  	pool = gen_pool_create(min_alloc_order, nid);
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
709
710
711
712
713
714
  	if (!pool)
  		goto free_devres;
  
  	*ptr = pool;
  	pool->name = pool_name;
  	devres_add(dev, ptr);
9375db07a   Philipp Zabel   genalloc: add dev...
715
716
  
  	return pool;
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
717
718
719
720
721
722
723
  
  free_devres:
  	devres_free(ptr);
  free_pool_name:
  	kfree_const(pool_name);
  
  	return ERR_PTR(-ENOMEM);
9375db07a   Philipp Zabel   genalloc: add dev...
724
  }
b724aa213   Michal Simek   lib/genalloc.c: e...
725
  EXPORT_SYMBOL(devm_gen_pool_create);
9375db07a   Philipp Zabel   genalloc: add dev...
726

9375db07a   Philipp Zabel   genalloc: add dev...
727
728
  #ifdef CONFIG_OF
  /**
abdd4a702   Vladimir Zapolskiy   genalloc: rename ...
729
   * of_gen_pool_get - find a pool by phandle property
9375db07a   Philipp Zabel   genalloc: add dev...
730
731
732
733
734
735
736
737
   * @np: device node
   * @propname: property name containing phandle(s)
   * @index: index into the phandle array
   *
   * Returns the pool that contains the chunk starting at the physical
   * address of the device tree node pointed at by the phandle property,
   * or NULL if not found.
   */
abdd4a702   Vladimir Zapolskiy   genalloc: rename ...
738
  struct gen_pool *of_gen_pool_get(struct device_node *np,
9375db07a   Philipp Zabel   genalloc: add dev...
739
740
741
  	const char *propname, int index)
  {
  	struct platform_device *pdev;
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
742
743
744
  	struct device_node *np_pool, *parent;
  	const char *name = NULL;
  	struct gen_pool *pool = NULL;
9375db07a   Philipp Zabel   genalloc: add dev...
745
746
747
748
  
  	np_pool = of_parse_phandle(np, propname, index);
  	if (!np_pool)
  		return NULL;
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
749

9375db07a   Philipp Zabel   genalloc: add dev...
750
  	pdev = of_find_device_by_node(np_pool);
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
751
752
753
754
755
756
757
758
759
760
761
762
  	if (!pdev) {
  		/* Check if named gen_pool is created by parent node device */
  		parent = of_get_parent(np_pool);
  		pdev = of_find_device_by_node(parent);
  		of_node_put(parent);
  
  		of_property_read_string(np_pool, "label", &name);
  		if (!name)
  			name = np_pool->name;
  	}
  	if (pdev)
  		pool = gen_pool_get(&pdev->dev, name);
6f3aabd18   Vladimir Zapolskiy   genalloc: fix dev...
763
  	of_node_put(np_pool);
c98c36355   Vladimir Zapolskiy   genalloc: add sup...
764
765
  
  	return pool;
9375db07a   Philipp Zabel   genalloc: add dev...
766
  }
abdd4a702   Vladimir Zapolskiy   genalloc: rename ...
767
  EXPORT_SYMBOL_GPL(of_gen_pool_get);
9375db07a   Philipp Zabel   genalloc: add dev...
768
  #endif /* CONFIG_OF */