Blame view

lib/idr.c 19.3 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
6
7
8
  /*
   * 2002-10-18  written by Jim Houston jim.houston@ccur.com
   *	Copyright (C) 2002 by Concurrent Computer Corporation
   *	Distributed under the GNU GPL license version 2.
   *
   * Modified by George Anzinger to reuse immediately and to use
   * find bit instructions.  Also removed _irq on spinlocks.
   *
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
9
   * Small id to pointer translation service.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
10
   *
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
11
   * It uses a radix tree like structure as a sparse array indexed
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
12
   * by the id to obtain the pointer.  The bitmap makes allocating
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
13
   * a new id quick.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
14
15
16
17
18
   *
   * You call it to allocate an id (an int) an associate with that id a
   * pointer or what ever, we treat it as a (void *).  You can pass this
   * id to a user for him to pass back at a later time.  You then pass
   * that id to this code and it returns your pointer.
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
19
   * You can release ids at any time. When all ids are released, most of
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
20
   * the memory is returned (we keep IDR_FREE_MAX) in a local pool so we
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
21
   * don't need to go to the memory "store" during an id allocate, just
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
22
23
24
25
26
27
28
29
30
   * so you don't need to be too concerned about locking and conflicts
   * with the slab allocator.
   */
  
  #ifndef TEST                        // to test in user space...
  #include <linux/slab.h>
  #include <linux/init.h>
  #include <linux/module.h>
  #endif
5806f07cd   Jeff Mahoney   [PATCH] lib: add ...
31
  #include <linux/err.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
32
33
  #include <linux/string.h>
  #include <linux/idr.h>
e18b890bb   Christoph Lameter   [PATCH] slab: rem...
34
  static struct kmem_cache *idr_layer_cache;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
35
36
37
38
  
  static struct idr_layer *alloc_layer(struct idr *idp)
  {
  	struct idr_layer *p;
c259cc281   Roland Dreier   [PATCH] Convert i...
39
  	unsigned long flags;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
40

c259cc281   Roland Dreier   [PATCH] Convert i...
41
  	spin_lock_irqsave(&idp->lock, flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
42
43
44
45
46
  	if ((p = idp->id_free)) {
  		idp->id_free = p->ary[0];
  		idp->id_free_cnt--;
  		p->ary[0] = NULL;
  	}
c259cc281   Roland Dreier   [PATCH] Convert i...
47
  	spin_unlock_irqrestore(&idp->lock, flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
48
49
  	return(p);
  }
1eec00565   Sonny Rao   [PATCH] fix race ...
50
51
52
53
54
55
56
  /* only called when idp->lock is held */
  static void __free_layer(struct idr *idp, struct idr_layer *p)
  {
  	p->ary[0] = idp->id_free;
  	idp->id_free = p;
  	idp->id_free_cnt++;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
57
58
  static void free_layer(struct idr *idp, struct idr_layer *p)
  {
c259cc281   Roland Dreier   [PATCH] Convert i...
59
  	unsigned long flags;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
60
61
62
  	/*
  	 * Depends on the return element being zeroed.
  	 */
c259cc281   Roland Dreier   [PATCH] Convert i...
63
  	spin_lock_irqsave(&idp->lock, flags);
1eec00565   Sonny Rao   [PATCH] fix race ...
64
  	__free_layer(idp, p);
c259cc281   Roland Dreier   [PATCH] Convert i...
65
  	spin_unlock_irqrestore(&idp->lock, flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
66
  }
e33ac8bdb   Tejun Heo   idr: separate out...
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
  static void idr_mark_full(struct idr_layer **pa, int id)
  {
  	struct idr_layer *p = pa[0];
  	int l = 0;
  
  	__set_bit(id & IDR_MASK, &p->bitmap);
  	/*
  	 * If this layer is full mark the bit in the layer above to
  	 * show that this part of the radix tree is full.  This may
  	 * complete the layer above and require walking up the radix
  	 * tree.
  	 */
  	while (p->bitmap == IDR_FULL) {
  		if (!(p = pa[++l]))
  			break;
  		id = id >> IDR_BITS;
  		__set_bit((id & IDR_MASK), &p->bitmap);
  	}
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
86
87
88
89
90
91
92
93
94
95
96
97
  /**
   * idr_pre_get - reserver resources for idr allocation
   * @idp:	idr handle
   * @gfp_mask:	memory allocation flags
   *
   * This function should be called prior to locking and calling the
   * following function.  It preallocates enough memory to satisfy
   * the worst possible allocation.
   *
   * If the system is REALLY out of memory this function returns 0,
   * otherwise 1.
   */
fd4f2df24   Al Viro   [PATCH] gfp_t: lib/*
98
  int idr_pre_get(struct idr *idp, gfp_t gfp_mask)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
99
100
101
102
  {
  	while (idp->id_free_cnt < IDR_FREE_MAX) {
  		struct idr_layer *new;
  		new = kmem_cache_alloc(idr_layer_cache, gfp_mask);
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
103
  		if (new == NULL)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
104
105
106
107
108
109
  			return (0);
  		free_layer(idp, new);
  	}
  	return 1;
  }
  EXPORT_SYMBOL(idr_pre_get);
e33ac8bdb   Tejun Heo   idr: separate out...
110
  static int sub_alloc(struct idr *idp, int *starting_id, struct idr_layer **pa)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
111
112
113
  {
  	int n, m, sh;
  	struct idr_layer *p, *new;
7aae6dd80   Tejun Heo   idr: fix obscure ...
114
  	int l, id, oid;
5ba253313   Al Viro   more low-hanging ...
115
  	unsigned long bm;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
116
117
  
  	id = *starting_id;
7aae6dd80   Tejun Heo   idr: fix obscure ...
118
   restart:
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
119
120
121
122
123
124
125
126
127
128
129
130
131
  	p = idp->top;
  	l = idp->layers;
  	pa[l--] = NULL;
  	while (1) {
  		/*
  		 * We run around this while until we reach the leaf node...
  		 */
  		n = (id >> (IDR_BITS*l)) & IDR_MASK;
  		bm = ~p->bitmap;
  		m = find_next_bit(&bm, IDR_SIZE, n);
  		if (m == IDR_SIZE) {
  			/* no space available go back to previous layer. */
  			l++;
7aae6dd80   Tejun Heo   idr: fix obscure ...
132
  			oid = id;
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
133
  			id = (id | ((1 << (IDR_BITS * l)) - 1)) + 1;
7aae6dd80   Tejun Heo   idr: fix obscure ...
134
135
  
  			/* if already at the top layer, we need to grow */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
136
137
138
139
  			if (!(p = pa[l])) {
  				*starting_id = id;
  				return -2;
  			}
7aae6dd80   Tejun Heo   idr: fix obscure ...
140
141
142
143
144
145
146
147
148
  
  			/* If we need to go up one layer, continue the
  			 * loop; otherwise, restart from the top.
  			 */
  			sh = IDR_BITS * (l + 1);
  			if (oid >> sh == id >> sh)
  				continue;
  			else
  				goto restart;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
  		}
  		if (m != n) {
  			sh = IDR_BITS*l;
  			id = ((id >> sh) ^ n ^ m) << sh;
  		}
  		if ((id >= MAX_ID_BIT) || (id < 0))
  			return -3;
  		if (l == 0)
  			break;
  		/*
  		 * Create the layer below if it is missing.
  		 */
  		if (!p->ary[m]) {
  			if (!(new = alloc_layer(idp)))
  				return -1;
  			p->ary[m] = new;
  			p->count++;
  		}
  		pa[l--] = p;
  		p = p->ary[m];
  	}
e33ac8bdb   Tejun Heo   idr: separate out...
170
171
172
  
  	pa[l] = p;
  	return id;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
173
  }
e33ac8bdb   Tejun Heo   idr: separate out...
174
175
  static int idr_get_empty_slot(struct idr *idp, int starting_id,
  			      struct idr_layer **pa)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
176
177
178
  {
  	struct idr_layer *p, *new;
  	int layers, v, id;
c259cc281   Roland Dreier   [PATCH] Convert i...
179
  	unsigned long flags;
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
180

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
181
182
183
184
185
186
187
188
189
190
191
192
193
  	id = starting_id;
  build_up:
  	p = idp->top;
  	layers = idp->layers;
  	if (unlikely(!p)) {
  		if (!(p = alloc_layer(idp)))
  			return -1;
  		layers = 1;
  	}
  	/*
  	 * Add a new layer to the top of the tree if the requested
  	 * id is larger than the currently allocated space.
  	 */
589777eab   Zaur Kambarov   [PATCH] coverity:...
194
  	while ((layers < (MAX_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
195
196
197
198
199
200
201
202
  		layers++;
  		if (!p->count)
  			continue;
  		if (!(new = alloc_layer(idp))) {
  			/*
  			 * The allocation failed.  If we built part of
  			 * the structure tear it down.
  			 */
c259cc281   Roland Dreier   [PATCH] Convert i...
203
  			spin_lock_irqsave(&idp->lock, flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
204
205
206
207
  			for (new = p; p && p != idp->top; new = p) {
  				p = p->ary[0];
  				new->ary[0] = NULL;
  				new->bitmap = new->count = 0;
1eec00565   Sonny Rao   [PATCH] fix race ...
208
  				__free_layer(idp, new);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
209
  			}
c259cc281   Roland Dreier   [PATCH] Convert i...
210
  			spin_unlock_irqrestore(&idp->lock, flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
211
212
213
214
215
216
217
218
219
220
  			return -1;
  		}
  		new->ary[0] = p;
  		new->count = 1;
  		if (p->bitmap == IDR_FULL)
  			__set_bit(0, &new->bitmap);
  		p = new;
  	}
  	idp->top = p;
  	idp->layers = layers;
e33ac8bdb   Tejun Heo   idr: separate out...
221
  	v = sub_alloc(idp, &id, pa);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
222
223
224
225
  	if (v == -2)
  		goto build_up;
  	return(v);
  }
e33ac8bdb   Tejun Heo   idr: separate out...
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
  static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id)
  {
  	struct idr_layer *pa[MAX_LEVEL];
  	int id;
  
  	id = idr_get_empty_slot(idp, starting_id, pa);
  	if (id >= 0) {
  		/*
  		 * Successfully found an empty slot.  Install the user
  		 * pointer and mark the slot full.
  		 */
  		pa[0]->ary[id & IDR_MASK] = (struct idr_layer *)ptr;
  		pa[0]->count++;
  		idr_mark_full(pa, id);
  	}
  
  	return id;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
244
  /**
7c657f2f2   John McCutchan   [PATCH] Document ...
245
   * idr_get_new_above - allocate new idr entry above or equal to a start id
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
   * @idp: idr handle
   * @ptr: pointer you want associated with the ide
   * @start_id: id to start search at
   * @id: pointer to the allocated handle
   *
   * This is the allocate id function.  It should be called with any
   * required locks.
   *
   * If memory is required, it will return -EAGAIN, you should unlock
   * and go back to the idr_pre_get() call.  If the idr is full, it will
   * return -ENOSPC.
   *
   * @id returns a value in the range 0 ... 0x7fffffff
   */
  int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id)
  {
  	int rv;
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
263

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
  	rv = idr_get_new_above_int(idp, ptr, starting_id);
  	/*
  	 * This is a cheap hack until the IDR code can be fixed to
  	 * return proper error values.
  	 */
  	if (rv < 0) {
  		if (rv == -1)
  			return -EAGAIN;
  		else /* Will be -3 */
  			return -ENOSPC;
  	}
  	*id = rv;
  	return 0;
  }
  EXPORT_SYMBOL(idr_get_new_above);
  
  /**
   * idr_get_new - allocate new idr entry
   * @idp: idr handle
   * @ptr: pointer you want associated with the ide
   * @id: pointer to the allocated handle
   *
   * This is the allocate id function.  It should be called with any
   * required locks.
   *
   * If memory is required, it will return -EAGAIN, you should unlock
   * and go back to the idr_pre_get() call.  If the idr is full, it will
   * return -ENOSPC.
   *
   * @id returns a value in the range 0 ... 0x7fffffff
   */
  int idr_get_new(struct idr *idp, void *ptr, int *id)
  {
  	int rv;
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
298

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
  	rv = idr_get_new_above_int(idp, ptr, 0);
  	/*
  	 * This is a cheap hack until the IDR code can be fixed to
  	 * return proper error values.
  	 */
  	if (rv < 0) {
  		if (rv == -1)
  			return -EAGAIN;
  		else /* Will be -3 */
  			return -ENOSPC;
  	}
  	*id = rv;
  	return 0;
  }
  EXPORT_SYMBOL(idr_get_new);
  
  static void idr_remove_warning(int id)
  {
  	printk("idr_remove called for id=%d which is not allocated.
  ", id);
  	dump_stack();
  }
  
  static void sub_remove(struct idr *idp, int shift, int id)
  {
  	struct idr_layer *p = idp->top;
  	struct idr_layer **pa[MAX_LEVEL];
  	struct idr_layer ***paa = &pa[0];
  	int n;
  
  	*paa = NULL;
  	*++paa = &idp->top;
  
  	while ((shift > 0) && p) {
  		n = (id >> shift) & IDR_MASK;
  		__clear_bit(n, &p->bitmap);
  		*++paa = &p->ary[n];
  		p = p->ary[n];
  		shift -= IDR_BITS;
  	}
  	n = id & IDR_MASK;
  	if (likely(p != NULL && test_bit(n, &p->bitmap))){
  		__clear_bit(n, &p->bitmap);
  		p->ary[n] = NULL;
  		while(*paa && ! --((**paa)->count)){
  			free_layer(idp, **paa);
  			**paa-- = NULL;
  		}
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
347
  		if (!*paa)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
348
  			idp->layers = 0;
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
349
  	} else
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
350
  		idr_remove_warning(id);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
351
352
353
354
  }
  
  /**
   * idr_remove - remove the given id and free it's slot
72fd4a35a   Robert P. J. Day   [PATCH] Numerous ...
355
356
   * @idp: idr handle
   * @id: unique key
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
357
358
359
360
361
362
363
364
365
   */
  void idr_remove(struct idr *idp, int id)
  {
  	struct idr_layer *p;
  
  	/* Mask off upper bits we don't use for the search. */
  	id &= MAX_ID_MASK;
  
  	sub_remove(idp, (idp->layers - 1) * IDR_BITS, id);
e15ae2dd3   Jesper Juhl   [PATCH] Whitespac...
366
367
  	if (idp->top && idp->top->count == 1 && (idp->layers > 1) &&
  	    idp->top->ary[0]) {  // We can drop a layer
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
368
369
370
371
372
373
374
375
  
  		p = idp->top->ary[0];
  		idp->top->bitmap = idp->top->count = 0;
  		free_layer(idp, idp->top);
  		idp->top = p;
  		--idp->layers;
  	}
  	while (idp->id_free_cnt >= IDR_FREE_MAX) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
376
377
  		p = alloc_layer(idp);
  		kmem_cache_free(idr_layer_cache, p);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
378
  	}
af8e2a4cb   Nadia Derbey   idr: fix idr_remo...
379
  	return;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
380
381
382
383
  }
  EXPORT_SYMBOL(idr_remove);
  
  /**
23936cc0b   Kristian Hoegsberg   lib: add idr_remo...
384
385
386
387
388
389
390
391
392
393
394
395
396
397
   * idr_remove_all - remove all ids from the given idr tree
   * @idp: idr handle
   *
   * idr_destroy() only frees up unused, cached idp_layers, but this
   * function will remove all id mappings and leave all idp_layers
   * unused.
   *
   * A typical clean-up sequence for objects stored in an idr tree, will
   * use idr_for_each() to free all objects, if necessay, then
   * idr_remove_all() to remove all ids, and idr_destroy() to free
   * up the cached idr_layers.
   */
  void idr_remove_all(struct idr *idp)
  {
6ace06dc6   Oleg Nesterov   idr_remove_all: k...
398
  	int n, id, max;
23936cc0b   Kristian Hoegsberg   lib: add idr_remo...
399
400
401
402
403
404
405
406
407
  	struct idr_layer *p;
  	struct idr_layer *pa[MAX_LEVEL];
  	struct idr_layer **paa = &pa[0];
  
  	n = idp->layers * IDR_BITS;
  	p = idp->top;
  	max = 1 << n;
  
  	id = 0;
6ace06dc6   Oleg Nesterov   idr_remove_all: k...
408
  	while (id < max) {
23936cc0b   Kristian Hoegsberg   lib: add idr_remo...
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
  		while (n > IDR_BITS && p) {
  			n -= IDR_BITS;
  			*paa++ = p;
  			p = p->ary[(id >> n) & IDR_MASK];
  		}
  
  		id += 1 << n;
  		while (n < fls(id)) {
  			if (p) {
  				memset(p, 0, sizeof *p);
  				free_layer(idp, p);
  			}
  			n += IDR_BITS;
  			p = *--paa;
  		}
  	}
  	idp->top = NULL;
  	idp->layers = 0;
  }
  EXPORT_SYMBOL(idr_remove_all);
  
  /**
8d3b35914   Andrew Morton   [PATCH] inotify/i...
431
432
433
434
435
436
437
438
439
440
441
442
443
   * idr_destroy - release all cached layers within an idr tree
   * idp: idr handle
   */
  void idr_destroy(struct idr *idp)
  {
  	while (idp->id_free_cnt) {
  		struct idr_layer *p = alloc_layer(idp);
  		kmem_cache_free(idr_layer_cache, p);
  	}
  }
  EXPORT_SYMBOL(idr_destroy);
  
  /**
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
   * idr_find - return pointer for given id
   * @idp: idr handle
   * @id: lookup key
   *
   * Return the pointer given the id it has been registered with.  A %NULL
   * return indicates that @id is not valid or you passed %NULL in
   * idr_get_new().
   *
   * The caller must serialize idr_find() vs idr_get_new() and idr_remove().
   */
  void *idr_find(struct idr *idp, int id)
  {
  	int n;
  	struct idr_layer *p;
  
  	n = idp->layers * IDR_BITS;
  	p = idp->top;
  
  	/* Mask off upper bits we don't use for the search. */
  	id &= MAX_ID_MASK;
  
  	if (id >= (1 << n))
  		return NULL;
  
  	while (n > 0 && p) {
  		n -= IDR_BITS;
  		p = p->ary[(id >> n) & IDR_MASK];
  	}
  	return((void *)p);
  }
  EXPORT_SYMBOL(idr_find);
5806f07cd   Jeff Mahoney   [PATCH] lib: add ...
475
  /**
96d7fa421   Kristian Hoegsberg   lib: add idr_for_...
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
   * idr_for_each - iterate through all stored pointers
   * @idp: idr handle
   * @fn: function to be called for each pointer
   * @data: data passed back to callback function
   *
   * Iterate over the pointers registered with the given idr.  The
   * callback function will be called for each pointer currently
   * registered, passing the id, the pointer and the data pointer passed
   * to this function.  It is not safe to modify the idr tree while in
   * the callback, so functions such as idr_get_new and idr_remove are
   * not allowed.
   *
   * We check the return of @fn each time. If it returns anything other
   * than 0, we break out and return that value.
   *
   * The caller must serialize idr_for_each() vs idr_get_new() and idr_remove().
   */
  int idr_for_each(struct idr *idp,
  		 int (*fn)(int id, void *p, void *data), void *data)
  {
  	int n, id, max, error = 0;
  	struct idr_layer *p;
  	struct idr_layer *pa[MAX_LEVEL];
  	struct idr_layer **paa = &pa[0];
  
  	n = idp->layers * IDR_BITS;
  	p = idp->top;
  	max = 1 << n;
  
  	id = 0;
  	while (id < max) {
  		while (n > 0 && p) {
  			n -= IDR_BITS;
  			*paa++ = p;
  			p = p->ary[(id >> n) & IDR_MASK];
  		}
  
  		if (p) {
  			error = fn(id, (void *)p, data);
  			if (error)
  				break;
  		}
  
  		id += 1 << n;
  		while (n < fls(id)) {
  			n += IDR_BITS;
  			p = *--paa;
  		}
  	}
  
  	return error;
  }
  EXPORT_SYMBOL(idr_for_each);
  
  /**
5806f07cd   Jeff Mahoney   [PATCH] lib: add ...
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
   * idr_replace - replace pointer for given id
   * @idp: idr handle
   * @ptr: pointer you want associated with the id
   * @id: lookup key
   *
   * Replace the pointer registered with an id and return the old value.
   * A -ENOENT return indicates that @id was not found.
   * A -EINVAL return indicates that @id was not within valid constraints.
   *
   * The caller must serialize vs idr_find(), idr_get_new(), and idr_remove().
   */
  void *idr_replace(struct idr *idp, void *ptr, int id)
  {
  	int n;
  	struct idr_layer *p, *old_p;
  
  	n = idp->layers * IDR_BITS;
  	p = idp->top;
  
  	id &= MAX_ID_MASK;
  
  	if (id >= (1 << n))
  		return ERR_PTR(-EINVAL);
  
  	n -= IDR_BITS;
  	while ((n > 0) && p) {
  		p = p->ary[(id >> n) & IDR_MASK];
  		n -= IDR_BITS;
  	}
  
  	n = id & IDR_MASK;
  	if (unlikely(p == NULL || !test_bit(n, &p->bitmap)))
  		return ERR_PTR(-ENOENT);
  
  	old_p = p->ary[n];
  	p->ary[n] = ptr;
  
  	return old_p;
  }
  EXPORT_SYMBOL(idr_replace);
4ba9b9d0b   Christoph Lameter   Slab API: remove ...
571
  static void idr_cache_ctor(struct kmem_cache *idr_layer_cache, void *idr_layer)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
572
573
574
  {
  	memset(idr_layer, 0, sizeof(struct idr_layer));
  }
199f0ca51   Akinobu Mita   idr: create idr_l...
575
  void __init idr_init_cache(void)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
576
  {
199f0ca51   Akinobu Mita   idr: create idr_l...
577
578
579
  	idr_layer_cache = kmem_cache_create("idr_layer_cache",
  				sizeof(struct idr_layer), 0, SLAB_PANIC,
  				idr_cache_ctor);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
580
581
582
583
584
585
586
587
588
589
590
  }
  
  /**
   * idr_init - initialize idr handle
   * @idp:	idr handle
   *
   * This function is use to set up the handle (@idp) that you will pass
   * to the rest of the functions.
   */
  void idr_init(struct idr *idp)
  {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
591
592
593
594
  	memset(idp, 0, sizeof(struct idr));
  	spin_lock_init(&idp->lock);
  }
  EXPORT_SYMBOL(idr_init);
72dba584b   Tejun Heo   ida: implement id...
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
  
  
  /*
   * IDA - IDR based ID allocator
   *
   * this is id allocator without id -> pointer translation.  Memory
   * usage is much lower than full blown idr because each id only
   * occupies a bit.  ida uses a custom leaf node which contains
   * IDA_BITMAP_BITS slots.
   *
   * 2007-04-25  written by Tejun Heo <htejun@gmail.com>
   */
  
  static void free_bitmap(struct ida *ida, struct ida_bitmap *bitmap)
  {
  	unsigned long flags;
  
  	if (!ida->free_bitmap) {
  		spin_lock_irqsave(&ida->idr.lock, flags);
  		if (!ida->free_bitmap) {
  			ida->free_bitmap = bitmap;
  			bitmap = NULL;
  		}
  		spin_unlock_irqrestore(&ida->idr.lock, flags);
  	}
  
  	kfree(bitmap);
  }
  
  /**
   * ida_pre_get - reserve resources for ida allocation
   * @ida:	ida handle
   * @gfp_mask:	memory allocation flag
   *
   * This function should be called prior to locking and calling the
   * following function.  It preallocates enough memory to satisfy the
   * worst possible allocation.
   *
   * If the system is REALLY out of memory this function returns 0,
   * otherwise 1.
   */
  int ida_pre_get(struct ida *ida, gfp_t gfp_mask)
  {
  	/* allocate idr_layers */
  	if (!idr_pre_get(&ida->idr, gfp_mask))
  		return 0;
  
  	/* allocate free_bitmap */
  	if (!ida->free_bitmap) {
  		struct ida_bitmap *bitmap;
  
  		bitmap = kmalloc(sizeof(struct ida_bitmap), gfp_mask);
  		if (!bitmap)
  			return 0;
  
  		free_bitmap(ida, bitmap);
  	}
  
  	return 1;
  }
  EXPORT_SYMBOL(ida_pre_get);
  
  /**
   * ida_get_new_above - allocate new ID above or equal to a start id
   * @ida:	ida handle
   * @staring_id:	id to start search at
   * @p_id:	pointer to the allocated handle
   *
   * Allocate new ID above or equal to @ida.  It should be called with
   * any required locks.
   *
   * If memory is required, it will return -EAGAIN, you should unlock
   * and go back to the ida_pre_get() call.  If the ida is full, it will
   * return -ENOSPC.
   *
   * @p_id returns a value in the range 0 ... 0x7fffffff.
   */
  int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
  {
  	struct idr_layer *pa[MAX_LEVEL];
  	struct ida_bitmap *bitmap;
  	unsigned long flags;
  	int idr_id = starting_id / IDA_BITMAP_BITS;
  	int offset = starting_id % IDA_BITMAP_BITS;
  	int t, id;
  
   restart:
  	/* get vacant slot */
  	t = idr_get_empty_slot(&ida->idr, idr_id, pa);
  	if (t < 0) {
  		if (t == -1)
  			return -EAGAIN;
  		else /* will be -3 */
  			return -ENOSPC;
  	}
  
  	if (t * IDA_BITMAP_BITS >= MAX_ID_BIT)
  		return -ENOSPC;
  
  	if (t != idr_id)
  		offset = 0;
  	idr_id = t;
  
  	/* if bitmap isn't there, create a new one */
  	bitmap = (void *)pa[0]->ary[idr_id & IDR_MASK];
  	if (!bitmap) {
  		spin_lock_irqsave(&ida->idr.lock, flags);
  		bitmap = ida->free_bitmap;
  		ida->free_bitmap = NULL;
  		spin_unlock_irqrestore(&ida->idr.lock, flags);
  
  		if (!bitmap)
  			return -EAGAIN;
  
  		memset(bitmap, 0, sizeof(struct ida_bitmap));
  		pa[0]->ary[idr_id & IDR_MASK] = (void *)bitmap;
  		pa[0]->count++;
  	}
  
  	/* lookup for empty slot */
  	t = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, offset);
  	if (t == IDA_BITMAP_BITS) {
  		/* no empty slot after offset, continue to the next chunk */
  		idr_id++;
  		offset = 0;
  		goto restart;
  	}
  
  	id = idr_id * IDA_BITMAP_BITS + t;
  	if (id >= MAX_ID_BIT)
  		return -ENOSPC;
  
  	__set_bit(t, bitmap->bitmap);
  	if (++bitmap->nr_busy == IDA_BITMAP_BITS)
  		idr_mark_full(pa, idr_id);
  
  	*p_id = id;
  
  	/* Each leaf node can handle nearly a thousand slots and the
  	 * whole idea of ida is to have small memory foot print.
  	 * Throw away extra resources one by one after each successful
  	 * allocation.
  	 */
  	if (ida->idr.id_free_cnt || ida->free_bitmap) {
  		struct idr_layer *p = alloc_layer(&ida->idr);
  		if (p)
  			kmem_cache_free(idr_layer_cache, p);
  	}
  
  	return 0;
  }
  EXPORT_SYMBOL(ida_get_new_above);
  
  /**
   * ida_get_new - allocate new ID
   * @ida:	idr handle
   * @p_id:	pointer to the allocated handle
   *
   * Allocate new ID.  It should be called with any required locks.
   *
   * If memory is required, it will return -EAGAIN, you should unlock
   * and go back to the idr_pre_get() call.  If the idr is full, it will
   * return -ENOSPC.
   *
   * @id returns a value in the range 0 ... 0x7fffffff.
   */
  int ida_get_new(struct ida *ida, int *p_id)
  {
  	return ida_get_new_above(ida, 0, p_id);
  }
  EXPORT_SYMBOL(ida_get_new);
  
  /**
   * ida_remove - remove the given ID
   * @ida:	ida handle
   * @id:		ID to free
   */
  void ida_remove(struct ida *ida, int id)
  {
  	struct idr_layer *p = ida->idr.top;
  	int shift = (ida->idr.layers - 1) * IDR_BITS;
  	int idr_id = id / IDA_BITMAP_BITS;
  	int offset = id % IDA_BITMAP_BITS;
  	int n;
  	struct ida_bitmap *bitmap;
  
  	/* clear full bits while looking up the leaf idr_layer */
  	while ((shift > 0) && p) {
  		n = (idr_id >> shift) & IDR_MASK;
  		__clear_bit(n, &p->bitmap);
  		p = p->ary[n];
  		shift -= IDR_BITS;
  	}
  
  	if (p == NULL)
  		goto err;
  
  	n = idr_id & IDR_MASK;
  	__clear_bit(n, &p->bitmap);
  
  	bitmap = (void *)p->ary[n];
  	if (!test_bit(offset, bitmap->bitmap))
  		goto err;
  
  	/* update bitmap and remove it if empty */
  	__clear_bit(offset, bitmap->bitmap);
  	if (--bitmap->nr_busy == 0) {
  		__set_bit(n, &p->bitmap);	/* to please idr_remove() */
  		idr_remove(&ida->idr, idr_id);
  		free_bitmap(ida, bitmap);
  	}
  
  	return;
  
   err:
  	printk(KERN_WARNING
  	       "ida_remove called for id=%d which is not allocated.
  ", id);
  }
  EXPORT_SYMBOL(ida_remove);
  
  /**
   * ida_destroy - release all cached layers within an ida tree
   * ida:		ida handle
   */
  void ida_destroy(struct ida *ida)
  {
  	idr_destroy(&ida->idr);
  	kfree(ida->free_bitmap);
  }
  EXPORT_SYMBOL(ida_destroy);
  
  /**
   * ida_init - initialize ida handle
   * @ida:	ida handle
   *
   * This function is use to set up the handle (@ida) that you will pass
   * to the rest of the functions.
   */
  void ida_init(struct ida *ida)
  {
  	memset(ida, 0, sizeof(struct ida));
  	idr_init(&ida->idr);
  
  }
  EXPORT_SYMBOL(ida_init);