Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 859ddf09743a8cc680af33f7259ccd0fd36bfe9d

Authored by Tejun Heo
Committed by Linus Torvalds
1 parent 1a45dcfe25
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

idr: fix a critical misallocation bug 

Eric Paris located a bug in idr.  With IDR_BITS of 6, it grows to three
layers when id 4096 is first allocated.  When that happens, idr wraps
incorrectly and searches the idr array ignoring the high bits.  The
following test code from Eric demonstrates the bug nicely.

#include <linux/idr.h>
#include <linux/kernel.h>
#include <linux/module.h>

static DEFINE_IDR(test_idr);

int init_module(void)
{
	int ret, forty95, forty96;
	void *addr;

	/* add 2 entries both with 4095 as the start address */
again1:
	if (!idr_pre_get(&test_idr, GFP_KERNEL))
		return -ENOMEM;
	ret = idr_get_new_above(&test_idr, (void *)4095, 4095, &forty95);
	if (ret) {
		if (ret == -EAGAIN)
			goto again1;
		return ret;
	}
	if (forty95 != 4095)
		printk(KERN_ERR "hmmm, forty95=%d\n", forty95);

again2:
	if (!idr_pre_get(&test_idr, GFP_KERNEL))
		return -ENOMEM;
	ret = idr_get_new_above(&test_idr, (void *)4096, 4095, &forty96);
	if (ret) {
		if (ret == -EAGAIN)
			goto again2;
		return ret;
	}
	if (forty96 != 4096)
		printk(KERN_ERR "hmmm, forty96=%d\n", forty96);

	/* try to find the 2 entries, noticing that 4096 broke */
	addr = idr_find(&test_idr, forty95);
	if ((int)addr != forty95)
		printk(KERN_ERR "hmmm, after find forty95=%d addr=%d\n", forty95, (int)addr);
	addr = idr_find(&test_idr, forty96);
	if ((int)addr != forty96)
		printk(KERN_ERR "hmmm, after find forty96=%d addr=%d\n", forty96, (int)addr);
	/* really weird, the entry which should be at 4096 is actually at 0!! */
	addr = idr_find(&test_idr, 0);
	if ((int)addr)
		printk(KERN_ERR "found an entry at id=0 for addr=%d\n", (int)addr);

	idr_remove(&test_idr, forty95);
	idr_remove(&test_idr, forty96);

	return 0;
}

void cleanup_module(void)
{
}

MODULE_AUTHOR("Eric Paris <eparis@redhat.com>");
MODULE_DESCRIPTION("Simple idr test");
MODULE_LICENSE("GPL");

This happens because when sub_alloc() back tracks it doesn't always do it
step-by-step while the over-the-limit detection assumes step-by-step
backtracking.  The logic in sub_alloc() looks like the following.

  restart:
    clear pa[top level + 1] for end cond detection
    l = top level
    while (true) {
	search for empty slot at this level
	if (not found) {
	    push id to the next possible value
	    l++
A:	    if (pa[l] is clear)
	        failed, return asking caller to grow the tree
	    if (going up 1 level gives more slots to search)
	        continue the while loop above with the incremented l
	    else
C:	        goto restart
	}
	adjust id accordingly to the found slot
	if (l == 0)
	    return found id;
	create lower level if not there yet
	record pa[l] and l--
    }

Test A is the fail exit condition but this assumes that failure is
propagated upwared one level at a time but the B optimization path breaks
the assumption and restarts the whole thing with a start value which is
above the possible limit with the current layers.  sub_alloc() assumes the
start id value is inside the limit when called and test A is the only exit
condition check, so it ends up searching for empty slot while ignoring
high set bit.

So, for 4095->4096 test, level0 search fails but pa[1] contains a valid
pointer.  However, going up 1 level wouldn't give any more empty slot so
it takes C and when the whole thing restarts nobody notices the high bit
set beyond the top level.

This patch fixes the bug by changing the fail exit condition check to full
id limit check.

Based-on-patch-from: Eric Paris <eparis@redhat.com>
Reported-by: Eric Paris <eparis@redhat.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: <stable@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 1 changed file with 3 additions and 4 deletions Inline Diff

1 /* 1 /*
2 * 2002-10-18 written by Jim Houston jim.houston@ccur.com 2 * 2002-10-18 written by Jim Houston jim.houston@ccur.com
3 * Copyright (C) 2002 by Concurrent Computer Corporation 3 * Copyright (C) 2002 by Concurrent Computer Corporation
4 * Distributed under the GNU GPL license version 2. 4 * Distributed under the GNU GPL license version 2.
5 * 5 *
6 * Modified by George Anzinger to reuse immediately and to use 6 * Modified by George Anzinger to reuse immediately and to use
7 * find bit instructions. Also removed _irq on spinlocks. 7 * find bit instructions. Also removed _irq on spinlocks.
8 * 8 *
9 * Modified by Nadia Derbey to make it RCU safe. 9 * Modified by Nadia Derbey to make it RCU safe.
10 * 10 *
11 * Small id to pointer translation service. 11 * Small id to pointer translation service.
12 * 12 *
13 * It uses a radix tree like structure as a sparse array indexed 13 * It uses a radix tree like structure as a sparse array indexed
14 * by the id to obtain the pointer. The bitmap makes allocating 14 * by the id to obtain the pointer. The bitmap makes allocating
15 * a new id quick. 15 * a new id quick.
16 * 16 *
17 * You call it to allocate an id (an int) an associate with that id a 17 * You call it to allocate an id (an int) an associate with that id a
18 * pointer or what ever, we treat it as a (void *). You can pass this 18 * pointer or what ever, we treat it as a (void *). You can pass this
19 * id to a user for him to pass back at a later time. You then pass 19 * id to a user for him to pass back at a later time. You then pass
20 * that id to this code and it returns your pointer. 20 * that id to this code and it returns your pointer.
21 21
22 * You can release ids at any time. When all ids are released, most of 22 * You can release ids at any time. When all ids are released, most of
23 * the memory is returned (we keep IDR_FREE_MAX) in a local pool so we 23 * the memory is returned (we keep IDR_FREE_MAX) in a local pool so we
24 * don't need to go to the memory "store" during an id allocate, just 24 * don't need to go to the memory "store" during an id allocate, just
25 * so you don't need to be too concerned about locking and conflicts 25 * so you don't need to be too concerned about locking and conflicts
26 * with the slab allocator. 26 * with the slab allocator.
27 */ 27 */
28 28
29 #ifndef TEST // to test in user space... 29 #ifndef TEST // to test in user space...
30 #include <linux/slab.h> 30 #include <linux/slab.h>
31 #include <linux/init.h> 31 #include <linux/init.h>
32 #include <linux/module.h> 32 #include <linux/module.h>
33 #endif 33 #endif
34 #include <linux/err.h> 34 #include <linux/err.h>
35 #include <linux/string.h> 35 #include <linux/string.h>
36 #include <linux/idr.h> 36 #include <linux/idr.h>
37 37
38 static struct kmem_cache *idr_layer_cache; 38 static struct kmem_cache *idr_layer_cache;
39 39
40 static struct idr_layer *get_from_free_list(struct idr *idp) 40 static struct idr_layer *get_from_free_list(struct idr *idp)
41 { 41 {
42 struct idr_layer *p; 42 struct idr_layer *p;
43 unsigned long flags; 43 unsigned long flags;
44 44
45 spin_lock_irqsave(&idp->lock, flags); 45 spin_lock_irqsave(&idp->lock, flags);
46 if ((p = idp->id_free)) { 46 if ((p = idp->id_free)) {
47 idp->id_free = p->ary[0]; 47 idp->id_free = p->ary[0];
48 idp->id_free_cnt--; 48 idp->id_free_cnt--;
49 p->ary[0] = NULL; 49 p->ary[0] = NULL;
50 } 50 }
51 spin_unlock_irqrestore(&idp->lock, flags); 51 spin_unlock_irqrestore(&idp->lock, flags);
52 return(p); 52 return(p);
53 } 53 }
54 54
55 static void idr_layer_rcu_free(struct rcu_head *head) 55 static void idr_layer_rcu_free(struct rcu_head *head)
56 { 56 {
57 struct idr_layer *layer; 57 struct idr_layer *layer;
58 58
59 layer = container_of(head, struct idr_layer, rcu_head); 59 layer = container_of(head, struct idr_layer, rcu_head);
60 kmem_cache_free(idr_layer_cache, layer); 60 kmem_cache_free(idr_layer_cache, layer);
61 } 61 }
62 62
63 static inline void free_layer(struct idr_layer *p) 63 static inline void free_layer(struct idr_layer *p)
64 { 64 {
65 call_rcu(&p->rcu_head, idr_layer_rcu_free); 65 call_rcu(&p->rcu_head, idr_layer_rcu_free);
66 } 66 }
67 67
68 /* only called when idp->lock is held */ 68 /* only called when idp->lock is held */
69 static void __move_to_free_list(struct idr *idp, struct idr_layer *p) 69 static void __move_to_free_list(struct idr *idp, struct idr_layer *p)
70 { 70 {
71 p->ary[0] = idp->id_free; 71 p->ary[0] = idp->id_free;
72 idp->id_free = p; 72 idp->id_free = p;
73 idp->id_free_cnt++; 73 idp->id_free_cnt++;
74 } 74 }
75 75
76 static void move_to_free_list(struct idr *idp, struct idr_layer *p) 76 static void move_to_free_list(struct idr *idp, struct idr_layer *p)
77 { 77 {
78 unsigned long flags; 78 unsigned long flags;
79 79
80 /* 80 /*
81 * Depends on the return element being zeroed. 81 * Depends on the return element being zeroed.
82 */ 82 */
83 spin_lock_irqsave(&idp->lock, flags); 83 spin_lock_irqsave(&idp->lock, flags);
84 __move_to_free_list(idp, p); 84 __move_to_free_list(idp, p);
85 spin_unlock_irqrestore(&idp->lock, flags); 85 spin_unlock_irqrestore(&idp->lock, flags);
86 } 86 }
87 87
88 static void idr_mark_full(struct idr_layer **pa, int id) 88 static void idr_mark_full(struct idr_layer **pa, int id)
89 { 89 {
90 struct idr_layer *p = pa[0]; 90 struct idr_layer *p = pa[0];
91 int l = 0; 91 int l = 0;
92 92
93 __set_bit(id & IDR_MASK, &p->bitmap); 93 __set_bit(id & IDR_MASK, &p->bitmap);
94 /* 94 /*
95 * If this layer is full mark the bit in the layer above to 95 * If this layer is full mark the bit in the layer above to
96 * show that this part of the radix tree is full. This may 96 * show that this part of the radix tree is full. This may
97 * complete the layer above and require walking up the radix 97 * complete the layer above and require walking up the radix
98 * tree. 98 * tree.
99 */ 99 */
100 while (p->bitmap == IDR_FULL) { 100 while (p->bitmap == IDR_FULL) {
101 if (!(p = pa[++l])) 101 if (!(p = pa[++l]))
102 break; 102 break;
103 id = id >> IDR_BITS; 103 id = id >> IDR_BITS;
104 __set_bit((id & IDR_MASK), &p->bitmap); 104 __set_bit((id & IDR_MASK), &p->bitmap);
105 } 105 }
106 } 106 }
107 107
108 /** 108 /**
109 * idr_pre_get - reserver resources for idr allocation 109 * idr_pre_get - reserver resources for idr allocation
110 * @idp: idr handle 110 * @idp: idr handle
111 * @gfp_mask: memory allocation flags 111 * @gfp_mask: memory allocation flags
112 * 112 *
113 * This function should be called prior to locking and calling the 113 * This function should be called prior to locking and calling the
114 * idr_get_new* functions. It preallocates enough memory to satisfy 114 * idr_get_new* functions. It preallocates enough memory to satisfy
115 * the worst possible allocation. 115 * the worst possible allocation.
116 * 116 *
117 * If the system is REALLY out of memory this function returns 0, 117 * If the system is REALLY out of memory this function returns 0,
118 * otherwise 1. 118 * otherwise 1.
119 */ 119 */
120 int idr_pre_get(struct idr *idp, gfp_t gfp_mask) 120 int idr_pre_get(struct idr *idp, gfp_t gfp_mask)
121 { 121 {
122 while (idp->id_free_cnt < IDR_FREE_MAX) { 122 while (idp->id_free_cnt < IDR_FREE_MAX) {
123 struct idr_layer *new; 123 struct idr_layer *new;
124 new = kmem_cache_zalloc(idr_layer_cache, gfp_mask); 124 new = kmem_cache_zalloc(idr_layer_cache, gfp_mask);
125 if (new == NULL) 125 if (new == NULL)
126 return (0); 126 return (0);
127 move_to_free_list(idp, new); 127 move_to_free_list(idp, new);
128 } 128 }
129 return 1; 129 return 1;
130 } 130 }
131 EXPORT_SYMBOL(idr_pre_get); 131 EXPORT_SYMBOL(idr_pre_get);
132 132
133 static int sub_alloc(struct idr *idp, int *starting_id, struct idr_layer **pa) 133 static int sub_alloc(struct idr *idp, int *starting_id, struct idr_layer **pa)
134 { 134 {
135 int n, m, sh; 135 int n, m, sh;
136 struct idr_layer *p, *new; 136 struct idr_layer *p, *new;
137 int l, id, oid; 137 int l, id, oid;
138 unsigned long bm; 138 unsigned long bm;
139 139
140 id = *starting_id; 140 id = *starting_id;
141 restart: 141 restart:
142 p = idp->top; 142 p = idp->top;
143 l = idp->layers; 143 l = p->layer;
144 pa[l--] = NULL;
145 while (1) { 144 while (1) {
146 /* 145 /*
147 * We run around this while until we reach the leaf node... 146 * We run around this while until we reach the leaf node...
148 */ 147 */
149 n = (id >> (IDR_BITS*l)) & IDR_MASK; 148 n = (id >> (IDR_BITS*l)) & IDR_MASK;
150 bm = ~p->bitmap; 149 bm = ~p->bitmap;
151 m = find_next_bit(&bm, IDR_SIZE, n); 150 m = find_next_bit(&bm, IDR_SIZE, n);
152 if (m == IDR_SIZE) { 151 if (m == IDR_SIZE) {
153 /* no space available go back to previous layer. */ 152 /* no space available go back to previous layer. */
154 l++; 153 l++;
155 oid = id; 154 oid = id;
156 id = (id | ((1 << (IDR_BITS * l)) - 1)) + 1; 155 id = (id | ((1 << (IDR_BITS * l)) - 1)) + 1;
157 156
158 /* if already at the top layer, we need to grow */ 157 /* did id go over the limit? */
159 if (!(p = pa[l])) { 158 if (id >= (1 << (idp->layers * IDR_BITS))) {
160 *starting_id = id; 159 *starting_id = id;
161 return IDR_NEED_TO_GROW; 160 return IDR_NEED_TO_GROW;
162 } 161 }
163 162
164 /* If we need to go up one layer, continue the 163 /* If we need to go up one layer, continue the
165 * loop; otherwise, restart from the top. 164 * loop; otherwise, restart from the top.
166 */ 165 */
167 sh = IDR_BITS * (l + 1); 166 sh = IDR_BITS * (l + 1);
168 if (oid >> sh == id >> sh) 167 if (oid >> sh == id >> sh)
169 continue; 168 continue;
170 else 169 else
171 goto restart; 170 goto restart;
172 } 171 }
173 if (m != n) { 172 if (m != n) {
174 sh = IDR_BITS*l; 173 sh = IDR_BITS*l;
175 id = ((id >> sh) ^ n ^ m) << sh; 174 id = ((id >> sh) ^ n ^ m) << sh;
176 } 175 }
177 if ((id >= MAX_ID_BIT) || (id < 0)) 176 if ((id >= MAX_ID_BIT) || (id < 0))
178 return IDR_NOMORE_SPACE; 177 return IDR_NOMORE_SPACE;
179 if (l == 0) 178 if (l == 0)
180 break; 179 break;
181 /* 180 /*
182 * Create the layer below if it is missing. 181 * Create the layer below if it is missing.
183 */ 182 */
184 if (!p->ary[m]) { 183 if (!p->ary[m]) {
185 new = get_from_free_list(idp); 184 new = get_from_free_list(idp);
186 if (!new) 185 if (!new)
187 return -1; 186 return -1;
188 new->layer = l-1; 187 new->layer = l-1;
189 rcu_assign_pointer(p->ary[m], new); 188 rcu_assign_pointer(p->ary[m], new);
190 p->count++; 189 p->count++;
191 } 190 }
192 pa[l--] = p; 191 pa[l--] = p;
193 p = p->ary[m]; 192 p = p->ary[m];
194 } 193 }
195 194
196 pa[l] = p; 195 pa[l] = p;
197 return id; 196 return id;
198 } 197 }
199 198
200 static int idr_get_empty_slot(struct idr *idp, int starting_id, 199 static int idr_get_empty_slot(struct idr *idp, int starting_id,
201 struct idr_layer **pa) 200 struct idr_layer **pa)
202 { 201 {
203 struct idr_layer *p, *new; 202 struct idr_layer *p, *new;
204 int layers, v, id; 203 int layers, v, id;
205 unsigned long flags; 204 unsigned long flags;
206 205
207 id = starting_id; 206 id = starting_id;
208 build_up: 207 build_up:
209 p = idp->top; 208 p = idp->top;
210 layers = idp->layers; 209 layers = idp->layers;
211 if (unlikely(!p)) { 210 if (unlikely(!p)) {
212 if (!(p = get_from_free_list(idp))) 211 if (!(p = get_from_free_list(idp)))
213 return -1; 212 return -1;
214 p->layer = 0; 213 p->layer = 0;
215 layers = 1; 214 layers = 1;
216 } 215 }
217 /* 216 /*
218 * Add a new layer to the top of the tree if the requested 217 * Add a new layer to the top of the tree if the requested
219 * id is larger than the currently allocated space. 218 * id is larger than the currently allocated space.
220 */ 219 */
221 while ((layers < (MAX_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) { 220 while ((layers < (MAX_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) {
222 layers++; 221 layers++;
223 if (!p->count) { 222 if (!p->count) {
224 /* special case: if the tree is currently empty, 223 /* special case: if the tree is currently empty,
225 * then we grow the tree by moving the top node 224 * then we grow the tree by moving the top node
226 * upwards. 225 * upwards.
227 */ 226 */
228 p->layer++; 227 p->layer++;
229 continue; 228 continue;
230 } 229 }
231 if (!(new = get_from_free_list(idp))) { 230 if (!(new = get_from_free_list(idp))) {
232 /* 231 /*
233 * The allocation failed. If we built part of 232 * The allocation failed. If we built part of
234 * the structure tear it down. 233 * the structure tear it down.
235 */ 234 */
236 spin_lock_irqsave(&idp->lock, flags); 235 spin_lock_irqsave(&idp->lock, flags);
237 for (new = p; p && p != idp->top; new = p) { 236 for (new = p; p && p != idp->top; new = p) {
238 p = p->ary[0]; 237 p = p->ary[0];
239 new->ary[0] = NULL; 238 new->ary[0] = NULL;
240 new->bitmap = new->count = 0; 239 new->bitmap = new->count = 0;
241 __move_to_free_list(idp, new); 240 __move_to_free_list(idp, new);
242 } 241 }
243 spin_unlock_irqrestore(&idp->lock, flags); 242 spin_unlock_irqrestore(&idp->lock, flags);
244 return -1; 243 return -1;
245 } 244 }
246 new->ary[0] = p; 245 new->ary[0] = p;
247 new->count = 1; 246 new->count = 1;
248 new->layer = layers-1; 247 new->layer = layers-1;
249 if (p->bitmap == IDR_FULL) 248 if (p->bitmap == IDR_FULL)
250 __set_bit(0, &new->bitmap); 249 __set_bit(0, &new->bitmap);
251 p = new; 250 p = new;
252 } 251 }
253 rcu_assign_pointer(idp->top, p); 252 rcu_assign_pointer(idp->top, p);
254 idp->layers = layers; 253 idp->layers = layers;
255 v = sub_alloc(idp, &id, pa); 254 v = sub_alloc(idp, &id, pa);
256 if (v == IDR_NEED_TO_GROW) 255 if (v == IDR_NEED_TO_GROW)
257 goto build_up; 256 goto build_up;
258 return(v); 257 return(v);
259 } 258 }
260 259
261 static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id) 260 static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id)
262 { 261 {
263 struct idr_layer *pa[MAX_LEVEL]; 262 struct idr_layer *pa[MAX_LEVEL];
264 int id; 263 int id;
265 264
266 id = idr_get_empty_slot(idp, starting_id, pa); 265 id = idr_get_empty_slot(idp, starting_id, pa);
267 if (id >= 0) { 266 if (id >= 0) {
268 /* 267 /*
269 * Successfully found an empty slot. Install the user 268 * Successfully found an empty slot. Install the user
270 * pointer and mark the slot full. 269 * pointer and mark the slot full.
271 */ 270 */
272 rcu_assign_pointer(pa[0]->ary[id & IDR_MASK], 271 rcu_assign_pointer(pa[0]->ary[id & IDR_MASK],
273 (struct idr_layer *)ptr); 272 (struct idr_layer *)ptr);
274 pa[0]->count++; 273 pa[0]->count++;
275 idr_mark_full(pa, id); 274 idr_mark_full(pa, id);
276 } 275 }
277 276
278 return id; 277 return id;
279 } 278 }
280 279
281 /** 280 /**
282 * idr_get_new_above - allocate new idr entry above or equal to a start id 281 * idr_get_new_above - allocate new idr entry above or equal to a start id
283 * @idp: idr handle 282 * @idp: idr handle
284 * @ptr: pointer you want associated with the id 283 * @ptr: pointer you want associated with the id
285 * @start_id: id to start search at 284 * @start_id: id to start search at
286 * @id: pointer to the allocated handle 285 * @id: pointer to the allocated handle
287 * 286 *
288 * This is the allocate id function. It should be called with any 287 * This is the allocate id function. It should be called with any
289 * required locks. 288 * required locks.
290 * 289 *
291 * If memory is required, it will return -EAGAIN, you should unlock 290 * If memory is required, it will return -EAGAIN, you should unlock
292 * and go back to the idr_pre_get() call. If the idr is full, it will 291 * and go back to the idr_pre_get() call. If the idr is full, it will
293 * return -ENOSPC. 292 * return -ENOSPC.
294 * 293 *
295 * @id returns a value in the range @starting_id ... 0x7fffffff 294 * @id returns a value in the range @starting_id ... 0x7fffffff
296 */ 295 */
297 int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id) 296 int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id)
298 { 297 {
299 int rv; 298 int rv;
300 299
301 rv = idr_get_new_above_int(idp, ptr, starting_id); 300 rv = idr_get_new_above_int(idp, ptr, starting_id);
302 /* 301 /*
303 * This is a cheap hack until the IDR code can be fixed to 302 * This is a cheap hack until the IDR code can be fixed to
304 * return proper error values. 303 * return proper error values.
305 */ 304 */
306 if (rv < 0) 305 if (rv < 0)
307 return _idr_rc_to_errno(rv); 306 return _idr_rc_to_errno(rv);
308 *id = rv; 307 *id = rv;
309 return 0; 308 return 0;
310 } 309 }
311 EXPORT_SYMBOL(idr_get_new_above); 310 EXPORT_SYMBOL(idr_get_new_above);
312 311
313 /** 312 /**
314 * idr_get_new - allocate new idr entry 313 * idr_get_new - allocate new idr entry
315 * @idp: idr handle 314 * @idp: idr handle
316 * @ptr: pointer you want associated with the id 315 * @ptr: pointer you want associated with the id
317 * @id: pointer to the allocated handle 316 * @id: pointer to the allocated handle
318 * 317 *
319 * This is the allocate id function. It should be called with any 318 * This is the allocate id function. It should be called with any
320 * required locks. 319 * required locks.
321 * 320 *
322 * If memory is required, it will return -EAGAIN, you should unlock 321 * If memory is required, it will return -EAGAIN, you should unlock
323 * and go back to the idr_pre_get() call. If the idr is full, it will 322 * and go back to the idr_pre_get() call. If the idr is full, it will
324 * return -ENOSPC. 323 * return -ENOSPC.
325 * 324 *
326 * @id returns a value in the range 0 ... 0x7fffffff 325 * @id returns a value in the range 0 ... 0x7fffffff
327 */ 326 */
328 int idr_get_new(struct idr *idp, void *ptr, int *id) 327 int idr_get_new(struct idr *idp, void *ptr, int *id)
329 { 328 {
330 int rv; 329 int rv;
331 330
332 rv = idr_get_new_above_int(idp, ptr, 0); 331 rv = idr_get_new_above_int(idp, ptr, 0);
333 /* 332 /*
334 * This is a cheap hack until the IDR code can be fixed to 333 * This is a cheap hack until the IDR code can be fixed to
335 * return proper error values. 334 * return proper error values.
336 */ 335 */
337 if (rv < 0) 336 if (rv < 0)
338 return _idr_rc_to_errno(rv); 337 return _idr_rc_to_errno(rv);
339 *id = rv; 338 *id = rv;
340 return 0; 339 return 0;
341 } 340 }
342 EXPORT_SYMBOL(idr_get_new); 341 EXPORT_SYMBOL(idr_get_new);
343 342
344 static void idr_remove_warning(int id) 343 static void idr_remove_warning(int id)
345 { 344 {
346 printk(KERN_WARNING 345 printk(KERN_WARNING
347 "idr_remove called for id=%d which is not allocated.\n", id); 346 "idr_remove called for id=%d which is not allocated.\n", id);
348 dump_stack(); 347 dump_stack();
349 } 348 }
350 349
351 static void sub_remove(struct idr *idp, int shift, int id) 350 static void sub_remove(struct idr *idp, int shift, int id)
352 { 351 {
353 struct idr_layer *p = idp->top; 352 struct idr_layer *p = idp->top;
354 struct idr_layer **pa[MAX_LEVEL]; 353 struct idr_layer **pa[MAX_LEVEL];
355 struct idr_layer ***paa = &pa[0]; 354 struct idr_layer ***paa = &pa[0];
356 struct idr_layer *to_free; 355 struct idr_layer *to_free;
357 int n; 356 int n;
358 357
359 *paa = NULL; 358 *paa = NULL;
360 *++paa = &idp->top; 359 *++paa = &idp->top;
361 360
362 while ((shift > 0) && p) { 361 while ((shift > 0) && p) {
363 n = (id >> shift) & IDR_MASK; 362 n = (id >> shift) & IDR_MASK;
364 __clear_bit(n, &p->bitmap); 363 __clear_bit(n, &p->bitmap);
365 *++paa = &p->ary[n]; 364 *++paa = &p->ary[n];
366 p = p->ary[n]; 365 p = p->ary[n];
367 shift -= IDR_BITS; 366 shift -= IDR_BITS;
368 } 367 }
369 n = id & IDR_MASK; 368 n = id & IDR_MASK;
370 if (likely(p != NULL && test_bit(n, &p->bitmap))){ 369 if (likely(p != NULL && test_bit(n, &p->bitmap))){
371 __clear_bit(n, &p->bitmap); 370 __clear_bit(n, &p->bitmap);
372 rcu_assign_pointer(p->ary[n], NULL); 371 rcu_assign_pointer(p->ary[n], NULL);
373 to_free = NULL; 372 to_free = NULL;
374 while(*paa && ! --((**paa)->count)){ 373 while(*paa && ! --((**paa)->count)){
375 if (to_free) 374 if (to_free)
376 free_layer(to_free); 375 free_layer(to_free);
377 to_free = **paa; 376 to_free = **paa;
378 **paa-- = NULL; 377 **paa-- = NULL;
379 } 378 }
380 if (!*paa) 379 if (!*paa)
381 idp->layers = 0; 380 idp->layers = 0;
382 if (to_free) 381 if (to_free)
383 free_layer(to_free); 382 free_layer(to_free);
384 } else 383 } else
385 idr_remove_warning(id); 384 idr_remove_warning(id);
386 } 385 }
387 386
388 /** 387 /**
389 * idr_remove - remove the given id and free it's slot 388 * idr_remove - remove the given id and free it's slot
390 * @idp: idr handle 389 * @idp: idr handle
391 * @id: unique key 390 * @id: unique key
392 */ 391 */
393 void idr_remove(struct idr *idp, int id) 392 void idr_remove(struct idr *idp, int id)
394 { 393 {
395 struct idr_layer *p; 394 struct idr_layer *p;
396 struct idr_layer *to_free; 395 struct idr_layer *to_free;
397 396
398 /* Mask off upper bits we don't use for the search. */ 397 /* Mask off upper bits we don't use for the search. */
399 id &= MAX_ID_MASK; 398 id &= MAX_ID_MASK;
400 399
401 sub_remove(idp, (idp->layers - 1) * IDR_BITS, id); 400 sub_remove(idp, (idp->layers - 1) * IDR_BITS, id);
402 if (idp->top && idp->top->count == 1 && (idp->layers > 1) && 401 if (idp->top && idp->top->count == 1 && (idp->layers > 1) &&
403 idp->top->ary[0]) { 402 idp->top->ary[0]) {
404 /* 403 /*
405 * Single child at leftmost slot: we can shrink the tree. 404 * Single child at leftmost slot: we can shrink the tree.
406 * This level is not needed anymore since when layers are 405 * This level is not needed anymore since when layers are
407 * inserted, they are inserted at the top of the existing 406 * inserted, they are inserted at the top of the existing
408 * tree. 407 * tree.
409 */ 408 */
410 to_free = idp->top; 409 to_free = idp->top;
411 p = idp->top->ary[0]; 410 p = idp->top->ary[0];
412 rcu_assign_pointer(idp->top, p); 411 rcu_assign_pointer(idp->top, p);
413 --idp->layers; 412 --idp->layers;
414 to_free->bitmap = to_free->count = 0; 413 to_free->bitmap = to_free->count = 0;
415 free_layer(to_free); 414 free_layer(to_free);
416 } 415 }
417 while (idp->id_free_cnt >= IDR_FREE_MAX) { 416 while (idp->id_free_cnt >= IDR_FREE_MAX) {
418 p = get_from_free_list(idp); 417 p = get_from_free_list(idp);
419 /* 418 /*
420 * Note: we don't call the rcu callback here, since the only 419 * Note: we don't call the rcu callback here, since the only
421 * layers that fall into the freelist are those that have been 420 * layers that fall into the freelist are those that have been
422 * preallocated. 421 * preallocated.
423 */ 422 */
424 kmem_cache_free(idr_layer_cache, p); 423 kmem_cache_free(idr_layer_cache, p);
425 } 424 }
426 return; 425 return;
427 } 426 }
428 EXPORT_SYMBOL(idr_remove); 427 EXPORT_SYMBOL(idr_remove);
429 428
430 /** 429 /**
431 * idr_remove_all - remove all ids from the given idr tree 430 * idr_remove_all - remove all ids from the given idr tree
432 * @idp: idr handle 431 * @idp: idr handle
433 * 432 *
434 * idr_destroy() only frees up unused, cached idp_layers, but this 433 * idr_destroy() only frees up unused, cached idp_layers, but this
435 * function will remove all id mappings and leave all idp_layers 434 * function will remove all id mappings and leave all idp_layers
436 * unused. 435 * unused.
437 * 436 *
438 * A typical clean-up sequence for objects stored in an idr tree, will 437 * A typical clean-up sequence for objects stored in an idr tree, will
439 * use idr_for_each() to free all objects, if necessay, then 438 * use idr_for_each() to free all objects, if necessay, then
440 * idr_remove_all() to remove all ids, and idr_destroy() to free 439 * idr_remove_all() to remove all ids, and idr_destroy() to free
441 * up the cached idr_layers. 440 * up the cached idr_layers.
442 */ 441 */
443 void idr_remove_all(struct idr *idp) 442 void idr_remove_all(struct idr *idp)
444 { 443 {
445 int n, id, max; 444 int n, id, max;
446 struct idr_layer *p; 445 struct idr_layer *p;
447 struct idr_layer *pa[MAX_LEVEL]; 446 struct idr_layer *pa[MAX_LEVEL];
448 struct idr_layer **paa = &pa[0]; 447 struct idr_layer **paa = &pa[0];
449 448
450 n = idp->layers * IDR_BITS; 449 n = idp->layers * IDR_BITS;
451 p = idp->top; 450 p = idp->top;
452 rcu_assign_pointer(idp->top, NULL); 451 rcu_assign_pointer(idp->top, NULL);
453 max = 1 << n; 452 max = 1 << n;
454 453
455 id = 0; 454 id = 0;
456 while (id < max) { 455 while (id < max) {
457 while (n > IDR_BITS && p) { 456 while (n > IDR_BITS && p) {
458 n -= IDR_BITS; 457 n -= IDR_BITS;
459 *paa++ = p; 458 *paa++ = p;
460 p = p->ary[(id >> n) & IDR_MASK]; 459 p = p->ary[(id >> n) & IDR_MASK];
461 } 460 }
462 461
463 id += 1 << n; 462 id += 1 << n;
464 while (n < fls(id)) { 463 while (n < fls(id)) {
465 if (p) 464 if (p)
466 free_layer(p); 465 free_layer(p);
467 n += IDR_BITS; 466 n += IDR_BITS;
468 p = *--paa; 467 p = *--paa;
469 } 468 }
470 } 469 }
471 idp->layers = 0; 470 idp->layers = 0;
472 } 471 }
473 EXPORT_SYMBOL(idr_remove_all); 472 EXPORT_SYMBOL(idr_remove_all);
474 473
475 /** 474 /**
476 * idr_destroy - release all cached layers within an idr tree 475 * idr_destroy - release all cached layers within an idr tree
477 * idp: idr handle 476 * idp: idr handle
478 */ 477 */
479 void idr_destroy(struct idr *idp) 478 void idr_destroy(struct idr *idp)
480 { 479 {
481 while (idp->id_free_cnt) { 480 while (idp->id_free_cnt) {
482 struct idr_layer *p = get_from_free_list(idp); 481 struct idr_layer *p = get_from_free_list(idp);
483 kmem_cache_free(idr_layer_cache, p); 482 kmem_cache_free(idr_layer_cache, p);
484 } 483 }
485 } 484 }
486 EXPORT_SYMBOL(idr_destroy); 485 EXPORT_SYMBOL(idr_destroy);
487 486
488 /** 487 /**
489 * idr_find - return pointer for given id 488 * idr_find - return pointer for given id
490 * @idp: idr handle 489 * @idp: idr handle
491 * @id: lookup key 490 * @id: lookup key
492 * 491 *
493 * Return the pointer given the id it has been registered with. A %NULL 492 * Return the pointer given the id it has been registered with. A %NULL
494 * return indicates that @id is not valid or you passed %NULL in 493 * return indicates that @id is not valid or you passed %NULL in
495 * idr_get_new(). 494 * idr_get_new().
496 * 495 *
497 * This function can be called under rcu_read_lock(), given that the leaf 496 * This function can be called under rcu_read_lock(), given that the leaf
498 * pointers lifetimes are correctly managed. 497 * pointers lifetimes are correctly managed.
499 */ 498 */
500 void *idr_find(struct idr *idp, int id) 499 void *idr_find(struct idr *idp, int id)
501 { 500 {
502 int n; 501 int n;
503 struct idr_layer *p; 502 struct idr_layer *p;
504 503
505 p = rcu_dereference(idp->top); 504 p = rcu_dereference(idp->top);
506 if (!p) 505 if (!p)
507 return NULL; 506 return NULL;
508 n = (p->layer+1) * IDR_BITS; 507 n = (p->layer+1) * IDR_BITS;
509 508
510 /* Mask off upper bits we don't use for the search. */ 509 /* Mask off upper bits we don't use for the search. */
511 id &= MAX_ID_MASK; 510 id &= MAX_ID_MASK;
512 511
513 if (id >= (1 << n)) 512 if (id >= (1 << n))
514 return NULL; 513 return NULL;
515 BUG_ON(n == 0); 514 BUG_ON(n == 0);
516 515
517 while (n > 0 && p) { 516 while (n > 0 && p) {
518 n -= IDR_BITS; 517 n -= IDR_BITS;
519 BUG_ON(n != p->layer*IDR_BITS); 518 BUG_ON(n != p->layer*IDR_BITS);
520 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]); 519 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]);
521 } 520 }
522 return((void *)p); 521 return((void *)p);
523 } 522 }
524 EXPORT_SYMBOL(idr_find); 523 EXPORT_SYMBOL(idr_find);
525 524
526 /** 525 /**
527 * idr_for_each - iterate through all stored pointers 526 * idr_for_each - iterate through all stored pointers
528 * @idp: idr handle 527 * @idp: idr handle
529 * @fn: function to be called for each pointer 528 * @fn: function to be called for each pointer
530 * @data: data passed back to callback function 529 * @data: data passed back to callback function
531 * 530 *
532 * Iterate over the pointers registered with the given idr. The 531 * Iterate over the pointers registered with the given idr. The
533 * callback function will be called for each pointer currently 532 * callback function will be called for each pointer currently
534 * registered, passing the id, the pointer and the data pointer passed 533 * registered, passing the id, the pointer and the data pointer passed
535 * to this function. It is not safe to modify the idr tree while in 534 * to this function. It is not safe to modify the idr tree while in
536 * the callback, so functions such as idr_get_new and idr_remove are 535 * the callback, so functions such as idr_get_new and idr_remove are
537 * not allowed. 536 * not allowed.
538 * 537 *
539 * We check the return of @fn each time. If it returns anything other 538 * We check the return of @fn each time. If it returns anything other
540 * than 0, we break out and return that value. 539 * than 0, we break out and return that value.
541 * 540 *
542 * The caller must serialize idr_for_each() vs idr_get_new() and idr_remove(). 541 * The caller must serialize idr_for_each() vs idr_get_new() and idr_remove().
543 */ 542 */
544 int idr_for_each(struct idr *idp, 543 int idr_for_each(struct idr *idp,
545 int (*fn)(int id, void *p, void *data), void *data) 544 int (*fn)(int id, void *p, void *data), void *data)
546 { 545 {
547 int n, id, max, error = 0; 546 int n, id, max, error = 0;
548 struct idr_layer *p; 547 struct idr_layer *p;
549 struct idr_layer *pa[MAX_LEVEL]; 548 struct idr_layer *pa[MAX_LEVEL];
550 struct idr_layer **paa = &pa[0]; 549 struct idr_layer **paa = &pa[0];
551 550
552 n = idp->layers * IDR_BITS; 551 n = idp->layers * IDR_BITS;
553 p = rcu_dereference(idp->top); 552 p = rcu_dereference(idp->top);
554 max = 1 << n; 553 max = 1 << n;
555 554
556 id = 0; 555 id = 0;
557 while (id < max) { 556 while (id < max) {
558 while (n > 0 && p) { 557 while (n > 0 && p) {
559 n -= IDR_BITS; 558 n -= IDR_BITS;
560 *paa++ = p; 559 *paa++ = p;
561 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]); 560 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]);
562 } 561 }
563 562
564 if (p) { 563 if (p) {
565 error = fn(id, (void *)p, data); 564 error = fn(id, (void *)p, data);
566 if (error) 565 if (error)
567 break; 566 break;
568 } 567 }
569 568
570 id += 1 << n; 569 id += 1 << n;
571 while (n < fls(id)) { 570 while (n < fls(id)) {
572 n += IDR_BITS; 571 n += IDR_BITS;
573 p = *--paa; 572 p = *--paa;
574 } 573 }
575 } 574 }
576 575
577 return error; 576 return error;
578 } 577 }
579 EXPORT_SYMBOL(idr_for_each); 578 EXPORT_SYMBOL(idr_for_each);
580 579
581 /** 580 /**
582 * idr_get_next - lookup next object of id to given id. 581 * idr_get_next - lookup next object of id to given id.
583 * @idp: idr handle 582 * @idp: idr handle
584 * @id: pointer to lookup key 583 * @id: pointer to lookup key
585 * 584 *
586 * Returns pointer to registered object with id, which is next number to 585 * Returns pointer to registered object with id, which is next number to
587 * given id. 586 * given id.
588 */ 587 */
589 588
590 void *idr_get_next(struct idr *idp, int *nextidp) 589 void *idr_get_next(struct idr *idp, int *nextidp)
591 { 590 {
592 struct idr_layer *p, *pa[MAX_LEVEL]; 591 struct idr_layer *p, *pa[MAX_LEVEL];
593 struct idr_layer **paa = &pa[0]; 592 struct idr_layer **paa = &pa[0];
594 int id = *nextidp; 593 int id = *nextidp;
595 int n, max; 594 int n, max;
596 595
597 /* find first ent */ 596 /* find first ent */
598 n = idp->layers * IDR_BITS; 597 n = idp->layers * IDR_BITS;
599 max = 1 << n; 598 max = 1 << n;
600 p = rcu_dereference(idp->top); 599 p = rcu_dereference(idp->top);
601 if (!p) 600 if (!p)
602 return NULL; 601 return NULL;
603 602
604 while (id < max) { 603 while (id < max) {
605 while (n > 0 && p) { 604 while (n > 0 && p) {
606 n -= IDR_BITS; 605 n -= IDR_BITS;
607 *paa++ = p; 606 *paa++ = p;
608 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]); 607 p = rcu_dereference(p->ary[(id >> n) & IDR_MASK]);
609 } 608 }
610 609
611 if (p) { 610 if (p) {
612 *nextidp = id; 611 *nextidp = id;
613 return p; 612 return p;
614 } 613 }
615 614
616 id += 1 << n; 615 id += 1 << n;
617 while (n < fls(id)) { 616 while (n < fls(id)) {
618 n += IDR_BITS; 617 n += IDR_BITS;
619 p = *--paa; 618 p = *--paa;
620 } 619 }
621 } 620 }
622 return NULL; 621 return NULL;
623 } 622 }
624 623
625 624
626 625
627 /** 626 /**
628 * idr_replace - replace pointer for given id 627 * idr_replace - replace pointer for given id
629 * @idp: idr handle 628 * @idp: idr handle
630 * @ptr: pointer you want associated with the id 629 * @ptr: pointer you want associated with the id
631 * @id: lookup key 630 * @id: lookup key
632 * 631 *
633 * Replace the pointer registered with an id and return the old value. 632 * Replace the pointer registered with an id and return the old value.
634 * A -ENOENT return indicates that @id was not found. 633 * A -ENOENT return indicates that @id was not found.
635 * A -EINVAL return indicates that @id was not within valid constraints. 634 * A -EINVAL return indicates that @id was not within valid constraints.
636 * 635 *
637 * The caller must serialize with writers. 636 * The caller must serialize with writers.
638 */ 637 */
639 void *idr_replace(struct idr *idp, void *ptr, int id) 638 void *idr_replace(struct idr *idp, void *ptr, int id)
640 { 639 {
641 int n; 640 int n;
642 struct idr_layer *p, *old_p; 641 struct idr_layer *p, *old_p;
643 642
644 p = idp->top; 643 p = idp->top;
645 if (!p) 644 if (!p)
646 return ERR_PTR(-EINVAL); 645 return ERR_PTR(-EINVAL);
647 646
648 n = (p->layer+1) * IDR_BITS; 647 n = (p->layer+1) * IDR_BITS;
649 648
650 id &= MAX_ID_MASK; 649 id &= MAX_ID_MASK;
651 650
652 if (id >= (1 << n)) 651 if (id >= (1 << n))
653 return ERR_PTR(-EINVAL); 652 return ERR_PTR(-EINVAL);
654 653
655 n -= IDR_BITS; 654 n -= IDR_BITS;
656 while ((n > 0) && p) { 655 while ((n > 0) && p) {
657 p = p->ary[(id >> n) & IDR_MASK]; 656 p = p->ary[(id >> n) & IDR_MASK];
658 n -= IDR_BITS; 657 n -= IDR_BITS;
659 } 658 }
660 659
661 n = id & IDR_MASK; 660 n = id & IDR_MASK;
662 if (unlikely(p == NULL || !test_bit(n, &p->bitmap))) 661 if (unlikely(p == NULL || !test_bit(n, &p->bitmap)))
663 return ERR_PTR(-ENOENT); 662 return ERR_PTR(-ENOENT);
664 663
665 old_p = p->ary[n]; 664 old_p = p->ary[n];
666 rcu_assign_pointer(p->ary[n], ptr); 665 rcu_assign_pointer(p->ary[n], ptr);
667 666
668 return old_p; 667 return old_p;
669 } 668 }
670 EXPORT_SYMBOL(idr_replace); 669 EXPORT_SYMBOL(idr_replace);
671 670
672 void __init idr_init_cache(void) 671 void __init idr_init_cache(void)
673 { 672 {
674 idr_layer_cache = kmem_cache_create("idr_layer_cache", 673 idr_layer_cache = kmem_cache_create("idr_layer_cache",
675 sizeof(struct idr_layer), 0, SLAB_PANIC, NULL); 674 sizeof(struct idr_layer), 0, SLAB_PANIC, NULL);
676 } 675 }
677 676
678 /** 677 /**
679 * idr_init - initialize idr handle 678 * idr_init - initialize idr handle
680 * @idp: idr handle 679 * @idp: idr handle
681 * 680 *
682 * This function is use to set up the handle (@idp) that you will pass 681 * This function is use to set up the handle (@idp) that you will pass
683 * to the rest of the functions. 682 * to the rest of the functions.
684 */ 683 */
685 void idr_init(struct idr *idp) 684 void idr_init(struct idr *idp)
686 { 685 {
687 memset(idp, 0, sizeof(struct idr)); 686 memset(idp, 0, sizeof(struct idr));
688 spin_lock_init(&idp->lock); 687 spin_lock_init(&idp->lock);
689 } 688 }
690 EXPORT_SYMBOL(idr_init); 689 EXPORT_SYMBOL(idr_init);
691 690
692 691
693 /* 692 /*
694 * IDA - IDR based ID allocator 693 * IDA - IDR based ID allocator
695 * 694 *
696 * this is id allocator without id -> pointer translation. Memory 695 * this is id allocator without id -> pointer translation. Memory
697 * usage is much lower than full blown idr because each id only 696 * usage is much lower than full blown idr because each id only
698 * occupies a bit. ida uses a custom leaf node which contains 697 * occupies a bit. ida uses a custom leaf node which contains
699 * IDA_BITMAP_BITS slots. 698 * IDA_BITMAP_BITS slots.
700 * 699 *
701 * 2007-04-25 written by Tejun Heo <htejun@gmail.com> 700 * 2007-04-25 written by Tejun Heo <htejun@gmail.com>
702 */ 701 */
703 702
704 static void free_bitmap(struct ida *ida, struct ida_bitmap *bitmap) 703 static void free_bitmap(struct ida *ida, struct ida_bitmap *bitmap)
705 { 704 {
706 unsigned long flags; 705 unsigned long flags;
707 706
708 if (!ida->free_bitmap) { 707 if (!ida->free_bitmap) {
709 spin_lock_irqsave(&ida->idr.lock, flags); 708 spin_lock_irqsave(&ida->idr.lock, flags);
710 if (!ida->free_bitmap) { 709 if (!ida->free_bitmap) {
711 ida->free_bitmap = bitmap; 710 ida->free_bitmap = bitmap;
712 bitmap = NULL; 711 bitmap = NULL;
713 } 712 }
714 spin_unlock_irqrestore(&ida->idr.lock, flags); 713 spin_unlock_irqrestore(&ida->idr.lock, flags);
715 } 714 }
716 715
717 kfree(bitmap); 716 kfree(bitmap);
718 } 717 }
719 718
720 /** 719 /**
721 * ida_pre_get - reserve resources for ida allocation 720 * ida_pre_get - reserve resources for ida allocation
722 * @ida: ida handle 721 * @ida: ida handle
723 * @gfp_mask: memory allocation flag 722 * @gfp_mask: memory allocation flag
724 * 723 *
725 * This function should be called prior to locking and calling the 724 * This function should be called prior to locking and calling the
726 * following function. It preallocates enough memory to satisfy the 725 * following function. It preallocates enough memory to satisfy the
727 * worst possible allocation. 726 * worst possible allocation.
728 * 727 *
729 * If the system is REALLY out of memory this function returns 0, 728 * If the system is REALLY out of memory this function returns 0,
730 * otherwise 1. 729 * otherwise 1.
731 */ 730 */
732 int ida_pre_get(struct ida *ida, gfp_t gfp_mask) 731 int ida_pre_get(struct ida *ida, gfp_t gfp_mask)
733 { 732 {
734 /* allocate idr_layers */ 733 /* allocate idr_layers */
735 if (!idr_pre_get(&ida->idr, gfp_mask)) 734 if (!idr_pre_get(&ida->idr, gfp_mask))
736 return 0; 735 return 0;
737 736
738 /* allocate free_bitmap */ 737 /* allocate free_bitmap */
739 if (!ida->free_bitmap) { 738 if (!ida->free_bitmap) {
740 struct ida_bitmap *bitmap; 739 struct ida_bitmap *bitmap;
741 740
742 bitmap = kmalloc(sizeof(struct ida_bitmap), gfp_mask); 741 bitmap = kmalloc(sizeof(struct ida_bitmap), gfp_mask);
743 if (!bitmap) 742 if (!bitmap)
744 return 0; 743 return 0;
745 744
746 free_bitmap(ida, bitmap); 745 free_bitmap(ida, bitmap);
747 } 746 }
748 747
749 return 1; 748 return 1;
750 } 749 }
751 EXPORT_SYMBOL(ida_pre_get); 750 EXPORT_SYMBOL(ida_pre_get);
752 751
753 /** 752 /**
754 * ida_get_new_above - allocate new ID above or equal to a start id 753 * ida_get_new_above - allocate new ID above or equal to a start id
755 * @ida: ida handle 754 * @ida: ida handle
756 * @staring_id: id to start search at 755 * @staring_id: id to start search at
757 * @p_id: pointer to the allocated handle 756 * @p_id: pointer to the allocated handle
758 * 757 *
759 * Allocate new ID above or equal to @ida. It should be called with 758 * Allocate new ID above or equal to @ida. It should be called with
760 * any required locks. 759 * any required locks.
761 * 760 *
762 * If memory is required, it will return -EAGAIN, you should unlock 761 * If memory is required, it will return -EAGAIN, you should unlock
763 * and go back to the ida_pre_get() call. If the ida is full, it will 762 * and go back to the ida_pre_get() call. If the ida is full, it will
764 * return -ENOSPC. 763 * return -ENOSPC.
765 * 764 *
766 * @p_id returns a value in the range @starting_id ... 0x7fffffff. 765 * @p_id returns a value in the range @starting_id ... 0x7fffffff.
767 */ 766 */
768 int ida_get_new_above(struct ida *ida, int starting_id, int *p_id) 767 int ida_get_new_above(struct ida *ida, int starting_id, int *p_id)
769 { 768 {
770 struct idr_layer *pa[MAX_LEVEL]; 769 struct idr_layer *pa[MAX_LEVEL];
771 struct ida_bitmap *bitmap; 770 struct ida_bitmap *bitmap;
772 unsigned long flags; 771 unsigned long flags;
773 int idr_id = starting_id / IDA_BITMAP_BITS; 772 int idr_id = starting_id / IDA_BITMAP_BITS;
774 int offset = starting_id % IDA_BITMAP_BITS; 773 int offset = starting_id % IDA_BITMAP_BITS;
775 int t, id; 774 int t, id;
776 775
777 restart: 776 restart:
778 /* get vacant slot */ 777 /* get vacant slot */
779 t = idr_get_empty_slot(&ida->idr, idr_id, pa); 778 t = idr_get_empty_slot(&ida->idr, idr_id, pa);
780 if (t < 0) 779 if (t < 0)
781 return _idr_rc_to_errno(t); 780 return _idr_rc_to_errno(t);
782 781
783 if (t * IDA_BITMAP_BITS >= MAX_ID_BIT) 782 if (t * IDA_BITMAP_BITS >= MAX_ID_BIT)
784 return -ENOSPC; 783 return -ENOSPC;
785 784
786 if (t != idr_id) 785 if (t != idr_id)
787 offset = 0; 786 offset = 0;
788 idr_id = t; 787 idr_id = t;
789 788
790 /* if bitmap isn't there, create a new one */ 789 /* if bitmap isn't there, create a new one */
791 bitmap = (void *)pa[0]->ary[idr_id & IDR_MASK]; 790 bitmap = (void *)pa[0]->ary[idr_id & IDR_MASK];
792 if (!bitmap) { 791 if (!bitmap) {
793 spin_lock_irqsave(&ida->idr.lock, flags); 792 spin_lock_irqsave(&ida->idr.lock, flags);
794 bitmap = ida->free_bitmap; 793 bitmap = ida->free_bitmap;
795 ida->free_bitmap = NULL; 794 ida->free_bitmap = NULL;
796 spin_unlock_irqrestore(&ida->idr.lock, flags); 795 spin_unlock_irqrestore(&ida->idr.lock, flags);
797 796
798 if (!bitmap) 797 if (!bitmap)
799 return -EAGAIN; 798 return -EAGAIN;
800 799
801 memset(bitmap, 0, sizeof(struct ida_bitmap)); 800 memset(bitmap, 0, sizeof(struct ida_bitmap));
802 rcu_assign_pointer(pa[0]->ary[idr_id & IDR_MASK], 801 rcu_assign_pointer(pa[0]->ary[idr_id & IDR_MASK],
803 (void *)bitmap); 802 (void *)bitmap);
804 pa[0]->count++; 803 pa[0]->count++;
805 } 804 }
806 805
807 /* lookup for empty slot */ 806 /* lookup for empty slot */
808 t = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, offset); 807 t = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, offset);
809 if (t == IDA_BITMAP_BITS) { 808 if (t == IDA_BITMAP_BITS) {
810 /* no empty slot after offset, continue to the next chunk */ 809 /* no empty slot after offset, continue to the next chunk */
811 idr_id++; 810 idr_id++;
812 offset = 0; 811 offset = 0;
813 goto restart; 812 goto restart;
814 } 813 }
815 814
816 id = idr_id * IDA_BITMAP_BITS + t; 815 id = idr_id * IDA_BITMAP_BITS + t;
817 if (id >= MAX_ID_BIT) 816 if (id >= MAX_ID_BIT)
818 return -ENOSPC; 817 return -ENOSPC;
819 818
820 __set_bit(t, bitmap->bitmap); 819 __set_bit(t, bitmap->bitmap);
821 if (++bitmap->nr_busy == IDA_BITMAP_BITS) 820 if (++bitmap->nr_busy == IDA_BITMAP_BITS)
822 idr_mark_full(pa, idr_id); 821 idr_mark_full(pa, idr_id);
823 822
824 *p_id = id; 823 *p_id = id;
825 824
826 /* Each leaf node can handle nearly a thousand slots and the 825 /* Each leaf node can handle nearly a thousand slots and the
827 * whole idea of ida is to have small memory foot print. 826 * whole idea of ida is to have small memory foot print.
828 * Throw away extra resources one by one after each successful 827 * Throw away extra resources one by one after each successful
829 * allocation. 828 * allocation.
830 */ 829 */
831 if (ida->idr.id_free_cnt || ida->free_bitmap) { 830 if (ida->idr.id_free_cnt || ida->free_bitmap) {
832 struct idr_layer *p = get_from_free_list(&ida->idr); 831 struct idr_layer *p = get_from_free_list(&ida->idr);
833 if (p) 832 if (p)
834 kmem_cache_free(idr_layer_cache, p); 833 kmem_cache_free(idr_layer_cache, p);
835 } 834 }
836 835
837 return 0; 836 return 0;
838 } 837 }
839 EXPORT_SYMBOL(ida_get_new_above); 838 EXPORT_SYMBOL(ida_get_new_above);
840 839
841 /** 840 /**
842 * ida_get_new - allocate new ID 841 * ida_get_new - allocate new ID
843 * @ida: idr handle 842 * @ida: idr handle
844 * @p_id: pointer to the allocated handle 843 * @p_id: pointer to the allocated handle
845 * 844 *
846 * Allocate new ID. It should be called with any required locks. 845 * Allocate new ID. It should be called with any required locks.
847 * 846 *
848 * If memory is required, it will return -EAGAIN, you should unlock 847 * If memory is required, it will return -EAGAIN, you should unlock
849 * and go back to the idr_pre_get() call. If the idr is full, it will 848 * and go back to the idr_pre_get() call. If the idr is full, it will
850 * return -ENOSPC. 849 * return -ENOSPC.
851 * 850 *
852 * @id returns a value in the range 0 ... 0x7fffffff. 851 * @id returns a value in the range 0 ... 0x7fffffff.
853 */ 852 */
854 int ida_get_new(struct ida *ida, int *p_id) 853 int ida_get_new(struct ida *ida, int *p_id)
855 { 854 {
856 return ida_get_new_above(ida, 0, p_id); 855 return ida_get_new_above(ida, 0, p_id);
857 } 856 }
858 EXPORT_SYMBOL(ida_get_new); 857 EXPORT_SYMBOL(ida_get_new);
859 858
860 /** 859 /**
861 * ida_remove - remove the given ID 860 * ida_remove - remove the given ID
862 * @ida: ida handle 861 * @ida: ida handle
863 * @id: ID to free 862 * @id: ID to free
864 */ 863 */
865 void ida_remove(struct ida *ida, int id) 864 void ida_remove(struct ida *ida, int id)
866 { 865 {
867 struct idr_layer *p = ida->idr.top; 866 struct idr_layer *p = ida->idr.top;
868 int shift = (ida->idr.layers - 1) * IDR_BITS; 867 int shift = (ida->idr.layers - 1) * IDR_BITS;
869 int idr_id = id / IDA_BITMAP_BITS; 868 int idr_id = id / IDA_BITMAP_BITS;
870 int offset = id % IDA_BITMAP_BITS; 869 int offset = id % IDA_BITMAP_BITS;
871 int n; 870 int n;
872 struct ida_bitmap *bitmap; 871 struct ida_bitmap *bitmap;
873 872
874 /* clear full bits while looking up the leaf idr_layer */ 873 /* clear full bits while looking up the leaf idr_layer */
875 while ((shift > 0) && p) { 874 while ((shift > 0) && p) {
876 n = (idr_id >> shift) & IDR_MASK; 875 n = (idr_id >> shift) & IDR_MASK;
877 __clear_bit(n, &p->bitmap); 876 __clear_bit(n, &p->bitmap);
878 p = p->ary[n]; 877 p = p->ary[n];
879 shift -= IDR_BITS; 878 shift -= IDR_BITS;
880 } 879 }
881 880
882 if (p == NULL) 881 if (p == NULL)
883 goto err; 882 goto err;
884 883
885 n = idr_id & IDR_MASK; 884 n = idr_id & IDR_MASK;
886 __clear_bit(n, &p->bitmap); 885 __clear_bit(n, &p->bitmap);
887 886
888 bitmap = (void *)p->ary[n]; 887 bitmap = (void *)p->ary[n];
889 if (!test_bit(offset, bitmap->bitmap)) 888 if (!test_bit(offset, bitmap->bitmap))
890 goto err; 889 goto err;
891 890
892 /* update bitmap and remove it if empty */ 891 /* update bitmap and remove it if empty */
893 __clear_bit(offset, bitmap->bitmap); 892 __clear_bit(offset, bitmap->bitmap);
894 if (--bitmap->nr_busy == 0) { 893 if (--bitmap->nr_busy == 0) {
895 __set_bit(n, &p->bitmap); /* to please idr_remove() */ 894 __set_bit(n, &p->bitmap); /* to please idr_remove() */
896 idr_remove(&ida->idr, idr_id); 895 idr_remove(&ida->idr, idr_id);
897 free_bitmap(ida, bitmap); 896 free_bitmap(ida, bitmap);
898 } 897 }
899 898
900 return; 899 return;
901 900
902 err: 901 err:
903 printk(KERN_WARNING 902 printk(KERN_WARNING
904 "ida_remove called for id=%d which is not allocated.\n", id); 903 "ida_remove called for id=%d which is not allocated.\n", id);
905 } 904 }
906 EXPORT_SYMBOL(ida_remove); 905 EXPORT_SYMBOL(ida_remove);
907 906
908 /** 907 /**
909 * ida_destroy - release all cached layers within an ida tree 908 * ida_destroy - release all cached layers within an ida tree
910 * ida: ida handle 909 * ida: ida handle
911 */ 910 */
912 void ida_destroy(struct ida *ida) 911 void ida_destroy(struct ida *ida)
913 { 912 {
914 idr_destroy(&ida->idr); 913 idr_destroy(&ida->idr);
915 kfree(ida->free_bitmap); 914 kfree(ida->free_bitmap);
916 } 915 }
917 EXPORT_SYMBOL(ida_destroy); 916 EXPORT_SYMBOL(ida_destroy);
918 917
919 /** 918 /**
920 * ida_init - initialize ida handle 919 * ida_init - initialize ida handle
921 * @ida: ida handle 920 * @ida: ida handle
922 * 921 *
923 * This function is use to set up the handle (@ida) that you will pass 922 * This function is use to set up the handle (@ida) that you will pass
924 * to the rest of the functions. 923 * to the rest of the functions.
925 */ 924 */
926 void ida_init(struct ida *ida) 925 void ida_init(struct ida *ida)
927 { 926 {
928 memset(ida, 0, sizeof(struct ida)); 927 memset(ida, 0, sizeof(struct ida));
929 idr_init(&ida->idr); 928 idr_init(&ida->idr);
930 929
931 } 930 }
932 EXPORT_SYMBOL(ida_init); 931 EXPORT_SYMBOL(ida_init);
933 932