Commit a35a3455142976e3fffdf27027f3082cbaba6e8c
1 parent
6182a0943a
Exists in
master
and in
4 other branches
Change dmapool free block management
Use a list of free blocks within a page instead of using a bitmap. Update documentation to reflect this. As well as being a slight reduction in memory allocation, locked ops and lines of code, it speeds up a transaction processing benchmark by 0.4%. Signed-off-by: Matthew Wilcox <willy@linux.intel.com>
Showing 1 changed file with 58 additions and 61 deletions Side-by-side Diff
mm/dmapool.c
... | ... | @@ -17,7 +17,9 @@ |
17 | 17 | * The current design of this allocator is fairly simple. The pool is |
18 | 18 | * represented by the 'struct dma_pool' which keeps a doubly-linked list of |
19 | 19 | * allocated pages. Each page in the page_list is split into blocks of at |
20 | - * least 'size' bytes. | |
20 | + * least 'size' bytes. Free blocks are tracked in an unsorted singly-linked | |
21 | + * list of free blocks within the page. Used blocks aren't tracked, but we | |
22 | + * keep a count of how many are currently allocated from each page. | |
21 | 23 | */ |
22 | 24 | |
23 | 25 | #include <linux/device.h> |
... | ... | @@ -38,7 +40,6 @@ |
38 | 40 | struct dma_pool { /* the pool */ |
39 | 41 | struct list_head page_list; |
40 | 42 | spinlock_t lock; |
41 | - size_t blocks_per_page; | |
42 | 43 | size_t size; |
43 | 44 | struct device *dev; |
44 | 45 | size_t allocation; |
... | ... | @@ -51,8 +52,8 @@ |
51 | 52 | struct list_head page_list; |
52 | 53 | void *vaddr; |
53 | 54 | dma_addr_t dma; |
54 | - unsigned in_use; | |
55 | - unsigned long bitmap[0]; | |
55 | + unsigned int in_use; | |
56 | + unsigned int offset; | |
56 | 57 | }; |
57 | 58 | |
58 | 59 | #define POOL_TIMEOUT_JIFFIES ((100 /* msec */ * HZ) / 1000) |
... | ... | @@ -87,8 +88,8 @@ |
87 | 88 | |
88 | 89 | /* per-pool info, no real statistics yet */ |
89 | 90 | temp = scnprintf(next, size, "%-16s %4u %4Zu %4Zu %2u\n", |
90 | - pool->name, | |
91 | - blocks, pages * pool->blocks_per_page, | |
91 | + pool->name, blocks, | |
92 | + pages * (pool->allocation / pool->size), | |
92 | 93 | pool->size, pages); |
93 | 94 | size -= temp; |
94 | 95 | next += temp; |
95 | 96 | |
... | ... | @@ -132,8 +133,11 @@ |
132 | 133 | return NULL; |
133 | 134 | } |
134 | 135 | |
135 | - if (size == 0) | |
136 | + if (size == 0) { | |
136 | 137 | return NULL; |
138 | + } else if (size < 4) { | |
139 | + size = 4; | |
140 | + } | |
137 | 141 | |
138 | 142 | if ((size % align) != 0) |
139 | 143 | size = ALIGN(size, align); |
... | ... | @@ -160,7 +164,6 @@ |
160 | 164 | spin_lock_init(&retval->lock); |
161 | 165 | retval->size = size; |
162 | 166 | retval->allocation = allocation; |
163 | - retval->blocks_per_page = allocation / size; | |
164 | 167 | init_waitqueue_head(&retval->waitq); |
165 | 168 | |
166 | 169 | if (dev) { |
167 | 170 | |
168 | 171 | |
169 | 172 | |
170 | 173 | |
171 | 174 | |
172 | 175 | |
... | ... | @@ -186,28 +189,36 @@ |
186 | 189 | } |
187 | 190 | EXPORT_SYMBOL(dma_pool_create); |
188 | 191 | |
192 | +static void pool_initialise_page(struct dma_pool *pool, struct dma_page *page) | |
193 | +{ | |
194 | + unsigned int offset = 0; | |
195 | + | |
196 | + do { | |
197 | + unsigned int next = offset + pool->size; | |
198 | + if (unlikely((next + pool->size) >= pool->allocation)) | |
199 | + next = pool->allocation; | |
200 | + *(int *)(page->vaddr + offset) = next; | |
201 | + offset = next; | |
202 | + } while (offset < pool->allocation); | |
203 | +} | |
204 | + | |
189 | 205 | static struct dma_page *pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags) |
190 | 206 | { |
191 | 207 | struct dma_page *page; |
192 | - int mapsize; | |
193 | 208 | |
194 | - mapsize = pool->blocks_per_page; | |
195 | - mapsize = (mapsize + BITS_PER_LONG - 1) / BITS_PER_LONG; | |
196 | - mapsize *= sizeof(long); | |
197 | - | |
198 | - page = kmalloc(mapsize + sizeof *page, mem_flags); | |
209 | + page = kmalloc(sizeof(*page), mem_flags); | |
199 | 210 | if (!page) |
200 | 211 | return NULL; |
201 | - page->vaddr = dma_alloc_coherent(pool->dev, | |
202 | - pool->allocation, | |
212 | + page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation, | |
203 | 213 | &page->dma, mem_flags); |
204 | 214 | if (page->vaddr) { |
205 | - memset(page->bitmap, 0xff, mapsize); /* bit set == free */ | |
206 | 215 | #ifdef CONFIG_DEBUG_SLAB |
207 | 216 | memset(page->vaddr, POOL_POISON_FREED, pool->allocation); |
208 | 217 | #endif |
218 | + pool_initialise_page(pool, page); | |
209 | 219 | list_add(&page->page_list, &pool->page_list); |
210 | 220 | page->in_use = 0; |
221 | + page->offset = 0; | |
211 | 222 | } else { |
212 | 223 | kfree(page); |
213 | 224 | page = NULL; |
214 | 225 | |
... | ... | @@ -215,14 +226,9 @@ |
215 | 226 | return page; |
216 | 227 | } |
217 | 228 | |
218 | -static inline int is_page_busy(int blocks, unsigned long *bitmap) | |
229 | +static inline int is_page_busy(struct dma_page *page) | |
219 | 230 | { |
220 | - while (blocks > 0) { | |
221 | - if (*bitmap++ != ~0UL) | |
222 | - return 1; | |
223 | - blocks -= BITS_PER_LONG; | |
224 | - } | |
225 | - return 0; | |
231 | + return page->in_use != 0; | |
226 | 232 | } |
227 | 233 | |
228 | 234 | static void pool_free_page(struct dma_pool *pool, struct dma_page *page) |
... | ... | @@ -257,7 +263,7 @@ |
257 | 263 | struct dma_page *page; |
258 | 264 | page = list_entry(pool->page_list.next, |
259 | 265 | struct dma_page, page_list); |
260 | - if (is_page_busy(pool->blocks_per_page, page->bitmap)) { | |
266 | + if (is_page_busy(page)) { | |
261 | 267 | if (pool->dev) |
262 | 268 | dev_err(pool->dev, |
263 | 269 | "dma_pool_destroy %s, %p busy\n", |
264 | 270 | |
... | ... | @@ -292,27 +298,14 @@ |
292 | 298 | { |
293 | 299 | unsigned long flags; |
294 | 300 | struct dma_page *page; |
295 | - int map, block; | |
296 | 301 | size_t offset; |
297 | 302 | void *retval; |
298 | 303 | |
299 | 304 | spin_lock_irqsave(&pool->lock, flags); |
300 | 305 | restart: |
301 | 306 | list_for_each_entry(page, &pool->page_list, page_list) { |
302 | - int i; | |
303 | - /* only cachable accesses here ... */ | |
304 | - for (map = 0, i = 0; | |
305 | - i < pool->blocks_per_page; i += BITS_PER_LONG, map++) { | |
306 | - if (page->bitmap[map] == 0) | |
307 | - continue; | |
308 | - block = ffz(~page->bitmap[map]); | |
309 | - if ((i + block) < pool->blocks_per_page) { | |
310 | - clear_bit(block, &page->bitmap[map]); | |
311 | - offset = (BITS_PER_LONG * map) + block; | |
312 | - offset *= pool->size; | |
313 | - goto ready; | |
314 | - } | |
315 | - } | |
307 | + if (page->offset < pool->allocation) | |
308 | + goto ready; | |
316 | 309 | } |
317 | 310 | page = pool_alloc_page(pool, GFP_ATOMIC); |
318 | 311 | if (!page) { |
319 | 312 | |
... | ... | @@ -333,10 +326,10 @@ |
333 | 326 | goto done; |
334 | 327 | } |
335 | 328 | |
336 | - clear_bit(0, &page->bitmap[0]); | |
337 | - offset = 0; | |
338 | 329 | ready: |
339 | 330 | page->in_use++; |
331 | + offset = page->offset; | |
332 | + page->offset = *(int *)(page->vaddr + offset); | |
340 | 333 | retval = offset + page->vaddr; |
341 | 334 | *handle = offset + page->dma; |
342 | 335 | #ifdef CONFIG_DEBUG_SLAB |
... | ... | @@ -379,7 +372,7 @@ |
379 | 372 | { |
380 | 373 | struct dma_page *page; |
381 | 374 | unsigned long flags; |
382 | - int map, block; | |
375 | + unsigned int offset; | |
383 | 376 | |
384 | 377 | page = pool_find_page(pool, dma); |
385 | 378 | if (!page) { |
386 | 379 | |
... | ... | @@ -393,13 +386,9 @@ |
393 | 386 | return; |
394 | 387 | } |
395 | 388 | |
396 | - block = dma - page->dma; | |
397 | - block /= pool->size; | |
398 | - map = block / BITS_PER_LONG; | |
399 | - block %= BITS_PER_LONG; | |
400 | - | |
389 | + offset = vaddr - page->vaddr; | |
401 | 390 | #ifdef CONFIG_DEBUG_SLAB |
402 | - if (((dma - page->dma) + (void *)page->vaddr) != vaddr) { | |
391 | + if ((dma - page->dma) != offset) { | |
403 | 392 | if (pool->dev) |
404 | 393 | dev_err(pool->dev, |
405 | 394 | "dma_pool_free %s, %p (bad vaddr)/%Lx\n", |
406 | 395 | |
407 | 396 | |
... | ... | @@ -410,28 +399,36 @@ |
410 | 399 | pool->name, vaddr, (unsigned long long)dma); |
411 | 400 | return; |
412 | 401 | } |
413 | - if (page->bitmap[map] & (1UL << block)) { | |
414 | - if (pool->dev) | |
415 | - dev_err(pool->dev, | |
416 | - "dma_pool_free %s, dma %Lx already free\n", | |
417 | - pool->name, (unsigned long long)dma); | |
418 | - else | |
419 | - printk(KERN_ERR | |
420 | - "dma_pool_free %s, dma %Lx already free\n", | |
421 | - pool->name, (unsigned long long)dma); | |
422 | - return; | |
402 | + { | |
403 | + unsigned int chain = page->offset; | |
404 | + while (chain < pool->allocation) { | |
405 | + if (chain != offset) { | |
406 | + chain = *(int *)(page->vaddr + chain); | |
407 | + continue; | |
408 | + } | |
409 | + if (pool->dev) | |
410 | + dev_err(pool->dev, "dma_pool_free %s, dma %Lx " | |
411 | + "already free\n", pool->name, | |
412 | + (unsigned long long)dma); | |
413 | + else | |
414 | + printk(KERN_ERR "dma_pool_free %s, dma %Lx " | |
415 | + "already free\n", pool->name, | |
416 | + (unsigned long long)dma); | |
417 | + return; | |
418 | + } | |
423 | 419 | } |
424 | 420 | memset(vaddr, POOL_POISON_FREED, pool->size); |
425 | 421 | #endif |
426 | 422 | |
427 | 423 | spin_lock_irqsave(&pool->lock, flags); |
428 | 424 | page->in_use--; |
429 | - set_bit(block, &page->bitmap[map]); | |
425 | + *(int *)vaddr = page->offset; | |
426 | + page->offset = offset; | |
430 | 427 | if (waitqueue_active(&pool->waitq)) |
431 | 428 | wake_up_locked(&pool->waitq); |
432 | 429 | /* |
433 | 430 | * Resist a temptation to do |
434 | - * if (!is_page_busy(bpp, page->bitmap)) pool_free_page(pool, page); | |
431 | + * if (!is_page_busy(page)) pool_free_page(pool, page); | |
435 | 432 | * Better have a few empty pages hang around. |
436 | 433 | */ |
437 | 434 | spin_unlock_irqrestore(&pool->lock, flags); |