Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 9f060e2231ca96ca94f2ffcff730acd72606b280

Authored by Kent Overstreet
1 parent 6fda981caf
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

block: Convert integrity to bvec_alloc_bs() 

This adds a pointer to the bvec array to struct bio_integrity_payload,
instead of the bvecs always being inline; then the bvecs are allocated
with bvec_alloc_bs().

Changed bvec_alloc_bs() and bvec_free_bs() to take a pointer to a
mempool instead of the bioset, so that bio integrity can use a different
mempool for its bvecs, and thus avoid a potential deadlock.

This is eventually for immutable bio vecs - immutable bvecs aren't
useful if we still have to copy them, hence the need for the pointer.
Less code is always nice too, though.

Also, bio_integrity_alloc() was using fs_bio_set if no bio_set was
specified. This was wrong - using the bio_set doesn't protect us from
memory allocation failures, because we just used kmalloc for the
bio_integrity_payload. But it does introduce the possibility of
deadlock, if for some reason we weren't supposed to be using fs_bio_set.

Signed-off-by: Kent Overstreet <koverstreet@google.com>
CC: Jens Axboe <axboe@kernel.dk>
CC: Martin K. Petersen <martin.petersen@oracle.com>

Showing 3 changed files with 68 additions and 108 deletions Side-by-side Diff

... ... @@ -27,48 +27,11 @@
27 27 #include <linux/workqueue.h>
28 28 #include <linux/slab.h>
29 29  
30   -struct integrity_slab {
31   - struct kmem_cache *slab;
32   - unsigned short nr_vecs;
33   - char name[8];
34   -};
  30 +#define BIP_INLINE_VECS 4
35 31  
36   -#define IS(x) { .nr_vecs = x, .name = "bip-"__stringify(x) }
37   -struct integrity_slab bip_slab[BIOVEC_NR_POOLS] __read_mostly = {
38   - IS(1), IS(4), IS(16), IS(64), IS(128), IS(BIO_MAX_PAGES),
39   -};
40   -#undef IS
41   -
  32 +static struct kmem_cache *bip_slab;
42 33 static struct workqueue_struct *kintegrityd_wq;
43 34  
44   -static inline unsigned int vecs_to_idx(unsigned int nr)
45   -{
46   - switch (nr) {
47   - case 1:
48   - return 0;
49   - case 2 ... 4:
50   - return 1;
51   - case 5 ... 16:
52   - return 2;
53   - case 17 ... 64:
54   - return 3;
55   - case 65 ... 128:
56   - return 4;
57   - case 129 ... BIO_MAX_PAGES:
58   - return 5;
59   - default:
60   - BUG();
61   - }
62   -}
63   -
64   -static inline int use_bip_pool(unsigned int idx)
65   -{
66   - if (idx == BIOVEC_MAX_IDX)
67   - return 1;
68   -
69   - return 0;
70   -}
71   -
72 35 /**
73 36 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
74 37 * @bio: bio to attach integrity metadata to
75 38  
76 39  
77 40  
78 41  
79 42  
80 43  
81 44  
... ... @@ -84,38 +47,41 @@
84 47 unsigned int nr_vecs)
85 48 {
86 49 struct bio_integrity_payload *bip;
87   - unsigned int idx = vecs_to_idx(nr_vecs);
88 50 struct bio_set *bs = bio->bi_pool;
  51 + unsigned long idx = BIO_POOL_NONE;
  52 + unsigned inline_vecs;
89 53  
90   - if (!bs)
91   - bs = fs_bio_set;
92   -
93   - BUG_ON(bio == NULL);
94   - bip = NULL;
95   -
96   - /* Lower order allocations come straight from slab */
97   - if (!use_bip_pool(idx))
98   - bip = kmem_cache_alloc(bip_slab[idx].slab, gfp_mask);
99   -
100   - /* Use mempool if lower order alloc failed or max vecs were requested */
101   - if (bip == NULL) {
102   - idx = BIOVEC_MAX_IDX; /* so we free the payload properly later */
  54 + if (!bs) {
  55 + bip = kmalloc(sizeof(struct bio_integrity_payload) +
  56 + sizeof(struct bio_vec) * nr_vecs, gfp_mask);
  57 + inline_vecs = nr_vecs;
  58 + } else {
103 59 bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
104   -
105   - if (unlikely(bip == NULL)) {
106   - printk(KERN_ERR "%s: could not alloc bip\n", __func__);
107   - return NULL;
108   - }
  60 + inline_vecs = BIP_INLINE_VECS;
109 61 }
110 62  
  63 + if (unlikely(!bip))
  64 + return NULL;
  65 +
111 66 memset(bip, 0, sizeof(*bip));
112 67  
  68 + if (nr_vecs > inline_vecs) {
  69 + bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
  70 + bs->bvec_integrity_pool);
  71 + if (!bip->bip_vec)
  72 + goto err;
  73 + } else {
  74 + bip->bip_vec = bip->bip_inline_vecs;
  75 + }
  76 +
113 77 bip->bip_slab = idx;
114 78 bip->bip_bio = bio;
115   - bip->bip_vec = bip->bip_inline_vecs;
116 79 bio->bi_integrity = bip;
117 80  
118 81 return bip;
  82 +err:
  83 + mempool_free(bip, bs->bio_integrity_pool);
  84 + return NULL;
119 85 }
120 86 EXPORT_SYMBOL(bio_integrity_alloc);
121 87  
122 88  
123 89  
... ... @@ -131,20 +97,20 @@
131 97 struct bio_integrity_payload *bip = bio->bi_integrity;
132 98 struct bio_set *bs = bio->bi_pool;
133 99  
134   - if (!bs)
135   - bs = fs_bio_set;
136   -
137   - BUG_ON(bip == NULL);
138   -
139 100 /* A cloned bio doesn't own the integrity metadata */
140 101 if (!bio_flagged(bio, BIO_CLONED) && !bio_flagged(bio, BIO_FS_INTEGRITY)
141 102 && bip->bip_buf != NULL)
142 103 kfree(bip->bip_buf);
143 104  
144   - if (use_bip_pool(bip->bip_slab))
  105 + if (bs) {
  106 + if (bip->bip_slab != BIO_POOL_NONE)
  107 + bvec_free(bs->bvec_integrity_pool, bip->bip_vec,
  108 + bip->bip_slab);
  109 +
145 110 mempool_free(bip, bs->bio_integrity_pool);
146   - else
147   - kmem_cache_free(bip_slab[bip->bip_slab].slab, bip);
  111 + } else {
  112 + kfree(bip);
  113 + }
148 114  
149 115 bio->bi_integrity = NULL;
150 116 }
151 117  
152 118  
... ... @@ -747,14 +713,15 @@
747 713  
748 714 int bioset_integrity_create(struct bio_set *bs, int pool_size)
749 715 {
750   - unsigned int max_slab = vecs_to_idx(BIO_MAX_PAGES);
751   -
752 716 if (bs->bio_integrity_pool)
753 717 return 0;
754 718  
755   - bs->bio_integrity_pool =
756   - mempool_create_slab_pool(pool_size, bip_slab[max_slab].slab);
  719 + bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);
757 720  
  721 + bs->bvec_integrity_pool = biovec_create_pool(bs, pool_size);
  722 + if (!bs->bvec_integrity_pool)
  723 + return -1;
  724 +
758 725 if (!bs->bio_integrity_pool)
759 726 return -1;
760 727  
761 728  
... ... @@ -766,13 +733,14 @@
766 733 {
767 734 if (bs->bio_integrity_pool)
768 735 mempool_destroy(bs->bio_integrity_pool);
  736 +
  737 + if (bs->bvec_integrity_pool)
  738 + mempool_destroy(bs->bio_integrity_pool);
769 739 }
770 740 EXPORT_SYMBOL(bioset_integrity_free);
771 741  
772 742 void __init bio_integrity_init(void)
773 743 {
774   - unsigned int i;
775   -
776 744 /*
777 745 * kintegrityd won't block much but may burn a lot of CPU cycles.
778 746 * Make it highpri CPU intensive wq with max concurrency of 1.
... ... @@ -782,15 +750,11 @@
782 750 if (!kintegrityd_wq)
783 751 panic("Failed to create kintegrityd\n");
784 752  
785   - for (i = 0 ; i < BIOVEC_NR_POOLS ; i++) {
786   - unsigned int size;
787   -
788   - size = sizeof(struct bio_integrity_payload)
789   - + bip_slab[i].nr_vecs * sizeof(struct bio_vec);
790   -
791   - bip_slab[i].slab =
792   - kmem_cache_create(bip_slab[i].name, size, 0,
793   - SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
794   - }
  753 + bip_slab = kmem_cache_create("bio_integrity_payload",
  754 + sizeof(struct bio_integrity_payload) +
  755 + sizeof(struct bio_vec) * BIP_INLINE_VECS,
  756 + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  757 + if (!bip_slab)
  758 + panic("Failed to create slab\n");
795 759 }
... ... @@ -160,12 +160,12 @@
160 160 return bvec_slabs[idx].nr_vecs;
161 161 }
162 162  
163   -void bvec_free_bs(struct bio_set *bs, struct bio_vec *bv, unsigned int idx)
  163 +void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
164 164 {
165 165 BIO_BUG_ON(idx >= BIOVEC_NR_POOLS);
166 166  
167 167 if (idx == BIOVEC_MAX_IDX)
168   - mempool_free(bv, bs->bvec_pool);
  168 + mempool_free(bv, pool);
169 169 else {
170 170 struct biovec_slab *bvs = bvec_slabs + idx;
171 171  
... ... @@ -173,8 +173,8 @@
173 173 }
174 174 }
175 175  
176   -struct bio_vec *bvec_alloc_bs(gfp_t gfp_mask, int nr, unsigned long *idx,
177   - struct bio_set *bs)
  176 +struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
  177 + mempool_t *pool)
178 178 {
179 179 struct bio_vec *bvl;
180 180  
... ... @@ -210,7 +210,7 @@
210 210 */
211 211 if (*idx == BIOVEC_MAX_IDX) {
212 212 fallback:
213   - bvl = mempool_alloc(bs->bvec_pool, gfp_mask);
  213 + bvl = mempool_alloc(pool, gfp_mask);
214 214 } else {
215 215 struct biovec_slab *bvs = bvec_slabs + *idx;
216 216 gfp_t __gfp_mask = gfp_mask & ~(__GFP_WAIT | __GFP_IO);
... ... @@ -253,7 +253,7 @@
253 253  
254 254 if (bs) {
255 255 if (bio_has_allocated_vec(bio))
256   - bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
  256 + bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));
257 257  
258 258 /*
259 259 * If we have front padding, adjust the bio pointer before freeing
260 260  
... ... @@ -442,11 +442,11 @@
442 442 bio_init(bio);
443 443  
444 444 if (nr_iovecs > inline_vecs) {
445   - bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
  445 + bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
446 446 if (!bvl && gfp_mask != saved_gfp) {
447 447 punt_bios_to_rescuer(bs);
448 448 gfp_mask = saved_gfp;
449   - bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
  449 + bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
450 450 }
451 451  
452 452 if (unlikely(!bvl))
453 453  
454 454  
... ... @@ -1661,22 +1661,13 @@
1661 1661 * create memory pools for biovec's in a bio_set.
1662 1662 * use the global biovec slabs created for general use.
1663 1663 */
1664   -static int biovec_create_pools(struct bio_set *bs, int pool_entries)
  1664 +mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries)
1665 1665 {
1666 1666 struct biovec_slab *bp = bvec_slabs + BIOVEC_MAX_IDX;
1667 1667  
1668   - bs->bvec_pool = mempool_create_slab_pool(pool_entries, bp->slab);
1669   - if (!bs->bvec_pool)
1670   - return -ENOMEM;
1671   -
1672   - return 0;
  1668 + return mempool_create_slab_pool(pool_entries, bp->slab);
1673 1669 }
1674 1670  
1675   -static void biovec_free_pools(struct bio_set *bs)
1676   -{
1677   - mempool_destroy(bs->bvec_pool);
1678   -}
1679   -
1680 1671 void bioset_free(struct bio_set *bs)
1681 1672 {
1682 1673 if (bs->rescue_workqueue)
1683 1674  
... ... @@ -1685,8 +1676,10 @@
1685 1676 if (bs->bio_pool)
1686 1677 mempool_destroy(bs->bio_pool);
1687 1678  
  1679 + if (bs->bvec_pool)
  1680 + mempool_destroy(bs->bvec_pool);
  1681 +
1688 1682 bioset_integrity_free(bs);
1689   - biovec_free_pools(bs);
1690 1683 bio_put_slab(bs);
1691 1684  
1692 1685 kfree(bs);
... ... @@ -1731,7 +1724,8 @@
1731 1724 if (!bs->bio_pool)
1732 1725 goto bad;
1733 1726  
1734   - if (biovec_create_pools(bs, pool_size))
  1727 + bs->bvec_pool = biovec_create_pool(bs, pool_size);
  1728 + if (!bs->bvec_pool)
1735 1729 goto bad;
1736 1730  
1737 1731 bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
... ... @@ -213,6 +213,7 @@
213 213  
214 214 extern struct bio_set *bioset_create(unsigned int, unsigned int);
215 215 extern void bioset_free(struct bio_set *);
  216 +extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries);
216 217  
217 218 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
218 219 extern void bio_put(struct bio *);
... ... @@ -288,8 +289,8 @@
288 289 int, int, gfp_t);
289 290 extern int bio_uncopy_user(struct bio *);
290 291 void zero_fill_bio(struct bio *bio);
291   -extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);
292   -extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int);
  292 +extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
  293 +extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
293 294 extern unsigned int bvec_nr_vecs(unsigned short idx);
294 295  
295 296 #ifdef CONFIG_BLK_CGROUP
296 297  
297 298  
... ... @@ -511,10 +512,11 @@
511 512 unsigned int front_pad;
512 513  
513 514 mempool_t *bio_pool;
  515 + mempool_t *bvec_pool;
514 516 #if defined(CONFIG_BLK_DEV_INTEGRITY)
515 517 mempool_t *bio_integrity_pool;
  518 + mempool_t *bvec_integrity_pool;
516 519 #endif
517   - mempool_t *bvec_pool;
518 520  
519 521 /*
520 522 * Deadlock avoidance for stacking block drivers: see comments in