Eric Lee / linux-smarc-t335x-v3.2

Download as
Browse Code ยป

Commit 9817064b68fef7e4580c6df1ea597e106b9ff88b

Authored by Jens Axboe
Committed by Jens Axboe
1 parent 4aff5e2333
Exists in master and in 4 other branches v3.2_AM335xPSP_04.06.00.11, v3.2_NEWT335xPSP_04.06.00.11, v3.2_SBC_SMARTMEN, v3.2_SMARCT335xPSP_04.06.00.11

[PATCH] elevator: move the backmerging logic into the elevator core 

Right now, every IO scheduler implements its own backmerging (except for
noop, which does no merging). That results in duplicated code for
essentially the same operation, which is never a good thing. This patch
moves the backmerging out of the io schedulers and into the elevator
core. We save 1.6kb of text and as a bonus get backmerging for noop as
well. Win-win!

Signed-off-by: Jens Axboe <axboe@suse.de>

Showing 7 changed files with 146 additions and 375 deletions Side-by-side Diff

... ... @@ -14,7 +14,6 @@
14 14 #include <linux/slab.h>
15 15 #include <linux/init.h>
16 16 #include <linux/compiler.h>
17   -#include <linux/hash.h>
18 17 #include <linux/rbtree.h>
19 18 #include <linux/interrupt.h>
20 19  
... ... @@ -95,7 +94,6 @@
95 94  
96 95 struct as_rq *next_arq[2]; /* next in sort order */
97 96 sector_t last_sector[2]; /* last REQ_SYNC & REQ_ASYNC sectors */
98   - struct hlist_head *hash; /* request hash */
99 97  
100 98 unsigned long exit_prob; /* probability a task will exit while
101 99 being waited on */
... ... @@ -162,11 +160,6 @@
162 160 struct io_context *io_context; /* The submitting task */
163 161  
164 162 /*
165   - * request hash, key is the ending offset (for back merge lookup)
166   - */
167   - struct hlist_node hash;
168   -
169   - /*
170 163 * expire fifo
171 164 */
172 165 struct list_head fifo;
... ... @@ -273,77 +266,6 @@
273 266 }
274 267  
275 268 /*
276   - * the back merge hash support functions
277   - */
278   -static const int as_hash_shift = 6;
279   -#define AS_HASH_BLOCK(sec) ((sec) >> 3)
280   -#define AS_HASH_FN(sec) (hash_long(AS_HASH_BLOCK((sec)), as_hash_shift))
281   -#define AS_HASH_ENTRIES (1 << as_hash_shift)
282   -#define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors)
283   -
284   -static inline void __as_del_arq_hash(struct as_rq *arq)
285   -{
286   - hlist_del_init(&arq->hash);
287   -}
288   -
289   -static inline void as_del_arq_hash(struct as_rq *arq)
290   -{
291   - if (!hlist_unhashed(&arq->hash))
292   - __as_del_arq_hash(arq);
293   -}
294   -
295   -static void as_add_arq_hash(struct as_data *ad, struct as_rq *arq)
296   -{
297   - struct request *rq = arq->request;
298   -
299   - BUG_ON(!hlist_unhashed(&arq->hash));
300   -
301   - hlist_add_head(&arq->hash, &ad->hash[AS_HASH_FN(rq_hash_key(rq))]);
302   -}
303   -
304   -/*
305   - * move hot entry to front of chain
306   - */
307   -static inline void as_hot_arq_hash(struct as_data *ad, struct as_rq *arq)
308   -{
309   - struct request *rq = arq->request;
310   - struct hlist_head *head = &ad->hash[AS_HASH_FN(rq_hash_key(rq))];
311   -
312   - if (hlist_unhashed(&arq->hash)) {
313   - WARN_ON(1);
314   - return;
315   - }
316   -
317   - if (&arq->hash != head->first) {
318   - hlist_del(&arq->hash);
319   - hlist_add_head(&arq->hash, head);
320   - }
321   -}
322   -
323   -static struct request *as_find_arq_hash(struct as_data *ad, sector_t offset)
324   -{
325   - struct hlist_head *hash_list = &ad->hash[AS_HASH_FN(offset)];
326   - struct hlist_node *entry, *next;
327   - struct as_rq *arq;
328   -
329   - hlist_for_each_entry_safe(arq, entry, next, hash_list, hash) {
330   - struct request *__rq = arq->request;
331   -
332   - BUG_ON(hlist_unhashed(&arq->hash));
333   -
334   - if (!rq_mergeable(__rq)) {
335   - as_del_arq_hash(arq);
336   - continue;
337   - }
338   -
339   - if (rq_hash_key(__rq) == offset)
340   - return __rq;
341   - }
342   -
343   - return NULL;
344   -}
345   -
346   -/*
347 269 * rb tree support functions
348 270 */
349 271 #define rb_entry_arq(node) rb_entry((node), struct as_rq, rb_node)
... ... @@ -1060,7 +982,6 @@
1060 982 ad->next_arq[data_dir] = as_find_next_arq(ad, arq);
1061 983  
1062 984 list_del_init(&arq->fifo);
1063   - as_del_arq_hash(arq);
1064 985 as_del_arq_rb(ad, arq);
1065 986 }
1066 987  
... ... @@ -1349,8 +1270,6 @@
1349 1270 }
1350 1271  
1351 1272 as_add_arq_rb(ad, arq);
1352   - if (rq_mergeable(arq->request))
1353   - as_add_arq_hash(ad, arq);
1354 1273  
1355 1274 /*
1356 1275 * set expire time (only used for reads) and add to fifo list
1357 1276  
1358 1277  
1359 1278  
... ... @@ -1428,42 +1347,17 @@
1428 1347 struct as_data *ad = q->elevator->elevator_data;
1429 1348 sector_t rb_key = bio->bi_sector + bio_sectors(bio);
1430 1349 struct request *__rq;
1431   - int ret;
1432 1350  
1433 1351 /*
1434   - * see if the merge hash can satisfy a back merge
1435   - */
1436   - __rq = as_find_arq_hash(ad, bio->bi_sector);
1437   - if (__rq) {
1438   - BUG_ON(__rq->sector + __rq->nr_sectors != bio->bi_sector);
1439   -
1440   - if (elv_rq_merge_ok(__rq, bio)) {
1441   - ret = ELEVATOR_BACK_MERGE;
1442   - goto out;
1443   - }
1444   - }
1445   -
1446   - /*
1447 1352 * check for front merge
1448 1353 */
1449 1354 __rq = as_find_arq_rb(ad, rb_key, bio_data_dir(bio));
1450   - if (__rq) {
1451   - BUG_ON(rb_key != rq_rb_key(__rq));
1452   -
1453   - if (elv_rq_merge_ok(__rq, bio)) {
1454   - ret = ELEVATOR_FRONT_MERGE;
1455   - goto out;
1456   - }
  1355 + if (__rq && elv_rq_merge_ok(__rq, bio)) {
  1356 + *req = __rq;
  1357 + return ELEVATOR_FRONT_MERGE;
1457 1358 }
1458 1359  
1459 1360 return ELEVATOR_NO_MERGE;
1460   -out:
1461   - if (ret) {
1462   - if (rq_mergeable(__rq))
1463   - as_hot_arq_hash(ad, RQ_DATA(__rq));
1464   - }
1465   - *req = __rq;
1466   - return ret;
1467 1361 }
1468 1362  
1469 1363 static void as_merged_request(request_queue_t *q, struct request *req)
... ... @@ -1472,12 +1366,6 @@
1472 1366 struct as_rq *arq = RQ_DATA(req);
1473 1367  
1474 1368 /*
1475   - * hash always needs to be repositioned, key is end sector
1476   - */
1477   - as_del_arq_hash(arq);
1478   - as_add_arq_hash(ad, arq);
1479   -
1480   - /*
1481 1369 * if the merge was a front merge, we need to reposition request
1482 1370 */
1483 1371 if (rq_rb_key(req) != arq->rb_key) {
... ... @@ -1501,13 +1389,6 @@
1501 1389 BUG_ON(!arq);
1502 1390 BUG_ON(!anext);
1503 1391  
1504   - /*
1505   - * reposition arq (this is the merged request) in hash, and in rbtree
1506   - * in case of a front merge
1507   - */
1508   - as_del_arq_hash(arq);
1509   - as_add_arq_hash(ad, arq);
1510   -
1511 1392 if (rq_rb_key(req) != arq->rb_key) {
1512 1393 as_del_arq_rb(ad, arq);
1513 1394 as_add_arq_rb(ad, arq);
... ... @@ -1591,7 +1472,6 @@
1591 1472 arq->request = rq;
1592 1473 arq->state = AS_RQ_PRESCHED;
1593 1474 arq->io_context = NULL;
1594   - INIT_HLIST_NODE(&arq->hash);
1595 1475 INIT_LIST_HEAD(&arq->fifo);
1596 1476 rq->elevator_private = arq;
1597 1477 return 0;
... ... @@ -1628,7 +1508,6 @@
1628 1508  
1629 1509 mempool_destroy(ad->arq_pool);
1630 1510 put_io_context(ad->io_context);
1631   - kfree(ad->hash);
1632 1511 kfree(ad);
1633 1512 }
1634 1513  
... ... @@ -1639,7 +1518,6 @@
1639 1518 static void *as_init_queue(request_queue_t *q, elevator_t *e)
1640 1519 {
1641 1520 struct as_data *ad;
1642   - int i;
1643 1521  
1644 1522 if (!arq_pool)
1645 1523 return NULL;
1646 1524  
... ... @@ -1651,17 +1529,9 @@
1651 1529  
1652 1530 ad->q = q; /* Identify what queue the data belongs to */
1653 1531  
1654   - ad->hash = kmalloc_node(sizeof(struct hlist_head)*AS_HASH_ENTRIES,
1655   - GFP_KERNEL, q->node);
1656   - if (!ad->hash) {
1657   - kfree(ad);
1658   - return NULL;
1659   - }
1660   -
1661 1532 ad->arq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
1662 1533 mempool_free_slab, arq_pool, q->node);
1663 1534 if (!ad->arq_pool) {
1664   - kfree(ad->hash);
1665 1535 kfree(ad);
1666 1536 return NULL;
1667 1537 }
... ... @@ -1671,9 +1541,6 @@
1671 1541 ad->antic_timer.data = (unsigned long)q;
1672 1542 init_timer(&ad->antic_timer);
1673 1543 INIT_WORK(&ad->antic_work, as_work_handler, q);
1674   -
1675   - for (i = 0; i < AS_HASH_ENTRIES; i++)
1676   - INIT_HLIST_HEAD(&ad->hash[i]);
1677 1544  
1678 1545 INIT_LIST_HEAD(&ad->fifo_list[REQ_SYNC]);
1679 1546 INIT_LIST_HEAD(&ad->fifo_list[REQ_ASYNC]);
... ... @@ -41,16 +41,6 @@
41 41 #define CFQ_QHASH_ENTRIES (1 << CFQ_QHASH_SHIFT)
42 42 #define list_entry_qhash(entry) hlist_entry((entry), struct cfq_queue, cfq_hash)
43 43  
44   -/*
45   - * for the hash of crq inside the cfqq
46   - */
47   -#define CFQ_MHASH_SHIFT 6
48   -#define CFQ_MHASH_BLOCK(sec) ((sec) >> 3)
49   -#define CFQ_MHASH_ENTRIES (1 << CFQ_MHASH_SHIFT)
50   -#define CFQ_MHASH_FN(sec) hash_long(CFQ_MHASH_BLOCK(sec), CFQ_MHASH_SHIFT)
51   -#define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors)
52   -#define list_entry_hash(ptr) hlist_entry((ptr), struct cfq_rq, hash)
53   -
54 44 #define list_entry_cfqq(ptr) list_entry((ptr), struct cfq_queue, cfq_list)
55 45 #define list_entry_fifo(ptr) list_entry((ptr), struct request, queuelist)
56 46  
... ... @@ -112,11 +102,6 @@
112 102 */
113 103 struct hlist_head *cfq_hash;
114 104  
115   - /*
116   - * global crq hash for all queues
117   - */
118   - struct hlist_head *crq_hash;
119   -
120 105 mempool_t *crq_pool;
121 106  
122 107 int rq_in_driver;
... ... @@ -203,7 +188,6 @@
203 188 struct rb_node rb_node;
204 189 sector_t rb_key;
205 190 struct request *request;
206   - struct hlist_node hash;
207 191  
208 192 struct cfq_queue *cfq_queue;
209 193 struct cfq_io_context *io_context;
... ... @@ -272,42 +256,6 @@
272 256 static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, unsigned int key, struct task_struct *tsk, gfp_t gfp_mask);
273 257  
274 258 /*
275   - * lots of deadline iosched dupes, can be abstracted later...
276   - */
277   -static inline void cfq_del_crq_hash(struct cfq_rq *crq)
278   -{
279   - hlist_del_init(&crq->hash);
280   -}
281   -
282   -static inline void cfq_add_crq_hash(struct cfq_data *cfqd, struct cfq_rq *crq)
283   -{
284   - const int hash_idx = CFQ_MHASH_FN(rq_hash_key(crq->request));
285   -
286   - hlist_add_head(&crq->hash, &cfqd->crq_hash[hash_idx]);
287   -}
288   -
289   -static struct request *cfq_find_rq_hash(struct cfq_data *cfqd, sector_t offset)
290   -{
291   - struct hlist_head *hash_list = &cfqd->crq_hash[CFQ_MHASH_FN(offset)];
292   - struct hlist_node *entry, *next;
293   -
294   - hlist_for_each_safe(entry, next, hash_list) {
295   - struct cfq_rq *crq = list_entry_hash(entry);
296   - struct request *__rq = crq->request;
297   -
298   - if (!rq_mergeable(__rq)) {
299   - cfq_del_crq_hash(crq);
300   - continue;
301   - }
302   -
303   - if (rq_hash_key(__rq) == offset)
304   - return __rq;
305   - }
306   -
307   - return NULL;
308   -}
309   -
310   -/*
311 259 * scheduler run of queue, if there are requests pending and no one in the
312 260 * driver that will restart queueing
313 261 */
... ... @@ -677,7 +625,6 @@
677 625  
678 626 list_del_init(&rq->queuelist);
679 627 cfq_del_crq_rb(crq);
680   - cfq_del_crq_hash(crq);
681 628 }
682 629  
683 630 static int
684 631  
685 632  
686 633  
687 634  
688 635  
... ... @@ -685,34 +632,20 @@
685 632 {
686 633 struct cfq_data *cfqd = q->elevator->elevator_data;
687 634 struct request *__rq;
688   - int ret;
689 635  
690   - __rq = cfq_find_rq_hash(cfqd, bio->bi_sector);
691   - if (__rq && elv_rq_merge_ok(__rq, bio)) {
692   - ret = ELEVATOR_BACK_MERGE;
693   - goto out;
694   - }
695   -
696 636 __rq = cfq_find_rq_fmerge(cfqd, bio);
697 637 if (__rq && elv_rq_merge_ok(__rq, bio)) {
698   - ret = ELEVATOR_FRONT_MERGE;
699   - goto out;
  638 + *req = __rq;
  639 + return ELEVATOR_FRONT_MERGE;
700 640 }
701 641  
702 642 return ELEVATOR_NO_MERGE;
703   -out:
704   - *req = __rq;
705   - return ret;
706 643 }
707 644  
708 645 static void cfq_merged_request(request_queue_t *q, struct request *req)
709 646 {
710   - struct cfq_data *cfqd = q->elevator->elevator_data;
711 647 struct cfq_rq *crq = RQ_DATA(req);
712 648  
713   - cfq_del_crq_hash(crq);
714   - cfq_add_crq_hash(cfqd, crq);
715   -
716 649 if (rq_rb_key(req) != crq->rb_key) {
717 650 struct cfq_queue *cfqq = crq->cfq_queue;
718 651  
... ... @@ -1825,9 +1758,6 @@
1825 1758  
1826 1759 list_add_tail(&rq->queuelist, &cfqq->fifo);
1827 1760  
1828   - if (rq_mergeable(rq))
1829   - cfq_add_crq_hash(cfqd, crq);
1830   -
1831 1761 cfq_crq_enqueued(cfqd, cfqq, crq);
1832 1762 }
1833 1763  
... ... @@ -2055,7 +1985,6 @@
2055 1985 RB_CLEAR_NODE(&crq->rb_node);
2056 1986 crq->rb_key = 0;
2057 1987 crq->request = rq;
2058   - INIT_HLIST_NODE(&crq->hash);
2059 1988 crq->cfq_queue = cfqq;
2060 1989 crq->io_context = cic;
2061 1990  
... ... @@ -2221,7 +2150,6 @@
2221 2150 cfq_shutdown_timer_wq(cfqd);
2222 2151  
2223 2152 mempool_destroy(cfqd->crq_pool);
2224   - kfree(cfqd->crq_hash);
2225 2153 kfree(cfqd->cfq_hash);
2226 2154 kfree(cfqd);
2227 2155 }
2228 2156  
2229 2157  
... ... @@ -2246,20 +2174,14 @@
2246 2174 INIT_LIST_HEAD(&cfqd->empty_list);
2247 2175 INIT_LIST_HEAD(&cfqd->cic_list);
2248 2176  
2249   - cfqd->crq_hash = kmalloc(sizeof(struct hlist_head) * CFQ_MHASH_ENTRIES, GFP_KERNEL);
2250   - if (!cfqd->crq_hash)
2251   - goto out_crqhash;
2252   -
2253 2177 cfqd->cfq_hash = kmalloc(sizeof(struct hlist_head) * CFQ_QHASH_ENTRIES, GFP_KERNEL);
2254 2178 if (!cfqd->cfq_hash)
2255   - goto out_cfqhash;
  2179 + goto out_crqhash;
2256 2180  
2257 2181 cfqd->crq_pool = mempool_create_slab_pool(BLKDEV_MIN_RQ, crq_pool);
2258 2182 if (!cfqd->crq_pool)
2259 2183 goto out_crqpool;
2260 2184  
2261   - for (i = 0; i < CFQ_MHASH_ENTRIES; i++)
2262   - INIT_HLIST_HEAD(&cfqd->crq_hash[i]);
2263 2185 for (i = 0; i < CFQ_QHASH_ENTRIES; i++)
2264 2186 INIT_HLIST_HEAD(&cfqd->cfq_hash[i]);
2265 2187  
... ... @@ -2289,8 +2211,6 @@
2289 2211 return cfqd;
2290 2212 out_crqpool:
2291 2213 kfree(cfqd->cfq_hash);
2292   -out_cfqhash:
2293   - kfree(cfqd->crq_hash);
2294 2214 out_crqhash:
2295 2215 kfree(cfqd);
2296 2216 return NULL;
block/deadline-iosched.c
Diff comments   View file @ 9817064
... ... @@ -12,7 +12,6 @@
12 12 #include <linux/slab.h>
13 13 #include <linux/init.h>
14 14 #include <linux/compiler.h>
15   -#include <linux/hash.h>
16 15 #include <linux/rbtree.h>
17 16  
18 17 /*
... ... @@ -24,13 +23,6 @@
24 23 static const int fifo_batch = 16; /* # of sequential requests treated as one
25 24 by the above parameters. For throughput. */
26 25  
27   -static const int deadline_hash_shift = 5;
28   -#define DL_HASH_BLOCK(sec) ((sec) >> 3)
29   -#define DL_HASH_FN(sec) (hash_long(DL_HASH_BLOCK((sec)), deadline_hash_shift))
30   -#define DL_HASH_ENTRIES (1 << deadline_hash_shift)
31   -#define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors)
32   -#define ON_HASH(drq) (!hlist_unhashed(&(drq)->hash))
33   -
34 26 struct deadline_data {
35 27 /*
36 28 * run time data
... ... @@ -46,7 +38,6 @@
46 38 * next in sort order. read, write or both are NULL
47 39 */
48 40 struct deadline_rq *next_drq[2];
49   - struct hlist_head *hash; /* request hash */
50 41 unsigned int batching; /* number of sequential requests made */
51 42 sector_t last_sector; /* head position */
52 43 unsigned int starved; /* times reads have starved writes */
... ... @@ -75,11 +66,6 @@
75 66 struct request *request;
76 67  
77 68 /*
78   - * request hash, key is the ending offset (for back merge lookup)
79   - */
80   - struct hlist_node hash;
81   -
82   - /*
83 69 * expire fifo
84 70 */
85 71 struct list_head fifo;
... ... @@ -93,69 +79,6 @@
93 79 #define RQ_DATA(rq) ((struct deadline_rq *) (rq)->elevator_private)
94 80  
95 81 /*
96   - * the back merge hash support functions
97   - */
98   -static inline void __deadline_del_drq_hash(struct deadline_rq *drq)
99   -{
100   - hlist_del_init(&drq->hash);
101   -}
102   -
103   -static inline void deadline_del_drq_hash(struct deadline_rq *drq)
104   -{
105   - if (ON_HASH(drq))
106   - __deadline_del_drq_hash(drq);
107   -}
108   -
109   -static inline void
110   -deadline_add_drq_hash(struct deadline_data *dd, struct deadline_rq *drq)
111   -{
112   - struct request *rq = drq->request;
113   -
114   - BUG_ON(ON_HASH(drq));
115   -
116   - hlist_add_head(&drq->hash, &dd->hash[DL_HASH_FN(rq_hash_key(rq))]);
117   -}
118   -
119   -/*
120   - * move hot entry to front of chain
121   - */
122   -static inline void
123   -deadline_hot_drq_hash(struct deadline_data *dd, struct deadline_rq *drq)
124   -{
125   - struct request *rq = drq->request;
126   - struct hlist_head *head = &dd->hash[DL_HASH_FN(rq_hash_key(rq))];
127   -
128   - if (ON_HASH(drq) && &drq->hash != head->first) {
129   - hlist_del(&drq->hash);
130   - hlist_add_head(&drq->hash, head);
131   - }
132   -}
133   -
134   -static struct request *
135   -deadline_find_drq_hash(struct deadline_data *dd, sector_t offset)
136   -{
137   - struct hlist_head *hash_list = &dd->hash[DL_HASH_FN(offset)];
138   - struct hlist_node *entry, *next;
139   - struct deadline_rq *drq;
140   -
141   - hlist_for_each_entry_safe(drq, entry, next, hash_list, hash) {
142   - struct request *__rq = drq->request;
143   -
144   - BUG_ON(!ON_HASH(drq));
145   -
146   - if (!rq_mergeable(__rq)) {
147   - __deadline_del_drq_hash(drq);
148   - continue;
149   - }
150   -
151   - if (rq_hash_key(__rq) == offset)
152   - return __rq;
153   - }
154   -
155   - return NULL;
156   -}
157   -
158   -/*
159 82 * rb tree support functions
160 83 */
161 84 #define rb_entry_drq(node) rb_entry((node), struct deadline_rq, rb_node)
162 85  
163 86  
164 87  
... ... @@ -267,22 +190,19 @@
267 190 {
268 191 struct deadline_data *dd = q->elevator->elevator_data;
269 192 struct deadline_rq *drq = RQ_DATA(rq);
270   -
271 193 const int data_dir = rq_data_dir(drq->request);
272 194  
273 195 deadline_add_drq_rb(dd, drq);
  196 +
274 197 /*
275 198 * set expire time (only used for reads) and add to fifo list
276 199 */
277 200 drq->expires = jiffies + dd->fifo_expire[data_dir];
278 201 list_add_tail(&drq->fifo, &dd->fifo_list[data_dir]);
279   -
280   - if (rq_mergeable(rq))
281   - deadline_add_drq_hash(dd, drq);
282 202 }
283 203  
284 204 /*
285   - * remove rq from rbtree, fifo, and hash
  205 + * remove rq from rbtree and fifo.
286 206 */
287 207 static void deadline_remove_request(request_queue_t *q, struct request *rq)
288 208 {
... ... @@ -291,7 +211,6 @@
291 211  
292 212 list_del_init(&drq->fifo);
293 213 deadline_del_drq_rb(dd, drq);
294   - deadline_del_drq_hash(drq);
295 214 }
296 215  
297 216 static int
... ... @@ -302,19 +221,6 @@
302 221 int ret;
303 222  
304 223 /*
305   - * see if the merge hash can satisfy a back merge
306   - */
307   - __rq = deadline_find_drq_hash(dd, bio->bi_sector);
308   - if (__rq) {
309   - BUG_ON(__rq->sector + __rq->nr_sectors != bio->bi_sector);
310   -
311   - if (elv_rq_merge_ok(__rq, bio)) {
312   - ret = ELEVATOR_BACK_MERGE;
313   - goto out;
314   - }
315   - }
316   -
317   - /*
318 224 * check for front merge
319 225 */
320 226 if (dd->front_merges) {
... ... @@ -333,8 +239,6 @@
333 239  
334 240 return ELEVATOR_NO_MERGE;
335 241 out:
336   - if (ret)
337   - deadline_hot_drq_hash(dd, RQ_DATA(__rq));
338 242 *req = __rq;
339 243 return ret;
340 244 }
... ... @@ -345,12 +249,6 @@
345 249 struct deadline_rq *drq = RQ_DATA(req);
346 250  
347 251 /*
348   - * hash always needs to be repositioned, key is end sector
349   - */
350   - deadline_del_drq_hash(drq);
351   - deadline_add_drq_hash(dd, drq);
352   -
353   - /*
354 252 * if the merge was a front merge, we need to reposition request
355 253 */
356 254 if (rq_rb_key(req) != drq->rb_key) {
... ... @@ -370,13 +268,6 @@
370 268 BUG_ON(!drq);
371 269 BUG_ON(!dnext);
372 270  
373   - /*
374   - * reposition drq (this is the merged request) in hash, and in rbtree
375   - * in case of a front merge
376   - */
377   - deadline_del_drq_hash(drq);
378   - deadline_add_drq_hash(dd, drq);
379   -
380 271 if (rq_rb_key(req) != drq->rb_key) {
381 272 deadline_del_drq_rb(dd, drq);
382 273 deadline_add_drq_rb(dd, drq);
... ... @@ -594,7 +485,6 @@
594 485 BUG_ON(!list_empty(&dd->fifo_list[WRITE]));
595 486  
596 487 mempool_destroy(dd->drq_pool);
597   - kfree(dd->hash);
598 488 kfree(dd);
599 489 }
600 490  
... ... @@ -605,7 +495,6 @@
605 495 static void *deadline_init_queue(request_queue_t *q, elevator_t *e)
606 496 {
607 497 struct deadline_data *dd;
608   - int i;
609 498  
610 499 if (!drq_pool)
611 500 return NULL;
612 501  
613 502  
... ... @@ -615,24 +504,13 @@
615 504 return NULL;
616 505 memset(dd, 0, sizeof(*dd));
617 506  
618   - dd->hash = kmalloc_node(sizeof(struct hlist_head)*DL_HASH_ENTRIES,
619   - GFP_KERNEL, q->node);
620   - if (!dd->hash) {
621   - kfree(dd);
622   - return NULL;
623   - }
624   -
625 507 dd->drq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
626 508 mempool_free_slab, drq_pool, q->node);
627 509 if (!dd->drq_pool) {
628   - kfree(dd->hash);
629 510 kfree(dd);
630 511 return NULL;
631 512 }
632 513  
633   - for (i = 0; i < DL_HASH_ENTRIES; i++)
634   - INIT_HLIST_HEAD(&dd->hash[i]);
635   -
636 514 INIT_LIST_HEAD(&dd->fifo_list[READ]);
637 515 INIT_LIST_HEAD(&dd->fifo_list[WRITE]);
638 516 dd->sort_list[READ] = RB_ROOT;
... ... @@ -666,8 +544,6 @@
666 544 memset(drq, 0, sizeof(*drq));
667 545 RB_CLEAR_NODE(&drq->rb_node);
668 546 drq->request = rq;
669   -
670   - INIT_HLIST_NODE(&drq->hash);
671 547  
672 548 INIT_LIST_HEAD(&drq->fifo);
673 549  
... ... @@ -33,6 +33,7 @@
33 33 #include <linux/compiler.h>
34 34 #include <linux/delay.h>
35 35 #include <linux/blktrace_api.h>
  36 +#include <linux/hash.h>
36 37  
37 38 #include <asm/uaccess.h>
38 39  
... ... @@ -40,6 +41,16 @@
40 41 static LIST_HEAD(elv_list);
41 42  
42 43 /*
  44 + * Merge hash stuff.
  45 + */
  46 +static const int elv_hash_shift = 6;
  47 +#define ELV_HASH_BLOCK(sec) ((sec) >> 3)
  48 +#define ELV_HASH_FN(sec) (hash_long(ELV_HASH_BLOCK((sec)), elv_hash_shift))
  49 +#define ELV_HASH_ENTRIES (1 << elv_hash_shift)
  50 +#define rq_hash_key(rq) ((rq)->sector + (rq)->nr_sectors)
  51 +#define ELV_ON_HASH(rq) (!hlist_unhashed(&(rq)->hash))
  52 +
  53 +/*
43 54 * can we safely merge with this request?
44 55 */
45 56 inline int elv_rq_merge_ok(struct request *rq, struct bio *bio)
46 57  
47 58  
48 59  
... ... @@ -153,25 +164,41 @@
153 164  
154 165 static elevator_t *elevator_alloc(struct elevator_type *e)
155 166 {
156   - elevator_t *eq = kmalloc(sizeof(elevator_t), GFP_KERNEL);
157   - if (eq) {
158   - memset(eq, 0, sizeof(*eq));
159   - eq->ops = &e->ops;
160   - eq->elevator_type = e;
161   - kobject_init(&eq->kobj);
162   - snprintf(eq->kobj.name, KOBJ_NAME_LEN, "%s", "iosched");
163   - eq->kobj.ktype = &elv_ktype;
164   - mutex_init(&eq->sysfs_lock);
165   - } else {
166   - elevator_put(e);
167   - }
  167 + elevator_t *eq;
  168 + int i;
  169 +
  170 + eq = kmalloc(sizeof(elevator_t), GFP_KERNEL);
  171 + if (unlikely(!eq))
  172 + goto err;
  173 +
  174 + memset(eq, 0, sizeof(*eq));
  175 + eq->ops = &e->ops;
  176 + eq->elevator_type = e;
  177 + kobject_init(&eq->kobj);
  178 + snprintf(eq->kobj.name, KOBJ_NAME_LEN, "%s", "iosched");
  179 + eq->kobj.ktype = &elv_ktype;
  180 + mutex_init(&eq->sysfs_lock);
  181 +
  182 + eq->hash = kmalloc(sizeof(struct hlist_head) * ELV_HASH_ENTRIES, GFP_KERNEL);
  183 + if (!eq->hash)
  184 + goto err;
  185 +
  186 + for (i = 0; i < ELV_HASH_ENTRIES; i++)
  187 + INIT_HLIST_HEAD(&eq->hash[i]);
  188 +
168 189 return eq;
  190 +err:
  191 + kfree(eq);
  192 + elevator_put(e);
  193 + return NULL;
169 194 }
170 195  
171 196 static void elevator_release(struct kobject *kobj)
172 197 {
173 198 elevator_t *e = container_of(kobj, elevator_t, kobj);
  199 +
174 200 elevator_put(e->elevator_type);
  201 + kfree(e->hash);
175 202 kfree(e);
176 203 }
177 204  
... ... @@ -223,6 +250,53 @@
223 250 kobject_put(&e->kobj);
224 251 }
225 252  
  253 +static inline void __elv_rqhash_del(struct request *rq)
  254 +{
  255 + hlist_del_init(&rq->hash);
  256 +}
  257 +
  258 +static void elv_rqhash_del(request_queue_t *q, struct request *rq)
  259 +{
  260 + if (ELV_ON_HASH(rq))
  261 + __elv_rqhash_del(rq);
  262 +}
  263 +
  264 +static void elv_rqhash_add(request_queue_t *q, struct request *rq)
  265 +{
  266 + elevator_t *e = q->elevator;
  267 +
  268 + BUG_ON(ELV_ON_HASH(rq));
  269 + hlist_add_head(&rq->hash, &e->hash[ELV_HASH_FN(rq_hash_key(rq))]);
  270 +}
  271 +
  272 +static void elv_rqhash_reposition(request_queue_t *q, struct request *rq)
  273 +{
  274 + __elv_rqhash_del(rq);
  275 + elv_rqhash_add(q, rq);
  276 +}
  277 +
  278 +static struct request *elv_rqhash_find(request_queue_t *q, sector_t offset)
  279 +{
  280 + elevator_t *e = q->elevator;
  281 + struct hlist_head *hash_list = &e->hash[ELV_HASH_FN(offset)];
  282 + struct hlist_node *entry, *next;
  283 + struct request *rq;
  284 +
  285 + hlist_for_each_entry_safe(rq, entry, next, hash_list, hash) {
  286 + BUG_ON(!ELV_ON_HASH(rq));
  287 +
  288 + if (unlikely(!rq_mergeable(rq))) {
  289 + __elv_rqhash_del(rq);
  290 + continue;
  291 + }
  292 +
  293 + if (rq_hash_key(rq) == offset)
  294 + return rq;
  295 + }
  296 +
  297 + return NULL;
  298 +}
  299 +
226 300 /*
227 301 * Insert rq into dispatch queue of q. Queue lock must be held on
228 302 * entry. If sort != 0, rq is sort-inserted; otherwise, rq will be
... ... @@ -235,6 +309,9 @@
235 309  
236 310 if (q->last_merge == rq)
237 311 q->last_merge = NULL;
  312 +
  313 + elv_rqhash_del(q, rq);
  314 +
238 315 q->nr_sorted--;
239 316  
240 317 boundary = q->end_sector;
241 318  
242 319  
... ... @@ -258,11 +335,32 @@
258 335 list_add(&rq->queuelist, entry);
259 336 }
260 337  
  338 +/*
  339 + * This should be in elevator.h, but that requires pulling in rq and q
  340 + */
  341 +void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)
  342 +{
  343 + if (q->last_merge == rq)
  344 + q->last_merge = NULL;
  345 +
  346 + elv_rqhash_del(q, rq);
  347 +
  348 + q->nr_sorted--;
  349 +
  350 + q->end_sector = rq_end_sector(rq);
  351 + q->boundary_rq = rq;
  352 + list_add_tail(&rq->queuelist, &q->queue_head);
  353 +}
  354 +
261 355 int elv_merge(request_queue_t *q, struct request **req, struct bio *bio)
262 356 {
263 357 elevator_t *e = q->elevator;
  358 + struct request *__rq;
264 359 int ret;
265 360  
  361 + /*
  362 + * First try one-hit cache.
  363 + */
266 364 if (q->last_merge) {
267 365 ret = elv_try_merge(q->last_merge, bio);
268 366 if (ret != ELEVATOR_NO_MERGE) {
... ... @@ -271,6 +369,15 @@
271 369 }
272 370 }
273 371  
  372 + /*
  373 + * See if our hash lookup can find a potential backmerge.
  374 + */
  375 + __rq = elv_rqhash_find(q, bio->bi_sector);
  376 + if (__rq && elv_rq_merge_ok(__rq, bio)) {
  377 + *req = __rq;
  378 + return ELEVATOR_BACK_MERGE;
  379 + }
  380 +
274 381 if (e->ops->elevator_merge_fn)
275 382 return e->ops->elevator_merge_fn(q, req, bio);
276 383  
... ... @@ -284,6 +391,8 @@
284 391 if (e->ops->elevator_merged_fn)
285 392 e->ops->elevator_merged_fn(q, rq);
286 393  
  394 + elv_rqhash_reposition(q, rq);
  395 +
287 396 q->last_merge = rq;
288 397 }
289 398  
290 399  
... ... @@ -294,8 +403,11 @@
294 403  
295 404 if (e->ops->elevator_merge_req_fn)
296 405 e->ops->elevator_merge_req_fn(q, rq, next);
297   - q->nr_sorted--;
298 406  
  407 + elv_rqhash_reposition(q, rq);
  408 + elv_rqhash_del(q, next);
  409 +
  410 + q->nr_sorted--;
299 411 q->last_merge = rq;
300 412 }
301 413  
... ... @@ -371,8 +483,12 @@
371 483 BUG_ON(!blk_fs_request(rq));
372 484 rq->cmd_flags |= REQ_SORTED;
373 485 q->nr_sorted++;
374   - if (q->last_merge == NULL && rq_mergeable(rq))
375   - q->last_merge = rq;
  486 + if (rq_mergeable(rq)) {
  487 + elv_rqhash_add(q, rq);
  488 + if (!q->last_merge)
  489 + q->last_merge = rq;
  490 + }
  491 +
376 492 /*
377 493 * Some ioscheds (cfq) run q->request_fn directly, so
378 494 * rq cannot be accessed after calling
... ... @@ -557,6 +673,7 @@
557 673 void elv_dequeue_request(request_queue_t *q, struct request *rq)
558 674 {
559 675 BUG_ON(list_empty(&rq->queuelist));
  676 + BUG_ON(ELV_ON_HASH(rq));
560 677  
561 678 list_del_init(&rq->queuelist);
562 679  
... ... @@ -281,6 +281,7 @@
281 281 {
282 282 INIT_LIST_HEAD(&rq->queuelist);
283 283 INIT_LIST_HEAD(&rq->donelist);
  284 + INIT_HLIST_NODE(&rq->hash);
284 285  
285 286 rq->errors = 0;
286 287 rq->rq_status = RQ_ACTIVE;
... ... @@ -2700,6 +2701,7 @@
2700 2701 int priv = req->cmd_flags & REQ_ELVPRIV;
2701 2702  
2702 2703 BUG_ON(!list_empty(&req->queuelist));
  2704 + BUG_ON(!hlist_unhashed(&req->hash));
2703 2705  
2704 2706 blk_free_request(q, req);
2705 2707 freed_request(q, rw, priv);
include/linux/blkdev.h
Diff comments   View file @ 9817064
... ... @@ -229,6 +229,8 @@
229 229 struct bio *bio;
230 230 struct bio *biotail;
231 231  
  232 + struct hlist_node hash; /* merge hash */
  233 +
232 234 void *elevator_private;
233 235 void *completion_data;
234 236  
... ... @@ -694,21 +696,6 @@
694 696 static inline void blkdev_dequeue_request(struct request *req)
695 697 {
696 698 elv_dequeue_request(req->q, req);
697   -}
698   -
699   -/*
700   - * This should be in elevator.h, but that requires pulling in rq and q
701   - */
702   -static inline void elv_dispatch_add_tail(struct request_queue *q,
703   - struct request *rq)
704   -{
705   - if (q->last_merge == rq)
706   - q->last_merge = NULL;
707   - q->nr_sorted--;
708   -
709   - q->end_sector = rq_end_sector(rq);
710   - q->boundary_rq = rq;
711   - list_add_tail(&rq->queuelist, &q->queue_head);
712 699 }
713 700  
714 701 /*
include/linux/elevator.h
Diff comments   View file @ 9817064
... ... @@ -82,12 +82,14 @@
82 82 struct kobject kobj;
83 83 struct elevator_type *elevator_type;
84 84 struct mutex sysfs_lock;
  85 + struct hlist_head *hash;
85 86 };
86 87  
87 88 /*
88 89 * block elevator interface
89 90 */
90 91 extern void elv_dispatch_sort(request_queue_t *, struct request *);
  92 +extern void elv_dispatch_add_tail(request_queue_t *, struct request *);
91 93 extern void elv_add_request(request_queue_t *, struct request *, int, int);
92 94 extern void __elv_add_request(request_queue_t *, struct request *, int, int);
93 95 extern void elv_insert(request_queue_t *, struct request *, int);