block: drop queue lock before calling __blk_run_queue() for kblockd punt

If we know we are going to punt to kblockd, we can drop the queue lock before calling into __blk_run_queue() since it only does a safe bit test and a workqueue call. Since kblockd needs to grab this very lock as one of the first things it does, it's a good optimization to drop the lock before waking kblockd. Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

block: drop queue lock before calling __blk_run_queue() for kblockd punt
If we know we are going to punt to kblockd, we can drop the queue lock before calling into __blk_run_queue() since it only does a safe bit test and a workqueue call. Since kblockd needs to grab this very lock as one of the first things it does, it's a good optimization to drop the lock before waking kblockd. Signed-off-by: Jens Axboe <jaxboe@fusionio.com>
Jens Axboe
1 parent b4cb290e0a
Showing 1 changed file with 25 additions and 8 deletions Side-by-side Diff
block/blk-core.c
@@ -295,7 +295,8 @@
  *
  * Description:
  *    See @blk_run_queue. This variant must be called with the queue lock
- *    held and interrupts disabled.
+ *    held and interrupts disabled. If force_kblockd is true, then it is
+ *    safe to call this without holding the queue lock.
  *
  */
 void __blk_run_queue(struct request_queue *q, bool force_kblockd)
  
@@ -2671,9 +2672,23 @@
  */
 static void queue_unplugged(struct request_queue *q, unsigned int depth,
 			    bool from_schedule)
+	__releases(q->queue_lock)
 {
 	trace_block_unplug(q, depth, !from_schedule);
-	__blk_run_queue(q, from_schedule);
+
+	/*
+	 * If we are punting this to kblockd, then we can safely drop
+	 * the queue_lock before waking kblockd (which needs to take
+	 * this lock).
+	 */
+	if (from_schedule) {
+		spin_unlock(q->queue_lock);
+		__blk_run_queue(q, true);
+	} else {
+		__blk_run_queue(q, false);
+		spin_unlock(q->queue_lock);
+	}
+
 }
  
 static void flush_plug_callbacks(struct blk_plug *plug)
  
@@ -2729,10 +2744,11 @@
 		BUG_ON(!(rq->cmd_flags & REQ_ON_PLUG));
 		BUG_ON(!rq->q);
 		if (rq->q != q) {
-			if (q) {
+			/*
+			 * This drops the queue lock
+			 */
+			if (q)
 				queue_unplugged(q, depth, from_schedule);
-				spin_unlock(q->queue_lock);
-			}
 			q = rq->q;
 			depth = 0;
 			spin_lock(q->queue_lock);
  
@@ -2750,10 +2766,11 @@
 		depth++;
 	}
  
-	if (q) {
+	/*
+	 * This drops the queue lock
+	 */
+	if (q)
 		queue_unplugged(q, depth, from_schedule);
-		spin_unlock(q->queue_lock);
-	}
  
 	local_irq_restore(flags);
 }
...	...	@@ -295,7 +295,8 @@
295	295	*
296	296	* Description:
297	297	* See @blk_run_queue. This variant must be called with the queue lock
298		- * held and interrupts disabled.
	298	+ * held and interrupts disabled. If force_kblockd is true, then it is
	299	+ * safe to call this without holding the queue lock.
299	300	*
300	301	*/
301	302	void __blk_run_queue(struct request_queue *q, bool force_kblockd)
302	303
...	...	@@ -2671,9 +2672,23 @@
2671	2672	*/
2672	2673	static void queue_unplugged(struct request_queue *q, unsigned int depth,
2673	2674	bool from_schedule)
	2675	+ __releases(q->queue_lock)
2674	2676	{
2675	2677	trace_block_unplug(q, depth, !from_schedule);
2676		- __blk_run_queue(q, from_schedule);
	2678	+
	2679	+ /*
	2680	+ * If we are punting this to kblockd, then we can safely drop
	2681	+ * the queue_lock before waking kblockd (which needs to take
	2682	+ * this lock).
	2683	+ */
	2684	+ if (from_schedule) {
	2685	+ spin_unlock(q->queue_lock);
	2686	+ __blk_run_queue(q, true);
	2687	+ } else {
	2688	+ __blk_run_queue(q, false);
	2689	+ spin_unlock(q->queue_lock);
	2690	+ }
	2691	+
2677	2692	}
2678	2693
2679	2694	static void flush_plug_callbacks(struct blk_plug *plug)
2680	2695
...	...	@@ -2729,10 +2744,11 @@
2729	2744	BUG_ON(!(rq->cmd_flags & REQ_ON_PLUG));
2730	2745	BUG_ON(!rq->q);
2731	2746	if (rq->q != q) {
2732		- if (q) {
	2747	+ /*
	2748	+ * This drops the queue lock
	2749	+ */
	2750	+ if (q)
2733	2751	queue_unplugged(q, depth, from_schedule);
2734		- spin_unlock(q->queue_lock);
2735		- }
2736	2752	q = rq->q;
2737	2753	depth = 0;
2738	2754	spin_lock(q->queue_lock);
2739	2755
...	...	@@ -2750,10 +2766,11 @@
2750	2766	depth++;
2751	2767	}
2752	2768
2753		- if (q) {
	2769	+ /*
	2770	+ * This drops the queue lock
	2771	+ */
	2772	+ if (q)
2754	2773	queue_unplugged(q, depth, from_schedule);
2755		- spin_unlock(q->queue_lock);
2756		- }
2757	2774
2758	2775	local_irq_restore(flags);
2759	2776	}