blk-throttle: make throtl_pending_timer_fn() ready for hierarchy

throtl_pending_timer_fn() currently assumes that the parent_sq is the top level one and the bio's dispatched are ready to be issued; however, this assumption will be wrong with proper hierarchy support. This patch makes the following changes to make throtl_pending_timer_fn() ready for hiearchy. * If the parent_sq isn't the top-level one, update the parent throtl_grp's dispatch time and schedule the next dispatch as necessary. If the parent's dispatch time is now, repeat the function for the parent throtl_grp. * If the parent_sq is the top-level one, kick issue work_item as before. * The debug message printed by throtl_log() now prints out the service_queue's nr_queued[] instead of the total nr_queued as the latter becomes uninteresting and misleading with hierarchical dispatch. Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Vivek Goyal <vgoyal@redhat.com>

blk-throttle: make throtl_pending_timer_fn() ready for hierarchy
throtl_pending_timer_fn() currently assumes that the parent_sq is the top level one and the bio's dispatched are ready to be issued; however, this assumption will be wrong with proper hierarchy support. This patch makes the following changes to make throtl_pending_timer_fn() ready for hiearchy. * If the parent_sq isn't the top-level one, update the parent throtl_grp's dispatch time and schedule the next dispatch as necessary. If the parent's dispatch time is now, repeat the function for the parent throtl_grp. * If the parent_sq is the top-level one, kick issue work_item as before. * The debug message printed by throtl_log() now prints out the service_queue's nr_queued[] instead of the total nr_queued as the latter becomes uninteresting and misleading with hierarchical dispatch. Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Vivek Goyal <vgoyal@redhat.com>
Tejun Heo
1 parent 6bc9c2b464
Showing 1 changed file with 33 additions and 7 deletions Side-by-side Diff
block/blk-throttle.c
@@ -952,23 +952,33 @@
  * This timer is armed when a child throtl_grp with active bio's become
  * pending and queued on the service_queue's pending_tree and expires when
  * the first child throtl_grp should be dispatched.  This function
- * dispatches bio's from the children throtl_grps and kicks
- * throtl_data->dispatch_work if there are bio's ready to be issued.
+ * dispatches bio's from the children throtl_grps to the parent
+ * service_queue.
+ *
+ * If the parent's parent is another throtl_grp, dispatching is propagated
+ * by either arming its pending_timer or repeating dispatch directly.  If
+ * the top-level service_tree is reached, throtl_data->dispatch_work is
+ * kicked so that the ready bio's are issued.
  */
 static void throtl_pending_timer_fn(unsigned long arg)
 {
 	struct throtl_service_queue *sq = (void *)arg;
+	struct throtl_grp *tg = sq_to_tg(sq);
 	struct throtl_data *td = sq_to_td(sq);
 	struct request_queue *q = td->queue;
-	bool dispatched = false;
+	struct throtl_service_queue *parent_sq;
+	bool dispatched;
 	int ret;
  
 	spin_lock_irq(q->queue_lock);
+again:
+	parent_sq = sq->parent_sq;
+	dispatched = false;
  
 	while (true) {
 		throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",
-			   td->nr_queued[READ] + td->nr_queued[WRITE],
-			   td->nr_queued[READ], td->nr_queued[WRITE]);
+			   sq->nr_queued[READ] + sq->nr_queued[WRITE],
+			   sq->nr_queued[READ], sq->nr_queued[WRITE]);
  
 		ret = throtl_select_dispatch(sq);
 		if (ret) {
  
@@ -985,9 +995,25 @@
 		spin_lock_irq(q->queue_lock);
 	}
  
-	if (dispatched)
-		queue_work(kthrotld_workqueue, &td->dispatch_work);
+	if (!dispatched)
+		goto out_unlock;
  
+	if (parent_sq) {
+		/* @parent_sq is another throl_grp, propagate dispatch */
+		if (tg->flags & THROTL_TG_WAS_EMPTY) {
+			tg_update_disptime(tg);
+			if (!throtl_schedule_next_dispatch(parent_sq, false)) {
+				/* window is already open, repeat dispatching */
+				sq = parent_sq;
+				tg = sq_to_tg(sq);
+				goto again;
+			}
+		}
+	} else {
+		/* reached the top-level, queue issueing */
+		queue_work(kthrotld_workqueue, &td->dispatch_work);
+	}
+out_unlock:
 	spin_unlock_irq(q->queue_lock);
 }
...	...	@@ -952,23 +952,33 @@
952	952	* This timer is armed when a child throtl_grp with active bio's become
953	953	* pending and queued on the service_queue's pending_tree and expires when
954	954	* the first child throtl_grp should be dispatched. This function
955		- * dispatches bio's from the children throtl_grps and kicks
956		- * throtl_data->dispatch_work if there are bio's ready to be issued.
	955	+ * dispatches bio's from the children throtl_grps to the parent
	956	+ * service_queue.
	957	+ *
	958	+ * If the parent's parent is another throtl_grp, dispatching is propagated
	959	+ * by either arming its pending_timer or repeating dispatch directly. If
	960	+ * the top-level service_tree is reached, throtl_data->dispatch_work is
	961	+ * kicked so that the ready bio's are issued.
957	962	*/
958	963	static void throtl_pending_timer_fn(unsigned long arg)
959	964	{
960	965	struct throtl_service_queue sq = (void )arg;
	966	+ struct throtl_grp *tg = sq_to_tg(sq);
961	967	struct throtl_data *td = sq_to_td(sq);
962	968	struct request_queue *q = td->queue;
963		- bool dispatched = false;
	969	+ struct throtl_service_queue *parent_sq;
	970	+ bool dispatched;
964	971	int ret;
965	972
966	973	spin_lock_irq(q->queue_lock);
	974	+again:
	975	+ parent_sq = sq->parent_sq;
	976	+ dispatched = false;
967	977
968	978	while (true) {
969	979	throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",
970		- td->nr_queued[READ] + td->nr_queued[WRITE],
971		- td->nr_queued[READ], td->nr_queued[WRITE]);
	980	+ sq->nr_queued[READ] + sq->nr_queued[WRITE],
	981	+ sq->nr_queued[READ], sq->nr_queued[WRITE]);
972	982
973	983	ret = throtl_select_dispatch(sq);
974	984	if (ret) {
975	985
...	...	@@ -985,9 +995,25 @@
985	995	spin_lock_irq(q->queue_lock);
986	996	}
987	997
988		- if (dispatched)
989		- queue_work(kthrotld_workqueue, &td->dispatch_work);
	998	+ if (!dispatched)
	999	+ goto out_unlock;
990	1000
	1001	+ if (parent_sq) {
	1002	+ /* @parent_sq is another throl_grp, propagate dispatch */
	1003	+ if (tg->flags & THROTL_TG_WAS_EMPTY) {
	1004	+ tg_update_disptime(tg);
	1005	+ if (!throtl_schedule_next_dispatch(parent_sq, false)) {
	1006	+ /* window is already open, repeat dispatching */
	1007	+ sq = parent_sq;
	1008	+ tg = sq_to_tg(sq);
	1009	+ goto again;
	1010	+ }
	1011	+ }
	1012	+ } else {
	1013	+ /* reached the top-level, queue issueing */
	1014	+ queue_work(kthrotld_workqueue, &td->dispatch_work);
	1015	+ }
	1016	+out_unlock:
991	1017	spin_unlock_irq(q->queue_lock);
992	1018	}
993	1019