scsi: implement .get_budget and .put_budget for blk-mq

We need to tell blk-mq to reserve resources before queuing one request, so implement these two callbacks. Then blk-mq can avoid to dequeue request too early, and IO merging can be improved a lot. Signed-off-by: Ming Lei <ming.lei@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>

scsi: implement .get_budget and .put_budget for blk-mq
We need to tell blk-mq to reserve resources before queuing one request, so implement these two callbacks. Then blk-mq can avoid to dequeue request too early, and IO merging can be improved a lot. Signed-off-by: Ming Lei <ming.lei@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Ming Lei · Jens Axboe
1 parent aeec77629a
Showing 1 changed file with 52 additions and 23 deletions Side-by-side Diff
drivers/scsi/scsi_lib.c
@@ -1946,25 +1946,32 @@
 	blk_mq_complete_request(cmd->request);
 }
  
-static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
-			 const struct blk_mq_queue_data *bd)
+static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
 {
-	struct request *req = bd->rq;
-	struct request_queue *q = req->q;
+	struct request_queue *q = hctx->queue;
 	struct scsi_device *sdev = q->queuedata;
 	struct Scsi_Host *shost = sdev->host;
-	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+
+	atomic_dec(&shost->host_busy);
+	if (scsi_target(sdev)->can_queue > 0)
+		atomic_dec(&scsi_target(sdev)->target_busy);
+	atomic_dec(&sdev->device_busy);
+	put_device(&sdev->sdev_gendev);
+}
+
+static blk_status_t scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
+{
+	struct request_queue *q = hctx->queue;
+	struct scsi_device *sdev = q->queuedata;
+	struct Scsi_Host *shost = sdev->host;
 	blk_status_t ret;
-	int reason;
  
-	ret = prep_to_mq(scsi_prep_state_check(sdev, req));
-	if (ret != BLK_STS_OK)
-		goto out;
+	ret = prep_to_mq(scsi_prep_state_check(sdev, NULL));
+	if (ret == BLK_STS_RESOURCE || ret != BLK_STS_OK)
+		return ret;
  
-	ret = BLK_STS_RESOURCE;
 	if (!get_device(&sdev->sdev_gendev))
 		goto out;
-
 	if (!scsi_dev_queue_ready(q, sdev))
 		goto out_put_device;
 	if (!scsi_target_queue_ready(shost, sdev))
  
@@ -1972,10 +1979,38 @@
 	if (!scsi_host_queue_ready(q, shost, sdev))
 		goto out_dec_target_busy;
  
+	return BLK_STS_OK;
+
+out_dec_target_busy:
+	if (scsi_target(sdev)->can_queue > 0)
+		atomic_dec(&scsi_target(sdev)->target_busy);
+out_dec_device_busy:
+	atomic_dec(&sdev->device_busy);
+out_put_device:
+	put_device(&sdev->sdev_gendev);
+out:
+	return BLK_STS_RESOURCE;
+}
+
+static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
+			 const struct blk_mq_queue_data *bd)
+{
+	struct request *req = bd->rq;
+	struct request_queue *q = req->q;
+	struct scsi_device *sdev = q->queuedata;
+	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
+	blk_status_t ret;
+	int reason;
+
+	ret = prep_to_mq(scsi_prep_state_check(sdev, req));
+	if (ret != BLK_STS_OK)
+		goto out_put_budget;
+
+	ret = BLK_STS_RESOURCE;
 	if (!(req->rq_flags & RQF_DONTPREP)) {
 		ret = prep_to_mq(scsi_mq_prep_fn(req));
 		if (ret != BLK_STS_OK)
-			goto out_dec_host_busy;
+			goto out_put_budget;
 		req->rq_flags |= RQF_DONTPREP;
 	} else {
 		blk_mq_start_request(req);
  
@@ -1993,21 +2028,13 @@
 	if (reason) {
 		scsi_set_blocked(cmd, reason);
 		ret = BLK_STS_RESOURCE;
-		goto out_dec_host_busy;
+		goto out_put_budget;
 	}
  
 	return BLK_STS_OK;
  
-out_dec_host_busy:
-	atomic_dec(&shost->host_busy);
-out_dec_target_busy:
-	if (scsi_target(sdev)->can_queue > 0)
-		atomic_dec(&scsi_target(sdev)->target_busy);
-out_dec_device_busy:
-	atomic_dec(&sdev->device_busy);
-out_put_device:
-	put_device(&sdev->sdev_gendev);
-out:
+out_put_budget:
+	scsi_mq_put_budget(hctx);
 	switch (ret) {
 	case BLK_STS_OK:
 		break;
@@ -2211,6 +2238,8 @@
 }
  
 static const struct blk_mq_ops scsi_mq_ops = {
+	.get_budget	= scsi_mq_get_budget,
+	.put_budget	= scsi_mq_put_budget,
 	.queue_rq	= scsi_queue_rq,
 	.complete	= scsi_softirq_done,
 	.timeout	= scsi_timeout,
...	...	@@ -1946,25 +1946,32 @@
1946	1946	blk_mq_complete_request(cmd->request);
1947	1947	}
1948	1948
1949		-static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
1950		- const struct blk_mq_queue_data *bd)
	1949	+static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
1951	1950	{
1952		- struct request *req = bd->rq;
1953		- struct request_queue *q = req->q;
	1951	+ struct request_queue *q = hctx->queue;
1954	1952	struct scsi_device *sdev = q->queuedata;
1955	1953	struct Scsi_Host *shost = sdev->host;
1956		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
	1954	+
	1955	+ atomic_dec(&shost->host_busy);
	1956	+ if (scsi_target(sdev)->can_queue > 0)
	1957	+ atomic_dec(&scsi_target(sdev)->target_busy);
	1958	+ atomic_dec(&sdev->device_busy);
	1959	+ put_device(&sdev->sdev_gendev);
	1960	+}
	1961	+
	1962	+static blk_status_t scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
	1963	+{
	1964	+ struct request_queue *q = hctx->queue;
	1965	+ struct scsi_device *sdev = q->queuedata;
	1966	+ struct Scsi_Host *shost = sdev->host;
1957	1967	blk_status_t ret;
1958		- int reason;
1959	1968
1960		- ret = prep_to_mq(scsi_prep_state_check(sdev, req));
1961		- if (ret != BLK_STS_OK)
1962		- goto out;
	1969	+ ret = prep_to_mq(scsi_prep_state_check(sdev, NULL));
	1970	+ if (ret == BLK_STS_RESOURCE \|\| ret != BLK_STS_OK)
	1971	+ return ret;
1963	1972
1964		- ret = BLK_STS_RESOURCE;
1965	1973	if (!get_device(&sdev->sdev_gendev))
1966	1974	goto out;
1967		-
1968	1975	if (!scsi_dev_queue_ready(q, sdev))
1969	1976	goto out_put_device;
1970	1977	if (!scsi_target_queue_ready(shost, sdev))
1971	1978
...	...	@@ -1972,10 +1979,38 @@
1972	1979	if (!scsi_host_queue_ready(q, shost, sdev))
1973	1980	goto out_dec_target_busy;
1974	1981
	1982	+ return BLK_STS_OK;
	1983	+
	1984	+out_dec_target_busy:
	1985	+ if (scsi_target(sdev)->can_queue > 0)
	1986	+ atomic_dec(&scsi_target(sdev)->target_busy);
	1987	+out_dec_device_busy:
	1988	+ atomic_dec(&sdev->device_busy);
	1989	+out_put_device:
	1990	+ put_device(&sdev->sdev_gendev);
	1991	+out:
	1992	+ return BLK_STS_RESOURCE;
	1993	+}
	1994	+
	1995	+static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
	1996	+ const struct blk_mq_queue_data *bd)
	1997	+{
	1998	+ struct request *req = bd->rq;
	1999	+ struct request_queue *q = req->q;
	2000	+ struct scsi_device *sdev = q->queuedata;
	2001	+ struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
	2002	+ blk_status_t ret;
	2003	+ int reason;
	2004	+
	2005	+ ret = prep_to_mq(scsi_prep_state_check(sdev, req));
	2006	+ if (ret != BLK_STS_OK)
	2007	+ goto out_put_budget;
	2008	+
	2009	+ ret = BLK_STS_RESOURCE;
1975	2010	if (!(req->rq_flags & RQF_DONTPREP)) {
1976	2011	ret = prep_to_mq(scsi_mq_prep_fn(req));
1977	2012	if (ret != BLK_STS_OK)
1978		- goto out_dec_host_busy;
	2013	+ goto out_put_budget;
1979	2014	req->rq_flags \|= RQF_DONTPREP;
1980	2015	} else {
1981	2016	blk_mq_start_request(req);
1982	2017
...	...	@@ -1993,21 +2028,13 @@
1993	2028	if (reason) {
1994	2029	scsi_set_blocked(cmd, reason);
1995	2030	ret = BLK_STS_RESOURCE;
1996		- goto out_dec_host_busy;
	2031	+ goto out_put_budget;
1997	2032	}
1998	2033
1999	2034	return BLK_STS_OK;
2000	2035
2001		-out_dec_host_busy:
2002		- atomic_dec(&shost->host_busy);
2003		-out_dec_target_busy:
2004		- if (scsi_target(sdev)->can_queue > 0)
2005		- atomic_dec(&scsi_target(sdev)->target_busy);
2006		-out_dec_device_busy:
2007		- atomic_dec(&sdev->device_busy);
2008		-out_put_device:
2009		- put_device(&sdev->sdev_gendev);
2010		-out:
	2036	+out_put_budget:
	2037	+ scsi_mq_put_budget(hctx);
2011	2038	switch (ret) {
2012	2039	case BLK_STS_OK:
2013	2040	break;
...	...	@@ -2211,6 +2238,8 @@
2211	2238	}
2212	2239
2213	2240	static const struct blk_mq_ops scsi_mq_ops = {
	2241	+ .get_budget = scsi_mq_get_budget,
	2242	+ .put_budget = scsi_mq_put_budget,
2214	2243	.queue_rq = scsi_queue_rq,
2215	2244	.complete = scsi_softirq_done,
2216	2245	.timeout = scsi_timeout,