block: fix inconsistency in I/O stat accounting code

This forces in_flight to be zero when turning off or on the I/O stat accounting and stops updating I/O stats in attempt_merge() when accounting is turned off. Signed-off-by: Jerome Marchand <jmarchan@redhat.com> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

block: fix inconsistency in I/O stat accounting code
This forces in_flight to be zero when turning off or on the I/O stat accounting and stops updating I/O stats in attempt_merge() when accounting is turned off. Signed-off-by: Jerome Marchand <jmarchan@redhat.com> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
Jerome Marchand · Jens Axboe
1 parent 6c7e8cee6a
Showing 6 changed files with 33 additions and 26 deletions Side-by-side Diff
block/blk-core.c
block/blk-merge.c
block/blk-sysfs.c
block/blk.h
block/elevator.c
include/linux/elevator.h
@@ -64,12 +64,11 @@
  
 static void drive_stat_acct(struct request *rq, int new_io)
 {
-	struct gendisk *disk = rq->rq_disk;
 	struct hd_struct *part;
 	int rw = rq_data_dir(rq);
 	int cpu;
  
-	if (!blk_fs_request(rq) || !disk || !blk_do_io_stat(disk->queue))
+	if (!blk_fs_request(rq) || !blk_do_io_stat(rq))
 		return;
  
 	cpu = part_stat_lock();
@@ -1675,9 +1674,7 @@
  
 static void blk_account_io_completion(struct request *req, unsigned int bytes)
 {
-	struct gendisk *disk = req->rq_disk;
-
-	if (!disk || !blk_do_io_stat(disk->queue))
+	if (!blk_do_io_stat(req))
 		return;
  
 	if (blk_fs_request(req)) {
@@ -1694,9 +1691,7 @@
  
 static void blk_account_io_done(struct request *req)
 {
-	struct gendisk *disk = req->rq_disk;
-
-	if (!disk || !blk_do_io_stat(disk->queue))
+	if (!blk_do_io_stat(req))
 		return;
  
 	/*
@@ -1711,7 +1706,7 @@
 		int cpu;
  
 		cpu = part_stat_lock();
-		part = disk_map_sector_rcu(disk, req->sector);
+		part = disk_map_sector_rcu(req->rq_disk, req->sector);
  
 		part_stat_inc(cpu, part, ios[rw]);
 		part_stat_add(cpu, part, ticks[rw], duration);
@@ -338,6 +338,22 @@
 	return 1;
 }
  
+static void blk_account_io_merge(struct request *req)
+{
+	if (blk_do_io_stat(req)) {
+		struct hd_struct *part;
+		int cpu;
+
+		cpu = part_stat_lock();
+		part = disk_map_sector_rcu(req->rq_disk, req->sector);
+
+		part_round_stats(cpu, part);
+		part_dec_in_flight(part);
+
+		part_stat_unlock();
+	}
+}
+
 /*
  * Has to be called with the request spinlock acquired
  */
@@ -386,18 +402,7 @@
  
 	elv_merge_requests(q, req, next);
  
-	if (req->rq_disk) {
-		struct hd_struct *part;
-		int cpu;
-
-		cpu = part_stat_lock();
-		part = disk_map_sector_rcu(req->rq_disk, req->sector);
-
-		part_round_stats(cpu, part);
-		part_dec_in_flight(part);
-
-		part_stat_unlock();
-	}
+	blk_account_io_merge(req);
  
 	req->ioprio = ioprio_best(req->ioprio, next->ioprio);
 	if (blk_rq_cpu_valid(next))
@@ -209,10 +209,14 @@
 	ssize_t ret = queue_var_store(&stats, page, count);
  
 	spin_lock_irq(q->queue_lock);
+	elv_quisce_start(q);
+
 	if (stats)
 		queue_flag_set(QUEUE_FLAG_IO_STAT, q);
 	else
 		queue_flag_clear(QUEUE_FLAG_IO_STAT, q);
+
+	elv_quisce_end(q);
 	spin_unlock_irq(q->queue_lock);
  
 	return ret;
@@ -112,12 +112,14 @@
 #endif
 }
  
-static inline int blk_do_io_stat(struct request_queue *q)
+static inline int blk_do_io_stat(struct request *rq)
 {
-	if (q)
-		return blk_queue_io_stat(q);
+	struct gendisk *disk = rq->rq_disk;
  
-	return 0;
+	if (!disk || !disk->queue)
+		return 0;
+
+	return blk_queue_io_stat(disk->queue) && (rq->cmd_flags & REQ_ELVPRIV);
 }
  
 #endif
@@ -573,7 +573,7 @@
 	elv_insert(q, rq, ELEVATOR_INSERT_REQUEUE);
 }
  
-static void elv_drain_elevator(struct request_queue *q)
+void elv_drain_elevator(struct request_queue *q)
 {
 	static int printed;
 	while (q->elevator->ops->elevator_dispatch_fn(q, 1))
@@ -116,6 +116,7 @@
 extern void elv_completed_request(struct request_queue *, struct request *);
 extern int elv_set_request(struct request_queue *, struct request *, gfp_t);
 extern void elv_put_request(struct request_queue *, struct request *);
+extern void elv_drain_elevator(struct request_queue *);
  
 /*
  * io scheduler registration
...	...	@@ -64,12 +64,11 @@
64	64
65	65	static void drive_stat_acct(struct request *rq, int new_io)
66	66	{
67		- struct gendisk *disk = rq->rq_disk;
68	67	struct hd_struct *part;
69	68	int rw = rq_data_dir(rq);
70	69	int cpu;
71	70
72		- if (!blk_fs_request(rq) \|\| !disk \|\| !blk_do_io_stat(disk->queue))
	71	+ if (!blk_fs_request(rq) \|\| !blk_do_io_stat(rq))
73	72	return;
74	73
75	74	cpu = part_stat_lock();
...	...	@@ -1675,9 +1674,7 @@
1675	1674
1676	1675	static void blk_account_io_completion(struct request *req, unsigned int bytes)
1677	1676	{
1678		- struct gendisk *disk = req->rq_disk;
1679		-
1680		- if (!disk \|\| !blk_do_io_stat(disk->queue))
	1677	+ if (!blk_do_io_stat(req))
1681	1678	return;
1682	1679
1683	1680	if (blk_fs_request(req)) {
...	...	@@ -1694,9 +1691,7 @@
1694	1691
1695	1692	static void blk_account_io_done(struct request *req)
1696	1693	{
1697		- struct gendisk *disk = req->rq_disk;
1698		-
1699		- if (!disk \|\| !blk_do_io_stat(disk->queue))
	1694	+ if (!blk_do_io_stat(req))
1700	1695	return;
1701	1696
1702	1697	/*
...	...	@@ -1711,7 +1706,7 @@
1711	1706	int cpu;
1712	1707
1713	1708	cpu = part_stat_lock();
1714		- part = disk_map_sector_rcu(disk, req->sector);
	1709	+ part = disk_map_sector_rcu(req->rq_disk, req->sector);
1715	1710
1716	1711	part_stat_inc(cpu, part, ios[rw]);
1717	1712	part_stat_add(cpu, part, ticks[rw], duration);
...	...	@@ -338,6 +338,22 @@
338	338	return 1;
339	339	}
340	340
	341	+static void blk_account_io_merge(struct request *req)
	342	+{
	343	+ if (blk_do_io_stat(req)) {
	344	+ struct hd_struct *part;
	345	+ int cpu;
	346	+
	347	+ cpu = part_stat_lock();
	348	+ part = disk_map_sector_rcu(req->rq_disk, req->sector);
	349	+
	350	+ part_round_stats(cpu, part);
	351	+ part_dec_in_flight(part);
	352	+
	353	+ part_stat_unlock();
	354	+ }
	355	+}
	356	+
341	357	/*
342	358	* Has to be called with the request spinlock acquired
343	359	*/
...	...	@@ -386,18 +402,7 @@
386	402
387	403	elv_merge_requests(q, req, next);
388	404
389		- if (req->rq_disk) {
390		- struct hd_struct *part;
391		- int cpu;
392		-
393		- cpu = part_stat_lock();
394		- part = disk_map_sector_rcu(req->rq_disk, req->sector);
395		-
396		- part_round_stats(cpu, part);
397		- part_dec_in_flight(part);
398		-
399		- part_stat_unlock();
400		- }
	405	+ blk_account_io_merge(req);
401	406
402	407	req->ioprio = ioprio_best(req->ioprio, next->ioprio);
403	408	if (blk_rq_cpu_valid(next))
...	...	@@ -209,10 +209,14 @@
209	209	ssize_t ret = queue_var_store(&stats, page, count);
210	210
211	211	spin_lock_irq(q->queue_lock);
	212	+ elv_quisce_start(q);
	213	+
212	214	if (stats)
213	215	queue_flag_set(QUEUE_FLAG_IO_STAT, q);
214	216	else
215	217	queue_flag_clear(QUEUE_FLAG_IO_STAT, q);
	218	+
	219	+ elv_quisce_end(q);
216	220	spin_unlock_irq(q->queue_lock);
217	221
218	222	return ret;
...	...	@@ -112,12 +112,14 @@
112	112	#endif
113	113	}
114	114
115		-static inline int blk_do_io_stat(struct request_queue *q)
	115	+static inline int blk_do_io_stat(struct request *rq)
116	116	{
117		- if (q)
118		- return blk_queue_io_stat(q);
	117	+ struct gendisk *disk = rq->rq_disk;
119	118
120		- return 0;
	119	+ if (!disk \|\| !disk->queue)
	120	+ return 0;
	121	+
	122	+ return blk_queue_io_stat(disk->queue) && (rq->cmd_flags & REQ_ELVPRIV);
121	123	}
122	124
123	125	#endif
...	...	@@ -573,7 +573,7 @@
573	573	elv_insert(q, rq, ELEVATOR_INSERT_REQUEUE);
574	574	}
575	575
576		-static void elv_drain_elevator(struct request_queue *q)
	576	+void elv_drain_elevator(struct request_queue *q)
577	577	{
578	578	static int printed;
579	579	while (q->elevator->ops->elevator_dispatch_fn(q, 1))
...	...	@@ -116,6 +116,7 @@
116	116	extern void elv_completed_request(struct request_queue , struct request );
117	117	extern int elv_set_request(struct request_queue , struct request , gfp_t);
118	118	extern void elv_put_request(struct request_queue , struct request );
	119	+extern void elv_drain_elevator(struct request_queue *);
119	120
120	121	/*
121	122	* io scheduler registration