blk-cgroup: Make 64bit per cpu stats safe on 32bit arch

Some of the stats are 64bit and updation will be non atomic on 32bit architecture. Use sequence counters on 32bit arch to make reading of stats safe. Signed-off-by: Vivek Goyal <vgoyal@redhat.com> Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

blk-cgroup: Make 64bit per cpu stats safe on 32bit arch
Some of the stats are 64bit and updation will be non atomic on 32bit architecture. Use sequence counters on 32bit arch to make reading of stats safe. Signed-off-by: Vivek Goyal <vgoyal@redhat.com> Signed-off-by: Jens Axboe <jaxboe@fusionio.com>
Vivek Goyal · Jens Axboe
1 parent 5624a4e445
Showing 2 changed files with 24 additions and 5 deletions Side-by-side Diff
block/blk-cgroup.c
block/blk-cgroup.h
@@ -400,14 +400,25 @@
 				uint64_t bytes, bool direction, bool sync)
 {
 	struct blkio_group_stats_cpu *stats_cpu;
+	unsigned long flags;
  
+	/*
+	 * Disabling interrupts to provide mutual exclusion between two
+	 * writes on same cpu. It probably is not needed for 64bit. Not
+	 * optimizing that case yet.
+	 */
+	local_irq_save(flags);
+
 	stats_cpu = this_cpu_ptr(blkg->stats_cpu);
  
+	u64_stats_update_begin(&stats_cpu->syncp);
 	stats_cpu->sectors += bytes >> 9;
 	blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
 			1, direction, sync);
 	blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
 			bytes, direction, sync);
+	u64_stats_update_end(&stats_cpu->syncp);
+	local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
  
  
  
@@ -622,15 +633,21 @@
 {
 	int cpu;
 	struct blkio_group_stats_cpu *stats_cpu;
-	uint64_t val = 0;
+	u64 val = 0, tval;
  
 	for_each_possible_cpu(cpu) {
+		unsigned int start;
 		stats_cpu  = per_cpu_ptr(blkg->stats_cpu, cpu);
  
-		if (type == BLKIO_STAT_CPU_SECTORS)
-			val += stats_cpu->sectors;
-		else
-			val += stats_cpu->stat_arr_cpu[type][sub_type];
+		do {
+			start = u64_stats_fetch_begin(&stats_cpu->syncp);
+			if (type == BLKIO_STAT_CPU_SECTORS)
+				tval = stats_cpu->sectors;
+			else
+				tval = stats_cpu->stat_arr_cpu[type][sub_type];
+		} while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
+
+		val += tval;
 	}
  
 	return val;
@@ -14,6 +14,7 @@
  */
  
 #include <linux/cgroup.h>
+#include <linux/u64_stats_sync.h>
  
 enum blkio_policy_id {
 	BLKIO_POLICY_PROP = 0,		/* Proportional Bandwidth division */
@@ -154,6 +155,7 @@
 struct blkio_group_stats_cpu {
 	uint64_t sectors;
 	uint64_t stat_arr_cpu[BLKIO_STAT_CPU_NR][BLKIO_STAT_TOTAL];
+	struct u64_stats_sync syncp;
 };
  
 struct blkio_group {
...	...	@@ -400,14 +400,25 @@
400	400	uint64_t bytes, bool direction, bool sync)
401	401	{
402	402	struct blkio_group_stats_cpu *stats_cpu;
	403	+ unsigned long flags;
403	404
	405	+ /*
	406	+ * Disabling interrupts to provide mutual exclusion between two
	407	+ * writes on same cpu. It probably is not needed for 64bit. Not
	408	+ * optimizing that case yet.
	409	+ */
	410	+ local_irq_save(flags);
	411	+
404	412	stats_cpu = this_cpu_ptr(blkg->stats_cpu);
405	413
	414	+ u64_stats_update_begin(&stats_cpu->syncp);
406	415	stats_cpu->sectors += bytes >> 9;
407	416	blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICED],
408	417	1, direction, sync);
409	418	blkio_add_stat(stats_cpu->stat_arr_cpu[BLKIO_STAT_CPU_SERVICE_BYTES],
410	419	bytes, direction, sync);
	420	+ u64_stats_update_end(&stats_cpu->syncp);
	421	+ local_irq_restore(flags);
411	422	}
412	423	EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
413	424
414	425
415	426
...	...	@@ -622,15 +633,21 @@
622	633	{
623	634	int cpu;
624	635	struct blkio_group_stats_cpu *stats_cpu;
625		- uint64_t val = 0;
	636	+ u64 val = 0, tval;
626	637
627	638	for_each_possible_cpu(cpu) {
	639	+ unsigned int start;
628	640	stats_cpu = per_cpu_ptr(blkg->stats_cpu, cpu);
629	641
630		- if (type == BLKIO_STAT_CPU_SECTORS)
631		- val += stats_cpu->sectors;
632		- else
633		- val += stats_cpu->stat_arr_cpu[type][sub_type];
	642	+ do {
	643	+ start = u64_stats_fetch_begin(&stats_cpu->syncp);
	644	+ if (type == BLKIO_STAT_CPU_SECTORS)
	645	+ tval = stats_cpu->sectors;
	646	+ else
	647	+ tval = stats_cpu->stat_arr_cpu[type][sub_type];
	648	+ } while(u64_stats_fetch_retry(&stats_cpu->syncp, start));
	649	+
	650	+ val += tval;
634	651	}
635	652
636	653	return val;
...	...	@@ -14,6 +14,7 @@
14	14	*/
15	15
16	16	#include <linux/cgroup.h>
	17	+#include <linux/u64_stats_sync.h>
17	18
18	19	enum blkio_policy_id {
19	20	BLKIO_POLICY_PROP = 0, /* Proportional Bandwidth division */
...	...	@@ -154,6 +155,7 @@
154	155	struct blkio_group_stats_cpu {
155	156	uint64_t sectors;
156	157	uint64_t stat_arr_cpu[BLKIO_STAT_CPU_NR][BLKIO_STAT_TOTAL];
	158	+ struct u64_stats_sync syncp;
157	159	};
158	160
159	161	struct blkio_group {