blkcg: embed struct blkg_policy_data in policy specific data

Currently blkg_policy_data carries policy specific data as char flex array instead of being embedded in policy specific data. This was forced by oddities around blkg allocation which are all gone now. This patch makes blkg_policy_data embedded in policy specific data - throtl_grp and cfq_group so that it's more conventional and consistent with how io_cq is handled. * blkcg_policy->pdata_size is renamed to ->pd_size. * Functions which used to take void *pdata now takes struct blkg_policy_data *pd. * blkg_to_pdata/pdata_to_blkg() updated to blkg_to_pd/pd_to_blkg(). * Dummy struct blkg_policy_data definition added. Dummy pdata_to_blkg() definition was unused and inconsistent with the non-dummy version - correct dummy pd_to_blkg() added. * throtl and cfq updated accordingly. * As dummy blkg_to_pd/pd_to_blkg() are provided, blkg_to_cfqg/cfqg_to_blkg() don't need to be ifdef'd. Moved outside ifdef block. This patch doesn't introduce any functional change. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Vivek Goyal <vgoyal@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>

blkcg: embed struct blkg_policy_data in policy specific data
Currently blkg_policy_data carries policy specific data as char flex array instead of being embedded in policy specific data. This was forced by oddities around blkg allocation which are all gone now. This patch makes blkg_policy_data embedded in policy specific data - throtl_grp and cfq_group so that it's more conventional and consistent with how io_cq is handled. * blkcg_policy->pdata_size is renamed to ->pd_size. * Functions which used to take void *pdata now takes struct blkg_policy_data *pd. * blkg_to_pdata/pdata_to_blkg() updated to blkg_to_pd/pd_to_blkg(). * Dummy struct blkg_policy_data definition added. Dummy pdata_to_blkg() definition was unused and inconsistent with the non-dummy version - correct dummy pd_to_blkg() added. * throtl and cfq updated accordingly. * As dummy blkg_to_pd/pd_to_blkg() are provided, blkg_to_cfqg/cfqg_to_blkg() don't need to be ifdef'd. Moved outside ifdef block. This patch doesn't introduce any functional change. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Vivek Goyal <vgoyal@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Tejun Heo · Jens Axboe
1 parent 3c798398e3
Showing 4 changed files with 112 additions and 83 deletions Side-by-side Diff
block/blk-cgroup.c
block/blk-cgroup.h
block/blk-throttle.c
block/cfq-iosched.c
@@ -58,11 +58,6 @@
 	return pol && test_bit(pol->plid, q->blkcg_pols);
 }
  
-static size_t blkg_pd_size(const struct blkcg_policy *pol)
-{
-	return sizeof(struct blkg_policy_data) + pol->pdata_size;
-}
-
 /**
  * blkg_free - free a blkg
  * @blkg: blkg to free
@@ -122,7 +117,7 @@
 			continue;
  
 		/* alloc per-policy data and attach it to blkg */
-		pd = kzalloc_node(blkg_pd_size(pol), GFP_ATOMIC, q->node);
+		pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node);
 		if (!pd) {
 			blkg_free(blkg);
 			return NULL;
@@ -346,7 +341,8 @@
  * cftype->read_seq_string method.
  */
 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
-		       u64 (*prfill)(struct seq_file *, void *, int),
+		       u64 (*prfill)(struct seq_file *,
+				     struct blkg_policy_data *, int),
 		       const struct blkcg_policy *pol, int data,
 		       bool show_total)
 {
@@ -357,7 +353,7 @@
 	spin_lock_irq(&blkcg->lock);
 	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
 		if (blkcg_policy_enabled(blkg->q, pol))
-			total += prfill(sf, blkg->pd[pol->plid]->pdata, data);
+			total += prfill(sf, blkg->pd[pol->plid], data);
 	spin_unlock_irq(&blkcg->lock);
  
 	if (show_total)
  
  
  
@@ -368,14 +364,14 @@
 /**
  * __blkg_prfill_u64 - prfill helper for a single u64 value
  * @sf: seq_file to print to
- * @pdata: policy private data of interest
+ * @pd: policy private data of interest
  * @v: value to print
  *
- * Print @v to @sf for the device assocaited with @pdata.
+ * Print @v to @sf for the device assocaited with @pd.
  */
-u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v)
+u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
 {
-	const char *dname = blkg_dev_name(pdata_to_blkg(pdata));
+	const char *dname = blkg_dev_name(pd->blkg);
  
 	if (!dname)
 		return 0;
  
  
@@ -388,12 +384,12 @@
 /**
  * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
  * @sf: seq_file to print to
- * @pdata: policy private data of interest
+ * @pd: policy private data of interest
  * @rwstat: rwstat to print
  *
- * Print @rwstat to @sf for the device assocaited with @pdata.
+ * Print @rwstat to @sf for the device assocaited with @pd.
  */
-u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 			 const struct blkg_rwstat *rwstat)
 {
 	static const char *rwstr[] = {
@@ -402,7 +398,7 @@
 		[BLKG_RWSTAT_SYNC]	= "Sync",
 		[BLKG_RWSTAT_ASYNC]	= "Async",
 	};
-	const char *dname = blkg_dev_name(pdata_to_blkg(pdata));
+	const char *dname = blkg_dev_name(pd->blkg);
 	u64 v;
 	int i;
  
  
  
  
  
  
  
@@ -421,30 +417,31 @@
 /**
  * blkg_prfill_stat - prfill callback for blkg_stat
  * @sf: seq_file to print to
- * @pdata: policy private data of interest
- * @off: offset to the blkg_stat in @pdata
+ * @pd: policy private data of interest
+ * @off: offset to the blkg_stat in @pd
  *
  * prfill callback for printing a blkg_stat.
  */
-u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off)
+u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
 {
-	return __blkg_prfill_u64(sf, pdata, blkg_stat_read(pdata + off));
+	return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
 }
 EXPORT_SYMBOL_GPL(blkg_prfill_stat);
  
 /**
  * blkg_prfill_rwstat - prfill callback for blkg_rwstat
  * @sf: seq_file to print to
- * @pdata: policy private data of interest
- * @off: offset to the blkg_rwstat in @pdata
+ * @pd: policy private data of interest
+ * @off: offset to the blkg_rwstat in @pd
  *
  * prfill callback for printing a blkg_rwstat.
  */
-u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off)
+u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
+		       int off)
 {
-	struct blkg_rwstat rwstat = blkg_rwstat_read(pdata + off);
+	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
  
-	return __blkg_prfill_rwstat(sf, pdata, &rwstat);
+	return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
 EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
  
@@ -733,7 +730,7 @@
  
 	/* allocate policy_data for all existing blkgs */
 	while (cnt--) {
-		pd = kzalloc_node(blkg_pd_size(pol), GFP_KERNEL, q->node);
+		pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
 		if (!pd) {
 			ret = -ENOMEM;
 			goto out_free;
@@ -831,6 +828,9 @@
 int blkcg_policy_register(struct blkcg_policy *pol)
 {
 	int i, ret;
+
+	if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data)))
+		return -EINVAL;
  
 	mutex_lock(&blkcg_pol_mutex);
  
@@ -59,16 +59,25 @@
 	uint64_t			cnt[BLKG_RWSTAT_NR];
 };
  
-/* per-blkg per-policy data */
+/*
+ * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
+ * request_queue (q).  This is used by blkcg policies which need to track
+ * information per blkcg - q pair.
+ *
+ * There can be multiple active blkcg policies and each has its private
+ * data on each blkg, the size of which is determined by
+ * blkcg_policy->pd_size.  blkcg core allocates and frees such areas
+ * together with blkg and invokes pd_init/exit_fn() methods.
+ *
+ * Such private data must embed struct blkg_policy_data (pd) at the
+ * beginning and pd_size can't be smaller than pd.
+ */
 struct blkg_policy_data {
 	/* the blkg this per-policy data belongs to */
 	struct blkcg_gq			*blkg;
  
 	/* used during policy activation */
 	struct list_head		alloc_node;
-
-	/* pol->pdata_size bytes of private data used by policy impl */
-	char				pdata[] __aligned(__alignof__(unsigned long long));
 };
  
 /* association between a blk cgroup and a request queue */
@@ -100,7 +109,7 @@
 	struct blkcg_policy_ops		ops;
 	int				plid;
 	/* policy specific private data size */
-	size_t				pdata_size;
+	size_t				pd_size;
 	/* cgroup files for the policy */
 	struct cftype			*cftypes;
 };
  
  
@@ -125,14 +134,16 @@
 			     const struct blkcg_policy *pol);
  
 void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
-		       u64 (*prfill)(struct seq_file *, void *, int),
+		       u64 (*prfill)(struct seq_file *,
+				     struct blkg_policy_data *, int),
 		       const struct blkcg_policy *pol, int data,
 		       bool show_total);
-u64 __blkg_prfill_u64(struct seq_file *sf, void *pdata, u64 v);
-u64 __blkg_prfill_rwstat(struct seq_file *sf, void *pdata,
+u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v);
+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
 			 const struct blkg_rwstat *rwstat);
-u64 blkg_prfill_stat(struct seq_file *sf, void *pdata, int off);
-u64 blkg_prfill_rwstat(struct seq_file *sf, void *pdata, int off);
+u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off);
+u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
+		       int off);
  
 struct blkg_conf_ctx {
 	struct gendisk			*disk;
  
  
  
  
  
@@ -152,26 +163,21 @@
  *
  * Return pointer to private data associated with the @blkg-@pol pair.
  */
-static inline void *blkg_to_pdata(struct blkcg_gq *blkg,
-				  struct blkcg_policy *pol)
+static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
+						  struct blkcg_policy *pol)
 {
-	return blkg ? blkg->pd[pol->plid]->pdata : NULL;
+	return blkg ? blkg->pd[pol->plid] : NULL;
 }
  
 /**
  * pdata_to_blkg - get blkg associated with policy private data
- * @pdata: policy private data of interest
+ * @pd: policy private data of interest
  *
- * @pdata is policy private data.  Determine the blkg it's associated with.
+ * @pd is policy private data.  Determine the blkg it's associated with.
  */
-static inline struct blkcg_gq *pdata_to_blkg(void *pdata)
+static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd)
 {
-	if (pdata) {
-		struct blkg_policy_data *pd =
-			container_of(pdata, struct blkg_policy_data, pdata);
-		return pd->blkg;
-	}
-	return NULL;
+	return pd ? pd->blkg : NULL;
 }
  
 /**
@@ -342,6 +348,9 @@
  
 struct cgroup;
  
+struct blkg_policy_data {
+};
+
 struct blkcg_gq {
 };
  
@@ -361,10 +370,9 @@
 static inline void blkcg_deactivate_policy(struct request_queue *q,
 					   const struct blkcg_policy *pol) { }
  
-static inline void *blkg_to_pdata(struct blkcg_gq *blkg,
-				  struct blkcg_policy *pol) { return NULL; }
-static inline struct blkcg_gq *pdata_to_blkg(void *pdata,
-				  struct blkcg_policy *pol) { return NULL; }
+static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
+						  struct blkcg_policy *pol) { return NULL; }
+static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; }
 static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; }
 static inline void blkg_get(struct blkcg_gq *blkg) { }
 static inline void blkg_put(struct blkcg_gq *blkg) { }
@@ -49,6 +49,9 @@
 };
  
 struct throtl_grp {
+	/* must be the first member */
+	struct blkg_policy_data pd;
+
 	/* active throtl group service_tree member */
 	struct rb_node rb_node;
  
  
  
@@ -120,14 +123,19 @@
 static void tg_stats_alloc_fn(struct work_struct *);
 static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
  
+static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
+{
+	return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
+}
+
 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
 {
-	return blkg_to_pdata(blkg, &blkcg_policy_throtl);
+	return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
 }
  
 static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)
 {
-	return pdata_to_blkg(tg);
+	return pd_to_blkg(&tg->pd);
 }
  
 static inline struct throtl_grp *td_root_tg(struct throtl_data *td)
  
@@ -931,9 +939,10 @@
 	}
 }
  
-static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, void *pdata, int off)
+static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
+				struct blkg_policy_data *pd, int off)
 {
-	struct throtl_grp *tg = pdata;
+	struct throtl_grp *tg = pd_to_tg(pd);
 	struct blkg_rwstat rwstat = { }, tmp;
 	int i, cpu;
  
@@ -945,7 +954,7 @@
 			rwstat.cnt[i] += tmp.cnt[i];
 	}
  
-	return __blkg_prfill_rwstat(sf, pdata, &rwstat);
+	return __blkg_prfill_rwstat(sf, pd, &rwstat);
 }
  
 static int tg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
  
  
  
  
  
@@ -958,22 +967,26 @@
 	return 0;
 }
  
-static u64 tg_prfill_conf_u64(struct seq_file *sf, void *pdata, int off)
+static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd,
+			      int off)
 {
-	u64 v = *(u64 *)(pdata + off);
+	struct throtl_grp *tg = pd_to_tg(pd);
+	u64 v = *(u64 *)((void *)tg + off);
  
 	if (v == -1)
 		return 0;
-	return __blkg_prfill_u64(sf, pdata, v);
+	return __blkg_prfill_u64(sf, pd, v);
 }
  
-static u64 tg_prfill_conf_uint(struct seq_file *sf, void *pdata, int off)
+static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd,
+			       int off)
 {
-	unsigned int v = *(unsigned int *)(pdata + off);
+	struct throtl_grp *tg = pd_to_tg(pd);
+	unsigned int v = *(unsigned int *)((void *)tg + off);
  
 	if (v == -1)
 		return 0;
-	return __blkg_prfill_u64(sf, pdata, v);
+	return __blkg_prfill_u64(sf, pd, v);
 }
  
 static int tg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
@@ -1092,7 +1105,7 @@
 		.pd_exit_fn		= throtl_pd_exit,
 		.pd_reset_stats_fn	= throtl_pd_reset_stats,
 	},
-	.pdata_size = sizeof(struct throtl_grp),
+	.pd_size = sizeof(struct throtl_grp),
 	.cftypes = throtl_files,
 };
  
@@ -217,6 +217,9 @@
  
 /* This is per cgroup per device grouping structure */
 struct cfq_group {
+	/* must be the first member */
+	struct blkg_policy_data pd;
+
 	/* group service_tree member */
 	struct rb_node rb_node;
  
@@ -409,6 +412,21 @@
 CFQ_CFQQ_FNS(wait_busy);
 #undef CFQ_CFQQ_FNS
  
+static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
+{
+	return pd ? container_of(pd, struct cfq_group, pd) : NULL;
+}
+
+static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
+{
+	return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
+}
+
+static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
+{
+	return pd_to_blkg(&cfqg->pd);
+}
+
 #if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
  
 /* cfqg stats flags */
@@ -553,16 +571,6 @@
  
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
  
-static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
-{
-	return blkg_to_pdata(blkg, &blkcg_policy_cfq);
-}
-
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
-{
-	return pdata_to_blkg(cfqg);
-}
-
 static inline void cfqg_get(struct cfq_group *cfqg)
 {
 	return blkg_get(cfqg_to_blkg(cfqg));
@@ -662,8 +670,6 @@
  
 #else	/* CONFIG_CFQ_GROUP_IOSCHED */
  
-static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg) { return NULL; }
-static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) { return NULL; }
 static inline void cfqg_get(struct cfq_group *cfqg) { }
 static inline void cfqg_put(struct cfq_group *cfqg) { }
  
  
  
@@ -1374,13 +1380,14 @@
 	cfqg_get(cfqg);
 }
  
-static u64 cfqg_prfill_weight_device(struct seq_file *sf, void *pdata, int off)
+static u64 cfqg_prfill_weight_device(struct seq_file *sf,
+				     struct blkg_policy_data *pd, int off)
 {
-	struct cfq_group *cfqg = pdata;
+	struct cfq_group *cfqg = pd_to_cfqg(pd);
  
 	if (!cfqg->dev_weight)
 		return 0;
-	return __blkg_prfill_u64(sf, pdata, cfqg->dev_weight);
+	return __blkg_prfill_u64(sf, pd, cfqg->dev_weight);
 }
  
 static int cfqg_print_weight_device(struct cgroup *cgrp, struct cftype *cft,
  
@@ -1467,9 +1474,10 @@
 }
  
 #ifdef CONFIG_DEBUG_BLK_CGROUP
-static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, void *pdata, int off)
+static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf,
+				      struct blkg_policy_data *pd, int off)
 {
-	struct cfq_group *cfqg = pdata;
+	struct cfq_group *cfqg = pd_to_cfqg(pd);
 	u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);
 	u64 v = 0;
  
@@ -1477,7 +1485,7 @@
 		v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);
 		do_div(v, samples);
 	}
-	__blkg_prfill_u64(sf, pdata, v);
+	__blkg_prfill_u64(sf, pd, v);
 	return 0;
 }
  
@@ -4161,7 +4169,7 @@
 		.pd_init_fn		= cfq_pd_init,
 		.pd_reset_stats_fn	= cfq_pd_reset_stats,
 	},
-	.pdata_size = sizeof(struct cfq_group),
+	.pd_size = sizeof(struct cfq_group),
 	.cftypes = cfq_blkcg_files,
 };
 #endif
...	...	@@ -58,11 +58,6 @@
58	58	return pol && test_bit(pol->plid, q->blkcg_pols);
59	59	}
60	60
61		-static size_t blkg_pd_size(const struct blkcg_policy *pol)
62		-{
63		- return sizeof(struct blkg_policy_data) + pol->pdata_size;
64		-}
65		-
66	61	/**
67	62	* blkg_free - free a blkg
68	63	* @blkg: blkg to free
...	...	@@ -122,7 +117,7 @@
122	117	continue;
123	118
124	119	/* alloc per-policy data and attach it to blkg */
125		- pd = kzalloc_node(blkg_pd_size(pol), GFP_ATOMIC, q->node);
	120	+ pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node);
126	121	if (!pd) {
127	122	blkg_free(blkg);
128	123	return NULL;
...	...	@@ -346,7 +341,8 @@
346	341	* cftype->read_seq_string method.
347	342	*/
348	343	void blkcg_print_blkgs(struct seq_file sf, struct blkcg blkcg,
349		- u64 (prfill)(struct seq_file , void *, int),
	344	+ u64 (prfill)(struct seq_file ,
	345	+ struct blkg_policy_data *, int),
350	346	const struct blkcg_policy *pol, int data,
351	347	bool show_total)
352	348	{
...	...	@@ -357,7 +353,7 @@
357	353	spin_lock_irq(&blkcg->lock);
358	354	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
359	355	if (blkcg_policy_enabled(blkg->q, pol))
360		- total += prfill(sf, blkg->pd[pol->plid]->pdata, data);
	356	+ total += prfill(sf, blkg->pd[pol->plid], data);
361	357	spin_unlock_irq(&blkcg->lock);
362	358
363	359	if (show_total)
364	360
365	361
366	362
...	...	@@ -368,14 +364,14 @@
368	364	/**
369	365	* __blkg_prfill_u64 - prfill helper for a single u64 value
370	366	* @sf: seq_file to print to
371		- * @pdata: policy private data of interest
	367	+ * @pd: policy private data of interest
372	368	* @v: value to print
373	369	*
374		- * Print @v to @sf for the device assocaited with @pdata.
	370	+ * Print @v to @sf for the device assocaited with @pd.
375	371	*/
376		-u64 __blkg_prfill_u64(struct seq_file sf, void pdata, u64 v)
	372	+u64 __blkg_prfill_u64(struct seq_file sf, struct blkg_policy_data pd, u64 v)
377	373	{
378		- const char *dname = blkg_dev_name(pdata_to_blkg(pdata));
	374	+ const char *dname = blkg_dev_name(pd->blkg);
379	375
380	376	if (!dname)
381	377	return 0;
382	378
383	379
...	...	@@ -388,12 +384,12 @@
388	384	/**
389	385	* __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
390	386	* @sf: seq_file to print to
391		- * @pdata: policy private data of interest
	387	+ * @pd: policy private data of interest
392	388	* @rwstat: rwstat to print
393	389	*
394		- * Print @rwstat to @sf for the device assocaited with @pdata.
	390	+ * Print @rwstat to @sf for the device assocaited with @pd.
395	391	*/
396		-u64 __blkg_prfill_rwstat(struct seq_file sf, void pdata,
	392	+u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
397	393	const struct blkg_rwstat *rwstat)
398	394	{
399	395	static const char *rwstr[] = {
...	...	@@ -402,7 +398,7 @@
402	398	[BLKG_RWSTAT_SYNC] = "Sync",
403	399	[BLKG_RWSTAT_ASYNC] = "Async",
404	400	};
405		- const char *dname = blkg_dev_name(pdata_to_blkg(pdata));
	401	+ const char *dname = blkg_dev_name(pd->blkg);
406	402	u64 v;
407	403	int i;
408	404
409	405
410	406
411	407
412	408
413	409
414	410
...	...	@@ -421,30 +417,31 @@
421	417	/**
422	418	* blkg_prfill_stat - prfill callback for blkg_stat
423	419	* @sf: seq_file to print to
424		- * @pdata: policy private data of interest
425		- * @off: offset to the blkg_stat in @pdata
	420	+ * @pd: policy private data of interest
	421	+ * @off: offset to the blkg_stat in @pd
426	422	*
427	423	* prfill callback for printing a blkg_stat.
428	424	*/
429		-u64 blkg_prfill_stat(struct seq_file sf, void pdata, int off)
	425	+u64 blkg_prfill_stat(struct seq_file sf, struct blkg_policy_data pd, int off)
430	426	{
431		- return __blkg_prfill_u64(sf, pdata, blkg_stat_read(pdata + off));
	427	+ return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
432	428	}
433	429	EXPORT_SYMBOL_GPL(blkg_prfill_stat);
434	430
435	431	/**
436	432	* blkg_prfill_rwstat - prfill callback for blkg_rwstat
437	433	* @sf: seq_file to print to
438		- * @pdata: policy private data of interest
439		- * @off: offset to the blkg_rwstat in @pdata
	434	+ * @pd: policy private data of interest
	435	+ * @off: offset to the blkg_rwstat in @pd
440	436	*
441	437	* prfill callback for printing a blkg_rwstat.
442	438	*/
443		-u64 blkg_prfill_rwstat(struct seq_file sf, void pdata, int off)
	439	+u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	440	+ int off)
444	441	{
445		- struct blkg_rwstat rwstat = blkg_rwstat_read(pdata + off);
	442	+ struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
446	443
447		- return __blkg_prfill_rwstat(sf, pdata, &rwstat);
	444	+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
448	445	}
449	446	EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
450	447
...	...	@@ -733,7 +730,7 @@
733	730
734	731	/* allocate policy_data for all existing blkgs */
735	732	while (cnt--) {
736		- pd = kzalloc_node(blkg_pd_size(pol), GFP_KERNEL, q->node);
	733	+ pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
737	734	if (!pd) {
738	735	ret = -ENOMEM;
739	736	goto out_free;
...	...	@@ -831,6 +828,9 @@
831	828	int blkcg_policy_register(struct blkcg_policy *pol)
832	829	{
833	830	int i, ret;
	831	+
	832	+ if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data)))
	833	+ return -EINVAL;
834	834
835	835	mutex_lock(&blkcg_pol_mutex);
836	836
...	...	@@ -59,16 +59,25 @@
59	59	uint64_t cnt[BLKG_RWSTAT_NR];
60	60	};
61	61
62		-/* per-blkg per-policy data */
	62	+/*
	63	+ * A blkcg_gq (blkg) is association between a block cgroup (blkcg) and a
	64	+ * request_queue (q). This is used by blkcg policies which need to track
	65	+ * information per blkcg - q pair.
	66	+ *
	67	+ * There can be multiple active blkcg policies and each has its private
	68	+ * data on each blkg, the size of which is determined by
	69	+ * blkcg_policy->pd_size. blkcg core allocates and frees such areas
	70	+ * together with blkg and invokes pd_init/exit_fn() methods.
	71	+ *
	72	+ * Such private data must embed struct blkg_policy_data (pd) at the
	73	+ * beginning and pd_size can't be smaller than pd.
	74	+ */
63	75	struct blkg_policy_data {
64	76	/* the blkg this per-policy data belongs to */
65	77	struct blkcg_gq *blkg;
66	78
67	79	/* used during policy activation */
68	80	struct list_head alloc_node;
69		-
70		- /* pol->pdata_size bytes of private data used by policy impl */
71		- char pdata[] __aligned(__alignof__(unsigned long long));
72	81	};
73	82
74	83	/* association between a blk cgroup and a request queue */
...	...	@@ -100,7 +109,7 @@
100	109	struct blkcg_policy_ops ops;
101	110	int plid;
102	111	/* policy specific private data size */
103		- size_t pdata_size;
	112	+ size_t pd_size;
104	113	/* cgroup files for the policy */
105	114	struct cftype *cftypes;
106	115	};
107	116
108	117
...	...	@@ -125,14 +134,16 @@
125	134	const struct blkcg_policy *pol);
126	135
127	136	void blkcg_print_blkgs(struct seq_file sf, struct blkcg blkcg,
128		- u64 (prfill)(struct seq_file , void *, int),
	137	+ u64 (prfill)(struct seq_file ,
	138	+ struct blkg_policy_data *, int),
129	139	const struct blkcg_policy *pol, int data,
130	140	bool show_total);
131		-u64 __blkg_prfill_u64(struct seq_file sf, void pdata, u64 v);
132		-u64 __blkg_prfill_rwstat(struct seq_file sf, void pdata,
	141	+u64 __blkg_prfill_u64(struct seq_file sf, struct blkg_policy_data pd, u64 v);
	142	+u64 __blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
133	143	const struct blkg_rwstat *rwstat);
134		-u64 blkg_prfill_stat(struct seq_file sf, void pdata, int off);
135		-u64 blkg_prfill_rwstat(struct seq_file sf, void pdata, int off);
	144	+u64 blkg_prfill_stat(struct seq_file sf, struct blkg_policy_data pd, int off);
	145	+u64 blkg_prfill_rwstat(struct seq_file sf, struct blkg_policy_data pd,
	146	+ int off);
136	147
137	148	struct blkg_conf_ctx {
138	149	struct gendisk *disk;
139	150
140	151
141	152
142	153
143	154
...	...	@@ -152,26 +163,21 @@
152	163	*
153	164	* Return pointer to private data associated with the @blkg-@pol pair.
154	165	*/
155		-static inline void blkg_to_pdata(struct blkcg_gq blkg,
156		- struct blkcg_policy *pol)
	166	+static inline struct blkg_policy_data blkg_to_pd(struct blkcg_gq blkg,
	167	+ struct blkcg_policy *pol)
157	168	{
158		- return blkg ? blkg->pd[pol->plid]->pdata : NULL;
	169	+ return blkg ? blkg->pd[pol->plid] : NULL;
159	170	}
160	171
161	172	/**
162	173	* pdata_to_blkg - get blkg associated with policy private data
163		- * @pdata: policy private data of interest
	174	+ * @pd: policy private data of interest
164	175	*
165		- * @pdata is policy private data. Determine the blkg it's associated with.
	176	+ * @pd is policy private data. Determine the blkg it's associated with.
166	177	*/
167		-static inline struct blkcg_gq pdata_to_blkg(void pdata)
	178	+static inline struct blkcg_gq pd_to_blkg(struct blkg_policy_data pd)
168	179	{
169		- if (pdata) {
170		- struct blkg_policy_data *pd =
171		- container_of(pdata, struct blkg_policy_data, pdata);
172		- return pd->blkg;
173		- }
174		- return NULL;
	180	+ return pd ? pd->blkg : NULL;
175	181	}
176	182
177	183	/**
...	...	@@ -342,6 +348,9 @@
342	348
343	349	struct cgroup;
344	350
	351	+struct blkg_policy_data {
	352	+};
	353	+
345	354	struct blkcg_gq {
346	355	};
347	356
...	...	@@ -361,10 +370,9 @@
361	370	static inline void blkcg_deactivate_policy(struct request_queue *q,
362	371	const struct blkcg_policy *pol) { }
363	372
364		-static inline void blkg_to_pdata(struct blkcg_gq blkg,
365		- struct blkcg_policy *pol) { return NULL; }
366		-static inline struct blkcg_gq pdata_to_blkg(void pdata,
367		- struct blkcg_policy *pol) { return NULL; }
	373	+static inline struct blkg_policy_data blkg_to_pd(struct blkcg_gq blkg,
	374	+ struct blkcg_policy *pol) { return NULL; }
	375	+static inline struct blkcg_gq pd_to_blkg(struct blkg_policy_data pd) { return NULL; }
368	376	static inline char blkg_path(struct blkcg_gq blkg) { return NULL; }
369	377	static inline void blkg_get(struct blkcg_gq *blkg) { }
370	378	static inline void blkg_put(struct blkcg_gq *blkg) { }
...	...	@@ -49,6 +49,9 @@
49	49	};
50	50
51	51	struct throtl_grp {
	52	+ /* must be the first member */
	53	+ struct blkg_policy_data pd;
	54	+
52	55	/* active throtl group service_tree member */
53	56	struct rb_node rb_node;
54	57
55	58
56	59
...	...	@@ -120,14 +123,19 @@
120	123	static void tg_stats_alloc_fn(struct work_struct *);
121	124	static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn);
122	125
	126	+static inline struct throtl_grp pd_to_tg(struct blkg_policy_data pd)
	127	+{
	128	+ return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
	129	+}
	130	+
123	131	static inline struct throtl_grp blkg_to_tg(struct blkcg_gq blkg)
124	132	{
125		- return blkg_to_pdata(blkg, &blkcg_policy_throtl);
	133	+ return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
126	134	}
127	135
128	136	static inline struct blkcg_gq tg_to_blkg(struct throtl_grp tg)
129	137	{
130		- return pdata_to_blkg(tg);
	138	+ return pd_to_blkg(&tg->pd);
131	139	}
132	140
133	141	static inline struct throtl_grp td_root_tg(struct throtl_data td)
134	142
...	...	@@ -931,9 +939,10 @@
931	939	}
932	940	}
933	941
934		-static u64 tg_prfill_cpu_rwstat(struct seq_file sf, void pdata, int off)
	942	+static u64 tg_prfill_cpu_rwstat(struct seq_file *sf,
	943	+ struct blkg_policy_data *pd, int off)
935	944	{
936		- struct throtl_grp *tg = pdata;
	945	+ struct throtl_grp *tg = pd_to_tg(pd);
937	946	struct blkg_rwstat rwstat = { }, tmp;
938	947	int i, cpu;
939	948
...	...	@@ -945,7 +954,7 @@
945	954	rwstat.cnt[i] += tmp.cnt[i];
946	955	}
947	956
948		- return __blkg_prfill_rwstat(sf, pdata, &rwstat);
	957	+ return __blkg_prfill_rwstat(sf, pd, &rwstat);
949	958	}
950	959
951	960	static int tg_print_cpu_rwstat(struct cgroup cgrp, struct cftype cft,
952	961
953	962
954	963
955	964
956	965
...	...	@@ -958,22 +967,26 @@
958	967	return 0;
959	968	}
960	969
961		-static u64 tg_prfill_conf_u64(struct seq_file sf, void pdata, int off)
	970	+static u64 tg_prfill_conf_u64(struct seq_file sf, struct blkg_policy_data pd,
	971	+ int off)
962	972	{
963		- u64 v = (u64 )(pdata + off);
	973	+ struct throtl_grp *tg = pd_to_tg(pd);
	974	+ u64 v = (u64 )((void *)tg + off);
964	975
965	976	if (v == -1)
966	977	return 0;
967		- return __blkg_prfill_u64(sf, pdata, v);
	978	+ return __blkg_prfill_u64(sf, pd, v);
968	979	}
969	980
970		-static u64 tg_prfill_conf_uint(struct seq_file sf, void pdata, int off)
	981	+static u64 tg_prfill_conf_uint(struct seq_file sf, struct blkg_policy_data pd,
	982	+ int off)
971	983	{
972		- unsigned int v = (unsigned int )(pdata + off);
	984	+ struct throtl_grp *tg = pd_to_tg(pd);
	985	+ unsigned int v = (unsigned int )((void *)tg + off);
973	986
974	987	if (v == -1)
975	988	return 0;
976		- return __blkg_prfill_u64(sf, pdata, v);
	989	+ return __blkg_prfill_u64(sf, pd, v);
977	990	}
978	991
979	992	static int tg_print_conf_u64(struct cgroup cgrp, struct cftype cft,
...	...	@@ -1092,7 +1105,7 @@
1092	1105	.pd_exit_fn = throtl_pd_exit,
1093	1106	.pd_reset_stats_fn = throtl_pd_reset_stats,
1094	1107	},
1095		- .pdata_size = sizeof(struct throtl_grp),
	1108	+ .pd_size = sizeof(struct throtl_grp),
1096	1109	.cftypes = throtl_files,
1097	1110	};
1098	1111
...	...	@@ -217,6 +217,9 @@
217	217
218	218	/* This is per cgroup per device grouping structure */
219	219	struct cfq_group {
	220	+ /* must be the first member */
	221	+ struct blkg_policy_data pd;
	222	+
220	223	/* group service_tree member */
221	224	struct rb_node rb_node;
222	225
...	...	@@ -409,6 +412,21 @@
409	412	CFQ_CFQQ_FNS(wait_busy);
410	413	#undef CFQ_CFQQ_FNS
411	414
	415	+static inline struct cfq_group pd_to_cfqg(struct blkg_policy_data pd)
	416	+{
	417	+ return pd ? container_of(pd, struct cfq_group, pd) : NULL;
	418	+}
	419	+
	420	+static inline struct cfq_group blkg_to_cfqg(struct blkcg_gq blkg)
	421	+{
	422	+ return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
	423	+}
	424	+
	425	+static inline struct blkcg_gq cfqg_to_blkg(struct cfq_group cfqg)
	426	+{
	427	+ return pd_to_blkg(&cfqg->pd);
	428	+}
	429	+
412	430	#if defined(CONFIG_CFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
413	431
414	432	/* cfqg stats flags */
...	...	@@ -553,16 +571,6 @@
553	571
554	572	#ifdef CONFIG_CFQ_GROUP_IOSCHED
555	573
556		-static inline struct cfq_group blkg_to_cfqg(struct blkcg_gq blkg)
557		-{
558		- return blkg_to_pdata(blkg, &blkcg_policy_cfq);
559		-}
560		-
561		-static inline struct blkcg_gq cfqg_to_blkg(struct cfq_group cfqg)
562		-{
563		- return pdata_to_blkg(cfqg);
564		-}
565		-
566	574	static inline void cfqg_get(struct cfq_group *cfqg)
567	575	{
568	576	return blkg_get(cfqg_to_blkg(cfqg));
...	...	@@ -662,8 +670,6 @@
662	670
663	671	#else /* CONFIG_CFQ_GROUP_IOSCHED */
664	672
665		-static inline struct cfq_group blkg_to_cfqg(struct blkcg_gq blkg) { return NULL; }
666		-static inline struct blkcg_gq cfqg_to_blkg(struct cfq_group cfqg) { return NULL; }
667	673	static inline void cfqg_get(struct cfq_group *cfqg) { }
668	674	static inline void cfqg_put(struct cfq_group *cfqg) { }
669	675
670	676
671	677
...	...	@@ -1374,13 +1380,14 @@
1374	1380	cfqg_get(cfqg);
1375	1381	}
1376	1382
1377		-static u64 cfqg_prfill_weight_device(struct seq_file sf, void pdata, int off)
	1383	+static u64 cfqg_prfill_weight_device(struct seq_file *sf,
	1384	+ struct blkg_policy_data *pd, int off)
1378	1385	{
1379		- struct cfq_group *cfqg = pdata;
	1386	+ struct cfq_group *cfqg = pd_to_cfqg(pd);
1380	1387
1381	1388	if (!cfqg->dev_weight)
1382	1389	return 0;
1383		- return __blkg_prfill_u64(sf, pdata, cfqg->dev_weight);
	1390	+ return __blkg_prfill_u64(sf, pd, cfqg->dev_weight);
1384	1391	}
1385	1392
1386	1393	static int cfqg_print_weight_device(struct cgroup cgrp, struct cftype cft,
1387	1394
...	...	@@ -1467,9 +1474,10 @@
1467	1474	}
1468	1475
1469	1476	#ifdef CONFIG_DEBUG_BLK_CGROUP
1470		-static u64 cfqg_prfill_avg_queue_size(struct seq_file sf, void pdata, int off)
	1477	+static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf,
	1478	+ struct blkg_policy_data *pd, int off)
1471	1479	{
1472		- struct cfq_group *cfqg = pdata;
	1480	+ struct cfq_group *cfqg = pd_to_cfqg(pd);
1473	1481	u64 samples = blkg_stat_read(&cfqg->stats.avg_queue_size_samples);
1474	1482	u64 v = 0;
1475	1483
...	...	@@ -1477,7 +1485,7 @@
1477	1485	v = blkg_stat_read(&cfqg->stats.avg_queue_size_sum);
1478	1486	do_div(v, samples);
1479	1487	}
1480		- __blkg_prfill_u64(sf, pdata, v);
	1488	+ __blkg_prfill_u64(sf, pd, v);
1481	1489	return 0;
1482	1490	}
1483	1491
...	...	@@ -4161,7 +4169,7 @@
4161	4169	.pd_init_fn = cfq_pd_init,
4162	4170	.pd_reset_stats_fn = cfq_pd_reset_stats,
4163	4171	},
4164		- .pdata_size = sizeof(struct cfq_group),
	4172	+ .pd_size = sizeof(struct cfq_group),
4165	4173	.cftypes = cfq_blkcg_files,
4166	4174	};
4167	4175	#endif