block, cfq: move icq cache management to block core

Let elevators set ->icq_size and ->icq_align in elevator_type and elv_register() and elv_unregister() respectively create and destroy kmem_cache for icq. * elv_register() now can return failure. All callers updated. * icq caches are automatically named "ELVNAME_io_cq". * cfq_slab_setup/kill() are collapsed into cfq_init/exit(). * While at it, minor indentation change for iosched_cfq.elevator_name for consistency. This will help moving icq management to block core. This doesn't introduce any functional change. Signed-off-by: Tejun Heo <tj@kernel.org> Signed-off-by: Jens Axboe <axboe@kernel.dk>

block, cfq: move icq cache management to block core
Let elevators set ->icq_size and ->icq_align in elevator_type and elv_register() and elv_unregister() respectively create and destroy kmem_cache for icq. * elv_register() now can return failure. All callers updated. * icq caches are automatically named "ELVNAME_io_cq". * cfq_slab_setup/kill() are collapsed into cfq_init/exit(). * While at it, minor indentation change for iosched_cfq.elevator_name for consistency. This will help moving icq management to block core. This doesn't introduce any functional change. Signed-off-by: Tejun Heo <tj@kernel.org> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Tejun Heo · Jens Axboe
1 parent 47fdd4ca96
Showing 5 changed files with 62 additions and 42 deletions Side-by-side Diff
block/cfq-iosched.c
block/deadline-iosched.c
block/elevator.c
block/noop-iosched.c
include/linux/elevator.h
@@ -3914,34 +3914,6 @@
 	return cfqd;
 }
  
-static void cfq_slab_kill(void)
-{
-	/*
-	 * Caller already ensured that pending RCU callbacks are completed,
-	 * so we should have no busy allocations at this point.
-	 */
-	if (cfq_pool)
-		kmem_cache_destroy(cfq_pool);
-	if (cfq_icq_pool)
-		kmem_cache_destroy(cfq_icq_pool);
-}
-
-static int __init cfq_slab_setup(void)
-{
-	cfq_pool = KMEM_CACHE(cfq_queue, 0);
-	if (!cfq_pool)
-		goto fail;
-
-	cfq_icq_pool = KMEM_CACHE(cfq_io_cq, 0);
-	if (!cfq_icq_pool)
-		goto fail;
-
-	return 0;
-fail:
-	cfq_slab_kill();
-	return -ENOMEM;
-}
-
 /*
  * sysfs parts below -->
  */
  
@@ -4053,8 +4025,10 @@
 		.elevator_init_fn =		cfq_init_queue,
 		.elevator_exit_fn =		cfq_exit_queue,
 	},
+	.icq_size	=	sizeof(struct cfq_io_cq),
+	.icq_align	=	__alignof__(struct cfq_io_cq),
 	.elevator_attrs =	cfq_attrs,
-	.elevator_name =	"cfq",
+	.elevator_name	=	"cfq",
 	.elevator_owner =	THIS_MODULE,
 };
  
@@ -4072,6 +4046,8 @@
  
 static int __init cfq_init(void)
 {
+	int ret;
+
 	/*
 	 * could be 0 on HZ < 1000 setups
 	 */
  
@@ -4086,10 +4062,17 @@
 #else
 		cfq_group_idle = 0;
 #endif
-	if (cfq_slab_setup())
+	cfq_pool = KMEM_CACHE(cfq_queue, 0);
+	if (!cfq_pool)
 		return -ENOMEM;
  
-	elv_register(&iosched_cfq);
+	ret = elv_register(&iosched_cfq);
+	if (ret) {
+		kmem_cache_destroy(cfq_pool);
+		return ret;
+	}
+	cfq_icq_pool = iosched_cfq.icq_cache;
+
 	blkio_policy_register(&blkio_policy_cfq);
  
 	return 0;
@@ -4099,8 +4082,7 @@
 {
 	blkio_policy_unregister(&blkio_policy_cfq);
 	elv_unregister(&iosched_cfq);
-	rcu_barrier();	/* make sure all cic RCU frees are complete */
-	cfq_slab_kill();
+	kmem_cache_destroy(cfq_pool);
 }
  
 module_init(cfq_init);
@@ -448,9 +448,7 @@
  
 static int __init deadline_init(void)
 {
-	elv_register(&iosched_deadline);
-
-	return 0;
+	return elv_register(&iosched_deadline);
 }
  
 static void __exit deadline_exit(void)
@@ -886,15 +886,36 @@
 }
 EXPORT_SYMBOL(elv_unregister_queue);
  
-void elv_register(struct elevator_type *e)
+int elv_register(struct elevator_type *e)
 {
 	char *def = "";
  
+	/* create icq_cache if requested */
+	if (e->icq_size) {
+		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
+		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))
+			return -EINVAL;
+
+		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
+			 "%s_io_cq", e->elevator_name);
+		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
+						 e->icq_align, 0, NULL);
+		if (!e->icq_cache)
+			return -ENOMEM;
+	}
+
+	/* register, don't allow duplicate names */
 	spin_lock(&elv_list_lock);
-	BUG_ON(elevator_find(e->elevator_name));
+	if (elevator_find(e->elevator_name)) {
+		spin_unlock(&elv_list_lock);
+		if (e->icq_cache)
+			kmem_cache_destroy(e->icq_cache);
+		return -EBUSY;
+	}
 	list_add_tail(&e->list, &elv_list);
 	spin_unlock(&elv_list_lock);
  
+	/* print pretty message */
 	if (!strcmp(e->elevator_name, chosen_elevator) ||
 			(!*chosen_elevator &&
 			 !strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
  
  
@@ -902,14 +923,26 @@
  
 	printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,
 								def);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(elv_register);
  
 void elv_unregister(struct elevator_type *e)
 {
+	/* unregister */
 	spin_lock(&elv_list_lock);
 	list_del_init(&e->list);
 	spin_unlock(&elv_list_lock);
+
+	/*
+	 * Destroy icq_cache if it exists.  icq's are RCU managed.  Make
+	 * sure all RCU operations are complete before proceeding.
+	 */
+	if (e->icq_cache) {
+		rcu_barrier();
+		kmem_cache_destroy(e->icq_cache);
+		e->icq_cache = NULL;
+	}
 }
 EXPORT_SYMBOL_GPL(elv_unregister);
  
@@ -94,9 +94,7 @@
  
 static int __init noop_init(void)
 {
-	elv_register(&elevator_noop);
-
-	return 0;
+	return elv_register(&elevator_noop);
 }
  
 static void __exit noop_exit(void)
@@ -78,10 +78,19 @@
  */
 struct elevator_type
 {
+	/* managed by elevator core */
+	struct kmem_cache *icq_cache;
+
+	/* fields provided by elevator implementation */
 	struct elevator_ops ops;
+	size_t icq_size;
+	size_t icq_align;
 	struct elv_fs_entry *elevator_attrs;
 	char elevator_name[ELV_NAME_MAX];
 	struct module *elevator_owner;
+
+	/* managed by elevator core */
+	char icq_cache_name[ELV_NAME_MAX + 5];	/* elvname + "_io_cq" */
 	struct list_head list;
 };
  
@@ -127,7 +136,7 @@
 /*
  * io scheduler registration
  */
-extern void elv_register(struct elevator_type *);
+extern int elv_register(struct elevator_type *);
 extern void elv_unregister(struct elevator_type *);
  
 /*
...	...	@@ -3914,34 +3914,6 @@
3914	3914	return cfqd;
3915	3915	}
3916	3916
3917		-static void cfq_slab_kill(void)
3918		-{
3919		- /*
3920		- * Caller already ensured that pending RCU callbacks are completed,
3921		- * so we should have no busy allocations at this point.
3922		- */
3923		- if (cfq_pool)
3924		- kmem_cache_destroy(cfq_pool);
3925		- if (cfq_icq_pool)
3926		- kmem_cache_destroy(cfq_icq_pool);
3927		-}
3928		-
3929		-static int __init cfq_slab_setup(void)
3930		-{
3931		- cfq_pool = KMEM_CACHE(cfq_queue, 0);
3932		- if (!cfq_pool)
3933		- goto fail;
3934		-
3935		- cfq_icq_pool = KMEM_CACHE(cfq_io_cq, 0);
3936		- if (!cfq_icq_pool)
3937		- goto fail;
3938		-
3939		- return 0;
3940		-fail:
3941		- cfq_slab_kill();
3942		- return -ENOMEM;
3943		-}
3944		-
3945	3917	/*
3946	3918	* sysfs parts below -->
3947	3919	*/
3948	3920
...	...	@@ -4053,8 +4025,10 @@
4053	4025	.elevator_init_fn = cfq_init_queue,
4054	4026	.elevator_exit_fn = cfq_exit_queue,
4055	4027	},
	4028	+ .icq_size = sizeof(struct cfq_io_cq),
	4029	+ .icq_align = __alignof__(struct cfq_io_cq),
4056	4030	.elevator_attrs = cfq_attrs,
4057		- .elevator_name = "cfq",
	4031	+ .elevator_name = "cfq",
4058	4032	.elevator_owner = THIS_MODULE,
4059	4033	};
4060	4034
...	...	@@ -4072,6 +4046,8 @@
4072	4046
4073	4047	static int __init cfq_init(void)
4074	4048	{
	4049	+ int ret;
	4050	+
4075	4051	/*
4076	4052	* could be 0 on HZ < 1000 setups
4077	4053	*/
4078	4054
...	...	@@ -4086,10 +4062,17 @@
4086	4062	#else
4087	4063	cfq_group_idle = 0;
4088	4064	#endif
4089		- if (cfq_slab_setup())
	4065	+ cfq_pool = KMEM_CACHE(cfq_queue, 0);
	4066	+ if (!cfq_pool)
4090	4067	return -ENOMEM;
4091	4068
4092		- elv_register(&iosched_cfq);
	4069	+ ret = elv_register(&iosched_cfq);
	4070	+ if (ret) {
	4071	+ kmem_cache_destroy(cfq_pool);
	4072	+ return ret;
	4073	+ }
	4074	+ cfq_icq_pool = iosched_cfq.icq_cache;
	4075	+
4093	4076	blkio_policy_register(&blkio_policy_cfq);
4094	4077
4095	4078	return 0;
...	...	@@ -4099,8 +4082,7 @@
4099	4082	{
4100	4083	blkio_policy_unregister(&blkio_policy_cfq);
4101	4084	elv_unregister(&iosched_cfq);
4102		- rcu_barrier(); /* make sure all cic RCU frees are complete */
4103		- cfq_slab_kill();
	4085	+ kmem_cache_destroy(cfq_pool);
4104	4086	}
4105	4087
4106	4088	module_init(cfq_init);
...	...	@@ -448,9 +448,7 @@
448	448
449	449	static int __init deadline_init(void)
450	450	{
451		- elv_register(&iosched_deadline);
452		-
453		- return 0;
	451	+ return elv_register(&iosched_deadline);
454	452	}
455	453
456	454	static void __exit deadline_exit(void)
...	...	@@ -886,15 +886,36 @@
886	886	}
887	887	EXPORT_SYMBOL(elv_unregister_queue);
888	888
889		-void elv_register(struct elevator_type *e)
	889	+int elv_register(struct elevator_type *e)
890	890	{
891	891	char *def = "";
892	892
	893	+ /* create icq_cache if requested */
	894	+ if (e->icq_size) {
	895	+ if (WARN_ON(e->icq_size < sizeof(struct io_cq)) \|\|
	896	+ WARN_ON(e->icq_align < __alignof__(struct io_cq)))
	897	+ return -EINVAL;
	898	+
	899	+ snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
	900	+ "%s_io_cq", e->elevator_name);
	901	+ e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
	902	+ e->icq_align, 0, NULL);
	903	+ if (!e->icq_cache)
	904	+ return -ENOMEM;
	905	+ }
	906	+
	907	+ /* register, don't allow duplicate names */
893	908	spin_lock(&elv_list_lock);
894		- BUG_ON(elevator_find(e->elevator_name));
	909	+ if (elevator_find(e->elevator_name)) {
	910	+ spin_unlock(&elv_list_lock);
	911	+ if (e->icq_cache)
	912	+ kmem_cache_destroy(e->icq_cache);
	913	+ return -EBUSY;
	914	+ }
895	915	list_add_tail(&e->list, &elv_list);
896	916	spin_unlock(&elv_list_lock);
897	917
	918	+ /* print pretty message */
898	919	if (!strcmp(e->elevator_name, chosen_elevator) \|\|
899	920	(!*chosen_elevator &&
900	921	!strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
901	922
902	923
...	...	@@ -902,14 +923,26 @@
902	923
903	924	printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,
904	925	def);
	926	+ return 0;
905	927	}
906	928	EXPORT_SYMBOL_GPL(elv_register);
907	929
908	930	void elv_unregister(struct elevator_type *e)
909	931	{
	932	+ /* unregister */
910	933	spin_lock(&elv_list_lock);
911	934	list_del_init(&e->list);
912	935	spin_unlock(&elv_list_lock);
	936	+
	937	+ /*
	938	+ * Destroy icq_cache if it exists. icq's are RCU managed. Make
	939	+ * sure all RCU operations are complete before proceeding.
	940	+ */
	941	+ if (e->icq_cache) {
	942	+ rcu_barrier();
	943	+ kmem_cache_destroy(e->icq_cache);
	944	+ e->icq_cache = NULL;
	945	+ }
913	946	}
914	947	EXPORT_SYMBOL_GPL(elv_unregister);
915	948
...	...	@@ -94,9 +94,7 @@
94	94
95	95	static int __init noop_init(void)
96	96	{
97		- elv_register(&elevator_noop);
98		-
99		- return 0;
	97	+ return elv_register(&elevator_noop);
100	98	}
101	99
102	100	static void __exit noop_exit(void)
...	...	@@ -78,10 +78,19 @@
78	78	*/
79	79	struct elevator_type
80	80	{
	81	+ /* managed by elevator core */
	82	+ struct kmem_cache *icq_cache;
	83	+
	84	+ /* fields provided by elevator implementation */
81	85	struct elevator_ops ops;
	86	+ size_t icq_size;
	87	+ size_t icq_align;
82	88	struct elv_fs_entry *elevator_attrs;
83	89	char elevator_name[ELV_NAME_MAX];
84	90	struct module *elevator_owner;
	91	+
	92	+ /* managed by elevator core */
	93	+ char icq_cache_name[ELV_NAME_MAX + 5]; /* elvname + "_io_cq" */
85	94	struct list_head list;
86	95	};
87	96
...	...	@@ -127,7 +136,7 @@
127	136	/*
128	137	* io scheduler registration
129	138	*/
130		-extern void elv_register(struct elevator_type *);
	139	+extern int elv_register(struct elevator_type *);
131	140	extern void elv_unregister(struct elevator_type *);
132	141
133	142	/*