target: Add percpu refcounting for se_lun access

This patch adds percpu refcounting for se_lun access that allows the association of an se_lun + se_cmd in transport_lookup_cmd_lun() to occur without an extra list_head for tracking outstanding I/O during se_lun shutdown. This effectively changes se_lun shutdown logic to wait for outstanding I/O percpu references to complete in transport_lun_remove_cmd() using se_lun->lun_ref_comp, instead of explicitly draining the per se_lun command list and waiting for individual se_cmd descriptor processing to complete. Cc: Kent Overstreet <kmo@daterainc.com> Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>

target: Add percpu refcounting for se_lun access
This patch adds percpu refcounting for se_lun access that allows the association of an se_lun + se_cmd in transport_lookup_cmd_lun() to occur without an extra list_head for tracking outstanding I/O during se_lun shutdown. This effectively changes se_lun shutdown logic to wait for outstanding I/O percpu references to complete in transport_lun_remove_cmd() using se_lun->lun_ref_comp, instead of explicitly draining the per se_lun command list and waiting for individual se_cmd descriptor processing to complete. Cc: Kent Overstreet <kmo@daterainc.com> Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
Nicholas Bellinger
1 parent c9e8d128fe
Showing 6 changed files with 45 additions and 18 deletions Side-by-side Diff
drivers/target/target_core_device.c
drivers/target/target_core_internal.h
drivers/target/target_core_tpg.c
drivers/target/target_core_transport.c
drivers/target/target_core_xcopy.c
include/target/target_core_base.h
@@ -92,6 +92,9 @@
 		se_cmd->pr_res_key = deve->pr_res_key;
 		se_cmd->orig_fe_lun = unpacked_lun;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
+
+		percpu_ref_get(&se_lun->lun_ref);
+		se_cmd->lun_ref_active = true;
 	}
 	spin_unlock_irqrestore(&se_sess->se_node_acl->device_list_lock, flags);
  
@@ -119,6 +122,9 @@
 		se_cmd->se_lun = &se_sess->se_tpg->tpg_virt_lun0;
 		se_cmd->orig_fe_lun = 0;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
+
+		percpu_ref_get(&se_lun->lun_ref);
+		se_cmd->lun_ref_active = true;
 	}
  
 	/* Directly associate cmd with se_dev */
@@ -133,10 +139,6 @@
 	else if (se_cmd->data_direction == DMA_FROM_DEVICE)
 		dev->read_bytes += se_cmd->data_length;
 	spin_unlock_irqrestore(&dev->stats_lock, flags);
-
-	spin_lock_irqsave(&se_lun->lun_cmd_lock, flags);
-	list_add_tail(&se_cmd->se_lun_node, &se_lun->lun_cmd_list);
-	spin_unlock_irqrestore(&se_lun->lun_cmd_lock, flags);
  
 	return 0;
 }
@@ -100,7 +100,7 @@
 int	transport_dump_vpd_ident_type(struct t10_vpd *, unsigned char *, int);
 int	transport_dump_vpd_ident(struct t10_vpd *, unsigned char *, int);
 bool	target_stop_cmd(struct se_cmd *cmd, unsigned long *flags);
-int	transport_clear_lun_from_sessions(struct se_lun *);
+int	transport_clear_lun_ref(struct se_lun *);
 void	transport_send_task_abort(struct se_cmd *);
 sense_reason_t	target_cmd_size_check(struct se_cmd *cmd, unsigned int size);
 void	target_qf_do_work(struct work_struct *work);
@@ -634,6 +634,13 @@
 }
 EXPORT_SYMBOL(core_tpg_set_initiator_node_tag);
  
+static void core_tpg_lun_ref_release(struct percpu_ref *ref)
+{
+	struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
+
+	complete(&lun->lun_ref_comp);
+}
+
 static int core_tpg_setup_virtual_lun0(struct se_portal_group *se_tpg)
 {
 	/* Set in core_dev_setup_virtual_lun0() */
  
  
@@ -651,11 +658,18 @@
 	spin_lock_init(&lun->lun_acl_lock);
 	spin_lock_init(&lun->lun_cmd_lock);
 	spin_lock_init(&lun->lun_sep_lock);
+	init_completion(&lun->lun_ref_comp);
  
-	ret = core_tpg_post_addlun(se_tpg, lun, lun_access, dev);
+	ret = percpu_ref_init(&lun->lun_ref, core_tpg_lun_ref_release);
 	if (ret < 0)
 		return ret;
  
+	ret = core_tpg_post_addlun(se_tpg, lun, lun_access, dev);
+	if (ret < 0) {
+		percpu_ref_cancel_init(&lun->lun_ref);
+		return ret;
+	}
+
 	return 0;
 }
  
@@ -696,6 +710,7 @@
 		spin_lock_init(&lun->lun_acl_lock);
 		spin_lock_init(&lun->lun_cmd_lock);
 		spin_lock_init(&lun->lun_sep_lock);
+		init_completion(&lun->lun_ref_comp);
 	}
  
 	se_tpg->se_tpg_type = se_tpg_type;
  
@@ -816,10 +831,16 @@
 {
 	int ret;
  
-	ret = core_dev_export(lun_ptr, tpg, lun);
+	ret = percpu_ref_init(&lun->lun_ref, core_tpg_lun_ref_release);
 	if (ret < 0)
 		return ret;
  
+	ret = core_dev_export(lun_ptr, tpg, lun);
+	if (ret < 0) {
+		percpu_ref_cancel_init(&lun->lun_ref);
+		return ret;
+	}
+
 	spin_lock(&tpg->tpg_lun_lock);
 	lun->lun_access = lun_access;
 	lun->lun_status = TRANSPORT_LUN_STATUS_ACTIVE;
@@ -833,7 +854,7 @@
 	struct se_lun *lun)
 {
 	core_clear_lun_from_tpg(lun, tpg);
-	transport_clear_lun_from_sessions(lun);
+	transport_clear_lun_ref(lun);
 }
  
 struct se_lun *core_tpg_pre_dellun(
@@ -575,15 +575,11 @@
 static void transport_lun_remove_cmd(struct se_cmd *cmd)
 {
 	struct se_lun *lun = cmd->se_lun;
-	unsigned long flags;
  
-	if (!lun)
+	if (!lun || !cmd->lun_ref_active)
 		return;
  
-	spin_lock_irqsave(&lun->lun_cmd_lock, flags);
-	if (!list_empty(&cmd->se_lun_node))
-		list_del_init(&cmd->se_lun_node);
-	spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
+	percpu_ref_put(&lun->lun_ref);
 }
  
 void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
  
  
  
@@ -2537,21 +2533,23 @@
 	spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
 }
  
-static int transport_clear_lun_thread(void *p)
+static int transport_clear_lun_ref_thread(void *p)
 {
 	struct se_lun *lun = p;
  
-	__transport_clear_lun_from_sessions(lun);
+	percpu_ref_kill(&lun->lun_ref);
+
+	wait_for_completion(&lun->lun_ref_comp);
 	complete(&lun->lun_shutdown_comp);
  
 	return 0;
 }
  
-int transport_clear_lun_from_sessions(struct se_lun *lun)
+int transport_clear_lun_ref(struct se_lun *lun)
 {
 	struct task_struct *kt;
  
-	kt = kthread_run(transport_clear_lun_thread, lun,
+	kt = kthread_run(transport_clear_lun_ref_thread, lun,
 			"tcm_cl_%u", lun->unpacked_lun);
 	if (IS_ERR(kt)) {
 		pr_err("Unable to start clear_lun thread\n");
@@ -579,6 +579,7 @@
 	spin_lock_init(&pt_cmd->se_lun->lun_acl_lock);
 	spin_lock_init(&pt_cmd->se_lun->lun_cmd_lock);
 	spin_lock_init(&pt_cmd->se_lun->lun_sep_lock);
+	init_completion(&pt_cmd->se_lun->lun_ref_comp);
  
 	pt_cmd->se_dev = se_dev;
  
@@ -499,6 +499,9 @@
  
 	/* backend private data */
 	void			*priv;
+
+	/* Used for lun->lun_ref counting */
+	bool			lun_ref_active;
 };
  
 struct se_ua {
@@ -757,6 +760,8 @@
 	struct se_port		*lun_sep;
 	struct config_group	lun_group;
 	struct se_port_stat_grps port_stat_grps;
+	struct completion	lun_ref_comp;
+	struct percpu_ref	lun_ref;
 };
  
 struct scsi_port_stats {
...	...	@@ -92,6 +92,9 @@
92	92	se_cmd->pr_res_key = deve->pr_res_key;
93	93	se_cmd->orig_fe_lun = unpacked_lun;
94	94	se_cmd->se_cmd_flags \|= SCF_SE_LUN_CMD;
	95	+
	96	+ percpu_ref_get(&se_lun->lun_ref);
	97	+ se_cmd->lun_ref_active = true;
95	98	}
96	99	spin_unlock_irqrestore(&se_sess->se_node_acl->device_list_lock, flags);
97	100
...	...	@@ -119,6 +122,9 @@
119	122	se_cmd->se_lun = &se_sess->se_tpg->tpg_virt_lun0;
120	123	se_cmd->orig_fe_lun = 0;
121	124	se_cmd->se_cmd_flags \|= SCF_SE_LUN_CMD;
	125	+
	126	+ percpu_ref_get(&se_lun->lun_ref);
	127	+ se_cmd->lun_ref_active = true;
122	128	}
123	129
124	130	/* Directly associate cmd with se_dev */
...	...	@@ -133,10 +139,6 @@
133	139	else if (se_cmd->data_direction == DMA_FROM_DEVICE)
134	140	dev->read_bytes += se_cmd->data_length;
135	141	spin_unlock_irqrestore(&dev->stats_lock, flags);
136		-
137		- spin_lock_irqsave(&se_lun->lun_cmd_lock, flags);
138		- list_add_tail(&se_cmd->se_lun_node, &se_lun->lun_cmd_list);
139		- spin_unlock_irqrestore(&se_lun->lun_cmd_lock, flags);
140	142
141	143	return 0;
142	144	}
...	...	@@ -100,7 +100,7 @@
100	100	int transport_dump_vpd_ident_type(struct t10_vpd , unsigned char , int);
101	101	int transport_dump_vpd_ident(struct t10_vpd , unsigned char , int);
102	102	bool target_stop_cmd(struct se_cmd cmd, unsigned long flags);
103		-int transport_clear_lun_from_sessions(struct se_lun *);
	103	+int transport_clear_lun_ref(struct se_lun *);
104	104	void transport_send_task_abort(struct se_cmd *);
105	105	sense_reason_t target_cmd_size_check(struct se_cmd *cmd, unsigned int size);
106	106	void target_qf_do_work(struct work_struct *work);
...	...	@@ -634,6 +634,13 @@
634	634	}
635	635	EXPORT_SYMBOL(core_tpg_set_initiator_node_tag);
636	636
	637	+static void core_tpg_lun_ref_release(struct percpu_ref *ref)
	638	+{
	639	+ struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
	640	+
	641	+ complete(&lun->lun_ref_comp);
	642	+}
	643	+
637	644	static int core_tpg_setup_virtual_lun0(struct se_portal_group *se_tpg)
638	645	{
639	646	/* Set in core_dev_setup_virtual_lun0() */
640	647
641	648
...	...	@@ -651,11 +658,18 @@
651	658	spin_lock_init(&lun->lun_acl_lock);
652	659	spin_lock_init(&lun->lun_cmd_lock);
653	660	spin_lock_init(&lun->lun_sep_lock);
	661	+ init_completion(&lun->lun_ref_comp);
654	662
655		- ret = core_tpg_post_addlun(se_tpg, lun, lun_access, dev);
	663	+ ret = percpu_ref_init(&lun->lun_ref, core_tpg_lun_ref_release);
656	664	if (ret < 0)
657	665	return ret;
658	666
	667	+ ret = core_tpg_post_addlun(se_tpg, lun, lun_access, dev);
	668	+ if (ret < 0) {
	669	+ percpu_ref_cancel_init(&lun->lun_ref);
	670	+ return ret;
	671	+ }
	672	+
659	673	return 0;
660	674	}
661	675
...	...	@@ -696,6 +710,7 @@
696	710	spin_lock_init(&lun->lun_acl_lock);
697	711	spin_lock_init(&lun->lun_cmd_lock);
698	712	spin_lock_init(&lun->lun_sep_lock);
	713	+ init_completion(&lun->lun_ref_comp);
699	714	}
700	715
701	716	se_tpg->se_tpg_type = se_tpg_type;
702	717
...	...	@@ -816,10 +831,16 @@
816	831	{
817	832	int ret;
818	833
819		- ret = core_dev_export(lun_ptr, tpg, lun);
	834	+ ret = percpu_ref_init(&lun->lun_ref, core_tpg_lun_ref_release);
820	835	if (ret < 0)
821	836	return ret;
822	837
	838	+ ret = core_dev_export(lun_ptr, tpg, lun);
	839	+ if (ret < 0) {
	840	+ percpu_ref_cancel_init(&lun->lun_ref);
	841	+ return ret;
	842	+ }
	843	+
823	844	spin_lock(&tpg->tpg_lun_lock);
824	845	lun->lun_access = lun_access;
825	846	lun->lun_status = TRANSPORT_LUN_STATUS_ACTIVE;
...	...	@@ -833,7 +854,7 @@
833	854	struct se_lun *lun)
834	855	{
835	856	core_clear_lun_from_tpg(lun, tpg);
836		- transport_clear_lun_from_sessions(lun);
	857	+ transport_clear_lun_ref(lun);
837	858	}
838	859
839	860	struct se_lun *core_tpg_pre_dellun(
...	...	@@ -575,15 +575,11 @@
575	575	static void transport_lun_remove_cmd(struct se_cmd *cmd)
576	576	{
577	577	struct se_lun *lun = cmd->se_lun;
578		- unsigned long flags;
579	578
580		- if (!lun)
	579	+ if (!lun \|\| !cmd->lun_ref_active)
581	580	return;
582	581
583		- spin_lock_irqsave(&lun->lun_cmd_lock, flags);
584		- if (!list_empty(&cmd->se_lun_node))
585		- list_del_init(&cmd->se_lun_node);
586		- spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
	582	+ percpu_ref_put(&lun->lun_ref);
587	583	}
588	584
589	585	void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
590	586
591	587
592	588
...	...	@@ -2537,21 +2533,23 @@
2537	2533	spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
2538	2534	}
2539	2535
2540		-static int transport_clear_lun_thread(void *p)
	2536	+static int transport_clear_lun_ref_thread(void *p)
2541	2537	{
2542	2538	struct se_lun *lun = p;
2543	2539
2544		- __transport_clear_lun_from_sessions(lun);
	2540	+ percpu_ref_kill(&lun->lun_ref);
	2541	+
	2542	+ wait_for_completion(&lun->lun_ref_comp);
2545	2543	complete(&lun->lun_shutdown_comp);
2546	2544
2547	2545	return 0;
2548	2546	}
2549	2547
2550		-int transport_clear_lun_from_sessions(struct se_lun *lun)
	2548	+int transport_clear_lun_ref(struct se_lun *lun)
2551	2549	{
2552	2550	struct task_struct *kt;
2553	2551
2554		- kt = kthread_run(transport_clear_lun_thread, lun,
	2552	+ kt = kthread_run(transport_clear_lun_ref_thread, lun,
2555	2553	"tcm_cl_%u", lun->unpacked_lun);
2556	2554	if (IS_ERR(kt)) {
2557	2555	pr_err("Unable to start clear_lun thread\n");
...	...	@@ -579,6 +579,7 @@
579	579	spin_lock_init(&pt_cmd->se_lun->lun_acl_lock);
580	580	spin_lock_init(&pt_cmd->se_lun->lun_cmd_lock);
581	581	spin_lock_init(&pt_cmd->se_lun->lun_sep_lock);
	582	+ init_completion(&pt_cmd->se_lun->lun_ref_comp);
582	583
583	584	pt_cmd->se_dev = se_dev;
584	585
...	...	@@ -499,6 +499,9 @@
499	499
500	500	/* backend private data */
501	501	void *priv;
	502	+
	503	+ /* Used for lun->lun_ref counting */
	504	+ bool lun_ref_active;
502	505	};
503	506
504	507	struct se_ua {
...	...	@@ -757,6 +760,8 @@
757	760	struct se_port *lun_sep;
758	761	struct config_group lun_group;
759	762	struct se_port_stat_grps port_stat_grps;
	763	+ struct completion lun_ref_comp;
	764	+ struct percpu_ref lun_ref;
760	765	};
761	766
762	767	struct scsi_port_stats {