libata: take advantage of cmwq and remove concurrency limitations

libata has two concurrency related limitations. a. ata_wq which is used for polling PIO has single thread per CPU. If there are multiple devices doing polling PIO on the same CPU, they can't be executed simultaneously. b. ata_aux_wq which is used for SCSI probing has single thread. In cases where SCSI probing is stalled for extended period of time which is possible for ATAPI devices, this will stall all probing. #a is solved by increasing maximum concurrency of ata_wq. Please note that polling PIO might be used under allocation path and thus needs to be served by a separate wq with a rescuer. #b is solved by using the default wq instead and achieving exclusion via per-port mutex. Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Jeff Garzik <jgarzik@pobox.com>

libata: take advantage of cmwq and remove concurrency limitations
libata has two concurrency related limitations. a. ata_wq which is used for polling PIO has single thread per CPU. If there are multiple devices doing polling PIO on the same CPU, they can't be executed simultaneously. b. ata_aux_wq which is used for SCSI probing has single thread. In cases where SCSI probing is stalled for extended period of time which is possible for ATAPI devices, this will stall all probing. #a is solved by increasing maximum concurrency of ata_wq. Please note that polling PIO might be used under allocation path and thus needs to be served by a separate wq with a rescuer. #b is solved by using the default wq instead and achieving exclusion via per-port mutex. Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Jeff Garzik <jgarzik@pobox.com>
Tejun Heo
1 parent d313dd85ad
Showing 6 changed files with 15 additions and 30 deletions Side-by-side Diff
drivers/ata/libata-core.c
drivers/ata/libata-eh.c
drivers/ata/libata-scsi.c
drivers/ata/libata-sff.c
drivers/ata/libata.h
include/linux/libata.h
@@ -98,8 +98,6 @@
  
 unsigned int ata_print_id = 1;
  
-struct workqueue_struct *ata_aux_wq;
-
 struct ata_force_param {
 	const char	*name;
 	unsigned int	cbl;
@@ -5611,6 +5609,7 @@
 	ap->msg_enable = ATA_MSG_DRV | ATA_MSG_ERR | ATA_MSG_WARN;
 #endif
  
+	mutex_init(&ap->scsi_scan_mutex);
 	INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug);
 	INIT_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan);
 	INIT_LIST_HEAD(&ap->eh_done_q);
  
  
  
@@ -6549,29 +6548,20 @@
  
 	ata_parse_force_param();
  
-	ata_aux_wq = create_singlethread_workqueue("ata_aux");
-	if (!ata_aux_wq)
-		goto fail;
-
 	rc = ata_sff_init();
-	if (rc)
-		goto fail;
+	if (rc) {
+		kfree(ata_force_tbl);
+		return rc;
+	}
  
 	printk(KERN_DEBUG "libata version " DRV_VERSION " loaded.\n");
 	return 0;
-
-fail:
-	kfree(ata_force_tbl);
-	if (ata_aux_wq)
-		destroy_workqueue(ata_aux_wq);
-	return rc;
 }
  
 static void __exit ata_exit(void)
 {
 	ata_sff_exit();
 	kfree(ata_force_tbl);
-	destroy_workqueue(ata_aux_wq);
 }
  
 subsys_initcall(ata_init);
@@ -727,7 +727,7 @@
 	if (ap->pflags & ATA_PFLAG_LOADING)
 		ap->pflags &= ~ATA_PFLAG_LOADING;
 	else if (ap->pflags & ATA_PFLAG_SCSI_HOTPLUG)
-		queue_delayed_work(ata_aux_wq, &ap->hotplug_task, 0);
+		schedule_delayed_work(&ap->hotplug_task, 0);
  
 	if (ap->pflags & ATA_PFLAG_RECOVERED)
 		ata_port_printk(ap, KERN_INFO, "EH complete\n");
@@ -2944,7 +2944,7 @@
 			ehc->i.flags |= ATA_EHI_SETMODE;
  
 			/* schedule the scsi_rescan_device() here */
-			queue_work(ata_aux_wq, &(ap->scsi_rescan_task));
+			schedule_work(&(ap->scsi_rescan_task));
 		} else if (dev->class == ATA_DEV_UNKNOWN &&
 			   ehc->tries[dev->devno] &&
 			   ata_class_enabled(ehc->classes[dev->devno])) {
@@ -3435,7 +3435,7 @@
 				"                  switching to async\n");
 	}
  
-	queue_delayed_work(ata_aux_wq, &ap->hotplug_task,
+	queue_delayed_work(system_long_wq, &ap->hotplug_task,
 			   round_jiffies_relative(HZ));
 }
  
@@ -3582,6 +3582,7 @@
 	}
  
 	DPRINTK("ENTER\n");
+	mutex_lock(&ap->scsi_scan_mutex);
  
 	/* Unplug detached devices.  We cannot use link iterator here
 	 * because PMP links have to be scanned even if PMP is
@@ -3595,6 +3596,7 @@
 	/* scan for new ones */
 	ata_scsi_scan_host(ap, 0);
  
+	mutex_unlock(&ap->scsi_scan_mutex);
 	DPRINTK("EXIT\n");
 }
  
@@ -3673,9 +3675,7 @@
  *	@work: Pointer to ATA port to perform scsi_rescan_device()
  *
  *	After ATA pass thru (SAT) commands are executed successfully,
- *	libata need to propagate the changes to SCSI layer.  This
- *	function must be executed from ata_aux_wq such that sdev
- *	attach/detach don't race with rescan.
+ *	libata need to propagate the changes to SCSI layer.
  *
  *	LOCKING:
  *	Kernel thread context (may sleep).
@@ -3688,6 +3688,7 @@
 	struct ata_device *dev;
 	unsigned long flags;
  
+	mutex_lock(&ap->scsi_scan_mutex);
 	spin_lock_irqsave(ap->lock, flags);
  
 	ata_for_each_link(link, ap, EDGE) {
@@ -3707,6 +3708,7 @@
 	}
  
 	spin_unlock_irqrestore(ap->lock, flags);
+	mutex_unlock(&ap->scsi_scan_mutex);
 }
  
 /**
@@ -3318,14 +3318,7 @@
  
 int __init ata_sff_init(void)
 {
-	/*
-	 * FIXME: In UP case, there is only one workqueue thread and if you
-	 * have more than one PIO device, latency is bloody awful, with
-	 * occasional multi-second "hiccups" as one PIO device waits for
-	 * another.  It's an ugly wart that users DO occasionally complain
-	 * about; luckily most users have at most one PIO polled device.
-	 */
-	ata_sff_wq = create_workqueue("ata_sff");
+	ata_sff_wq = alloc_workqueue("ata_sff", WQ_RESCUER, WQ_MAX_ACTIVE);
 	if (!ata_sff_wq)
 		return -ENOMEM;
  
@@ -54,7 +54,6 @@
 };
  
 extern unsigned int ata_print_id;
-extern struct workqueue_struct *ata_aux_wq;
 extern int atapi_passthru16;
 extern int libata_fua;
 extern int libata_noacpi;
@@ -751,6 +751,7 @@
 	struct ata_host		*host;
 	struct device 		*dev;
  
+	struct mutex		scsi_scan_mutex;
 	struct delayed_work	hotplug_task;
 	struct work_struct	scsi_rescan_task;
...	...	@@ -98,8 +98,6 @@
98	98
99	99	unsigned int ata_print_id = 1;
100	100
101		-struct workqueue_struct *ata_aux_wq;
102		-
103	101	struct ata_force_param {
104	102	const char *name;
105	103	unsigned int cbl;
...	...	@@ -5611,6 +5609,7 @@
5611	5609	ap->msg_enable = ATA_MSG_DRV \| ATA_MSG_ERR \| ATA_MSG_WARN;
5612	5610	#endif
5613	5611
	5612	+ mutex_init(&ap->scsi_scan_mutex);
5614	5613	INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug);
5615	5614	INIT_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan);
5616	5615	INIT_LIST_HEAD(&ap->eh_done_q);
5617	5616
5618	5617
5619	5618
...	...	@@ -6549,29 +6548,20 @@
6549	6548
6550	6549	ata_parse_force_param();
6551	6550
6552		- ata_aux_wq = create_singlethread_workqueue("ata_aux");
6553		- if (!ata_aux_wq)
6554		- goto fail;
6555		-
6556	6551	rc = ata_sff_init();
6557		- if (rc)
6558		- goto fail;
	6552	+ if (rc) {
	6553	+ kfree(ata_force_tbl);
	6554	+ return rc;
	6555	+ }
6559	6556
6560	6557	printk(KERN_DEBUG "libata version " DRV_VERSION " loaded.\n");
6561	6558	return 0;
6562		-
6563		-fail:
6564		- kfree(ata_force_tbl);
6565		- if (ata_aux_wq)
6566		- destroy_workqueue(ata_aux_wq);
6567		- return rc;
6568	6559	}
6569	6560
6570	6561	static void __exit ata_exit(void)
6571	6562	{
6572	6563	ata_sff_exit();
6573	6564	kfree(ata_force_tbl);
6574		- destroy_workqueue(ata_aux_wq);
6575	6565	}
6576	6566
6577	6567	subsys_initcall(ata_init);
...	...	@@ -727,7 +727,7 @@
727	727	if (ap->pflags & ATA_PFLAG_LOADING)
728	728	ap->pflags &= ~ATA_PFLAG_LOADING;
729	729	else if (ap->pflags & ATA_PFLAG_SCSI_HOTPLUG)
730		- queue_delayed_work(ata_aux_wq, &ap->hotplug_task, 0);
	730	+ schedule_delayed_work(&ap->hotplug_task, 0);
731	731
732	732	if (ap->pflags & ATA_PFLAG_RECOVERED)
733	733	ata_port_printk(ap, KERN_INFO, "EH complete\n");
...	...	@@ -2944,7 +2944,7 @@
2944	2944	ehc->i.flags \|= ATA_EHI_SETMODE;
2945	2945
2946	2946	/* schedule the scsi_rescan_device() here */
2947		- queue_work(ata_aux_wq, &(ap->scsi_rescan_task));
	2947	+ schedule_work(&(ap->scsi_rescan_task));
2948	2948	} else if (dev->class == ATA_DEV_UNKNOWN &&
2949	2949	ehc->tries[dev->devno] &&
2950	2950	ata_class_enabled(ehc->classes[dev->devno])) {
...	...	@@ -3435,7 +3435,7 @@
3435	3435	" switching to async\n");
3436	3436	}
3437	3437
3438		- queue_delayed_work(ata_aux_wq, &ap->hotplug_task,
	3438	+ queue_delayed_work(system_long_wq, &ap->hotplug_task,
3439	3439	round_jiffies_relative(HZ));
3440	3440	}
3441	3441
...	...	@@ -3582,6 +3582,7 @@
3582	3582	}
3583	3583
3584	3584	DPRINTK("ENTER\n");
	3585	+ mutex_lock(&ap->scsi_scan_mutex);
3585	3586
3586	3587	/* Unplug detached devices. We cannot use link iterator here
3587	3588	* because PMP links have to be scanned even if PMP is
...	...	@@ -3595,6 +3596,7 @@
3595	3596	/* scan for new ones */
3596	3597	ata_scsi_scan_host(ap, 0);
3597	3598
	3599	+ mutex_unlock(&ap->scsi_scan_mutex);
3598	3600	DPRINTK("EXIT\n");
3599	3601	}
3600	3602
...	...	@@ -3673,9 +3675,7 @@
3673	3675	* @work: Pointer to ATA port to perform scsi_rescan_device()
3674	3676	*
3675	3677	* After ATA pass thru (SAT) commands are executed successfully,
3676		- * libata need to propagate the changes to SCSI layer. This
3677		- * function must be executed from ata_aux_wq such that sdev
3678		- * attach/detach don't race with rescan.
	3678	+ * libata need to propagate the changes to SCSI layer.
3679	3679	*
3680	3680	* LOCKING:
3681	3681	* Kernel thread context (may sleep).
...	...	@@ -3688,6 +3688,7 @@
3688	3688	struct ata_device *dev;
3689	3689	unsigned long flags;
3690	3690
	3691	+ mutex_lock(&ap->scsi_scan_mutex);
3691	3692	spin_lock_irqsave(ap->lock, flags);
3692	3693
3693	3694	ata_for_each_link(link, ap, EDGE) {
...	...	@@ -3707,6 +3708,7 @@
3707	3708	}
3708	3709
3709	3710	spin_unlock_irqrestore(ap->lock, flags);
	3711	+ mutex_unlock(&ap->scsi_scan_mutex);
3710	3712	}
3711	3713
3712	3714	/**
...	...	@@ -3318,14 +3318,7 @@
3318	3318
3319	3319	int __init ata_sff_init(void)
3320	3320	{
3321		- /*
3322		- * FIXME: In UP case, there is only one workqueue thread and if you
3323		- * have more than one PIO device, latency is bloody awful, with
3324		- * occasional multi-second "hiccups" as one PIO device waits for
3325		- * another. It's an ugly wart that users DO occasionally complain
3326		- * about; luckily most users have at most one PIO polled device.
3327		- */
3328		- ata_sff_wq = create_workqueue("ata_sff");
	3321	+ ata_sff_wq = alloc_workqueue("ata_sff", WQ_RESCUER, WQ_MAX_ACTIVE);
3329	3322	if (!ata_sff_wq)
3330	3323	return -ENOMEM;
3331	3324
...	...	@@ -54,7 +54,6 @@
54	54	};
55	55
56	56	extern unsigned int ata_print_id;
57		-extern struct workqueue_struct *ata_aux_wq;
58	57	extern int atapi_passthru16;
59	58	extern int libata_fua;
60	59	extern int libata_noacpi;
...	...	@@ -751,6 +751,7 @@
751	751	struct ata_host *host;
752	752	struct device *dev;
753	753
	754	+ struct mutex scsi_scan_mutex;
754	755	struct delayed_work hotplug_task;
755	756	struct work_struct scsi_rescan_task;
756	757