Eric Lee / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 67acd8b4b7a3f1b183ae358e1dfdb8a80e170736

Authored by Linus Torvalds
2 parents b13d3720ec ad160d2319

Merge git://git.kernel.org/pub/scm/linux/kernel/git/arjan/linux-2.6-async 

* git://git.kernel.org/pub/scm/linux/kernel/git/arjan/linux-2.6-async:
  async: don't do the initcall stuff post boot
  bootchart: improve output based on Dave Jones' feedback
  async: make the final inode deletion an asynchronous event
  fastboot: Make libata initialization even more async
  fastboot: make the libata port scan asynchronous
  fastboot: make scsi probes asynchronous
  async: Asynchronous function calls to speed up kernel boot

Showing 13 changed files Side-by-side Diff

drivers/ata/libata-core.c
Diff comments   View file @ 67acd8b
... ... @@ -56,6 +56,7 @@
56 56 #include <linux/workqueue.h>
57 57 #include <linux/scatterlist.h>
58 58 #include <linux/io.h>
  59 +#include <linux/async.h>
59 60 #include <scsi/scsi.h>
60 61 #include <scsi/scsi_cmnd.h>
61 62 #include <scsi/scsi_host.h>
... ... @@ -5909,6 +5910,54 @@
5909 5910 host->ops = ops;
5910 5911 }
5911 5912  
  5913 +
  5914 +static void async_port_probe(void *data, async_cookie_t cookie)
  5915 +{
  5916 + int rc;
  5917 + struct ata_port *ap = data;
  5918 + /* probe */
  5919 + if (ap->ops->error_handler) {
  5920 + struct ata_eh_info *ehi = &ap->link.eh_info;
  5921 + unsigned long flags;
  5922 +
  5923 + ata_port_probe(ap);
  5924 +
  5925 + /* kick EH for boot probing */
  5926 + spin_lock_irqsave(ap->lock, flags);
  5927 +
  5928 + ehi->probe_mask |= ATA_ALL_DEVICES;
  5929 + ehi->action |= ATA_EH_RESET | ATA_EH_LPM;
  5930 + ehi->flags |= ATA_EHI_NO_AUTOPSY | ATA_EHI_QUIET;
  5931 +
  5932 + ap->pflags &= ~ATA_PFLAG_INITIALIZING;
  5933 + ap->pflags |= ATA_PFLAG_LOADING;
  5934 + ata_port_schedule_eh(ap);
  5935 +
  5936 + spin_unlock_irqrestore(ap->lock, flags);
  5937 +
  5938 + /* wait for EH to finish */
  5939 + ata_port_wait_eh(ap);
  5940 + } else {
  5941 + DPRINTK("ata%u: bus probe begin\n", ap->print_id);
  5942 + rc = ata_bus_probe(ap);
  5943 + DPRINTK("ata%u: bus probe end\n", ap->print_id);
  5944 +
  5945 + if (rc) {
  5946 + /* FIXME: do something useful here?
  5947 + * Current libata behavior will
  5948 + * tear down everything when
  5949 + * the module is removed
  5950 + * or the h/w is unplugged.
  5951 + */
  5952 + }
  5953 + }
  5954 +
  5955 + /* in order to keep device order, we need to synchronize at this point */
  5956 + async_synchronize_cookie(cookie);
  5957 +
  5958 + ata_scsi_scan_host(ap, 1);
  5959 +
  5960 +}
5912 5961 /**
5913 5962 * ata_host_register - register initialized ATA host
5914 5963 * @host: ATA host to register
5915 5964  
... ... @@ -5988,52 +6037,9 @@
5988 6037 DPRINTK("probe begin\n");
5989 6038 for (i = 0; i < host->n_ports; i++) {
5990 6039 struct ata_port *ap = host->ports[i];
5991   -
5992   - /* probe */
5993   - if (ap->ops->error_handler) {
5994   - struct ata_eh_info *ehi = &ap->link.eh_info;
5995   - unsigned long flags;
5996   -
5997   - ata_port_probe(ap);
5998   -
5999   - /* kick EH for boot probing */
6000   - spin_lock_irqsave(ap->lock, flags);
6001   -
6002   - ehi->probe_mask |= ATA_ALL_DEVICES;
6003   - ehi->action |= ATA_EH_RESET | ATA_EH_LPM;
6004   - ehi->flags |= ATA_EHI_NO_AUTOPSY | ATA_EHI_QUIET;
6005   -
6006   - ap->pflags &= ~ATA_PFLAG_INITIALIZING;
6007   - ap->pflags |= ATA_PFLAG_LOADING;
6008   - ata_port_schedule_eh(ap);
6009   -
6010   - spin_unlock_irqrestore(ap->lock, flags);
6011   -
6012   - /* wait for EH to finish */
6013   - ata_port_wait_eh(ap);
6014   - } else {
6015   - DPRINTK("ata%u: bus probe begin\n", ap->print_id);
6016   - rc = ata_bus_probe(ap);
6017   - DPRINTK("ata%u: bus probe end\n", ap->print_id);
6018   -
6019   - if (rc) {
6020   - /* FIXME: do something useful here?
6021   - * Current libata behavior will
6022   - * tear down everything when
6023   - * the module is removed
6024   - * or the h/w is unplugged.
6025   - */
6026   - }
6027   - }
  6040 + async_schedule(async_port_probe, ap);
6028 6041 }
6029   -
6030   - /* probes are done, now scan each port's disk(s) */
6031   - DPRINTK("host probe begin\n");
6032   - for (i = 0; i < host->n_ports; i++) {
6033   - struct ata_port *ap = host->ports[i];
6034   -
6035   - ata_scsi_scan_host(ap, 1);
6036   - }
  6042 + DPRINTK("probe end\n");
6037 6043  
6038 6044 return 0;
6039 6045 }
drivers/scsi/scsi_scan.c
Diff comments   View file @ 67acd8b
... ... @@ -32,6 +32,7 @@
32 32 #include <linux/delay.h>
33 33 #include <linux/kthread.h>
34 34 #include <linux/spinlock.h>
  35 +#include <linux/async.h>
35 36  
36 37 #include <scsi/scsi.h>
37 38 #include <scsi/scsi_cmnd.h>
... ... @@ -179,6 +180,8 @@
179 180 spin_unlock(&async_scan_lock);
180 181  
181 182 kfree(data);
  183 + /* Synchronize async operations globally */
  184 + async_synchronize_full();
182 185 return 0;
183 186 }
184 187  
... ... @@ -48,6 +48,7 @@
48 48 #include <linux/delay.h>
49 49 #include <linux/mutex.h>
50 50 #include <linux/string_helpers.h>
  51 +#include <linux/async.h>
51 52 #include <asm/uaccess.h>
52 53  
53 54 #include <scsi/scsi.h>
... ... @@ -1802,6 +1803,71 @@
1802 1803 return 0;
1803 1804 }
1804 1805  
  1806 +/*
  1807 + * The asynchronous part of sd_probe
  1808 + */
  1809 +static void sd_probe_async(void *data, async_cookie_t cookie)
  1810 +{
  1811 + struct scsi_disk *sdkp = data;
  1812 + struct scsi_device *sdp;
  1813 + struct gendisk *gd;
  1814 + u32 index;
  1815 + struct device *dev;
  1816 +
  1817 + sdp = sdkp->device;
  1818 + gd = sdkp->disk;
  1819 + index = sdkp->index;
  1820 + dev = &sdp->sdev_gendev;
  1821 +
  1822 + if (!sdp->request_queue->rq_timeout) {
  1823 + if (sdp->type != TYPE_MOD)
  1824 + blk_queue_rq_timeout(sdp->request_queue, SD_TIMEOUT);
  1825 + else
  1826 + blk_queue_rq_timeout(sdp->request_queue,
  1827 + SD_MOD_TIMEOUT);
  1828 + }
  1829 +
  1830 + device_initialize(&sdkp->dev);
  1831 + sdkp->dev.parent = &sdp->sdev_gendev;
  1832 + sdkp->dev.class = &sd_disk_class;
  1833 + strncpy(sdkp->dev.bus_id, sdp->sdev_gendev.bus_id, BUS_ID_SIZE);
  1834 +
  1835 + if (device_add(&sdkp->dev))
  1836 + goto out_free_index;
  1837 +
  1838 + get_device(&sdp->sdev_gendev);
  1839 +
  1840 + if (index < SD_MAX_DISKS) {
  1841 + gd->major = sd_major((index & 0xf0) >> 4);
  1842 + gd->first_minor = ((index & 0xf) << 4) | (index & 0xfff00);
  1843 + gd->minors = SD_MINORS;
  1844 + }
  1845 + gd->fops = &sd_fops;
  1846 + gd->private_data = &sdkp->driver;
  1847 + gd->queue = sdkp->device->request_queue;
  1848 +
  1849 + sd_revalidate_disk(gd);
  1850 +
  1851 + blk_queue_prep_rq(sdp->request_queue, sd_prep_fn);
  1852 +
  1853 + gd->driverfs_dev = &sdp->sdev_gendev;
  1854 + gd->flags = GENHD_FL_EXT_DEVT | GENHD_FL_DRIVERFS;
  1855 + if (sdp->removable)
  1856 + gd->flags |= GENHD_FL_REMOVABLE;
  1857 +
  1858 + dev_set_drvdata(dev, sdkp);
  1859 + add_disk(gd);
  1860 + sd_dif_config_host(sdkp);
  1861 +
  1862 + sd_printk(KERN_NOTICE, sdkp, "Attached SCSI %sdisk\n",
  1863 + sdp->removable ? "removable " : "");
  1864 +
  1865 + return;
  1866 +
  1867 + out_free_index:
  1868 + ida_remove(&sd_index_ida, index);
  1869 +}
  1870 +
1805 1871 /**
1806 1872 * sd_probe - called during driver initialization and whenever a
1807 1873 * new scsi device is attached to the system. It is called once
... ... @@ -1865,48 +1931,7 @@
1865 1931 sdkp->openers = 0;
1866 1932 sdkp->previous_state = 1;
1867 1933  
1868   - if (!sdp->request_queue->rq_timeout) {
1869   - if (sdp->type != TYPE_MOD)
1870   - blk_queue_rq_timeout(sdp->request_queue, SD_TIMEOUT);
1871   - else
1872   - blk_queue_rq_timeout(sdp->request_queue,
1873   - SD_MOD_TIMEOUT);
1874   - }
1875   -
1876   - device_initialize(&sdkp->dev);
1877   - sdkp->dev.parent = &sdp->sdev_gendev;
1878   - sdkp->dev.class = &sd_disk_class;
1879   - strncpy(sdkp->dev.bus_id, sdp->sdev_gendev.bus_id, BUS_ID_SIZE);
1880   -
1881   - if (device_add(&sdkp->dev))
1882   - goto out_free_index;
1883   -
1884   - get_device(&sdp->sdev_gendev);
1885   -
1886   - if (index < SD_MAX_DISKS) {
1887   - gd->major = sd_major((index & 0xf0) >> 4);
1888   - gd->first_minor = ((index & 0xf) << 4) | (index & 0xfff00);
1889   - gd->minors = SD_MINORS;
1890   - }
1891   - gd->fops = &sd_fops;
1892   - gd->private_data = &sdkp->driver;
1893   - gd->queue = sdkp->device->request_queue;
1894   -
1895   - sd_revalidate_disk(gd);
1896   -
1897   - blk_queue_prep_rq(sdp->request_queue, sd_prep_fn);
1898   -
1899   - gd->driverfs_dev = &sdp->sdev_gendev;
1900   - gd->flags = GENHD_FL_EXT_DEVT | GENHD_FL_DRIVERFS;
1901   - if (sdp->removable)
1902   - gd->flags |= GENHD_FL_REMOVABLE;
1903   -
1904   - dev_set_drvdata(dev, sdkp);
1905   - add_disk(gd);
1906   - sd_dif_config_host(sdkp);
1907   -
1908   - sd_printk(KERN_NOTICE, sdkp, "Attached SCSI %sdisk\n",
1909   - sdp->removable ? "removable " : "");
  1934 + async_schedule(sd_probe_async, sdkp);
1910 1935  
1911 1936 return 0;
1912 1937  
... ... @@ -22,6 +22,7 @@
22 22 #include <linux/bootmem.h>
23 23 #include <linux/inotify.h>
24 24 #include <linux/mount.h>
  25 +#include <linux/async.h>
25 26  
26 27 /*
27 28 * This is needed for the following functions:
28 29  
29 30  
... ... @@ -1138,16 +1139,11 @@
1138 1139 * I_FREEING is set so that no-one will take a new reference to the inode while
1139 1140 * it is being deleted.
1140 1141 */
1141   -void generic_delete_inode(struct inode *inode)
  1142 +static void generic_delete_inode_async(void *data, async_cookie_t cookie)
1142 1143 {
  1144 + struct inode *inode = data;
1143 1145 const struct super_operations *op = inode->i_sb->s_op;
1144 1146  
1145   - list_del_init(&inode->i_list);
1146   - list_del_init(&inode->i_sb_list);
1147   - inode->i_state |= I_FREEING;
1148   - inodes_stat.nr_inodes--;
1149   - spin_unlock(&inode_lock);
1150   -
1151 1147 security_inode_delete(inode);
1152 1148  
1153 1149 if (op->delete_inode) {
... ... @@ -1169,6 +1165,16 @@
1169 1165 wake_up_inode(inode);
1170 1166 BUG_ON(inode->i_state != I_CLEAR);
1171 1167 destroy_inode(inode);
  1168 +}
  1169 +
  1170 +void generic_delete_inode(struct inode *inode)
  1171 +{
  1172 + list_del_init(&inode->i_list);
  1173 + list_del_init(&inode->i_sb_list);
  1174 + inode->i_state |= I_FREEING;
  1175 + inodes_stat.nr_inodes--;
  1176 + spin_unlock(&inode_lock);
  1177 + async_schedule_special(generic_delete_inode_async, inode, &inode->i_sb->s_async_list);
1172 1178 }
1173 1179  
1174 1180 EXPORT_SYMBOL(generic_delete_inode);
... ... @@ -38,6 +38,7 @@
38 38 #include <linux/kobject.h>
39 39 #include <linux/mutex.h>
40 40 #include <linux/file.h>
  41 +#include <linux/async.h>
41 42 #include <asm/uaccess.h>
42 43 #include "internal.h"
43 44  
... ... @@ -71,6 +72,7 @@
71 72 INIT_HLIST_HEAD(&s->s_anon);
72 73 INIT_LIST_HEAD(&s->s_inodes);
73 74 INIT_LIST_HEAD(&s->s_dentry_lru);
  75 + INIT_LIST_HEAD(&s->s_async_list);
74 76 init_rwsem(&s->s_umount);
75 77 mutex_init(&s->s_lock);
76 78 lockdep_set_class(&s->s_umount, &type->s_umount_key);
77 79  
... ... @@ -289,11 +291,18 @@
289 291 {
290 292 const struct super_operations *sop = sb->s_op;
291 293  
  294 +
292 295 if (sb->s_root) {
293 296 shrink_dcache_for_umount(sb);
294 297 fsync_super(sb);
295 298 lock_super(sb);
296 299 sb->s_flags &= ~MS_ACTIVE;
  300 +
  301 + /*
  302 + * wait for asynchronous fs operations to finish before going further
  303 + */
  304 + async_synchronize_full_special(&sb->s_async_list);
  305 +
297 306 /* bad name - it should be evict_inodes() */
298 307 invalidate_inodes(sb);
299 308 lock_kernel();
... ... @@ -449,6 +458,7 @@
449 458 if (sb->s_flags & MS_RDONLY)
450 459 continue;
451 460 sb->s_need_sync_fs = 1;
  461 + async_synchronize_full_special(&sb->s_async_list);
452 462 }
453 463  
454 464 restart:
include/linux/async.h
Diff comments   View file @ 67acd8b
  1 +/*
  2 + * async.h: Asynchronous function calls for boot performance
  3 + *
  4 + * (C) Copyright 2009 Intel Corporation
  5 + * Author: Arjan van de Ven <arjan@linux.intel.com>
  6 + *
  7 + * This program is free software; you can redistribute it and/or
  8 + * modify it under the terms of the GNU General Public License
  9 + * as published by the Free Software Foundation; version 2
  10 + * of the License.
  11 + */
  12 +
  13 +#include <linux/types.h>
  14 +#include <linux/list.h>
  15 +
  16 +typedef u64 async_cookie_t;
  17 +typedef void (async_func_ptr) (void *data, async_cookie_t cookie);
  18 +
  19 +extern async_cookie_t async_schedule(async_func_ptr *ptr, void *data);
  20 +extern async_cookie_t async_schedule_special(async_func_ptr *ptr, void *data, struct list_head *list);
  21 +extern void async_synchronize_full(void);
  22 +extern void async_synchronize_full_special(struct list_head *list);
  23 +extern void async_synchronize_cookie(async_cookie_t cookie);
  24 +extern void async_synchronize_cookie_special(async_cookie_t cookie, struct list_head *list);
... ... @@ -1184,6 +1184,11 @@
1184 1184 * generic_show_options()
1185 1185 */
1186 1186 char *s_options;
  1187 +
  1188 + /*
  1189 + * storage for asynchronous operations
  1190 + */
  1191 + struct list_head s_async_list;
1187 1192 };
1188 1193  
1189 1194 extern struct timespec current_fs_time(struct super_block *sb);
... ... @@ -13,6 +13,7 @@
13 13 #include <linux/init.h>
14 14 #include <linux/fs.h>
15 15 #include <linux/initrd.h>
  16 +#include <linux/async.h>
16 17  
17 18 #include <linux/nfs_fs.h>
18 19 #include <linux/nfs_fs_sb.h>
... ... @@ -372,6 +373,7 @@
372 373 /* wait for the known devices to complete their probing */
373 374 while (driver_probe_done() != 0)
374 375 msleep(100);
  376 + async_synchronize_full();
375 377  
376 378 md_run_setup();
377 379  
... ... @@ -62,6 +62,7 @@
62 62 #include <linux/signal.h>
63 63 #include <linux/idr.h>
64 64 #include <linux/ftrace.h>
  65 +#include <linux/async.h>
65 66 #include <trace/boot.h>
66 67  
67 68 #include <asm/io.h>
... ... @@ -685,7 +686,7 @@
685 686 rest_init();
686 687 }
687 688  
688   -static int initcall_debug;
  689 +int initcall_debug;
689 690 core_param(initcall_debug, initcall_debug, bool, 0644);
690 691  
691 692 int do_one_initcall(initcall_t fn)
... ... @@ -786,6 +787,8 @@
786 787 */
787 788 static noinline int init_post(void)
788 789 {
  790 + /* need to finish all async __init code before freeing the memory */
  791 + async_synchronize_full();
789 792 free_initmem();
790 793 unlock_kernel();
791 794 mark_rodata_ro();
... ... @@ -9,7 +9,8 @@
9 9 rcupdate.o extable.o params.o posix-timers.o \
10 10 kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
11 11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
12   - notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o
  12 + notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
  13 + async.o
13 14  
14 15 ifdef CONFIG_FUNCTION_TRACER
15 16 # Do not trace debug files and internal ftrace files
  1 +/*
  2 + * async.c: Asynchronous function calls for boot performance
  3 + *
  4 + * (C) Copyright 2009 Intel Corporation
  5 + * Author: Arjan van de Ven <arjan@linux.intel.com>
  6 + *
  7 + * This program is free software; you can redistribute it and/or
  8 + * modify it under the terms of the GNU General Public License
  9 + * as published by the Free Software Foundation; version 2
  10 + * of the License.
  11 + */
  12 +
  13 +
  14 +/*
  15 +
  16 +Goals and Theory of Operation
  17 +
  18 +The primary goal of this feature is to reduce the kernel boot time,
  19 +by doing various independent hardware delays and discovery operations
  20 +decoupled and not strictly serialized.
  21 +
  22 +More specifically, the asynchronous function call concept allows
  23 +certain operations (primarily during system boot) to happen
  24 +asynchronously, out of order, while these operations still
  25 +have their externally visible parts happen sequentially and in-order.
  26 +(not unlike how out-of-order CPUs retire their instructions in order)
  27 +
  28 +Key to the asynchronous function call implementation is the concept of
  29 +a "sequence cookie" (which, although it has an abstracted type, can be
  30 +thought of as a monotonically incrementing number).
  31 +
  32 +The async core will assign each scheduled event such a sequence cookie and
  33 +pass this to the called functions.
  34 +
  35 +The asynchronously called function should before doing a globally visible
  36 +operation, such as registering device numbers, call the
  37 +async_synchronize_cookie() function and pass in its own cookie. The
  38 +async_synchronize_cookie() function will make sure that all asynchronous
  39 +operations that were scheduled prior to the operation corresponding with the
  40 +cookie have completed.
  41 +
  42 +Subsystem/driver initialization code that scheduled asynchronous probe
  43 +functions, but which shares global resources with other drivers/subsystems
  44 +that do not use the asynchronous call feature, need to do a full
  45 +synchronization with the async_synchronize_full() function, before returning
  46 +from their init function. This is to maintain strict ordering between the
  47 +asynchronous and synchronous parts of the kernel.
  48 +
  49 +*/
  50 +
  51 +#include <linux/async.h>
  52 +#include <linux/module.h>
  53 +#include <linux/wait.h>
  54 +#include <linux/sched.h>
  55 +#include <linux/init.h>
  56 +#include <linux/kthread.h>
  57 +#include <asm/atomic.h>
  58 +
  59 +static async_cookie_t next_cookie = 1;
  60 +
  61 +#define MAX_THREADS 256
  62 +#define MAX_WORK 32768
  63 +
  64 +static LIST_HEAD(async_pending);
  65 +static LIST_HEAD(async_running);
  66 +static DEFINE_SPINLOCK(async_lock);
  67 +
  68 +struct async_entry {
  69 + struct list_head list;
  70 + async_cookie_t cookie;
  71 + async_func_ptr *func;
  72 + void *data;
  73 + struct list_head *running;
  74 +};
  75 +
  76 +static DECLARE_WAIT_QUEUE_HEAD(async_done);
  77 +static DECLARE_WAIT_QUEUE_HEAD(async_new);
  78 +
  79 +static atomic_t entry_count;
  80 +static atomic_t thread_count;
  81 +
  82 +extern int initcall_debug;
  83 +
  84 +
  85 +/*
  86 + * MUST be called with the lock held!
  87 + */
  88 +static async_cookie_t __lowest_in_progress(struct list_head *running)
  89 +{
  90 + struct async_entry *entry;
  91 + if (!list_empty(&async_pending)) {
  92 + entry = list_first_entry(&async_pending,
  93 + struct async_entry, list);
  94 + return entry->cookie;
  95 + } else if (!list_empty(running)) {
  96 + entry = list_first_entry(running,
  97 + struct async_entry, list);
  98 + return entry->cookie;
  99 + } else {
  100 + /* nothing in progress... next_cookie is "infinity" */
  101 + return next_cookie;
  102 + }
  103 +
  104 +}
  105 +/*
  106 + * pick the first pending entry and run it
  107 + */
  108 +static void run_one_entry(void)
  109 +{
  110 + unsigned long flags;
  111 + struct async_entry *entry;
  112 + ktime_t calltime, delta, rettime;
  113 +
  114 + /* 1) pick one task from the pending queue */
  115 +
  116 + spin_lock_irqsave(&async_lock, flags);
  117 + if (list_empty(&async_pending))
  118 + goto out;
  119 + entry = list_first_entry(&async_pending, struct async_entry, list);
  120 +
  121 + /* 2) move it to the running queue */
  122 + list_del(&entry->list);
  123 + list_add_tail(&entry->list, &async_running);
  124 + spin_unlock_irqrestore(&async_lock, flags);
  125 +
  126 + /* 3) run it (and print duration)*/
  127 + if (initcall_debug && system_state == SYSTEM_BOOTING) {
  128 + printk("calling %lli_%pF @ %i\n", entry->cookie, entry->func, task_pid_nr(current));
  129 + calltime = ktime_get();
  130 + }
  131 + entry->func(entry->data, entry->cookie);
  132 + if (initcall_debug && system_state == SYSTEM_BOOTING) {
  133 + rettime = ktime_get();
  134 + delta = ktime_sub(rettime, calltime);
  135 + printk("initcall %lli_%pF returned 0 after %lld usecs\n", entry->cookie,
  136 + entry->func, ktime_to_ns(delta) >> 10);
  137 + }
  138 +
  139 + /* 4) remove it from the running queue */
  140 + spin_lock_irqsave(&async_lock, flags);
  141 + list_del(&entry->list);
  142 +
  143 + /* 5) free the entry */
  144 + kfree(entry);
  145 + atomic_dec(&entry_count);
  146 +
  147 + spin_unlock_irqrestore(&async_lock, flags);
  148 +
  149 + /* 6) wake up any waiters. */
  150 + wake_up(&async_done);
  151 + return;
  152 +
  153 +out:
  154 + spin_unlock_irqrestore(&async_lock, flags);
  155 +}
  156 +
  157 +
  158 +static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct list_head *running)
  159 +{
  160 + struct async_entry *entry;
  161 + unsigned long flags;
  162 + async_cookie_t newcookie;
  163 +
  164 +
  165 + /* allow irq-off callers */
  166 + entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
  167 +
  168 + /*
  169 + * If we're out of memory or if there's too much work
  170 + * pending already, we execute synchronously.
  171 + */
  172 + if (!entry || atomic_read(&entry_count) > MAX_WORK) {
  173 + kfree(entry);
  174 + spin_lock_irqsave(&async_lock, flags);
  175 + newcookie = next_cookie++;
  176 + spin_unlock_irqrestore(&async_lock, flags);
  177 +
  178 + /* low on memory.. run synchronously */
  179 + ptr(data, newcookie);
  180 + return newcookie;
  181 + }
  182 + entry->func = ptr;
  183 + entry->data = data;
  184 + entry->running = running;
  185 +
  186 + spin_lock_irqsave(&async_lock, flags);
  187 + newcookie = entry->cookie = next_cookie++;
  188 + list_add_tail(&entry->list, &async_pending);
  189 + atomic_inc(&entry_count);
  190 + spin_unlock_irqrestore(&async_lock, flags);
  191 + wake_up(&async_new);
  192 + return newcookie;
  193 +}
  194 +
  195 +async_cookie_t async_schedule(async_func_ptr *ptr, void *data)
  196 +{
  197 + return __async_schedule(ptr, data, &async_pending);
  198 +}
  199 +EXPORT_SYMBOL_GPL(async_schedule);
  200 +
  201 +async_cookie_t async_schedule_special(async_func_ptr *ptr, void *data, struct list_head *running)
  202 +{
  203 + return __async_schedule(ptr, data, running);
  204 +}
  205 +EXPORT_SYMBOL_GPL(async_schedule_special);
  206 +
  207 +void async_synchronize_full(void)
  208 +{
  209 + async_synchronize_cookie(next_cookie);
  210 +}
  211 +EXPORT_SYMBOL_GPL(async_synchronize_full);
  212 +
  213 +void async_synchronize_full_special(struct list_head *list)
  214 +{
  215 + async_synchronize_cookie_special(next_cookie, list);
  216 +}
  217 +EXPORT_SYMBOL_GPL(async_synchronize_full_special);
  218 +
  219 +void async_synchronize_cookie_special(async_cookie_t cookie, struct list_head *running)
  220 +{
  221 + ktime_t starttime, delta, endtime;
  222 +
  223 + if (initcall_debug && system_state == SYSTEM_BOOTING) {
  224 + printk("async_waiting @ %i\n", task_pid_nr(current));
  225 + starttime = ktime_get();
  226 + }
  227 +
  228 + wait_event(async_done, __lowest_in_progress(running) >= cookie);
  229 +
  230 + if (initcall_debug && system_state == SYSTEM_BOOTING) {
  231 + endtime = ktime_get();
  232 + delta = ktime_sub(endtime, starttime);
  233 +
  234 + printk("async_continuing @ %i after %lli usec\n",
  235 + task_pid_nr(current), ktime_to_ns(delta) >> 10);
  236 + }
  237 +}
  238 +EXPORT_SYMBOL_GPL(async_synchronize_cookie_special);
  239 +
  240 +void async_synchronize_cookie(async_cookie_t cookie)
  241 +{
  242 + async_synchronize_cookie_special(cookie, &async_running);
  243 +}
  244 +EXPORT_SYMBOL_GPL(async_synchronize_cookie);
  245 +
  246 +
  247 +static int async_thread(void *unused)
  248 +{
  249 + DECLARE_WAITQUEUE(wq, current);
  250 + add_wait_queue(&async_new, &wq);
  251 +
  252 + while (!kthread_should_stop()) {
  253 + int ret = HZ;
  254 + set_current_state(TASK_INTERRUPTIBLE);
  255 + /*
  256 + * check the list head without lock.. false positives
  257 + * are dealt with inside run_one_entry() while holding
  258 + * the lock.
  259 + */
  260 + rmb();
  261 + if (!list_empty(&async_pending))
  262 + run_one_entry();
  263 + else
  264 + ret = schedule_timeout(HZ);
  265 +
  266 + if (ret == 0) {
  267 + /*
  268 + * we timed out, this means we as thread are redundant.
  269 + * we sign off and die, but we to avoid any races there
  270 + * is a last-straw check to see if work snuck in.
  271 + */
  272 + atomic_dec(&thread_count);
  273 + wmb(); /* manager must see our departure first */
  274 + if (list_empty(&async_pending))
  275 + break;
  276 + /*
  277 + * woops work came in between us timing out and us
  278 + * signing off; we need to stay alive and keep working.
  279 + */
  280 + atomic_inc(&thread_count);
  281 + }
  282 + }
  283 + remove_wait_queue(&async_new, &wq);
  284 +
  285 + return 0;
  286 +}
  287 +
  288 +static int async_manager_thread(void *unused)
  289 +{
  290 + DECLARE_WAITQUEUE(wq, current);
  291 + add_wait_queue(&async_new, &wq);
  292 +
  293 + while (!kthread_should_stop()) {
  294 + int tc, ec;
  295 +
  296 + set_current_state(TASK_INTERRUPTIBLE);
  297 +
  298 + tc = atomic_read(&thread_count);
  299 + rmb();
  300 + ec = atomic_read(&entry_count);
  301 +
  302 + while (tc < ec && tc < MAX_THREADS) {
  303 + kthread_run(async_thread, NULL, "async/%i", tc);
  304 + atomic_inc(&thread_count);
  305 + tc++;
  306 + }
  307 +
  308 + schedule();
  309 + }
  310 + remove_wait_queue(&async_new, &wq);
  311 +
  312 + return 0;
  313 +}
  314 +
  315 +static int __init async_init(void)
  316 +{
  317 + kthread_run(async_manager_thread, NULL, "async/mgr");
  318 + return 0;
  319 +}
  320 +
  321 +core_initcall(async_init);
kernel/irq/autoprobe.c
Diff comments   View file @ 67acd8b
... ... @@ -10,6 +10,7 @@
10 10 #include <linux/module.h>
11 11 #include <linux/interrupt.h>
12 12 #include <linux/delay.h>
  13 +#include <linux/async.h>
13 14  
14 15 #include "internals.h"
15 16  
... ... @@ -34,6 +35,10 @@
34 35 unsigned int status;
35 36 int i;
36 37  
  38 + /*
  39 + * quiesce the kernel, or at least the asynchronous portion
  40 + */
  41 + async_synchronize_full();
37 42 mutex_lock(&probing_active);
38 43 /*
39 44 * something may have generated an irq long ago and we want to
... ... @@ -50,6 +50,7 @@
50 50 #include <asm/sections.h>
51 51 #include <linux/tracepoint.h>
52 52 #include <linux/ftrace.h>
  53 +#include <linux/async.h>
53 54  
54 55 #if 0
55 56 #define DEBUGP printk
... ... @@ -816,6 +817,7 @@
816 817 mod->exit();
817 818 blocking_notifier_call_chain(&module_notify_list,
818 819 MODULE_STATE_GOING, mod);
  820 + async_synchronize_full();
819 821 mutex_lock(&module_mutex);
820 822 /* Store the name of the last unloaded module for diagnostic purposes */
821 823 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));