Commit 8b8edefa2fffbff97f9eec8b70e78ae23abad1a0

Authored by Tejun Heo
1 parent e120153ddf

fscache: convert object to use workqueue instead of slow-work

Make fscache object state transition callbacks use workqueue instead
of slow-work.  New dedicated unbound CPU workqueue fscache_object_wq
is created.  get/put callbacks are renamed and modified to take
@object and called directly from the enqueue wrapper and the work
function.  While at it, make all open coded instances of get/put to
use fscache_get/put_object().

* Unbound workqueue is used.

* work_busy() output is printed instead of slow-work flags in object
  debugging outputs.  They mean basically the same thing bit-for-bit.

* sysctl fscache.object_max_active added to control concurrency.  The
  default value is nr_cpus clamped between 4 and
  WQ_UNBOUND_MAX_ACTIVE.

* slow_work_sleep_till_thread_needed() is replaced with fscache
  private implementation fscache_object_sleep_till_congested() which
  waits on fscache_object_wq congestion.

* debugfs support is dropped for now.  Tracing API based debug
  facility is planned to be added.

Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: David Howells <dhowells@redhat.com>

Showing 7 changed files with 158 additions and 74 deletions Side-by-side Diff

Documentation/filesystems/caching/fscache.txt
... ... @@ -343,8 +343,8 @@
343 343 [root@andromeda ~]# head /proc/fs/fscache/objects
344 344 OBJECT PARENT STAT CHLDN OPS OOP IPR EX READS EM EV F S | NETFS_COOKIE_DEF TY FL NETFS_DATA OBJECT_KEY, AUX_DATA
345 345 ======== ======== ==== ===== === === === == ===== == == = = | ================ == == ================ ================
346   - 17e4b 2 ACTV 0 0 0 0 0 0 7b 4 0 8 | NFS.fh DT 0 ffff88001dd82820 010006017edcf8bbc93b43298fdfbe71e50b57b13a172c0117f38472, e567634700000000000000000000000063f2404a000000000000000000000000c9030000000000000000000063f2404a
347   - 1693a 2 ACTV 0 0 0 0 0 0 7b 4 0 8 | NFS.fh DT 0 ffff88002db23380 010006017edcf8bbc93b43298fdfbe71e50b57b1e0162c01a2df0ea6, 420ebc4a000000000000000000000000420ebc4a0000000000000000000000000e1801000000000000000000420ebc4a
  346 + 17e4b 2 ACTV 0 0 0 0 0 0 7b 4 0 0 | NFS.fh DT 0 ffff88001dd82820 010006017edcf8bbc93b43298fdfbe71e50b57b13a172c0117f38472, e567634700000000000000000000000063f2404a000000000000000000000000c9030000000000000000000063f2404a
  347 + 1693a 2 ACTV 0 0 0 0 0 0 7b 4 0 0 | NFS.fh DT 0 ffff88002db23380 010006017edcf8bbc93b43298fdfbe71e50b57b1e0162c01a2df0ea6, 420ebc4a000000000000000000000000420ebc4a0000000000000000000000000e1801000000000000000000420ebc4a
348 348  
349 349 where the first set of columns before the '|' describe the object:
350 350  
... ... @@ -362,7 +362,7 @@
362 362 EM Object's event mask
363 363 EV Events raised on this object
364 364 F Object flags
365   - S Object slow-work work item flags
  365 + S Object work item busy state mask (1:pending 2:running)
366 366  
367 367 and the second set of columns describe the object's cookie, if present:
368 368  
... ... @@ -395,8 +395,8 @@
395 395 w Show objects that don't have pending writes
396 396 R Show objects that have outstanding reads
397 397 r Show objects that don't have outstanding reads
398   - S Show objects that have slow work queued
399   - s Show objects that don't have slow work queued
  398 + S Show objects that have work queued
  399 + s Show objects that don't have work queued
400 400  
401 401 If neither side of a letter pair is given, then both are implied. For example:
402 402  
fs/cachefiles/namei.c
... ... @@ -37,9 +37,9 @@
37 37  
38 38 printk(KERN_ERR "%sobject: OBJ%x\n",
39 39 prefix, object->fscache.debug_id);
40   - printk(KERN_ERR "%sobjstate=%s fl=%lx swfl=%lx ev=%lx[%lx]\n",
  40 + printk(KERN_ERR "%sobjstate=%s fl=%lx wbusy=%x ev=%lx[%lx]\n",
41 41 prefix, fscache_object_states[object->fscache.state],
42   - object->fscache.flags, object->fscache.work.flags,
  42 + object->fscache.flags, work_busy(&object->fscache.work),
43 43 object->fscache.events,
44 44 object->fscache.event_mask & FSCACHE_OBJECT_EVENTS_MASK);
45 45 printk(KERN_ERR "%sops=%u inp=%u exc=%u\n",
... ... @@ -212,7 +212,7 @@
212 212  
213 213 /* if the object we're waiting for is queued for processing,
214 214 * then just put ourselves on the queue behind it */
215   - if (slow_work_is_queued(&xobject->fscache.work)) {
  215 + if (work_pending(&xobject->fscache.work)) {
216 216 _debug("queue OBJ%x behind OBJ%x immediately",
217 217 object->fscache.debug_id,
218 218 xobject->fscache.debug_id);
... ... @@ -220,8 +220,7 @@
220 220 }
221 221  
222 222 /* otherwise we sleep until either the object we're waiting for
223   - * is done, or the slow-work facility wants the thread back to
224   - * do other work */
  223 + * is done, or the fscache_object is congested */
225 224 wq = bit_waitqueue(&xobject->flags, CACHEFILES_OBJECT_ACTIVE);
226 225 init_wait(&wait);
227 226 requeue = false;
... ... @@ -229,8 +228,8 @@
229 228 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
230 229 if (!test_bit(CACHEFILES_OBJECT_ACTIVE, &xobject->flags))
231 230 break;
232   - requeue = slow_work_sleep_till_thread_needed(
233   - &object->fscache.work, &timeout);
  231 +
  232 + requeue = fscache_object_sleep_till_congested(&timeout);
234 233 } while (timeout > 0 && !requeue);
235 234 finish_wait(wq, &wait);
236 235  
fs/fscache/internal.h
... ... @@ -82,6 +82,13 @@
82 82 extern unsigned fscache_defer_create;
83 83 extern unsigned fscache_debug;
84 84 extern struct kobject *fscache_root;
  85 +extern struct workqueue_struct *fscache_object_wq;
  86 +DECLARE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait);
  87 +
  88 +static inline bool fscache_object_congested(void)
  89 +{
  90 + return workqueue_congested(WORK_CPU_UNBOUND, fscache_object_wq);
  91 +}
85 92  
86 93 extern int fscache_wait_bit(void *);
87 94 extern int fscache_wait_bit_interruptible(void *);
... ... @@ -15,6 +15,7 @@
15 15 #include <linux/sched.h>
16 16 #include <linux/completion.h>
17 17 #include <linux/slab.h>
  18 +#include <linux/seq_file.h>
18 19 #include "internal.h"
19 20  
20 21 MODULE_DESCRIPTION("FS Cache Manager");
21 22  
22 23  
23 24  
24 25  
... ... @@ -40,22 +41,89 @@
40 41 "FS-Cache debugging mask");
41 42  
42 43 struct kobject *fscache_root;
  44 +struct workqueue_struct *fscache_object_wq;
43 45  
  46 +DEFINE_PER_CPU(wait_queue_head_t, fscache_object_cong_wait);
  47 +
  48 +/* these values serve as lower bounds, will be adjusted in fscache_init() */
  49 +static unsigned fscache_object_max_active = 4;
  50 +
  51 +#ifdef CONFIG_SYSCTL
  52 +static struct ctl_table_header *fscache_sysctl_header;
  53 +
  54 +static int fscache_max_active_sysctl(struct ctl_table *table, int write,
  55 + void __user *buffer,
  56 + size_t *lenp, loff_t *ppos)
  57 +{
  58 + struct workqueue_struct **wqp = table->extra1;
  59 + unsigned int *datap = table->data;
  60 + int ret;
  61 +
  62 + ret = proc_dointvec(table, write, buffer, lenp, ppos);
  63 + if (ret == 0)
  64 + workqueue_set_max_active(*wqp, *datap);
  65 + return ret;
  66 +}
  67 +
  68 +ctl_table fscache_sysctls[] = {
  69 + {
  70 + .procname = "object_max_active",
  71 + .data = &fscache_object_max_active,
  72 + .maxlen = sizeof(unsigned),
  73 + .mode = 0644,
  74 + .proc_handler = fscache_max_active_sysctl,
  75 + .extra1 = &fscache_object_wq,
  76 + },
  77 + {}
  78 +};
  79 +
  80 +ctl_table fscache_sysctls_root[] = {
  81 + {
  82 + .procname = "fscache",
  83 + .mode = 0555,
  84 + .child = fscache_sysctls,
  85 + },
  86 + {}
  87 +};
  88 +#endif
  89 +
44 90 /*
45 91 * initialise the fs caching module
46 92 */
47 93 static int __init fscache_init(void)
48 94 {
  95 + unsigned int nr_cpus = num_possible_cpus();
  96 + unsigned int cpu;
49 97 int ret;
50 98  
51 99 ret = slow_work_register_user(THIS_MODULE);
52 100 if (ret < 0)
53 101 goto error_slow_work;
54 102  
  103 + fscache_object_max_active =
  104 + clamp_val(nr_cpus,
  105 + fscache_object_max_active, WQ_UNBOUND_MAX_ACTIVE);
  106 +
  107 + ret = -ENOMEM;
  108 + fscache_object_wq = alloc_workqueue("fscache_object", WQ_UNBOUND,
  109 + fscache_object_max_active);
  110 + if (!fscache_object_wq)
  111 + goto error_object_wq;
  112 +
  113 + for_each_possible_cpu(cpu)
  114 + init_waitqueue_head(&per_cpu(fscache_object_cong_wait, cpu));
  115 +
55 116 ret = fscache_proc_init();
56 117 if (ret < 0)
57 118 goto error_proc;
58 119  
  120 +#ifdef CONFIG_SYSCTL
  121 + ret = -ENOMEM;
  122 + fscache_sysctl_header = register_sysctl_table(fscache_sysctls_root);
  123 + if (!fscache_sysctl_header)
  124 + goto error_sysctl;
  125 +#endif
  126 +
59 127 fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar",
60 128 sizeof(struct fscache_cookie),
61 129 0,
62 130  
... ... @@ -78,8 +146,14 @@
78 146 error_kobj:
79 147 kmem_cache_destroy(fscache_cookie_jar);
80 148 error_cookie_jar:
  149 +#ifdef CONFIG_SYSCTL
  150 + unregister_sysctl_table(fscache_sysctl_header);
  151 +error_sysctl:
  152 +#endif
81 153 fscache_proc_cleanup();
82 154 error_proc:
  155 + destroy_workqueue(fscache_object_wq);
  156 +error_object_wq:
83 157 slow_work_unregister_user(THIS_MODULE);
84 158 error_slow_work:
85 159 return ret;
86 160  
... ... @@ -96,7 +170,9 @@
96 170  
97 171 kobject_put(fscache_root);
98 172 kmem_cache_destroy(fscache_cookie_jar);
  173 + unregister_sysctl_table(fscache_sysctl_header);
99 174 fscache_proc_cleanup();
  175 + destroy_workqueue(fscache_object_wq);
100 176 slow_work_unregister_user(THIS_MODULE);
101 177 printk(KERN_NOTICE "FS-Cache: Unloaded\n");
102 178 }
fs/fscache/object-list.c
... ... @@ -34,8 +34,8 @@
34 34 #define FSCACHE_OBJLIST_CONFIG_NOREADS 0x00000200 /* show objects without active reads */
35 35 #define FSCACHE_OBJLIST_CONFIG_EVENTS 0x00000400 /* show objects with events */
36 36 #define FSCACHE_OBJLIST_CONFIG_NOEVENTS 0x00000800 /* show objects without no events */
37   -#define FSCACHE_OBJLIST_CONFIG_WORK 0x00001000 /* show objects with slow work */
38   -#define FSCACHE_OBJLIST_CONFIG_NOWORK 0x00002000 /* show objects without slow work */
  37 +#define FSCACHE_OBJLIST_CONFIG_WORK 0x00001000 /* show objects with work */
  38 +#define FSCACHE_OBJLIST_CONFIG_NOWORK 0x00002000 /* show objects without work */
39 39  
40 40 u8 buf[512]; /* key and aux data buffer */
41 41 };
42 42  
... ... @@ -231,12 +231,11 @@
231 231 READS, NOREADS);
232 232 FILTER(obj->events & obj->event_mask,
233 233 EVENTS, NOEVENTS);
234   - FILTER(obj->work.flags & ~(1UL << SLOW_WORK_VERY_SLOW),
235   - WORK, NOWORK);
  234 + FILTER(work_busy(&obj->work), WORK, NOWORK);
236 235 }
237 236  
238 237 seq_printf(m,
239   - "%8x %8x %s %5u %3u %3u %3u %2u %5u %2lx %2lx %1lx %1lx | ",
  238 + "%8x %8x %s %5u %3u %3u %3u %2u %5u %2lx %2lx %1lx %1x | ",
240 239 obj->debug_id,
241 240 obj->parent ? obj->parent->debug_id : -1,
242 241 fscache_object_states_short[obj->state],
... ... @@ -249,7 +248,7 @@
249 248 obj->event_mask & FSCACHE_OBJECT_EVENTS_MASK,
250 249 obj->events,
251 250 obj->flags,
252   - obj->work.flags);
  251 + work_busy(&obj->work));
253 252  
254 253 no_cookie = true;
255 254 keylen = auxlen = 0;
... ... @@ -14,7 +14,6 @@
14 14  
15 15 #define FSCACHE_DEBUG_LEVEL COOKIE
16 16 #include <linux/module.h>
17   -#include <linux/seq_file.h>
18 17 #include "internal.h"
19 18  
20 19 const char *fscache_object_states[FSCACHE_OBJECT__NSTATES] = {
... ... @@ -50,12 +49,8 @@
50 49 [FSCACHE_OBJECT_DEAD] = "DEAD",
51 50 };
52 51  
53   -static void fscache_object_slow_work_put_ref(struct slow_work *);
54   -static int fscache_object_slow_work_get_ref(struct slow_work *);
55   -static void fscache_object_slow_work_execute(struct slow_work *);
56   -#ifdef CONFIG_SLOW_WORK_DEBUG
57   -static void fscache_object_slow_work_desc(struct slow_work *, struct seq_file *);
58   -#endif
  52 +static int fscache_get_object(struct fscache_object *);
  53 +static void fscache_put_object(struct fscache_object *);
59 54 static void fscache_initialise_object(struct fscache_object *);
60 55 static void fscache_lookup_object(struct fscache_object *);
61 56 static void fscache_object_available(struct fscache_object *);
... ... @@ -64,17 +59,6 @@
64 59 static void fscache_enqueue_dependents(struct fscache_object *);
65 60 static void fscache_dequeue_object(struct fscache_object *);
66 61  
67   -const struct slow_work_ops fscache_object_slow_work_ops = {
68   - .owner = THIS_MODULE,
69   - .get_ref = fscache_object_slow_work_get_ref,
70   - .put_ref = fscache_object_slow_work_put_ref,
71   - .execute = fscache_object_slow_work_execute,
72   -#ifdef CONFIG_SLOW_WORK_DEBUG
73   - .desc = fscache_object_slow_work_desc,
74   -#endif
75   -};
76   -EXPORT_SYMBOL(fscache_object_slow_work_ops);
77   -
78 62 /*
79 63 * we need to notify the parent when an op completes that we had outstanding
80 64 * upon it
... ... @@ -345,7 +329,7 @@
345 329 /*
346 330 * execute an object
347 331 */
348   -static void fscache_object_slow_work_execute(struct slow_work *work)
  332 +void fscache_object_work_func(struct work_struct *work)
349 333 {
350 334 struct fscache_object *object =
351 335 container_of(work, struct fscache_object, work);
352 336  
353 337  
... ... @@ -359,25 +343,11 @@
359 343 if (object->events & object->event_mask)
360 344 fscache_enqueue_object(object);
361 345 clear_bit(FSCACHE_OBJECT_EV_REQUEUE, &object->events);
  346 + fscache_put_object(object);
362 347 }
  348 +EXPORT_SYMBOL(fscache_object_work_func);
363 349  
364 350 /*
365   - * describe an object for slow-work debugging
366   - */
367   -#ifdef CONFIG_SLOW_WORK_DEBUG
368   -static void fscache_object_slow_work_desc(struct slow_work *work,
369   - struct seq_file *m)
370   -{
371   - struct fscache_object *object =
372   - container_of(work, struct fscache_object, work);
373   -
374   - seq_printf(m, "FSC: OBJ%x: %s",
375   - object->debug_id,
376   - fscache_object_states_short[object->state]);
377   -}
378   -#endif
379   -
380   -/*
381 351 * initialise an object
382 352 * - check the specified object's parent to see if we can make use of it
383 353 * immediately to do a creation
... ... @@ -393,7 +363,6 @@
393 363 _enter("");
394 364 ASSERT(object->cookie != NULL);
395 365 ASSERT(object->cookie->parent != NULL);
396   - ASSERT(list_empty(&object->work.link));
397 366  
398 367 if (object->events & ((1 << FSCACHE_OBJECT_EV_ERROR) |
399 368 (1 << FSCACHE_OBJECT_EV_RELEASE) |
... ... @@ -671,10 +640,8 @@
671 640 object->parent = NULL;
672 641 }
673 642  
674   - /* this just shifts the object release to the slow work processor */
675   - fscache_stat(&fscache_n_cop_put_object);
676   - object->cache->ops->put_object(object);
677   - fscache_stat_d(&fscache_n_cop_put_object);
  643 + /* this just shifts the object release to the work processor */
  644 + fscache_put_object(object);
678 645  
679 646 _leave("");
680 647 }
681 648  
682 649  
... ... @@ -758,12 +725,10 @@
758 725 }
759 726  
760 727 /*
761   - * allow the slow work item processor to get a ref on an object
  728 + * get a ref on an object
762 729 */
763   -static int fscache_object_slow_work_get_ref(struct slow_work *work)
  730 +static int fscache_get_object(struct fscache_object *object)
764 731 {
765   - struct fscache_object *object =
766   - container_of(work, struct fscache_object, work);
767 732 int ret;
768 733  
769 734 fscache_stat(&fscache_n_cop_grab_object);
770 735  
771 736  
... ... @@ -773,13 +738,10 @@
773 738 }
774 739  
775 740 /*
776   - * allow the slow work item processor to discard a ref on a work item
  741 + * discard a ref on a work item
777 742 */
778   -static void fscache_object_slow_work_put_ref(struct slow_work *work)
  743 +static void fscache_put_object(struct fscache_object *object)
779 744 {
780   - struct fscache_object *object =
781   - container_of(work, struct fscache_object, work);
782   -
783 745 fscache_stat(&fscache_n_cop_put_object);
784 746 object->cache->ops->put_object(object);
785 747 fscache_stat_d(&fscache_n_cop_put_object);
786 748  
... ... @@ -792,9 +754,49 @@
792 754 {
793 755 _enter("{OBJ%x}", object->debug_id);
794 756  
795   - slow_work_enqueue(&object->work);
  757 + if (fscache_get_object(object) >= 0) {
  758 + wait_queue_head_t *cong_wq =
  759 + &get_cpu_var(fscache_object_cong_wait);
  760 +
  761 + if (queue_work(fscache_object_wq, &object->work)) {
  762 + if (fscache_object_congested())
  763 + wake_up(cong_wq);
  764 + } else
  765 + fscache_put_object(object);
  766 +
  767 + put_cpu_var(fscache_object_cong_wait);
  768 + }
796 769 }
797 770  
  771 +/**
  772 + * fscache_object_sleep_till_congested - Sleep until object wq is congested
  773 + * @timoutp: Scheduler sleep timeout
  774 + *
  775 + * Allow an object handler to sleep until the object workqueue is congested.
  776 + *
  777 + * The caller must set up a wake up event before calling this and must have set
  778 + * the appropriate sleep mode (such as TASK_UNINTERRUPTIBLE) and tested its own
  779 + * condition before calling this function as no test is made here.
  780 + *
  781 + * %true is returned if the object wq is congested, %false otherwise.
  782 + */
  783 +bool fscache_object_sleep_till_congested(signed long *timeoutp)
  784 +{
  785 + wait_queue_head_t *cong_wq = &__get_cpu_var(fscache_object_cong_wait);
  786 + DEFINE_WAIT(wait);
  787 +
  788 + if (fscache_object_congested())
  789 + return true;
  790 +
  791 + add_wait_queue_exclusive(cong_wq, &wait);
  792 + if (!fscache_object_congested())
  793 + *timeoutp = schedule_timeout(*timeoutp);
  794 + finish_wait(cong_wq, &wait);
  795 +
  796 + return fscache_object_congested();
  797 +}
  798 +EXPORT_SYMBOL_GPL(fscache_object_sleep_till_congested);
  799 +
798 800 /*
799 801 * enqueue the dependents of an object for metadata-type processing
800 802 * - the caller must hold the object's lock
... ... @@ -819,9 +821,7 @@
819 821  
820 822 /* sort onto appropriate lists */
821 823 fscache_enqueue_object(dep);
822   - fscache_stat(&fscache_n_cop_put_object);
823   - dep->cache->ops->put_object(dep);
824   - fscache_stat_d(&fscache_n_cop_put_object);
  824 + fscache_put_object(dep);
825 825  
826 826 if (!list_empty(&object->dependents))
827 827 cond_resched_lock(&object->lock);
include/linux/fscache-cache.h
... ... @@ -21,6 +21,7 @@
21 21 #include <linux/fscache.h>
22 22 #include <linux/sched.h>
23 23 #include <linux/slow-work.h>
  24 +#include <linux/workqueue.h>
24 25  
25 26 #define NR_MAXCACHES BITS_PER_LONG
26 27  
... ... @@ -389,7 +390,7 @@
389 390 struct fscache_cache *cache; /* cache that supplied this object */
390 391 struct fscache_cookie *cookie; /* netfs's file/index object */
391 392 struct fscache_object *parent; /* parent object */
392   - struct slow_work work; /* attention scheduling record */
  393 + struct work_struct work; /* attention scheduling record */
393 394 struct list_head dependents; /* FIFO of dependent objects */
394 395 struct list_head dep_link; /* link in parent's dependents list */
395 396 struct list_head pending_ops; /* unstarted operations on this object */
... ... @@ -411,7 +412,7 @@
411 412 (test_bit(FSCACHE_IOERROR, &(obj)->cache->flags) && \
412 413 (obj)->state >= FSCACHE_OBJECT_DYING)
413 414  
414   -extern const struct slow_work_ops fscache_object_slow_work_ops;
  415 +extern void fscache_object_work_func(struct work_struct *work);
415 416  
416 417 /**
417 418 * fscache_object_init - Initialise a cache object description
... ... @@ -433,7 +434,7 @@
433 434 spin_lock_init(&object->lock);
434 435 INIT_LIST_HEAD(&object->cache_link);
435 436 INIT_HLIST_NODE(&object->cookie_link);
436   - vslow_work_init(&object->work, &fscache_object_slow_work_ops);
  437 + INIT_WORK(&object->work, fscache_object_work_func);
437 438 INIT_LIST_HEAD(&object->dependents);
438 439 INIT_LIST_HEAD(&object->dep_link);
439 440 INIT_LIST_HEAD(&object->pending_ops);
... ... @@ -533,6 +534,8 @@
533 534  
534 535 extern void fscache_mark_pages_cached(struct fscache_retrieval *op,
535 536 struct pagevec *pagevec);
  537 +
  538 +extern bool fscache_object_sleep_till_congested(signed long *timeoutp);
536 539  
537 540 extern enum fscache_checkaux fscache_check_aux(struct fscache_object *object,
538 541 const void *data,