Commit 5160ee6fc891a9ca114be0e90fa6655647bb64b2

Authored by Eric Dumazet
Committed by Linus Torvalds
1 parent 21b6bf143d

[PATCH] shrink dentry struct

Some long time ago, dentry struct was carefully tuned so that on 32 bits
UP, sizeof(struct dentry) was exactly 128, ie a power of 2, and a multiple
of memory cache lines.

Then RCU was added and dentry struct enlarged by two pointers, with nice
results for SMP, but not so good on UP, because breaking the above tuning
(128 + 8 = 136 bytes)

This patch reverts this unwanted side effect, by using an union (d_u),
where d_rcu and d_child are placed so that these two fields can share their
memory needs.

At the time d_free() is called (and d_rcu is really used), d_child is known
to be empty and not touched by the dentry freeing.

Lockless lookups only access d_name, d_parent, d_lock, d_op, d_flags (so
the previous content of d_child is not needed if said dentry was unhashed
but still accessed by a CPU because of RCU constraints)

As dentry cache easily contains millions of entries, a size reduction is
worth the extra complexity of the ugly C union.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Cc: Dipankar Sarma <dipankar@in.ibm.com>
Cc: Maneesh Soni <maneesh@in.ibm.com>
Cc: Miklos Szeredi <miklos@szeredi.hu>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Cc: Ian Kent <raven@themaw.net>
Cc: Paul Jackson <pj@sgi.com>
Cc: Al Viro <viro@ftp.linux.org.uk>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Trond Myklebust <trond.myklebust@fys.uio.no>
Cc: Neil Brown <neilb@cse.unsw.edu.au>
Cc: James Morris <jmorris@namei.org>
Cc: Stephen Smalley <sds@epoch.ncsc.mil>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

Showing 15 changed files with 54 additions and 48 deletions Side-by-side Diff

drivers/usb/core/inode.c
... ... @@ -186,7 +186,7 @@
186 186  
187 187 down(&bus->d_inode->i_sem);
188 188  
189   - list_for_each_entry(dev, &bus->d_subdirs, d_child)
  189 + list_for_each_entry(dev, &bus->d_subdirs, d_u.d_child)
190 190 if (dev->d_inode)
191 191 update_dev(dev);
192 192  
... ... @@ -203,7 +203,7 @@
203 203  
204 204 down(&root->d_inode->i_sem);
205 205  
206   - list_for_each_entry(bus, &root->d_subdirs, d_child) {
  206 + list_for_each_entry(bus, &root->d_subdirs, d_u.d_child) {
207 207 if (bus->d_inode) {
208 208 switch (S_IFMT & bus->d_inode->i_mode) {
209 209 case S_IFDIR:
... ... @@ -319,7 +319,7 @@
319 319 spin_lock(&dcache_lock);
320 320  
321 321 list_for_each(list, &dentry->d_subdirs) {
322   - struct dentry *de = list_entry(list, struct dentry, d_child);
  322 + struct dentry *de = list_entry(list, struct dentry, d_u.d_child);
323 323 if (usbfs_positive(de)) {
324 324 spin_unlock(&dcache_lock);
325 325 return 0;
fs/autofs4/autofs_i.h
... ... @@ -209,7 +209,7 @@
209 209 struct dentry *child;
210 210 int ret = 0;
211 211  
212   - list_for_each_entry(child, &dentry->d_subdirs, d_child)
  212 + list_for_each_entry(child, &dentry->d_subdirs, d_u.d_child)
213 213 if (simple_positive(child))
214 214 goto out;
215 215 ret = 1;
... ... @@ -105,7 +105,7 @@
105 105 next = this_parent->d_subdirs.next;
106 106 resume:
107 107 while (next != &this_parent->d_subdirs) {
108   - struct dentry *dentry = list_entry(next, struct dentry, d_child);
  108 + struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
109 109  
110 110 /* Negative dentry - give up */
111 111 if (!simple_positive(dentry)) {
... ... @@ -138,7 +138,7 @@
138 138 }
139 139  
140 140 if (this_parent != top) {
141   - next = this_parent->d_child.next;
  141 + next = this_parent->d_u.d_child.next;
142 142 this_parent = this_parent->d_parent;
143 143 goto resume;
144 144 }
... ... @@ -163,7 +163,7 @@
163 163 next = this_parent->d_subdirs.next;
164 164 resume:
165 165 while (next != &this_parent->d_subdirs) {
166   - struct dentry *dentry = list_entry(next, struct dentry, d_child);
  166 + struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
167 167  
168 168 /* Negative dentry - give up */
169 169 if (!simple_positive(dentry)) {
... ... @@ -199,7 +199,7 @@
199 199 }
200 200  
201 201 if (this_parent != parent) {
202   - next = this_parent->d_child.next;
  202 + next = this_parent->d_u.d_child.next;
203 203 this_parent = this_parent->d_parent;
204 204 goto resume;
205 205 }
... ... @@ -238,7 +238,7 @@
238 238 /* On exit from the loop expire is set to a dgot dentry
239 239 * to expire or it's NULL */
240 240 while ( next != &root->d_subdirs ) {
241   - struct dentry *dentry = list_entry(next, struct dentry, d_child);
  241 + struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
242 242  
243 243 /* Negative dentry - give up */
244 244 if ( !simple_positive(dentry) ) {
... ... @@ -302,7 +302,7 @@
302 302 expired, (int)expired->d_name.len, expired->d_name.name);
303 303 spin_lock(&dcache_lock);
304 304 list_del(&expired->d_parent->d_subdirs);
305   - list_add(&expired->d_parent->d_subdirs, &expired->d_child);
  305 + list_add(&expired->d_parent->d_subdirs, &expired->d_u.d_child);
306 306 spin_unlock(&dcache_lock);
307 307 return expired;
308 308 }
... ... @@ -91,7 +91,7 @@
91 91 next = this_parent->d_subdirs.next;
92 92 resume:
93 93 while (next != &this_parent->d_subdirs) {
94   - struct dentry *dentry = list_entry(next, struct dentry, d_child);
  94 + struct dentry *dentry = list_entry(next, struct dentry, d_u.d_child);
95 95  
96 96 /* Negative dentry - don`t care */
97 97 if (!simple_positive(dentry)) {
... ... @@ -117,7 +117,7 @@
117 117 if (this_parent != sbi->root) {
118 118 struct dentry *dentry = this_parent;
119 119  
120   - next = this_parent->d_child.next;
  120 + next = this_parent->d_u.d_child.next;
121 121 this_parent = this_parent->d_parent;
122 122 spin_unlock(&dcache_lock);
123 123 DPRINTK("parent dentry %p %.*s",
... ... @@ -143,7 +143,8 @@
143 143 }
144 144  
145 145 while(1) {
146   - struct dentry *de = list_entry(list, struct dentry, d_child);
  146 + struct dentry *de = list_entry(list,
  147 + struct dentry, d_u.d_child);
147 148  
148 149 if (!d_unhashed(de) && de->d_inode) {
149 150 spin_unlock(&dcache_lock);
... ... @@ -93,7 +93,7 @@
93 93 spin_lock(&dcache_lock);
94 94 list_for_each(child, &parent->d_subdirs)
95 95 {
96   - de = list_entry(child, struct dentry, d_child);
  96 + de = list_entry(child, struct dentry, d_u.d_child);
97 97 /* don't know what to do with negative dentries */
98 98 if ( ! de->d_inode )
99 99 continue;
... ... @@ -71,7 +71,7 @@
71 71  
72 72 static void d_callback(struct rcu_head *head)
73 73 {
74   - struct dentry * dentry = container_of(head, struct dentry, d_rcu);
  74 + struct dentry * dentry = container_of(head, struct dentry, d_u.d_rcu);
75 75  
76 76 if (dname_external(dentry))
77 77 kfree(dentry->d_name.name);
... ... @@ -86,7 +86,7 @@
86 86 {
87 87 if (dentry->d_op && dentry->d_op->d_release)
88 88 dentry->d_op->d_release(dentry);
89   - call_rcu(&dentry->d_rcu, d_callback);
  89 + call_rcu(&dentry->d_u.d_rcu, d_callback);
90 90 }
91 91  
92 92 /*
... ... @@ -193,7 +193,7 @@
193 193 list_del(&dentry->d_lru);
194 194 dentry_stat.nr_unused--;
195 195 }
196   - list_del(&dentry->d_child);
  196 + list_del(&dentry->d_u.d_child);
197 197 dentry_stat.nr_dentry--; /* For d_free, below */
198 198 /*drops the locks, at that point nobody can reach this dentry */
199 199 dentry_iput(dentry);
... ... @@ -367,7 +367,7 @@
367 367 struct dentry * parent;
368 368  
369 369 __d_drop(dentry);
370   - list_del(&dentry->d_child);
  370 + list_del(&dentry->d_u.d_child);
371 371 dentry_stat.nr_dentry--; /* For d_free, below */
372 372 dentry_iput(dentry);
373 373 parent = dentry->d_parent;
... ... @@ -518,7 +518,7 @@
518 518 resume:
519 519 while (next != &this_parent->d_subdirs) {
520 520 struct list_head *tmp = next;
521   - struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
  521 + struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
522 522 next = tmp->next;
523 523 /* Have we found a mount point ? */
524 524 if (d_mountpoint(dentry))
... ... @@ -532,7 +532,7 @@
532 532 * All done at this level ... ascend and resume the search.
533 533 */
534 534 if (this_parent != parent) {
535   - next = this_parent->d_child.next;
  535 + next = this_parent->d_u.d_child.next;
536 536 this_parent = this_parent->d_parent;
537 537 goto resume;
538 538 }
... ... @@ -569,7 +569,7 @@
569 569 resume:
570 570 while (next != &this_parent->d_subdirs) {
571 571 struct list_head *tmp = next;
572   - struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
  572 + struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
573 573 next = tmp->next;
574 574  
575 575 if (!list_empty(&dentry->d_lru)) {
... ... @@ -610,7 +610,7 @@
610 610 * All done at this level ... ascend and resume the search.
611 611 */
612 612 if (this_parent != parent) {
613   - next = this_parent->d_child.next;
  613 + next = this_parent->d_u.d_child.next;
614 614 this_parent = this_parent->d_parent;
615 615 #ifdef DCACHE_DEBUG
616 616 printk(KERN_DEBUG "select_parent: ascending to %s/%s, found=%d\n",
617 617  
... ... @@ -753,12 +753,12 @@
753 753 dentry->d_parent = dget(parent);
754 754 dentry->d_sb = parent->d_sb;
755 755 } else {
756   - INIT_LIST_HEAD(&dentry->d_child);
  756 + INIT_LIST_HEAD(&dentry->d_u.d_child);
757 757 }
758 758  
759 759 spin_lock(&dcache_lock);
760 760 if (parent)
761   - list_add(&dentry->d_child, &parent->d_subdirs);
  761 + list_add(&dentry->d_u.d_child, &parent->d_subdirs);
762 762 dentry_stat.nr_dentry++;
763 763 spin_unlock(&dcache_lock);
764 764  
... ... @@ -1310,8 +1310,8 @@
1310 1310 /* Unhash the target: dput() will then get rid of it */
1311 1311 __d_drop(target);
1312 1312  
1313   - list_del(&dentry->d_child);
1314   - list_del(&target->d_child);
  1313 + list_del(&dentry->d_u.d_child);
  1314 + list_del(&target->d_u.d_child);
1315 1315  
1316 1316 /* Switch the names.. */
1317 1317 switch_names(dentry, target);
1318 1318  
1319 1319  
... ... @@ -1322,15 +1322,15 @@
1322 1322 if (IS_ROOT(dentry)) {
1323 1323 dentry->d_parent = target->d_parent;
1324 1324 target->d_parent = target;
1325   - INIT_LIST_HEAD(&target->d_child);
  1325 + INIT_LIST_HEAD(&target->d_u.d_child);
1326 1326 } else {
1327 1327 do_switch(dentry->d_parent, target->d_parent);
1328 1328  
1329 1329 /* And add them back to the (new) parent lists */
1330   - list_add(&target->d_child, &target->d_parent->d_subdirs);
  1330 + list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
1331 1331 }
1332 1332  
1333   - list_add(&dentry->d_child, &dentry->d_parent->d_subdirs);
  1333 + list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
1334 1334 spin_unlock(&target->d_lock);
1335 1335 spin_unlock(&dentry->d_lock);
1336 1336 write_sequnlock(&rename_lock);
... ... @@ -1568,7 +1568,7 @@
1568 1568 resume:
1569 1569 while (next != &this_parent->d_subdirs) {
1570 1570 struct list_head *tmp = next;
1571   - struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
  1571 + struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
1572 1572 next = tmp->next;
1573 1573 if (d_unhashed(dentry)||!dentry->d_inode)
1574 1574 continue;
... ... @@ -1579,7 +1579,7 @@
1579 1579 atomic_dec(&dentry->d_count);
1580 1580 }
1581 1581 if (this_parent != root) {
1582   - next = this_parent->d_child.next;
  1582 + next = this_parent->d_u.d_child.next;
1583 1583 atomic_dec(&this_parent->d_count);
1584 1584 this_parent = this_parent->d_parent;
1585 1585 goto resume;
... ... @@ -93,16 +93,16 @@
93 93 loff_t n = file->f_pos - 2;
94 94  
95 95 spin_lock(&dcache_lock);
96   - list_del(&cursor->d_child);
  96 + list_del(&cursor->d_u.d_child);
97 97 p = file->f_dentry->d_subdirs.next;
98 98 while (n && p != &file->f_dentry->d_subdirs) {
99 99 struct dentry *next;
100   - next = list_entry(p, struct dentry, d_child);
  100 + next = list_entry(p, struct dentry, d_u.d_child);
101 101 if (!d_unhashed(next) && next->d_inode)
102 102 n--;
103 103 p = p->next;
104 104 }
105   - list_add_tail(&cursor->d_child, p);
  105 + list_add_tail(&cursor->d_u.d_child, p);
106 106 spin_unlock(&dcache_lock);
107 107 }
108 108 }
... ... @@ -126,7 +126,7 @@
126 126 {
127 127 struct dentry *dentry = filp->f_dentry;
128 128 struct dentry *cursor = filp->private_data;
129   - struct list_head *p, *q = &cursor->d_child;
  129 + struct list_head *p, *q = &cursor->d_u.d_child;
130 130 ino_t ino;
131 131 int i = filp->f_pos;
132 132  
... ... @@ -153,7 +153,7 @@
153 153 }
154 154 for (p=q->next; p != &dentry->d_subdirs; p=p->next) {
155 155 struct dentry *next;
156   - next = list_entry(p, struct dentry, d_child);
  156 + next = list_entry(p, struct dentry, d_u.d_child);
157 157 if (d_unhashed(next) || !next->d_inode)
158 158 continue;
159 159  
... ... @@ -261,7 +261,7 @@
261 261 int ret = 0;
262 262  
263 263 spin_lock(&dcache_lock);
264   - list_for_each_entry(child, &dentry->d_subdirs, d_child)
  264 + list_for_each_entry(child, &dentry->d_subdirs, d_u.d_child)
265 265 if (simple_positive(child))
266 266 goto out;
267 267 ret = 1;
... ... @@ -365,7 +365,7 @@
365 365 spin_lock(&dcache_lock);
366 366 next = parent->d_subdirs.next;
367 367 while (next != &parent->d_subdirs) {
368   - dent = list_entry(next, struct dentry, d_child);
  368 + dent = list_entry(next, struct dentry, d_u.d_child);
369 369 if ((unsigned long)dent->d_fsdata == fpos) {
370 370 if (dent->d_inode)
371 371 dget_locked(dent);
fs/ncpfs/ncplib_kernel.h
... ... @@ -196,7 +196,7 @@
196 196 spin_lock(&dcache_lock);
197 197 next = parent->d_subdirs.next;
198 198 while (next != &parent->d_subdirs) {
199   - dentry = list_entry(next, struct dentry, d_child);
  199 + dentry = list_entry(next, struct dentry, d_u.d_child);
200 200  
201 201 if (dentry->d_fsdata == NULL)
202 202 ncp_age_dentry(server, dentry);
... ... @@ -218,7 +218,7 @@
218 218 spin_lock(&dcache_lock);
219 219 next = parent->d_subdirs.next;
220 220 while (next != &parent->d_subdirs) {
221   - dentry = list_entry(next, struct dentry, d_child);
  221 + dentry = list_entry(next, struct dentry, d_u.d_child);
222 222 dentry->d_fsdata = NULL;
223 223 ncp_age_dentry(server, dentry);
224 224 next = next->next;
... ... @@ -66,7 +66,7 @@
66 66 spin_lock(&dcache_lock);
67 67 next = parent->d_subdirs.next;
68 68 while (next != &parent->d_subdirs) {
69   - dentry = list_entry(next, struct dentry, d_child);
  69 + dentry = list_entry(next, struct dentry, d_u.d_child);
70 70 dentry->d_fsdata = NULL;
71 71 smb_age_dentry(server, dentry);
72 72 next = next->next;
... ... @@ -100,7 +100,7 @@
100 100 spin_lock(&dcache_lock);
101 101 next = parent->d_subdirs.next;
102 102 while (next != &parent->d_subdirs) {
103   - dent = list_entry(next, struct dentry, d_child);
  103 + dent = list_entry(next, struct dentry, d_u.d_child);
104 104 if ((unsigned long)dent->d_fsdata == fpos) {
105 105 if (dent->d_inode)
106 106 dget_locked(dent);
include/linux/dcache.h
... ... @@ -95,14 +95,19 @@
95 95 struct qstr d_name;
96 96  
97 97 struct list_head d_lru; /* LRU list */
98   - struct list_head d_child; /* child of parent list */
  98 + /*
  99 + * d_child and d_rcu can share memory
  100 + */
  101 + union {
  102 + struct list_head d_child; /* child of parent list */
  103 + struct rcu_head d_rcu;
  104 + } d_u;
99 105 struct list_head d_subdirs; /* our children */
100 106 struct list_head d_alias; /* inode alias list */
101 107 unsigned long d_time; /* used by d_revalidate */
102 108 struct dentry_operations *d_op;
103 109 struct super_block *d_sb; /* The root of the dentry tree */
104 110 void *d_fsdata; /* fs-specific data */
105   - struct rcu_head d_rcu;
106 111 struct dcookie_struct *d_cookie; /* cookie, if any */
107 112 int d_mounted;
108 113 unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */
... ... @@ -331,7 +331,7 @@
331 331 spin_lock(&dcache_lock);
332 332 node = dentry->d_subdirs.next;
333 333 while (node != &dentry->d_subdirs) {
334   - struct dentry *d = list_entry(node, struct dentry, d_child);
  334 + struct dentry *d = list_entry(node, struct dentry, d_u.d_child);
335 335 list_del_init(node);
336 336 if (d->d_inode) {
337 337 d = dget_locked(d);
... ... @@ -343,7 +343,7 @@
343 343 }
344 344 node = dentry->d_subdirs.next;
345 345 }
346   - list_del_init(&dentry->d_child);
  346 + list_del_init(&dentry->d_u.d_child);
347 347 spin_unlock(&dcache_lock);
348 348 remove_dir(dentry);
349 349 }
net/sunrpc/rpc_pipe.c
... ... @@ -495,7 +495,7 @@
495 495 repeat:
496 496 spin_lock(&dcache_lock);
497 497 list_for_each_safe(pos, next, &parent->d_subdirs) {
498   - dentry = list_entry(pos, struct dentry, d_child);
  498 + dentry = list_entry(pos, struct dentry, d_u.d_child);
499 499 spin_lock(&dentry->d_lock);
500 500 if (!d_unhashed(dentry)) {
501 501 dget_locked(dentry);
security/selinux/selinuxfs.c
... ... @@ -889,7 +889,7 @@
889 889 spin_lock(&dcache_lock);
890 890 node = de->d_subdirs.next;
891 891 while (node != &de->d_subdirs) {
892   - struct dentry *d = list_entry(node, struct dentry, d_child);
  892 + struct dentry *d = list_entry(node, struct dentry, d_u.d_child);
893 893 list_del_init(node);
894 894  
895 895 if (d->d_inode) {