Commit c2452f32786159ed85f0e4b21fec09258f822fc8

Authored by Nick Piggin
Committed by Al Viro
1 parent e2b689d82c

shrink struct dentry

struct dentry is one of the most critical structures in the kernel. So it's
sad to see it going neglected.

With CONFIG_PROFILING turned on (which is probably the common case at least
for distros and kernel developers), sizeof(struct dcache) == 208 here
(64-bit). This gives 19 objects per slab.

I packed d_mounted into a hole, and took another 4 bytes off the inline
name length to take the padding out from the end of the structure. This
shinks it to 200 bytes. I could have gone the other way and increased the
length to 40, but I'm aiming for a magic number, read on...

I then got rid of the d_cookie pointer. This shrinks it to 192 bytes. Rant:
why was this ever a good idea? The cookie system should increase its hash
size or use a tree or something if lookups are a problem. Also the "fast
dcookie lookups" in oprofile should be moved into the dcookie code -- how
can oprofile possibly care about the dcookie_mutex? It gets dropped after
get_dcookie() returns so it can't be providing any sort of protection.

At 192 bytes, 21 objects fit into a 4K page, saving about 3MB on my system
with ~140 000 entries allocated. 192 is also a multiple of 64, so we get
nice cacheline alignment on 64 and 32 byte line systems -- any given dentry
will now require 3 cachelines to touch all fields wheras previously it
would require 4.

I know the inline name size was chosen quite carefully, however with the
reduction in cacheline footprint, it should actually be just about as fast
to do a name lookup for a 36 character name as it was before the patch (and
faster for other sizes). The memory footprint savings for names which are
<= 32 or > 36 bytes long should more than make up for the memory cost for
33-36 byte names.

Performance is a feature...

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

Showing 5 changed files with 35 additions and 22 deletions Side-by-side Diff

arch/powerpc/oprofile/cell/spu_task_sync.c
... ... @@ -297,7 +297,7 @@
297 297 {
298 298 unsigned long cookie;
299 299  
300   - if (path->dentry->d_cookie)
  300 + if (path->dentry->d_flags & DCACHE_COOKIE)
301 301 return (unsigned long)path->dentry;
302 302 get_dcookie(path, &cookie);
303 303 return cookie;
drivers/oprofile/buffer_sync.c
... ... @@ -200,7 +200,7 @@
200 200 {
201 201 unsigned long cookie;
202 202  
203   - if (path->dentry->d_cookie)
  203 + if (path->dentry->d_flags & DCACHE_COOKIE)
204 204 return (unsigned long)path->dentry;
205 205 get_dcookie(path, &cookie);
206 206 return cookie;
... ... @@ -34,7 +34,6 @@
34 34 #include <linux/bootmem.h>
35 35 #include "internal.h"
36 36  
37   -
38 37 int sysctl_vfs_cache_pressure __read_mostly = 100;
39 38 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
40 39  
... ... @@ -948,9 +947,6 @@
948 947 dentry->d_op = NULL;
949 948 dentry->d_fsdata = NULL;
950 949 dentry->d_mounted = 0;
951   -#ifdef CONFIG_PROFILING
952   - dentry->d_cookie = NULL;
953   -#endif
954 950 INIT_HLIST_NODE(&dentry->d_hash);
955 951 INIT_LIST_HEAD(&dentry->d_lru);
956 952 INIT_LIST_HEAD(&dentry->d_subdirs);
... ... @@ -93,10 +93,15 @@
93 93 {
94 94 struct dcookie_struct *dcs = kmem_cache_alloc(dcookie_cache,
95 95 GFP_KERNEL);
  96 + struct dentry *d;
96 97 if (!dcs)
97 98 return NULL;
98 99  
99   - path->dentry->d_cookie = dcs;
  100 + d = path->dentry;
  101 + spin_lock(&d->d_lock);
  102 + d->d_flags |= DCACHE_COOKIE;
  103 + spin_unlock(&d->d_lock);
  104 +
100 105 dcs->path = *path;
101 106 path_get(path);
102 107 hash_dcookie(dcs);
103 108  
... ... @@ -119,14 +124,14 @@
119 124 goto out;
120 125 }
121 126  
122   - dcs = path->dentry->d_cookie;
123   -
124   - if (!dcs)
  127 + if (path->dentry->d_flags & DCACHE_COOKIE) {
  128 + dcs = find_dcookie((unsigned long)path->dentry);
  129 + } else {
125 130 dcs = alloc_dcookie(path);
126   -
127   - if (!dcs) {
128   - err = -ENOMEM;
129   - goto out;
  131 + if (!dcs) {
  132 + err = -ENOMEM;
  133 + goto out;
  134 + }
130 135 }
131 136  
132 137 *cookie = dcookie_value(dcs);
... ... @@ -251,7 +256,12 @@
251 256  
252 257 static void free_dcookie(struct dcookie_struct * dcs)
253 258 {
254   - dcs->path.dentry->d_cookie = NULL;
  259 + struct dentry *d = dcs->path.dentry;
  260 +
  261 + spin_lock(&d->d_lock);
  262 + d->d_flags &= ~DCACHE_COOKIE;
  263 + spin_unlock(&d->d_lock);
  264 +
255 265 path_put(&dcs->path);
256 266 kmem_cache_free(dcookie_cache, dcs);
257 267 }
include/linux/dcache.h
... ... @@ -75,14 +75,22 @@
75 75 return end_name_hash(hash);
76 76 }
77 77  
78   -struct dcookie_struct;
  78 +/*
  79 + * Try to keep struct dentry aligned on 64 byte cachelines (this will
  80 + * give reasonable cacheline footprint with larger lines without the
  81 + * large memory footprint increase).
  82 + */
  83 +#ifdef CONFIG_64BIT
  84 +#define DNAME_INLINE_LEN_MIN 32 /* 192 bytes */
  85 +#else
  86 +#define DNAME_INLINE_LEN_MIN 40 /* 128 bytes */
  87 +#endif
79 88  
80   -#define DNAME_INLINE_LEN_MIN 36
81   -
82 89 struct dentry {
83 90 atomic_t d_count;
84 91 unsigned int d_flags; /* protected by d_lock */
85 92 spinlock_t d_lock; /* per dentry lock */
  93 + int d_mounted;
86 94 struct inode *d_inode; /* Where the name belongs to - NULL is
87 95 * negative */
88 96 /*
... ... @@ -107,10 +115,7 @@
107 115 struct dentry_operations *d_op;
108 116 struct super_block *d_sb; /* The root of the dentry tree */
109 117 void *d_fsdata; /* fs-specific data */
110   -#ifdef CONFIG_PROFILING
111   - struct dcookie_struct *d_cookie; /* cookie, if any */
112   -#endif
113   - int d_mounted;
  118 +
114 119 unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */
115 120 };
116 121  
... ... @@ -176,6 +181,8 @@
176 181 #define DCACHE_UNHASHED 0x0010
177 182  
178 183 #define DCACHE_INOTIFY_PARENT_WATCHED 0x0020 /* Parent inode is watched */
  184 +
  185 +#define DCACHE_COOKIE 0x0040 /* For use by dcookie subsystem */
179 186  
180 187 extern spinlock_t dcache_lock;
181 188 extern seqlock_t rename_lock;