Commit c2452f32786159ed85f0e4b21fec09258f822fc8
Committed by
Al Viro
1 parent
e2b689d82c
Exists in
master
and in
4 other branches
shrink struct dentry
struct dentry is one of the most critical structures in the kernel. So it's sad to see it going neglected. With CONFIG_PROFILING turned on (which is probably the common case at least for distros and kernel developers), sizeof(struct dcache) == 208 here (64-bit). This gives 19 objects per slab. I packed d_mounted into a hole, and took another 4 bytes off the inline name length to take the padding out from the end of the structure. This shinks it to 200 bytes. I could have gone the other way and increased the length to 40, but I'm aiming for a magic number, read on... I then got rid of the d_cookie pointer. This shrinks it to 192 bytes. Rant: why was this ever a good idea? The cookie system should increase its hash size or use a tree or something if lookups are a problem. Also the "fast dcookie lookups" in oprofile should be moved into the dcookie code -- how can oprofile possibly care about the dcookie_mutex? It gets dropped after get_dcookie() returns so it can't be providing any sort of protection. At 192 bytes, 21 objects fit into a 4K page, saving about 3MB on my system with ~140 000 entries allocated. 192 is also a multiple of 64, so we get nice cacheline alignment on 64 and 32 byte line systems -- any given dentry will now require 3 cachelines to touch all fields wheras previously it would require 4. I know the inline name size was chosen quite carefully, however with the reduction in cacheline footprint, it should actually be just about as fast to do a name lookup for a 36 character name as it was before the patch (and faster for other sizes). The memory footprint savings for names which are <= 32 or > 36 bytes long should more than make up for the memory cost for 33-36 byte names. Performance is a feature... Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
Showing 5 changed files with 35 additions and 22 deletions Side-by-side Diff
arch/powerpc/oprofile/cell/spu_task_sync.c
drivers/oprofile/buffer_sync.c
fs/dcache.c
... | ... | @@ -34,7 +34,6 @@ |
34 | 34 | #include <linux/bootmem.h> |
35 | 35 | #include "internal.h" |
36 | 36 | |
37 | - | |
38 | 37 | int sysctl_vfs_cache_pressure __read_mostly = 100; |
39 | 38 | EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); |
40 | 39 | |
... | ... | @@ -948,9 +947,6 @@ |
948 | 947 | dentry->d_op = NULL; |
949 | 948 | dentry->d_fsdata = NULL; |
950 | 949 | dentry->d_mounted = 0; |
951 | -#ifdef CONFIG_PROFILING | |
952 | - dentry->d_cookie = NULL; | |
953 | -#endif | |
954 | 950 | INIT_HLIST_NODE(&dentry->d_hash); |
955 | 951 | INIT_LIST_HEAD(&dentry->d_lru); |
956 | 952 | INIT_LIST_HEAD(&dentry->d_subdirs); |
fs/dcookies.c
... | ... | @@ -93,10 +93,15 @@ |
93 | 93 | { |
94 | 94 | struct dcookie_struct *dcs = kmem_cache_alloc(dcookie_cache, |
95 | 95 | GFP_KERNEL); |
96 | + struct dentry *d; | |
96 | 97 | if (!dcs) |
97 | 98 | return NULL; |
98 | 99 | |
99 | - path->dentry->d_cookie = dcs; | |
100 | + d = path->dentry; | |
101 | + spin_lock(&d->d_lock); | |
102 | + d->d_flags |= DCACHE_COOKIE; | |
103 | + spin_unlock(&d->d_lock); | |
104 | + | |
100 | 105 | dcs->path = *path; |
101 | 106 | path_get(path); |
102 | 107 | hash_dcookie(dcs); |
103 | 108 | |
... | ... | @@ -119,14 +124,14 @@ |
119 | 124 | goto out; |
120 | 125 | } |
121 | 126 | |
122 | - dcs = path->dentry->d_cookie; | |
123 | - | |
124 | - if (!dcs) | |
127 | + if (path->dentry->d_flags & DCACHE_COOKIE) { | |
128 | + dcs = find_dcookie((unsigned long)path->dentry); | |
129 | + } else { | |
125 | 130 | dcs = alloc_dcookie(path); |
126 | - | |
127 | - if (!dcs) { | |
128 | - err = -ENOMEM; | |
129 | - goto out; | |
131 | + if (!dcs) { | |
132 | + err = -ENOMEM; | |
133 | + goto out; | |
134 | + } | |
130 | 135 | } |
131 | 136 | |
132 | 137 | *cookie = dcookie_value(dcs); |
... | ... | @@ -251,7 +256,12 @@ |
251 | 256 | |
252 | 257 | static void free_dcookie(struct dcookie_struct * dcs) |
253 | 258 | { |
254 | - dcs->path.dentry->d_cookie = NULL; | |
259 | + struct dentry *d = dcs->path.dentry; | |
260 | + | |
261 | + spin_lock(&d->d_lock); | |
262 | + d->d_flags &= ~DCACHE_COOKIE; | |
263 | + spin_unlock(&d->d_lock); | |
264 | + | |
255 | 265 | path_put(&dcs->path); |
256 | 266 | kmem_cache_free(dcookie_cache, dcs); |
257 | 267 | } |
include/linux/dcache.h
... | ... | @@ -75,14 +75,22 @@ |
75 | 75 | return end_name_hash(hash); |
76 | 76 | } |
77 | 77 | |
78 | -struct dcookie_struct; | |
78 | +/* | |
79 | + * Try to keep struct dentry aligned on 64 byte cachelines (this will | |
80 | + * give reasonable cacheline footprint with larger lines without the | |
81 | + * large memory footprint increase). | |
82 | + */ | |
83 | +#ifdef CONFIG_64BIT | |
84 | +#define DNAME_INLINE_LEN_MIN 32 /* 192 bytes */ | |
85 | +#else | |
86 | +#define DNAME_INLINE_LEN_MIN 40 /* 128 bytes */ | |
87 | +#endif | |
79 | 88 | |
80 | -#define DNAME_INLINE_LEN_MIN 36 | |
81 | - | |
82 | 89 | struct dentry { |
83 | 90 | atomic_t d_count; |
84 | 91 | unsigned int d_flags; /* protected by d_lock */ |
85 | 92 | spinlock_t d_lock; /* per dentry lock */ |
93 | + int d_mounted; | |
86 | 94 | struct inode *d_inode; /* Where the name belongs to - NULL is |
87 | 95 | * negative */ |
88 | 96 | /* |
... | ... | @@ -107,10 +115,7 @@ |
107 | 115 | struct dentry_operations *d_op; |
108 | 116 | struct super_block *d_sb; /* The root of the dentry tree */ |
109 | 117 | void *d_fsdata; /* fs-specific data */ |
110 | -#ifdef CONFIG_PROFILING | |
111 | - struct dcookie_struct *d_cookie; /* cookie, if any */ | |
112 | -#endif | |
113 | - int d_mounted; | |
118 | + | |
114 | 119 | unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */ |
115 | 120 | }; |
116 | 121 | |
... | ... | @@ -176,6 +181,8 @@ |
176 | 181 | #define DCACHE_UNHASHED 0x0010 |
177 | 182 | |
178 | 183 | #define DCACHE_INOTIFY_PARENT_WATCHED 0x0020 /* Parent inode is watched */ |
184 | + | |
185 | +#define DCACHE_COOKIE 0x0040 /* For use by dcookie subsystem */ | |
179 | 186 | |
180 | 187 | extern spinlock_t dcache_lock; |
181 | 188 | extern seqlock_t rename_lock; |