Eric Lee / smarc-ti-linux-kernel

Download as
Browse Code ยป

Commit e55fd011549eae01a230e3cace6f4d031b6a3453

Authored by Al Viro
1 parent 64fd72e0a4
Exists in ti-lsk-linux-4.1.y and in 12 other branches dlt-processor-sdk-linux-03.00.00.04, processor-sdk-linux-02.00.01, smarc-ti-linux-3.15.y, smarc-ti-lsk-linux-4.1.y, smarct3x-processor-sdk-04.01.00.06, smarct3x-processor-sdk-linux-02.00.01, smarct3x-processor-sdk-linux-03.00.00.04, smarct4x-800-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-04.01.00.06, smarct4x-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-linux-03.00.00.04, ti-linux-3.15.y

split dentry_kill() 

... into trylocks and everything else.  The latter (actual killing)
is __dentry_kill().

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

Showing 1 changed file with 36 additions and 26 deletions Inline Diff

1 /* 1 /*
2 * fs/dcache.c 2 * fs/dcache.c
3 * 3 *
4 * Complete reimplementation 4 * Complete reimplementation
5 * (C) 1997 Thomas Schoebel-Theuer, 5 * (C) 1997 Thomas Schoebel-Theuer,
6 * with heavy changes by Linus Torvalds 6 * with heavy changes by Linus Torvalds
7 */ 7 */
8 8
9 /* 9 /*
10 * Notes on the allocation strategy: 10 * Notes on the allocation strategy:
11 * 11 *
12 * The dcache is a master of the icache - whenever a dcache entry 12 * The dcache is a master of the icache - whenever a dcache entry
13 * exists, the inode will always exist. "iput()" is done either when 13 * exists, the inode will always exist. "iput()" is done either when
14 * the dcache entry is deleted or garbage collected. 14 * the dcache entry is deleted or garbage collected.
15 */ 15 */
16 16
17 #include <linux/syscalls.h> 17 #include <linux/syscalls.h>
18 #include <linux/string.h> 18 #include <linux/string.h>
19 #include <linux/mm.h> 19 #include <linux/mm.h>
20 #include <linux/fs.h> 20 #include <linux/fs.h>
21 #include <linux/fsnotify.h> 21 #include <linux/fsnotify.h>
22 #include <linux/slab.h> 22 #include <linux/slab.h>
23 #include <linux/init.h> 23 #include <linux/init.h>
24 #include <linux/hash.h> 24 #include <linux/hash.h>
25 #include <linux/cache.h> 25 #include <linux/cache.h>
26 #include <linux/export.h> 26 #include <linux/export.h>
27 #include <linux/mount.h> 27 #include <linux/mount.h>
28 #include <linux/file.h> 28 #include <linux/file.h>
29 #include <asm/uaccess.h> 29 #include <asm/uaccess.h>
30 #include <linux/security.h> 30 #include <linux/security.h>
31 #include <linux/seqlock.h> 31 #include <linux/seqlock.h>
32 #include <linux/swap.h> 32 #include <linux/swap.h>
33 #include <linux/bootmem.h> 33 #include <linux/bootmem.h>
34 #include <linux/fs_struct.h> 34 #include <linux/fs_struct.h>
35 #include <linux/hardirq.h> 35 #include <linux/hardirq.h>
36 #include <linux/bit_spinlock.h> 36 #include <linux/bit_spinlock.h>
37 #include <linux/rculist_bl.h> 37 #include <linux/rculist_bl.h>
38 #include <linux/prefetch.h> 38 #include <linux/prefetch.h>
39 #include <linux/ratelimit.h> 39 #include <linux/ratelimit.h>
40 #include <linux/list_lru.h> 40 #include <linux/list_lru.h>
41 #include "internal.h" 41 #include "internal.h"
42 #include "mount.h" 42 #include "mount.h"
43 43
44 /* 44 /*
45 * Usage: 45 * Usage:
46 * dcache->d_inode->i_lock protects: 46 * dcache->d_inode->i_lock protects:
47 * - i_dentry, d_alias, d_inode of aliases 47 * - i_dentry, d_alias, d_inode of aliases
48 * dcache_hash_bucket lock protects: 48 * dcache_hash_bucket lock protects:
49 * - the dcache hash table 49 * - the dcache hash table
50 * s_anon bl list spinlock protects: 50 * s_anon bl list spinlock protects:
51 * - the s_anon list (see __d_drop) 51 * - the s_anon list (see __d_drop)
52 * dentry->d_sb->s_dentry_lru_lock protects: 52 * dentry->d_sb->s_dentry_lru_lock protects:
53 * - the dcache lru lists and counters 53 * - the dcache lru lists and counters
54 * d_lock protects: 54 * d_lock protects:
55 * - d_flags 55 * - d_flags
56 * - d_name 56 * - d_name
57 * - d_lru 57 * - d_lru
58 * - d_count 58 * - d_count
59 * - d_unhashed() 59 * - d_unhashed()
60 * - d_parent and d_subdirs 60 * - d_parent and d_subdirs
61 * - childrens' d_child and d_parent 61 * - childrens' d_child and d_parent
62 * - d_alias, d_inode 62 * - d_alias, d_inode
63 * 63 *
64 * Ordering: 64 * Ordering:
65 * dentry->d_inode->i_lock 65 * dentry->d_inode->i_lock
66 * dentry->d_lock 66 * dentry->d_lock
67 * dentry->d_sb->s_dentry_lru_lock 67 * dentry->d_sb->s_dentry_lru_lock
68 * dcache_hash_bucket lock 68 * dcache_hash_bucket lock
69 * s_anon lock 69 * s_anon lock
70 * 70 *
71 * If there is an ancestor relationship: 71 * If there is an ancestor relationship:
72 * dentry->d_parent->...->d_parent->d_lock 72 * dentry->d_parent->...->d_parent->d_lock
73 * ... 73 * ...
74 * dentry->d_parent->d_lock 74 * dentry->d_parent->d_lock
75 * dentry->d_lock 75 * dentry->d_lock
76 * 76 *
77 * If no ancestor relationship: 77 * If no ancestor relationship:
78 * if (dentry1 < dentry2) 78 * if (dentry1 < dentry2)
79 * dentry1->d_lock 79 * dentry1->d_lock
80 * dentry2->d_lock 80 * dentry2->d_lock
81 */ 81 */
82 int sysctl_vfs_cache_pressure __read_mostly = 100; 82 int sysctl_vfs_cache_pressure __read_mostly = 100;
83 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); 83 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
84 84
85 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); 85 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
86 86
87 EXPORT_SYMBOL(rename_lock); 87 EXPORT_SYMBOL(rename_lock);
88 88
89 static struct kmem_cache *dentry_cache __read_mostly; 89 static struct kmem_cache *dentry_cache __read_mostly;
90 90
91 /* 91 /*
92 * This is the single most critical data structure when it comes 92 * This is the single most critical data structure when it comes
93 * to the dcache: the hashtable for lookups. Somebody should try 93 * to the dcache: the hashtable for lookups. Somebody should try
94 * to make this good - I've just made it work. 94 * to make this good - I've just made it work.
95 * 95 *
96 * This hash-function tries to avoid losing too many bits of hash 96 * This hash-function tries to avoid losing too many bits of hash
97 * information, yet avoid using a prime hash-size or similar. 97 * information, yet avoid using a prime hash-size or similar.
98 */ 98 */
99 99
100 static unsigned int d_hash_mask __read_mostly; 100 static unsigned int d_hash_mask __read_mostly;
101 static unsigned int d_hash_shift __read_mostly; 101 static unsigned int d_hash_shift __read_mostly;
102 102
103 static struct hlist_bl_head *dentry_hashtable __read_mostly; 103 static struct hlist_bl_head *dentry_hashtable __read_mostly;
104 104
105 static inline struct hlist_bl_head *d_hash(const struct dentry *parent, 105 static inline struct hlist_bl_head *d_hash(const struct dentry *parent,
106 unsigned int hash) 106 unsigned int hash)
107 { 107 {
108 hash += (unsigned long) parent / L1_CACHE_BYTES; 108 hash += (unsigned long) parent / L1_CACHE_BYTES;
109 hash = hash + (hash >> d_hash_shift); 109 hash = hash + (hash >> d_hash_shift);
110 return dentry_hashtable + (hash & d_hash_mask); 110 return dentry_hashtable + (hash & d_hash_mask);
111 } 111 }
112 112
113 /* Statistics gathering. */ 113 /* Statistics gathering. */
114 struct dentry_stat_t dentry_stat = { 114 struct dentry_stat_t dentry_stat = {
115 .age_limit = 45, 115 .age_limit = 45,
116 }; 116 };
117 117
118 static DEFINE_PER_CPU(long, nr_dentry); 118 static DEFINE_PER_CPU(long, nr_dentry);
119 static DEFINE_PER_CPU(long, nr_dentry_unused); 119 static DEFINE_PER_CPU(long, nr_dentry_unused);
120 120
121 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) 121 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
122 122
123 /* 123 /*
124 * Here we resort to our own counters instead of using generic per-cpu counters 124 * Here we resort to our own counters instead of using generic per-cpu counters
125 * for consistency with what the vfs inode code does. We are expected to harvest 125 * for consistency with what the vfs inode code does. We are expected to harvest
126 * better code and performance by having our own specialized counters. 126 * better code and performance by having our own specialized counters.
127 * 127 *
128 * Please note that the loop is done over all possible CPUs, not over all online 128 * Please note that the loop is done over all possible CPUs, not over all online
129 * CPUs. The reason for this is that we don't want to play games with CPUs going 129 * CPUs. The reason for this is that we don't want to play games with CPUs going
130 * on and off. If one of them goes off, we will just keep their counters. 130 * on and off. If one of them goes off, we will just keep their counters.
131 * 131 *
132 * glommer: See cffbc8a for details, and if you ever intend to change this, 132 * glommer: See cffbc8a for details, and if you ever intend to change this,
133 * please update all vfs counters to match. 133 * please update all vfs counters to match.
134 */ 134 */
135 static long get_nr_dentry(void) 135 static long get_nr_dentry(void)
136 { 136 {
137 int i; 137 int i;
138 long sum = 0; 138 long sum = 0;
139 for_each_possible_cpu(i) 139 for_each_possible_cpu(i)
140 sum += per_cpu(nr_dentry, i); 140 sum += per_cpu(nr_dentry, i);
141 return sum < 0 ? 0 : sum; 141 return sum < 0 ? 0 : sum;
142 } 142 }
143 143
144 static long get_nr_dentry_unused(void) 144 static long get_nr_dentry_unused(void)
145 { 145 {
146 int i; 146 int i;
147 long sum = 0; 147 long sum = 0;
148 for_each_possible_cpu(i) 148 for_each_possible_cpu(i)
149 sum += per_cpu(nr_dentry_unused, i); 149 sum += per_cpu(nr_dentry_unused, i);
150 return sum < 0 ? 0 : sum; 150 return sum < 0 ? 0 : sum;
151 } 151 }
152 152
153 int proc_nr_dentry(ctl_table *table, int write, void __user *buffer, 153 int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
154 size_t *lenp, loff_t *ppos) 154 size_t *lenp, loff_t *ppos)
155 { 155 {
156 dentry_stat.nr_dentry = get_nr_dentry(); 156 dentry_stat.nr_dentry = get_nr_dentry();
157 dentry_stat.nr_unused = get_nr_dentry_unused(); 157 dentry_stat.nr_unused = get_nr_dentry_unused();
158 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 158 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
159 } 159 }
160 #endif 160 #endif
161 161
162 /* 162 /*
163 * Compare 2 name strings, return 0 if they match, otherwise non-zero. 163 * Compare 2 name strings, return 0 if they match, otherwise non-zero.
164 * The strings are both count bytes long, and count is non-zero. 164 * The strings are both count bytes long, and count is non-zero.
165 */ 165 */
166 #ifdef CONFIG_DCACHE_WORD_ACCESS 166 #ifdef CONFIG_DCACHE_WORD_ACCESS
167 167
168 #include <asm/word-at-a-time.h> 168 #include <asm/word-at-a-time.h>
169 /* 169 /*
170 * NOTE! 'cs' and 'scount' come from a dentry, so it has a 170 * NOTE! 'cs' and 'scount' come from a dentry, so it has a
171 * aligned allocation for this particular component. We don't 171 * aligned allocation for this particular component. We don't
172 * strictly need the load_unaligned_zeropad() safety, but it 172 * strictly need the load_unaligned_zeropad() safety, but it
173 * doesn't hurt either. 173 * doesn't hurt either.
174 * 174 *
175 * In contrast, 'ct' and 'tcount' can be from a pathname, and do 175 * In contrast, 'ct' and 'tcount' can be from a pathname, and do
176 * need the careful unaligned handling. 176 * need the careful unaligned handling.
177 */ 177 */
178 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) 178 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
179 { 179 {
180 unsigned long a,b,mask; 180 unsigned long a,b,mask;
181 181
182 for (;;) { 182 for (;;) {
183 a = *(unsigned long *)cs; 183 a = *(unsigned long *)cs;
184 b = load_unaligned_zeropad(ct); 184 b = load_unaligned_zeropad(ct);
185 if (tcount < sizeof(unsigned long)) 185 if (tcount < sizeof(unsigned long))
186 break; 186 break;
187 if (unlikely(a != b)) 187 if (unlikely(a != b))
188 return 1; 188 return 1;
189 cs += sizeof(unsigned long); 189 cs += sizeof(unsigned long);
190 ct += sizeof(unsigned long); 190 ct += sizeof(unsigned long);
191 tcount -= sizeof(unsigned long); 191 tcount -= sizeof(unsigned long);
192 if (!tcount) 192 if (!tcount)
193 return 0; 193 return 0;
194 } 194 }
195 mask = bytemask_from_count(tcount); 195 mask = bytemask_from_count(tcount);
196 return unlikely(!!((a ^ b) & mask)); 196 return unlikely(!!((a ^ b) & mask));
197 } 197 }
198 198
199 #else 199 #else
200 200
201 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) 201 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
202 { 202 {
203 do { 203 do {
204 if (*cs != *ct) 204 if (*cs != *ct)
205 return 1; 205 return 1;
206 cs++; 206 cs++;
207 ct++; 207 ct++;
208 tcount--; 208 tcount--;
209 } while (tcount); 209 } while (tcount);
210 return 0; 210 return 0;
211 } 211 }
212 212
213 #endif 213 #endif
214 214
215 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount) 215 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
216 { 216 {
217 const unsigned char *cs; 217 const unsigned char *cs;
218 /* 218 /*
219 * Be careful about RCU walk racing with rename: 219 * Be careful about RCU walk racing with rename:
220 * use ACCESS_ONCE to fetch the name pointer. 220 * use ACCESS_ONCE to fetch the name pointer.
221 * 221 *
222 * NOTE! Even if a rename will mean that the length 222 * NOTE! Even if a rename will mean that the length
223 * was not loaded atomically, we don't care. The 223 * was not loaded atomically, we don't care. The
224 * RCU walk will check the sequence count eventually, 224 * RCU walk will check the sequence count eventually,
225 * and catch it. And we won't overrun the buffer, 225 * and catch it. And we won't overrun the buffer,
226 * because we're reading the name pointer atomically, 226 * because we're reading the name pointer atomically,
227 * and a dentry name is guaranteed to be properly 227 * and a dentry name is guaranteed to be properly
228 * terminated with a NUL byte. 228 * terminated with a NUL byte.
229 * 229 *
230 * End result: even if 'len' is wrong, we'll exit 230 * End result: even if 'len' is wrong, we'll exit
231 * early because the data cannot match (there can 231 * early because the data cannot match (there can
232 * be no NUL in the ct/tcount data) 232 * be no NUL in the ct/tcount data)
233 */ 233 */
234 cs = ACCESS_ONCE(dentry->d_name.name); 234 cs = ACCESS_ONCE(dentry->d_name.name);
235 smp_read_barrier_depends(); 235 smp_read_barrier_depends();
236 return dentry_string_cmp(cs, ct, tcount); 236 return dentry_string_cmp(cs, ct, tcount);
237 } 237 }
238 238
239 static void __d_free(struct rcu_head *head) 239 static void __d_free(struct rcu_head *head)
240 { 240 {
241 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); 241 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
242 242
243 WARN_ON(!hlist_unhashed(&dentry->d_alias)); 243 WARN_ON(!hlist_unhashed(&dentry->d_alias));
244 if (dname_external(dentry)) 244 if (dname_external(dentry))
245 kfree(dentry->d_name.name); 245 kfree(dentry->d_name.name);
246 kmem_cache_free(dentry_cache, dentry); 246 kmem_cache_free(dentry_cache, dentry);
247 } 247 }
248 248
249 static void dentry_free(struct dentry *dentry) 249 static void dentry_free(struct dentry *dentry)
250 { 250 {
251 /* if dentry was never visible to RCU, immediate free is OK */ 251 /* if dentry was never visible to RCU, immediate free is OK */
252 if (!(dentry->d_flags & DCACHE_RCUACCESS)) 252 if (!(dentry->d_flags & DCACHE_RCUACCESS))
253 __d_free(&dentry->d_u.d_rcu); 253 __d_free(&dentry->d_u.d_rcu);
254 else 254 else
255 call_rcu(&dentry->d_u.d_rcu, __d_free); 255 call_rcu(&dentry->d_u.d_rcu, __d_free);
256 } 256 }
257 257
258 /** 258 /**
259 * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups 259 * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
260 * @dentry: the target dentry 260 * @dentry: the target dentry
261 * After this call, in-progress rcu-walk path lookup will fail. This 261 * After this call, in-progress rcu-walk path lookup will fail. This
262 * should be called after unhashing, and after changing d_inode (if 262 * should be called after unhashing, and after changing d_inode (if
263 * the dentry has not already been unhashed). 263 * the dentry has not already been unhashed).
264 */ 264 */
265 static inline void dentry_rcuwalk_barrier(struct dentry *dentry) 265 static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
266 { 266 {
267 assert_spin_locked(&dentry->d_lock); 267 assert_spin_locked(&dentry->d_lock);
268 /* Go through a barrier */ 268 /* Go through a barrier */
269 write_seqcount_barrier(&dentry->d_seq); 269 write_seqcount_barrier(&dentry->d_seq);
270 } 270 }
271 271
272 /* 272 /*
273 * Release the dentry's inode, using the filesystem 273 * Release the dentry's inode, using the filesystem
274 * d_iput() operation if defined. Dentry has no refcount 274 * d_iput() operation if defined. Dentry has no refcount
275 * and is unhashed. 275 * and is unhashed.
276 */ 276 */
277 static void dentry_iput(struct dentry * dentry) 277 static void dentry_iput(struct dentry * dentry)
278 __releases(dentry->d_lock) 278 __releases(dentry->d_lock)
279 __releases(dentry->d_inode->i_lock) 279 __releases(dentry->d_inode->i_lock)
280 { 280 {
281 struct inode *inode = dentry->d_inode; 281 struct inode *inode = dentry->d_inode;
282 if (inode) { 282 if (inode) {
283 dentry->d_inode = NULL; 283 dentry->d_inode = NULL;
284 hlist_del_init(&dentry->d_alias); 284 hlist_del_init(&dentry->d_alias);
285 spin_unlock(&dentry->d_lock); 285 spin_unlock(&dentry->d_lock);
286 spin_unlock(&inode->i_lock); 286 spin_unlock(&inode->i_lock);
287 if (!inode->i_nlink) 287 if (!inode->i_nlink)
288 fsnotify_inoderemove(inode); 288 fsnotify_inoderemove(inode);
289 if (dentry->d_op && dentry->d_op->d_iput) 289 if (dentry->d_op && dentry->d_op->d_iput)
290 dentry->d_op->d_iput(dentry, inode); 290 dentry->d_op->d_iput(dentry, inode);
291 else 291 else
292 iput(inode); 292 iput(inode);
293 } else { 293 } else {
294 spin_unlock(&dentry->d_lock); 294 spin_unlock(&dentry->d_lock);
295 } 295 }
296 } 296 }
297 297
298 /* 298 /*
299 * Release the dentry's inode, using the filesystem 299 * Release the dentry's inode, using the filesystem
300 * d_iput() operation if defined. dentry remains in-use. 300 * d_iput() operation if defined. dentry remains in-use.
301 */ 301 */
302 static void dentry_unlink_inode(struct dentry * dentry) 302 static void dentry_unlink_inode(struct dentry * dentry)
303 __releases(dentry->d_lock) 303 __releases(dentry->d_lock)
304 __releases(dentry->d_inode->i_lock) 304 __releases(dentry->d_inode->i_lock)
305 { 305 {
306 struct inode *inode = dentry->d_inode; 306 struct inode *inode = dentry->d_inode;
307 __d_clear_type(dentry); 307 __d_clear_type(dentry);
308 dentry->d_inode = NULL; 308 dentry->d_inode = NULL;
309 hlist_del_init(&dentry->d_alias); 309 hlist_del_init(&dentry->d_alias);
310 dentry_rcuwalk_barrier(dentry); 310 dentry_rcuwalk_barrier(dentry);
311 spin_unlock(&dentry->d_lock); 311 spin_unlock(&dentry->d_lock);
312 spin_unlock(&inode->i_lock); 312 spin_unlock(&inode->i_lock);
313 if (!inode->i_nlink) 313 if (!inode->i_nlink)
314 fsnotify_inoderemove(inode); 314 fsnotify_inoderemove(inode);
315 if (dentry->d_op && dentry->d_op->d_iput) 315 if (dentry->d_op && dentry->d_op->d_iput)
316 dentry->d_op->d_iput(dentry, inode); 316 dentry->d_op->d_iput(dentry, inode);
317 else 317 else
318 iput(inode); 318 iput(inode);
319 } 319 }
320 320
321 /* 321 /*
322 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry 322 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
323 * is in use - which includes both the "real" per-superblock 323 * is in use - which includes both the "real" per-superblock
324 * LRU list _and_ the DCACHE_SHRINK_LIST use. 324 * LRU list _and_ the DCACHE_SHRINK_LIST use.
325 * 325 *
326 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is 326 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
327 * on the shrink list (ie not on the superblock LRU list). 327 * on the shrink list (ie not on the superblock LRU list).
328 * 328 *
329 * The per-cpu "nr_dentry_unused" counters are updated with 329 * The per-cpu "nr_dentry_unused" counters are updated with
330 * the DCACHE_LRU_LIST bit. 330 * the DCACHE_LRU_LIST bit.
331 * 331 *
332 * These helper functions make sure we always follow the 332 * These helper functions make sure we always follow the
333 * rules. d_lock must be held by the caller. 333 * rules. d_lock must be held by the caller.
334 */ 334 */
335 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x)) 335 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
336 static void d_lru_add(struct dentry *dentry) 336 static void d_lru_add(struct dentry *dentry)
337 { 337 {
338 D_FLAG_VERIFY(dentry, 0); 338 D_FLAG_VERIFY(dentry, 0);
339 dentry->d_flags |= DCACHE_LRU_LIST; 339 dentry->d_flags |= DCACHE_LRU_LIST;
340 this_cpu_inc(nr_dentry_unused); 340 this_cpu_inc(nr_dentry_unused);
341 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); 341 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
342 } 342 }
343 343
344 static void d_lru_del(struct dentry *dentry) 344 static void d_lru_del(struct dentry *dentry)
345 { 345 {
346 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); 346 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
347 dentry->d_flags &= ~DCACHE_LRU_LIST; 347 dentry->d_flags &= ~DCACHE_LRU_LIST;
348 this_cpu_dec(nr_dentry_unused); 348 this_cpu_dec(nr_dentry_unused);
349 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); 349 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
350 } 350 }
351 351
352 static void d_shrink_del(struct dentry *dentry) 352 static void d_shrink_del(struct dentry *dentry)
353 { 353 {
354 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); 354 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
355 list_del_init(&dentry->d_lru); 355 list_del_init(&dentry->d_lru);
356 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); 356 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
357 this_cpu_dec(nr_dentry_unused); 357 this_cpu_dec(nr_dentry_unused);
358 } 358 }
359 359
360 static void d_shrink_add(struct dentry *dentry, struct list_head *list) 360 static void d_shrink_add(struct dentry *dentry, struct list_head *list)
361 { 361 {
362 D_FLAG_VERIFY(dentry, 0); 362 D_FLAG_VERIFY(dentry, 0);
363 list_add(&dentry->d_lru, list); 363 list_add(&dentry->d_lru, list);
364 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; 364 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
365 this_cpu_inc(nr_dentry_unused); 365 this_cpu_inc(nr_dentry_unused);
366 } 366 }
367 367
368 /* 368 /*
369 * These can only be called under the global LRU lock, ie during the 369 * These can only be called under the global LRU lock, ie during the
370 * callback for freeing the LRU list. "isolate" removes it from the 370 * callback for freeing the LRU list. "isolate" removes it from the
371 * LRU lists entirely, while shrink_move moves it to the indicated 371 * LRU lists entirely, while shrink_move moves it to the indicated
372 * private list. 372 * private list.
373 */ 373 */
374 static void d_lru_isolate(struct dentry *dentry) 374 static void d_lru_isolate(struct dentry *dentry)
375 { 375 {
376 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); 376 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
377 dentry->d_flags &= ~DCACHE_LRU_LIST; 377 dentry->d_flags &= ~DCACHE_LRU_LIST;
378 this_cpu_dec(nr_dentry_unused); 378 this_cpu_dec(nr_dentry_unused);
379 list_del_init(&dentry->d_lru); 379 list_del_init(&dentry->d_lru);
380 } 380 }
381 381
382 static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list) 382 static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list)
383 { 383 {
384 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); 384 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
385 dentry->d_flags |= DCACHE_SHRINK_LIST; 385 dentry->d_flags |= DCACHE_SHRINK_LIST;
386 list_move_tail(&dentry->d_lru, list); 386 list_move_tail(&dentry->d_lru, list);
387 } 387 }
388 388
389 /* 389 /*
390 * dentry_lru_(add|del)_list) must be called with d_lock held. 390 * dentry_lru_(add|del)_list) must be called with d_lock held.
391 */ 391 */
392 static void dentry_lru_add(struct dentry *dentry) 392 static void dentry_lru_add(struct dentry *dentry)
393 { 393 {
394 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) 394 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
395 d_lru_add(dentry); 395 d_lru_add(dentry);
396 } 396 }
397 397
398 /** 398 /**
399 * d_drop - drop a dentry 399 * d_drop - drop a dentry
400 * @dentry: dentry to drop 400 * @dentry: dentry to drop
401 * 401 *
402 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't 402 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
403 * be found through a VFS lookup any more. Note that this is different from 403 * be found through a VFS lookup any more. Note that this is different from
404 * deleting the dentry - d_delete will try to mark the dentry negative if 404 * deleting the dentry - d_delete will try to mark the dentry negative if
405 * possible, giving a successful _negative_ lookup, while d_drop will 405 * possible, giving a successful _negative_ lookup, while d_drop will
406 * just make the cache lookup fail. 406 * just make the cache lookup fail.
407 * 407 *
408 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some 408 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
409 * reason (NFS timeouts or autofs deletes). 409 * reason (NFS timeouts or autofs deletes).
410 * 410 *
411 * __d_drop requires dentry->d_lock. 411 * __d_drop requires dentry->d_lock.
412 */ 412 */
413 void __d_drop(struct dentry *dentry) 413 void __d_drop(struct dentry *dentry)
414 { 414 {
415 if (!d_unhashed(dentry)) { 415 if (!d_unhashed(dentry)) {
416 struct hlist_bl_head *b; 416 struct hlist_bl_head *b;
417 /* 417 /*
418 * Hashed dentries are normally on the dentry hashtable, 418 * Hashed dentries are normally on the dentry hashtable,
419 * with the exception of those newly allocated by 419 * with the exception of those newly allocated by
420 * d_obtain_alias, which are always IS_ROOT: 420 * d_obtain_alias, which are always IS_ROOT:
421 */ 421 */
422 if (unlikely(IS_ROOT(dentry))) 422 if (unlikely(IS_ROOT(dentry)))
423 b = &dentry->d_sb->s_anon; 423 b = &dentry->d_sb->s_anon;
424 else 424 else
425 b = d_hash(dentry->d_parent, dentry->d_name.hash); 425 b = d_hash(dentry->d_parent, dentry->d_name.hash);
426 426
427 hlist_bl_lock(b); 427 hlist_bl_lock(b);
428 __hlist_bl_del(&dentry->d_hash); 428 __hlist_bl_del(&dentry->d_hash);
429 dentry->d_hash.pprev = NULL; 429 dentry->d_hash.pprev = NULL;
430 hlist_bl_unlock(b); 430 hlist_bl_unlock(b);
431 dentry_rcuwalk_barrier(dentry); 431 dentry_rcuwalk_barrier(dentry);
432 } 432 }
433 } 433 }
434 EXPORT_SYMBOL(__d_drop); 434 EXPORT_SYMBOL(__d_drop);
435 435
436 void d_drop(struct dentry *dentry) 436 void d_drop(struct dentry *dentry)
437 { 437 {
438 spin_lock(&dentry->d_lock); 438 spin_lock(&dentry->d_lock);
439 __d_drop(dentry); 439 __d_drop(dentry);
440 spin_unlock(&dentry->d_lock); 440 spin_unlock(&dentry->d_lock);
441 } 441 }
442 EXPORT_SYMBOL(d_drop); 442 EXPORT_SYMBOL(d_drop);
443 443
444 /* 444 static void __dentry_kill(struct dentry *dentry)
445 * Finish off a dentry we've decided to kill.
446 * dentry->d_lock must be held, returns with it unlocked.
447 * If ref is non-zero, then decrement the refcount too.
448 * Returns dentry requiring refcount drop, or NULL if we're done.
449 */
450 static struct dentry *
451 dentry_kill(struct dentry *dentry, int unlock_on_failure)
452 __releases(dentry->d_lock)
453 { 445 {
454 struct inode *inode;
455 struct dentry *parent = NULL; 446 struct dentry *parent = NULL;
456 bool can_free = true; 447 bool can_free = true;
457
458 inode = dentry->d_inode;
459 if (inode && !spin_trylock(&inode->i_lock)) {
460 relock:
461 if (unlock_on_failure) {
462 spin_unlock(&dentry->d_lock);
463 cpu_relax();
464 }
465 return dentry; /* try again with same dentry */
466 }
467 if (!IS_ROOT(dentry)) 448 if (!IS_ROOT(dentry))
468 parent = dentry->d_parent; 449 parent = dentry->d_parent;
469 if (parent && !spin_trylock(&parent->d_lock)) {
470 if (inode)
471 spin_unlock(&inode->i_lock);
472 goto relock;
473 }
474 450
475 /* 451 /*
476 * The dentry is now unrecoverably dead to the world. 452 * The dentry is now unrecoverably dead to the world.
477 */ 453 */
478 lockref_mark_dead(&dentry->d_lockref); 454 lockref_mark_dead(&dentry->d_lockref);
479 455
480 /* 456 /*
481 * inform the fs via d_prune that this dentry is about to be 457 * inform the fs via d_prune that this dentry is about to be
482 * unhashed and destroyed. 458 * unhashed and destroyed.
483 */ 459 */
484 if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry)) 460 if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
485 dentry->d_op->d_prune(dentry); 461 dentry->d_op->d_prune(dentry);
486 462
487 if (dentry->d_flags & DCACHE_LRU_LIST) { 463 if (dentry->d_flags & DCACHE_LRU_LIST) {
488 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) 464 if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
489 d_lru_del(dentry); 465 d_lru_del(dentry);
490 } 466 }
491 /* if it was on the hash then remove it */ 467 /* if it was on the hash then remove it */
492 __d_drop(dentry); 468 __d_drop(dentry);
493 list_del(&dentry->d_u.d_child); 469 list_del(&dentry->d_u.d_child);
494 /* 470 /*
495 * Inform d_walk() that we are no longer attached to the 471 * Inform d_walk() that we are no longer attached to the
496 * dentry tree 472 * dentry tree
497 */ 473 */
498 dentry->d_flags |= DCACHE_DENTRY_KILLED; 474 dentry->d_flags |= DCACHE_DENTRY_KILLED;
499 if (parent) 475 if (parent)
500 spin_unlock(&parent->d_lock); 476 spin_unlock(&parent->d_lock);
501 dentry_iput(dentry); 477 dentry_iput(dentry);
502 /* 478 /*
503 * dentry_iput drops the locks, at which point nobody (except 479 * dentry_iput drops the locks, at which point nobody (except
504 * transient RCU lookups) can reach this dentry. 480 * transient RCU lookups) can reach this dentry.
505 */ 481 */
506 BUG_ON((int)dentry->d_lockref.count > 0); 482 BUG_ON((int)dentry->d_lockref.count > 0);
507 this_cpu_dec(nr_dentry); 483 this_cpu_dec(nr_dentry);
508 if (dentry->d_op && dentry->d_op->d_release) 484 if (dentry->d_op && dentry->d_op->d_release)
509 dentry->d_op->d_release(dentry); 485 dentry->d_op->d_release(dentry);
510 486
511 spin_lock(&dentry->d_lock); 487 spin_lock(&dentry->d_lock);
512 if (dentry->d_flags & DCACHE_SHRINK_LIST) { 488 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
513 dentry->d_flags |= DCACHE_MAY_FREE; 489 dentry->d_flags |= DCACHE_MAY_FREE;
514 can_free = false; 490 can_free = false;
515 } 491 }
516 spin_unlock(&dentry->d_lock); 492 spin_unlock(&dentry->d_lock);
517 out:
518 if (likely(can_free)) 493 if (likely(can_free))
519 dentry_free(dentry); 494 dentry_free(dentry);
495 }
496
497 /*
498 * Finish off a dentry we've decided to kill.
499 * dentry->d_lock must be held, returns with it unlocked.
500 * If ref is non-zero, then decrement the refcount too.
501 * Returns dentry requiring refcount drop, or NULL if we're done.
502 */
503 static struct dentry *
504 dentry_kill(struct dentry *dentry, int unlock_on_failure)
505 __releases(dentry->d_lock)
506 {
507 struct inode *inode = dentry->d_inode;
508 struct dentry *parent = NULL;
509
510 if (inode && unlikely(!spin_trylock(&inode->i_lock)))
511 goto failed;
512
513 if (!IS_ROOT(dentry)) {
514 parent = dentry->d_parent;
515 if (unlikely(!spin_trylock(&parent->d_lock))) {
516 if (inode)
517 spin_unlock(&inode->i_lock);
518 goto failed;
519 }
520 }
521
522 __dentry_kill(dentry);
520 return parent; 523 return parent;
524
525 failed:
526 if (unlock_on_failure) {
527 spin_unlock(&dentry->d_lock);
528 cpu_relax();
529 }
530 return dentry; /* try again with same dentry */
521 } 531 }
522 532
523 /* 533 /*
524 * This is dput 534 * This is dput
525 * 535 *
526 * This is complicated by the fact that we do not want to put 536 * This is complicated by the fact that we do not want to put
527 * dentries that are no longer on any hash chain on the unused 537 * dentries that are no longer on any hash chain on the unused
528 * list: we'd much rather just get rid of them immediately. 538 * list: we'd much rather just get rid of them immediately.
529 * 539 *
530 * However, that implies that we have to traverse the dentry 540 * However, that implies that we have to traverse the dentry
531 * tree upwards to the parents which might _also_ now be 541 * tree upwards to the parents which might _also_ now be
532 * scheduled for deletion (it may have been only waiting for 542 * scheduled for deletion (it may have been only waiting for
533 * its last child to go away). 543 * its last child to go away).
534 * 544 *
535 * This tail recursion is done by hand as we don't want to depend 545 * This tail recursion is done by hand as we don't want to depend
536 * on the compiler to always get this right (gcc generally doesn't). 546 * on the compiler to always get this right (gcc generally doesn't).
537 * Real recursion would eat up our stack space. 547 * Real recursion would eat up our stack space.
538 */ 548 */
539 549
540 /* 550 /*
541 * dput - release a dentry 551 * dput - release a dentry
542 * @dentry: dentry to release 552 * @dentry: dentry to release
543 * 553 *
544 * Release a dentry. This will drop the usage count and if appropriate 554 * Release a dentry. This will drop the usage count and if appropriate
545 * call the dentry unlink method as well as removing it from the queues and 555 * call the dentry unlink method as well as removing it from the queues and
546 * releasing its resources. If the parent dentries were scheduled for release 556 * releasing its resources. If the parent dentries were scheduled for release
547 * they too may now get deleted. 557 * they too may now get deleted.
548 */ 558 */
549 void dput(struct dentry *dentry) 559 void dput(struct dentry *dentry)
550 { 560 {
551 if (unlikely(!dentry)) 561 if (unlikely(!dentry))
552 return; 562 return;
553 563
554 repeat: 564 repeat:
555 if (lockref_put_or_lock(&dentry->d_lockref)) 565 if (lockref_put_or_lock(&dentry->d_lockref))
556 return; 566 return;
557 567
558 /* Unreachable? Get rid of it */ 568 /* Unreachable? Get rid of it */
559 if (unlikely(d_unhashed(dentry))) 569 if (unlikely(d_unhashed(dentry)))
560 goto kill_it; 570 goto kill_it;
561 571
562 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) { 572 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
563 if (dentry->d_op->d_delete(dentry)) 573 if (dentry->d_op->d_delete(dentry))
564 goto kill_it; 574 goto kill_it;
565 } 575 }
566 576
567 if (!(dentry->d_flags & DCACHE_REFERENCED)) 577 if (!(dentry->d_flags & DCACHE_REFERENCED))
568 dentry->d_flags |= DCACHE_REFERENCED; 578 dentry->d_flags |= DCACHE_REFERENCED;
569 dentry_lru_add(dentry); 579 dentry_lru_add(dentry);
570 580
571 dentry->d_lockref.count--; 581 dentry->d_lockref.count--;
572 spin_unlock(&dentry->d_lock); 582 spin_unlock(&dentry->d_lock);
573 return; 583 return;
574 584
575 kill_it: 585 kill_it:
576 dentry = dentry_kill(dentry, 1); 586 dentry = dentry_kill(dentry, 1);
577 if (dentry) 587 if (dentry)
578 goto repeat; 588 goto repeat;
579 } 589 }
580 EXPORT_SYMBOL(dput); 590 EXPORT_SYMBOL(dput);
581 591
582 /** 592 /**
583 * d_invalidate - invalidate a dentry 593 * d_invalidate - invalidate a dentry
584 * @dentry: dentry to invalidate 594 * @dentry: dentry to invalidate
585 * 595 *
586 * Try to invalidate the dentry if it turns out to be 596 * Try to invalidate the dentry if it turns out to be
587 * possible. If there are other dentries that can be 597 * possible. If there are other dentries that can be
588 * reached through this one we can't delete it and we 598 * reached through this one we can't delete it and we
589 * return -EBUSY. On success we return 0. 599 * return -EBUSY. On success we return 0.
590 * 600 *
591 * no dcache lock. 601 * no dcache lock.
592 */ 602 */
593 603
594 int d_invalidate(struct dentry * dentry) 604 int d_invalidate(struct dentry * dentry)
595 { 605 {
596 /* 606 /*
597 * If it's already been dropped, return OK. 607 * If it's already been dropped, return OK.
598 */ 608 */
599 spin_lock(&dentry->d_lock); 609 spin_lock(&dentry->d_lock);
600 if (d_unhashed(dentry)) { 610 if (d_unhashed(dentry)) {
601 spin_unlock(&dentry->d_lock); 611 spin_unlock(&dentry->d_lock);
602 return 0; 612 return 0;
603 } 613 }
604 /* 614 /*
605 * Check whether to do a partial shrink_dcache 615 * Check whether to do a partial shrink_dcache
606 * to get rid of unused child entries. 616 * to get rid of unused child entries.
607 */ 617 */
608 if (!list_empty(&dentry->d_subdirs)) { 618 if (!list_empty(&dentry->d_subdirs)) {
609 spin_unlock(&dentry->d_lock); 619 spin_unlock(&dentry->d_lock);
610 shrink_dcache_parent(dentry); 620 shrink_dcache_parent(dentry);
611 spin_lock(&dentry->d_lock); 621 spin_lock(&dentry->d_lock);
612 } 622 }
613 623
614 /* 624 /*
615 * Somebody else still using it? 625 * Somebody else still using it?
616 * 626 *
617 * If it's a directory, we can't drop it 627 * If it's a directory, we can't drop it
618 * for fear of somebody re-populating it 628 * for fear of somebody re-populating it
619 * with children (even though dropping it 629 * with children (even though dropping it
620 * would make it unreachable from the root, 630 * would make it unreachable from the root,
621 * we might still populate it if it was a 631 * we might still populate it if it was a
622 * working directory or similar). 632 * working directory or similar).
623 * We also need to leave mountpoints alone, 633 * We also need to leave mountpoints alone,
624 * directory or not. 634 * directory or not.
625 */ 635 */
626 if (dentry->d_lockref.count > 1 && dentry->d_inode) { 636 if (dentry->d_lockref.count > 1 && dentry->d_inode) {
627 if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) { 637 if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
628 spin_unlock(&dentry->d_lock); 638 spin_unlock(&dentry->d_lock);
629 return -EBUSY; 639 return -EBUSY;
630 } 640 }
631 } 641 }
632 642
633 __d_drop(dentry); 643 __d_drop(dentry);
634 spin_unlock(&dentry->d_lock); 644 spin_unlock(&dentry->d_lock);
635 return 0; 645 return 0;
636 } 646 }
637 EXPORT_SYMBOL(d_invalidate); 647 EXPORT_SYMBOL(d_invalidate);
638 648
639 /* This must be called with d_lock held */ 649 /* This must be called with d_lock held */
640 static inline void __dget_dlock(struct dentry *dentry) 650 static inline void __dget_dlock(struct dentry *dentry)
641 { 651 {
642 dentry->d_lockref.count++; 652 dentry->d_lockref.count++;
643 } 653 }
644 654
645 static inline void __dget(struct dentry *dentry) 655 static inline void __dget(struct dentry *dentry)
646 { 656 {
647 lockref_get(&dentry->d_lockref); 657 lockref_get(&dentry->d_lockref);
648 } 658 }
649 659
650 struct dentry *dget_parent(struct dentry *dentry) 660 struct dentry *dget_parent(struct dentry *dentry)
651 { 661 {
652 int gotref; 662 int gotref;
653 struct dentry *ret; 663 struct dentry *ret;
654 664
655 /* 665 /*
656 * Do optimistic parent lookup without any 666 * Do optimistic parent lookup without any
657 * locking. 667 * locking.
658 */ 668 */
659 rcu_read_lock(); 669 rcu_read_lock();
660 ret = ACCESS_ONCE(dentry->d_parent); 670 ret = ACCESS_ONCE(dentry->d_parent);
661 gotref = lockref_get_not_zero(&ret->d_lockref); 671 gotref = lockref_get_not_zero(&ret->d_lockref);
662 rcu_read_unlock(); 672 rcu_read_unlock();
663 if (likely(gotref)) { 673 if (likely(gotref)) {
664 if (likely(ret == ACCESS_ONCE(dentry->d_parent))) 674 if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
665 return ret; 675 return ret;
666 dput(ret); 676 dput(ret);
667 } 677 }
668 678
669 repeat: 679 repeat:
670 /* 680 /*
671 * Don't need rcu_dereference because we re-check it was correct under 681 * Don't need rcu_dereference because we re-check it was correct under
672 * the lock. 682 * the lock.
673 */ 683 */
674 rcu_read_lock(); 684 rcu_read_lock();
675 ret = dentry->d_parent; 685 ret = dentry->d_parent;
676 spin_lock(&ret->d_lock); 686 spin_lock(&ret->d_lock);
677 if (unlikely(ret != dentry->d_parent)) { 687 if (unlikely(ret != dentry->d_parent)) {
678 spin_unlock(&ret->d_lock); 688 spin_unlock(&ret->d_lock);
679 rcu_read_unlock(); 689 rcu_read_unlock();
680 goto repeat; 690 goto repeat;
681 } 691 }
682 rcu_read_unlock(); 692 rcu_read_unlock();
683 BUG_ON(!ret->d_lockref.count); 693 BUG_ON(!ret->d_lockref.count);
684 ret->d_lockref.count++; 694 ret->d_lockref.count++;
685 spin_unlock(&ret->d_lock); 695 spin_unlock(&ret->d_lock);
686 return ret; 696 return ret;
687 } 697 }
688 EXPORT_SYMBOL(dget_parent); 698 EXPORT_SYMBOL(dget_parent);
689 699
690 /** 700 /**
691 * d_find_alias - grab a hashed alias of inode 701 * d_find_alias - grab a hashed alias of inode
692 * @inode: inode in question 702 * @inode: inode in question
693 * @want_discon: flag, used by d_splice_alias, to request 703 * @want_discon: flag, used by d_splice_alias, to request
694 * that only a DISCONNECTED alias be returned. 704 * that only a DISCONNECTED alias be returned.
695 * 705 *
696 * If inode has a hashed alias, or is a directory and has any alias, 706 * If inode has a hashed alias, or is a directory and has any alias,
697 * acquire the reference to alias and return it. Otherwise return NULL. 707 * acquire the reference to alias and return it. Otherwise return NULL.
698 * Notice that if inode is a directory there can be only one alias and 708 * Notice that if inode is a directory there can be only one alias and
699 * it can be unhashed only if it has no children, or if it is the root 709 * it can be unhashed only if it has no children, or if it is the root
700 * of a filesystem. 710 * of a filesystem.
701 * 711 *
702 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer 712 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
703 * any other hashed alias over that one unless @want_discon is set, 713 * any other hashed alias over that one unless @want_discon is set,
704 * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias. 714 * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
705 */ 715 */
706 static struct dentry *__d_find_alias(struct inode *inode, int want_discon) 716 static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
707 { 717 {
708 struct dentry *alias, *discon_alias; 718 struct dentry *alias, *discon_alias;
709 719
710 again: 720 again:
711 discon_alias = NULL; 721 discon_alias = NULL;
712 hlist_for_each_entry(alias, &inode->i_dentry, d_alias) { 722 hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
713 spin_lock(&alias->d_lock); 723 spin_lock(&alias->d_lock);
714 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { 724 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
715 if (IS_ROOT(alias) && 725 if (IS_ROOT(alias) &&
716 (alias->d_flags & DCACHE_DISCONNECTED)) { 726 (alias->d_flags & DCACHE_DISCONNECTED)) {
717 discon_alias = alias; 727 discon_alias = alias;
718 } else if (!want_discon) { 728 } else if (!want_discon) {
719 __dget_dlock(alias); 729 __dget_dlock(alias);
720 spin_unlock(&alias->d_lock); 730 spin_unlock(&alias->d_lock);
721 return alias; 731 return alias;
722 } 732 }
723 } 733 }
724 spin_unlock(&alias->d_lock); 734 spin_unlock(&alias->d_lock);
725 } 735 }
726 if (discon_alias) { 736 if (discon_alias) {
727 alias = discon_alias; 737 alias = discon_alias;
728 spin_lock(&alias->d_lock); 738 spin_lock(&alias->d_lock);
729 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { 739 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
730 if (IS_ROOT(alias) && 740 if (IS_ROOT(alias) &&
731 (alias->d_flags & DCACHE_DISCONNECTED)) { 741 (alias->d_flags & DCACHE_DISCONNECTED)) {
732 __dget_dlock(alias); 742 __dget_dlock(alias);
733 spin_unlock(&alias->d_lock); 743 spin_unlock(&alias->d_lock);
734 return alias; 744 return alias;
735 } 745 }
736 } 746 }
737 spin_unlock(&alias->d_lock); 747 spin_unlock(&alias->d_lock);
738 goto again; 748 goto again;
739 } 749 }
740 return NULL; 750 return NULL;
741 } 751 }
742 752
743 struct dentry *d_find_alias(struct inode *inode) 753 struct dentry *d_find_alias(struct inode *inode)
744 { 754 {
745 struct dentry *de = NULL; 755 struct dentry *de = NULL;
746 756
747 if (!hlist_empty(&inode->i_dentry)) { 757 if (!hlist_empty(&inode->i_dentry)) {
748 spin_lock(&inode->i_lock); 758 spin_lock(&inode->i_lock);
749 de = __d_find_alias(inode, 0); 759 de = __d_find_alias(inode, 0);
750 spin_unlock(&inode->i_lock); 760 spin_unlock(&inode->i_lock);
751 } 761 }
752 return de; 762 return de;
753 } 763 }
754 EXPORT_SYMBOL(d_find_alias); 764 EXPORT_SYMBOL(d_find_alias);
755 765
756 /* 766 /*
757 * Try to kill dentries associated with this inode. 767 * Try to kill dentries associated with this inode.
758 * WARNING: you must own a reference to inode. 768 * WARNING: you must own a reference to inode.
759 */ 769 */
760 void d_prune_aliases(struct inode *inode) 770 void d_prune_aliases(struct inode *inode)
761 { 771 {
762 struct dentry *dentry; 772 struct dentry *dentry;
763 restart: 773 restart:
764 spin_lock(&inode->i_lock); 774 spin_lock(&inode->i_lock);
765 hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) { 775 hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
766 spin_lock(&dentry->d_lock); 776 spin_lock(&dentry->d_lock);
767 if (!dentry->d_lockref.count) { 777 if (!dentry->d_lockref.count) {
768 /* 778 /*
769 * inform the fs via d_prune that this dentry 779 * inform the fs via d_prune that this dentry
770 * is about to be unhashed and destroyed. 780 * is about to be unhashed and destroyed.
771 */ 781 */
772 if ((dentry->d_flags & DCACHE_OP_PRUNE) && 782 if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
773 !d_unhashed(dentry)) 783 !d_unhashed(dentry))
774 dentry->d_op->d_prune(dentry); 784 dentry->d_op->d_prune(dentry);
775 785
776 __dget_dlock(dentry); 786 __dget_dlock(dentry);
777 __d_drop(dentry); 787 __d_drop(dentry);
778 spin_unlock(&dentry->d_lock); 788 spin_unlock(&dentry->d_lock);
779 spin_unlock(&inode->i_lock); 789 spin_unlock(&inode->i_lock);
780 dput(dentry); 790 dput(dentry);
781 goto restart; 791 goto restart;
782 } 792 }
783 spin_unlock(&dentry->d_lock); 793 spin_unlock(&dentry->d_lock);
784 } 794 }
785 spin_unlock(&inode->i_lock); 795 spin_unlock(&inode->i_lock);
786 } 796 }
787 EXPORT_SYMBOL(d_prune_aliases); 797 EXPORT_SYMBOL(d_prune_aliases);
788 798
789 static void shrink_dentry_list(struct list_head *list) 799 static void shrink_dentry_list(struct list_head *list)
790 { 800 {
791 struct dentry *dentry, *parent; 801 struct dentry *dentry, *parent;
792 802
793 while (!list_empty(list)) { 803 while (!list_empty(list)) {
794 dentry = list_entry(list->prev, struct dentry, d_lru); 804 dentry = list_entry(list->prev, struct dentry, d_lru);
795 spin_lock(&dentry->d_lock); 805 spin_lock(&dentry->d_lock);
796 /* 806 /*
797 * The dispose list is isolated and dentries are not accounted 807 * The dispose list is isolated and dentries are not accounted
798 * to the LRU here, so we can simply remove it from the list 808 * to the LRU here, so we can simply remove it from the list
799 * here regardless of whether it is referenced or not. 809 * here regardless of whether it is referenced or not.
800 */ 810 */
801 d_shrink_del(dentry); 811 d_shrink_del(dentry);
802 812
803 /* 813 /*
804 * We found an inuse dentry which was not removed from 814 * We found an inuse dentry which was not removed from
805 * the LRU because of laziness during lookup. Do not free it. 815 * the LRU because of laziness during lookup. Do not free it.
806 */ 816 */
807 if ((int)dentry->d_lockref.count > 0) { 817 if ((int)dentry->d_lockref.count > 0) {
808 spin_unlock(&dentry->d_lock); 818 spin_unlock(&dentry->d_lock);
809 continue; 819 continue;
810 } 820 }
811 821
812 822
813 if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) { 823 if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) {
814 bool can_free = dentry->d_flags & DCACHE_MAY_FREE; 824 bool can_free = dentry->d_flags & DCACHE_MAY_FREE;
815 spin_unlock(&dentry->d_lock); 825 spin_unlock(&dentry->d_lock);
816 if (can_free) 826 if (can_free)
817 dentry_free(dentry); 827 dentry_free(dentry);
818 continue; 828 continue;
819 } 829 }
820 830
821 parent = dentry_kill(dentry, 0); 831 parent = dentry_kill(dentry, 0);
822 /* 832 /*
823 * If dentry_kill returns NULL, we have nothing more to do. 833 * If dentry_kill returns NULL, we have nothing more to do.
824 */ 834 */
825 if (!parent) 835 if (!parent)
826 continue; 836 continue;
827 837
828 if (unlikely(parent == dentry)) { 838 if (unlikely(parent == dentry)) {
829 /* 839 /*
830 * trylocks have failed and d_lock has been held the 840 * trylocks have failed and d_lock has been held the
831 * whole time, so it could not have been added to any 841 * whole time, so it could not have been added to any
832 * other lists. Just add it back to the shrink list. 842 * other lists. Just add it back to the shrink list.
833 */ 843 */
834 d_shrink_add(dentry, list); 844 d_shrink_add(dentry, list);
835 spin_unlock(&dentry->d_lock); 845 spin_unlock(&dentry->d_lock);
836 continue; 846 continue;
837 } 847 }
838 /* 848 /*
839 * We need to prune ancestors too. This is necessary to prevent 849 * We need to prune ancestors too. This is necessary to prevent
840 * quadratic behavior of shrink_dcache_parent(), but is also 850 * quadratic behavior of shrink_dcache_parent(), but is also
841 * expected to be beneficial in reducing dentry cache 851 * expected to be beneficial in reducing dentry cache
842 * fragmentation. 852 * fragmentation.
843 */ 853 */
844 dentry = parent; 854 dentry = parent;
845 while (dentry && !lockref_put_or_lock(&dentry->d_lockref)) 855 while (dentry && !lockref_put_or_lock(&dentry->d_lockref))
846 dentry = dentry_kill(dentry, 1); 856 dentry = dentry_kill(dentry, 1);
847 } 857 }
848 } 858 }
849 859
850 static enum lru_status 860 static enum lru_status
851 dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg) 861 dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg)
852 { 862 {
853 struct list_head *freeable = arg; 863 struct list_head *freeable = arg;
854 struct dentry *dentry = container_of(item, struct dentry, d_lru); 864 struct dentry *dentry = container_of(item, struct dentry, d_lru);
855 865
856 866
857 /* 867 /*
858 * we are inverting the lru lock/dentry->d_lock here, 868 * we are inverting the lru lock/dentry->d_lock here,
859 * so use a trylock. If we fail to get the lock, just skip 869 * so use a trylock. If we fail to get the lock, just skip
860 * it 870 * it
861 */ 871 */
862 if (!spin_trylock(&dentry->d_lock)) 872 if (!spin_trylock(&dentry->d_lock))
863 return LRU_SKIP; 873 return LRU_SKIP;
864 874
865 /* 875 /*
866 * Referenced dentries are still in use. If they have active 876 * Referenced dentries are still in use. If they have active
867 * counts, just remove them from the LRU. Otherwise give them 877 * counts, just remove them from the LRU. Otherwise give them
868 * another pass through the LRU. 878 * another pass through the LRU.
869 */ 879 */
870 if (dentry->d_lockref.count) { 880 if (dentry->d_lockref.count) {
871 d_lru_isolate(dentry); 881 d_lru_isolate(dentry);
872 spin_unlock(&dentry->d_lock); 882 spin_unlock(&dentry->d_lock);
873 return LRU_REMOVED; 883 return LRU_REMOVED;
874 } 884 }
875 885
876 if (dentry->d_flags & DCACHE_REFERENCED) { 886 if (dentry->d_flags & DCACHE_REFERENCED) {
877 dentry->d_flags &= ~DCACHE_REFERENCED; 887 dentry->d_flags &= ~DCACHE_REFERENCED;
878 spin_unlock(&dentry->d_lock); 888 spin_unlock(&dentry->d_lock);
879 889
880 /* 890 /*
881 * The list move itself will be made by the common LRU code. At 891 * The list move itself will be made by the common LRU code. At
882 * this point, we've dropped the dentry->d_lock but keep the 892 * this point, we've dropped the dentry->d_lock but keep the
883 * lru lock. This is safe to do, since every list movement is 893 * lru lock. This is safe to do, since every list movement is
884 * protected by the lru lock even if both locks are held. 894 * protected by the lru lock even if both locks are held.
885 * 895 *
886 * This is guaranteed by the fact that all LRU management 896 * This is guaranteed by the fact that all LRU management
887 * functions are intermediated by the LRU API calls like 897 * functions are intermediated by the LRU API calls like
888 * list_lru_add and list_lru_del. List movement in this file 898 * list_lru_add and list_lru_del. List movement in this file
889 * only ever occur through this functions or through callbacks 899 * only ever occur through this functions or through callbacks
890 * like this one, that are called from the LRU API. 900 * like this one, that are called from the LRU API.
891 * 901 *
892 * The only exceptions to this are functions like 902 * The only exceptions to this are functions like
893 * shrink_dentry_list, and code that first checks for the 903 * shrink_dentry_list, and code that first checks for the
894 * DCACHE_SHRINK_LIST flag. Those are guaranteed to be 904 * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
895 * operating only with stack provided lists after they are 905 * operating only with stack provided lists after they are
896 * properly isolated from the main list. It is thus, always a 906 * properly isolated from the main list. It is thus, always a
897 * local access. 907 * local access.
898 */ 908 */
899 return LRU_ROTATE; 909 return LRU_ROTATE;
900 } 910 }
901 911
902 d_lru_shrink_move(dentry, freeable); 912 d_lru_shrink_move(dentry, freeable);
903 spin_unlock(&dentry->d_lock); 913 spin_unlock(&dentry->d_lock);
904 914
905 return LRU_REMOVED; 915 return LRU_REMOVED;
906 } 916 }
907 917
908 /** 918 /**
909 * prune_dcache_sb - shrink the dcache 919 * prune_dcache_sb - shrink the dcache
910 * @sb: superblock 920 * @sb: superblock
911 * @nr_to_scan : number of entries to try to free 921 * @nr_to_scan : number of entries to try to free
912 * @nid: which node to scan for freeable entities 922 * @nid: which node to scan for freeable entities
913 * 923 *
914 * Attempt to shrink the superblock dcache LRU by @nr_to_scan entries. This is 924 * Attempt to shrink the superblock dcache LRU by @nr_to_scan entries. This is
915 * done when we need more memory an called from the superblock shrinker 925 * done when we need more memory an called from the superblock shrinker
916 * function. 926 * function.
917 * 927 *
918 * This function may fail to free any resources if all the dentries are in 928 * This function may fail to free any resources if all the dentries are in
919 * use. 929 * use.
920 */ 930 */
921 long prune_dcache_sb(struct super_block *sb, unsigned long nr_to_scan, 931 long prune_dcache_sb(struct super_block *sb, unsigned long nr_to_scan,
922 int nid) 932 int nid)
923 { 933 {
924 LIST_HEAD(dispose); 934 LIST_HEAD(dispose);
925 long freed; 935 long freed;
926 936
927 freed = list_lru_walk_node(&sb->s_dentry_lru, nid, dentry_lru_isolate, 937 freed = list_lru_walk_node(&sb->s_dentry_lru, nid, dentry_lru_isolate,
928 &dispose, &nr_to_scan); 938 &dispose, &nr_to_scan);
929 shrink_dentry_list(&dispose); 939 shrink_dentry_list(&dispose);
930 return freed; 940 return freed;
931 } 941 }
932 942
933 static enum lru_status dentry_lru_isolate_shrink(struct list_head *item, 943 static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
934 spinlock_t *lru_lock, void *arg) 944 spinlock_t *lru_lock, void *arg)
935 { 945 {
936 struct list_head *freeable = arg; 946 struct list_head *freeable = arg;
937 struct dentry *dentry = container_of(item, struct dentry, d_lru); 947 struct dentry *dentry = container_of(item, struct dentry, d_lru);
938 948
939 /* 949 /*
940 * we are inverting the lru lock/dentry->d_lock here, 950 * we are inverting the lru lock/dentry->d_lock here,
941 * so use a trylock. If we fail to get the lock, just skip 951 * so use a trylock. If we fail to get the lock, just skip
942 * it 952 * it
943 */ 953 */
944 if (!spin_trylock(&dentry->d_lock)) 954 if (!spin_trylock(&dentry->d_lock))
945 return LRU_SKIP; 955 return LRU_SKIP;
946 956
947 d_lru_shrink_move(dentry, freeable); 957 d_lru_shrink_move(dentry, freeable);
948 spin_unlock(&dentry->d_lock); 958 spin_unlock(&dentry->d_lock);
949 959
950 return LRU_REMOVED; 960 return LRU_REMOVED;
951 } 961 }
952 962
953 963
954 /** 964 /**
955 * shrink_dcache_sb - shrink dcache for a superblock 965 * shrink_dcache_sb - shrink dcache for a superblock
956 * @sb: superblock 966 * @sb: superblock
957 * 967 *
958 * Shrink the dcache for the specified super block. This is used to free 968 * Shrink the dcache for the specified super block. This is used to free
959 * the dcache before unmounting a file system. 969 * the dcache before unmounting a file system.
960 */ 970 */
961 void shrink_dcache_sb(struct super_block *sb) 971 void shrink_dcache_sb(struct super_block *sb)
962 { 972 {
963 long freed; 973 long freed;
964 974
965 do { 975 do {
966 LIST_HEAD(dispose); 976 LIST_HEAD(dispose);
967 977
968 freed = list_lru_walk(&sb->s_dentry_lru, 978 freed = list_lru_walk(&sb->s_dentry_lru,
969 dentry_lru_isolate_shrink, &dispose, UINT_MAX); 979 dentry_lru_isolate_shrink, &dispose, UINT_MAX);
970 980
971 this_cpu_sub(nr_dentry_unused, freed); 981 this_cpu_sub(nr_dentry_unused, freed);
972 shrink_dentry_list(&dispose); 982 shrink_dentry_list(&dispose);
973 } while (freed > 0); 983 } while (freed > 0);
974 } 984 }
975 EXPORT_SYMBOL(shrink_dcache_sb); 985 EXPORT_SYMBOL(shrink_dcache_sb);
976 986
977 /** 987 /**
978 * enum d_walk_ret - action to talke during tree walk 988 * enum d_walk_ret - action to talke during tree walk
979 * @D_WALK_CONTINUE: contrinue walk 989 * @D_WALK_CONTINUE: contrinue walk
980 * @D_WALK_QUIT: quit walk 990 * @D_WALK_QUIT: quit walk
981 * @D_WALK_NORETRY: quit when retry is needed 991 * @D_WALK_NORETRY: quit when retry is needed
982 * @D_WALK_SKIP: skip this dentry and its children 992 * @D_WALK_SKIP: skip this dentry and its children
983 */ 993 */
984 enum d_walk_ret { 994 enum d_walk_ret {
985 D_WALK_CONTINUE, 995 D_WALK_CONTINUE,
986 D_WALK_QUIT, 996 D_WALK_QUIT,
987 D_WALK_NORETRY, 997 D_WALK_NORETRY,
988 D_WALK_SKIP, 998 D_WALK_SKIP,
989 }; 999 };
990 1000
991 /** 1001 /**
992 * d_walk - walk the dentry tree 1002 * d_walk - walk the dentry tree
993 * @parent: start of walk 1003 * @parent: start of walk
994 * @data: data passed to @enter() and @finish() 1004 * @data: data passed to @enter() and @finish()
995 * @enter: callback when first entering the dentry 1005 * @enter: callback when first entering the dentry
996 * @finish: callback when successfully finished the walk 1006 * @finish: callback when successfully finished the walk
997 * 1007 *
998 * The @enter() and @finish() callbacks are called with d_lock held. 1008 * The @enter() and @finish() callbacks are called with d_lock held.
999 */ 1009 */
1000 static void d_walk(struct dentry *parent, void *data, 1010 static void d_walk(struct dentry *parent, void *data,
1001 enum d_walk_ret (*enter)(void *, struct dentry *), 1011 enum d_walk_ret (*enter)(void *, struct dentry *),
1002 void (*finish)(void *)) 1012 void (*finish)(void *))
1003 { 1013 {
1004 struct dentry *this_parent; 1014 struct dentry *this_parent;
1005 struct list_head *next; 1015 struct list_head *next;
1006 unsigned seq = 0; 1016 unsigned seq = 0;
1007 enum d_walk_ret ret; 1017 enum d_walk_ret ret;
1008 bool retry = true; 1018 bool retry = true;
1009 1019
1010 again: 1020 again:
1011 read_seqbegin_or_lock(&rename_lock, &seq); 1021 read_seqbegin_or_lock(&rename_lock, &seq);
1012 this_parent = parent; 1022 this_parent = parent;
1013 spin_lock(&this_parent->d_lock); 1023 spin_lock(&this_parent->d_lock);
1014 1024
1015 ret = enter(data, this_parent); 1025 ret = enter(data, this_parent);
1016 switch (ret) { 1026 switch (ret) {
1017 case D_WALK_CONTINUE: 1027 case D_WALK_CONTINUE:
1018 break; 1028 break;
1019 case D_WALK_QUIT: 1029 case D_WALK_QUIT:
1020 case D_WALK_SKIP: 1030 case D_WALK_SKIP:
1021 goto out_unlock; 1031 goto out_unlock;
1022 case D_WALK_NORETRY: 1032 case D_WALK_NORETRY:
1023 retry = false; 1033 retry = false;
1024 break; 1034 break;
1025 } 1035 }
1026 repeat: 1036 repeat:
1027 next = this_parent->d_subdirs.next; 1037 next = this_parent->d_subdirs.next;
1028 resume: 1038 resume:
1029 while (next != &this_parent->d_subdirs) { 1039 while (next != &this_parent->d_subdirs) {
1030 struct list_head *tmp = next; 1040 struct list_head *tmp = next;
1031 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); 1041 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
1032 next = tmp->next; 1042 next = tmp->next;
1033 1043
1034 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); 1044 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1035 1045
1036 ret = enter(data, dentry); 1046 ret = enter(data, dentry);
1037 switch (ret) { 1047 switch (ret) {
1038 case D_WALK_CONTINUE: 1048 case D_WALK_CONTINUE:
1039 break; 1049 break;
1040 case D_WALK_QUIT: 1050 case D_WALK_QUIT:
1041 spin_unlock(&dentry->d_lock); 1051 spin_unlock(&dentry->d_lock);
1042 goto out_unlock; 1052 goto out_unlock;
1043 case D_WALK_NORETRY: 1053 case D_WALK_NORETRY:
1044 retry = false; 1054 retry = false;
1045 break; 1055 break;
1046 case D_WALK_SKIP: 1056 case D_WALK_SKIP:
1047 spin_unlock(&dentry->d_lock); 1057 spin_unlock(&dentry->d_lock);
1048 continue; 1058 continue;
1049 } 1059 }
1050 1060
1051 if (!list_empty(&dentry->d_subdirs)) { 1061 if (!list_empty(&dentry->d_subdirs)) {
1052 spin_unlock(&this_parent->d_lock); 1062 spin_unlock(&this_parent->d_lock);
1053 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_); 1063 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
1054 this_parent = dentry; 1064 this_parent = dentry;
1055 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_); 1065 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
1056 goto repeat; 1066 goto repeat;
1057 } 1067 }
1058 spin_unlock(&dentry->d_lock); 1068 spin_unlock(&dentry->d_lock);
1059 } 1069 }
1060 /* 1070 /*
1061 * All done at this level ... ascend and resume the search. 1071 * All done at this level ... ascend and resume the search.
1062 */ 1072 */
1063 if (this_parent != parent) { 1073 if (this_parent != parent) {
1064 struct dentry *child = this_parent; 1074 struct dentry *child = this_parent;
1065 this_parent = child->d_parent; 1075 this_parent = child->d_parent;
1066 1076
1067 rcu_read_lock(); 1077 rcu_read_lock();
1068 spin_unlock(&child->d_lock); 1078 spin_unlock(&child->d_lock);
1069 spin_lock(&this_parent->d_lock); 1079 spin_lock(&this_parent->d_lock);
1070 1080
1071 /* 1081 /*
1072 * might go back up the wrong parent if we have had a rename 1082 * might go back up the wrong parent if we have had a rename
1073 * or deletion 1083 * or deletion
1074 */ 1084 */
1075 if (this_parent != child->d_parent || 1085 if (this_parent != child->d_parent ||
1076 (child->d_flags & DCACHE_DENTRY_KILLED) || 1086 (child->d_flags & DCACHE_DENTRY_KILLED) ||
1077 need_seqretry(&rename_lock, seq)) { 1087 need_seqretry(&rename_lock, seq)) {
1078 spin_unlock(&this_parent->d_lock); 1088 spin_unlock(&this_parent->d_lock);
1079 rcu_read_unlock(); 1089 rcu_read_unlock();
1080 goto rename_retry; 1090 goto rename_retry;
1081 } 1091 }
1082 rcu_read_unlock(); 1092 rcu_read_unlock();
1083 next = child->d_u.d_child.next; 1093 next = child->d_u.d_child.next;
1084 goto resume; 1094 goto resume;
1085 } 1095 }
1086 if (need_seqretry(&rename_lock, seq)) { 1096 if (need_seqretry(&rename_lock, seq)) {
1087 spin_unlock(&this_parent->d_lock); 1097 spin_unlock(&this_parent->d_lock);
1088 goto rename_retry; 1098 goto rename_retry;
1089 } 1099 }
1090 if (finish) 1100 if (finish)
1091 finish(data); 1101 finish(data);
1092 1102
1093 out_unlock: 1103 out_unlock:
1094 spin_unlock(&this_parent->d_lock); 1104 spin_unlock(&this_parent->d_lock);
1095 done_seqretry(&rename_lock, seq); 1105 done_seqretry(&rename_lock, seq);
1096 return; 1106 return;
1097 1107
1098 rename_retry: 1108 rename_retry:
1099 if (!retry) 1109 if (!retry)
1100 return; 1110 return;
1101 seq = 1; 1111 seq = 1;
1102 goto again; 1112 goto again;
1103 } 1113 }
1104 1114
1105 /* 1115 /*
1106 * Search for at least 1 mount point in the dentry's subdirs. 1116 * Search for at least 1 mount point in the dentry's subdirs.
1107 * We descend to the next level whenever the d_subdirs 1117 * We descend to the next level whenever the d_subdirs
1108 * list is non-empty and continue searching. 1118 * list is non-empty and continue searching.
1109 */ 1119 */
1110 1120
1111 static enum d_walk_ret check_mount(void *data, struct dentry *dentry) 1121 static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
1112 { 1122 {
1113 int *ret = data; 1123 int *ret = data;
1114 if (d_mountpoint(dentry)) { 1124 if (d_mountpoint(dentry)) {
1115 *ret = 1; 1125 *ret = 1;
1116 return D_WALK_QUIT; 1126 return D_WALK_QUIT;
1117 } 1127 }
1118 return D_WALK_CONTINUE; 1128 return D_WALK_CONTINUE;
1119 } 1129 }
1120 1130
1121 /** 1131 /**
1122 * have_submounts - check for mounts over a dentry 1132 * have_submounts - check for mounts over a dentry
1123 * @parent: dentry to check. 1133 * @parent: dentry to check.
1124 * 1134 *
1125 * Return true if the parent or its subdirectories contain 1135 * Return true if the parent or its subdirectories contain
1126 * a mount point 1136 * a mount point
1127 */ 1137 */
1128 int have_submounts(struct dentry *parent) 1138 int have_submounts(struct dentry *parent)
1129 { 1139 {
1130 int ret = 0; 1140 int ret = 0;
1131 1141
1132 d_walk(parent, &ret, check_mount, NULL); 1142 d_walk(parent, &ret, check_mount, NULL);
1133 1143
1134 return ret; 1144 return ret;
1135 } 1145 }
1136 EXPORT_SYMBOL(have_submounts); 1146 EXPORT_SYMBOL(have_submounts);
1137 1147
1138 /* 1148 /*
1139 * Called by mount code to set a mountpoint and check if the mountpoint is 1149 * Called by mount code to set a mountpoint and check if the mountpoint is
1140 * reachable (e.g. NFS can unhash a directory dentry and then the complete 1150 * reachable (e.g. NFS can unhash a directory dentry and then the complete
1141 * subtree can become unreachable). 1151 * subtree can become unreachable).
1142 * 1152 *
1143 * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For 1153 * Only one of check_submounts_and_drop() and d_set_mounted() must succeed. For
1144 * this reason take rename_lock and d_lock on dentry and ancestors. 1154 * this reason take rename_lock and d_lock on dentry and ancestors.
1145 */ 1155 */
1146 int d_set_mounted(struct dentry *dentry) 1156 int d_set_mounted(struct dentry *dentry)
1147 { 1157 {
1148 struct dentry *p; 1158 struct dentry *p;
1149 int ret = -ENOENT; 1159 int ret = -ENOENT;
1150 write_seqlock(&rename_lock); 1160 write_seqlock(&rename_lock);
1151 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) { 1161 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
1152 /* Need exclusion wrt. check_submounts_and_drop() */ 1162 /* Need exclusion wrt. check_submounts_and_drop() */
1153 spin_lock(&p->d_lock); 1163 spin_lock(&p->d_lock);
1154 if (unlikely(d_unhashed(p))) { 1164 if (unlikely(d_unhashed(p))) {
1155 spin_unlock(&p->d_lock); 1165 spin_unlock(&p->d_lock);
1156 goto out; 1166 goto out;
1157 } 1167 }
1158 spin_unlock(&p->d_lock); 1168 spin_unlock(&p->d_lock);
1159 } 1169 }
1160 spin_lock(&dentry->d_lock); 1170 spin_lock(&dentry->d_lock);
1161 if (!d_unlinked(dentry)) { 1171 if (!d_unlinked(dentry)) {
1162 dentry->d_flags |= DCACHE_MOUNTED; 1172 dentry->d_flags |= DCACHE_MOUNTED;
1163 ret = 0; 1173 ret = 0;
1164 } 1174 }
1165 spin_unlock(&dentry->d_lock); 1175 spin_unlock(&dentry->d_lock);
1166 out: 1176 out:
1167 write_sequnlock(&rename_lock); 1177 write_sequnlock(&rename_lock);
1168 return ret; 1178 return ret;
1169 } 1179 }
1170 1180
1171 /* 1181 /*
1172 * Search the dentry child list of the specified parent, 1182 * Search the dentry child list of the specified parent,
1173 * and move any unused dentries to the end of the unused 1183 * and move any unused dentries to the end of the unused
1174 * list for prune_dcache(). We descend to the next level 1184 * list for prune_dcache(). We descend to the next level
1175 * whenever the d_subdirs list is non-empty and continue 1185 * whenever the d_subdirs list is non-empty and continue
1176 * searching. 1186 * searching.
1177 * 1187 *
1178 * It returns zero iff there are no unused children, 1188 * It returns zero iff there are no unused children,
1179 * otherwise it returns the number of children moved to 1189 * otherwise it returns the number of children moved to
1180 * the end of the unused list. This may not be the total 1190 * the end of the unused list. This may not be the total
1181 * number of unused children, because select_parent can 1191 * number of unused children, because select_parent can
1182 * drop the lock and return early due to latency 1192 * drop the lock and return early due to latency
1183 * constraints. 1193 * constraints.
1184 */ 1194 */
1185 1195
1186 struct select_data { 1196 struct select_data {
1187 struct dentry *start; 1197 struct dentry *start;
1188 struct list_head dispose; 1198 struct list_head dispose;
1189 int found; 1199 int found;
1190 }; 1200 };
1191 1201
1192 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) 1202 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
1193 { 1203 {
1194 struct select_data *data = _data; 1204 struct select_data *data = _data;
1195 enum d_walk_ret ret = D_WALK_CONTINUE; 1205 enum d_walk_ret ret = D_WALK_CONTINUE;
1196 1206
1197 if (data->start == dentry) 1207 if (data->start == dentry)
1198 goto out; 1208 goto out;
1199 1209
1200 if (dentry->d_flags & DCACHE_SHRINK_LIST) { 1210 if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1201 data->found++; 1211 data->found++;
1202 } else { 1212 } else {
1203 if (dentry->d_flags & DCACHE_LRU_LIST) 1213 if (dentry->d_flags & DCACHE_LRU_LIST)
1204 d_lru_del(dentry); 1214 d_lru_del(dentry);
1205 if (!dentry->d_lockref.count) { 1215 if (!dentry->d_lockref.count) {
1206 d_shrink_add(dentry, &data->dispose); 1216 d_shrink_add(dentry, &data->dispose);
1207 data->found++; 1217 data->found++;
1208 } 1218 }
1209 } 1219 }
1210 /* 1220 /*
1211 * We can return to the caller if we have found some (this 1221 * We can return to the caller if we have found some (this
1212 * ensures forward progress). We'll be coming back to find 1222 * ensures forward progress). We'll be coming back to find
1213 * the rest. 1223 * the rest.
1214 */ 1224 */
1215 if (!list_empty(&data->dispose)) 1225 if (!list_empty(&data->dispose))
1216 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY; 1226 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1217 out: 1227 out:
1218 return ret; 1228 return ret;
1219 } 1229 }
1220 1230
1221 /** 1231 /**
1222 * shrink_dcache_parent - prune dcache 1232 * shrink_dcache_parent - prune dcache
1223 * @parent: parent of entries to prune 1233 * @parent: parent of entries to prune
1224 * 1234 *
1225 * Prune the dcache to remove unused children of the parent dentry. 1235 * Prune the dcache to remove unused children of the parent dentry.
1226 */ 1236 */
1227 void shrink_dcache_parent(struct dentry *parent) 1237 void shrink_dcache_parent(struct dentry *parent)
1228 { 1238 {
1229 for (;;) { 1239 for (;;) {
1230 struct select_data data; 1240 struct select_data data;
1231 1241
1232 INIT_LIST_HEAD(&data.dispose); 1242 INIT_LIST_HEAD(&data.dispose);
1233 data.start = parent; 1243 data.start = parent;
1234 data.found = 0; 1244 data.found = 0;
1235 1245
1236 d_walk(parent, &data, select_collect, NULL); 1246 d_walk(parent, &data, select_collect, NULL);
1237 if (!data.found) 1247 if (!data.found)
1238 break; 1248 break;
1239 1249
1240 shrink_dentry_list(&data.dispose); 1250 shrink_dentry_list(&data.dispose);
1241 cond_resched(); 1251 cond_resched();
1242 } 1252 }
1243 } 1253 }
1244 EXPORT_SYMBOL(shrink_dcache_parent); 1254 EXPORT_SYMBOL(shrink_dcache_parent);
1245 1255
1246 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry) 1256 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
1247 { 1257 {
1248 /* it has busy descendents; complain about those instead */ 1258 /* it has busy descendents; complain about those instead */
1249 if (!list_empty(&dentry->d_subdirs)) 1259 if (!list_empty(&dentry->d_subdirs))
1250 return D_WALK_CONTINUE; 1260 return D_WALK_CONTINUE;
1251 1261
1252 /* root with refcount 1 is fine */ 1262 /* root with refcount 1 is fine */
1253 if (dentry == _data && dentry->d_lockref.count == 1) 1263 if (dentry == _data && dentry->d_lockref.count == 1)
1254 return D_WALK_CONTINUE; 1264 return D_WALK_CONTINUE;
1255 1265
1256 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} " 1266 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} "
1257 " still in use (%d) [unmount of %s %s]\n", 1267 " still in use (%d) [unmount of %s %s]\n",
1258 dentry, 1268 dentry,
1259 dentry->d_inode ? 1269 dentry->d_inode ?
1260 dentry->d_inode->i_ino : 0UL, 1270 dentry->d_inode->i_ino : 0UL,
1261 dentry, 1271 dentry,
1262 dentry->d_lockref.count, 1272 dentry->d_lockref.count,
1263 dentry->d_sb->s_type->name, 1273 dentry->d_sb->s_type->name,
1264 dentry->d_sb->s_id); 1274 dentry->d_sb->s_id);
1265 WARN_ON(1); 1275 WARN_ON(1);
1266 return D_WALK_CONTINUE; 1276 return D_WALK_CONTINUE;
1267 } 1277 }
1268 1278
1269 static void do_one_tree(struct dentry *dentry) 1279 static void do_one_tree(struct dentry *dentry)
1270 { 1280 {
1271 shrink_dcache_parent(dentry); 1281 shrink_dcache_parent(dentry);
1272 d_walk(dentry, dentry, umount_check, NULL); 1282 d_walk(dentry, dentry, umount_check, NULL);
1273 d_drop(dentry); 1283 d_drop(dentry);
1274 dput(dentry); 1284 dput(dentry);
1275 } 1285 }
1276 1286
1277 /* 1287 /*
1278 * destroy the dentries attached to a superblock on unmounting 1288 * destroy the dentries attached to a superblock on unmounting
1279 */ 1289 */
1280 void shrink_dcache_for_umount(struct super_block *sb) 1290 void shrink_dcache_for_umount(struct super_block *sb)
1281 { 1291 {
1282 struct dentry *dentry; 1292 struct dentry *dentry;
1283 1293
1284 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked"); 1294 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
1285 1295
1286 dentry = sb->s_root; 1296 dentry = sb->s_root;
1287 sb->s_root = NULL; 1297 sb->s_root = NULL;
1288 do_one_tree(dentry); 1298 do_one_tree(dentry);
1289 1299
1290 while (!hlist_bl_empty(&sb->s_anon)) { 1300 while (!hlist_bl_empty(&sb->s_anon)) {
1291 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash)); 1301 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash));
1292 do_one_tree(dentry); 1302 do_one_tree(dentry);
1293 } 1303 }
1294 } 1304 }
1295 1305
1296 static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry) 1306 static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
1297 { 1307 {
1298 struct select_data *data = _data; 1308 struct select_data *data = _data;
1299 1309
1300 if (d_mountpoint(dentry)) { 1310 if (d_mountpoint(dentry)) {
1301 data->found = -EBUSY; 1311 data->found = -EBUSY;
1302 return D_WALK_QUIT; 1312 return D_WALK_QUIT;
1303 } 1313 }
1304 1314
1305 return select_collect(_data, dentry); 1315 return select_collect(_data, dentry);
1306 } 1316 }
1307 1317
1308 static void check_and_drop(void *_data) 1318 static void check_and_drop(void *_data)
1309 { 1319 {
1310 struct select_data *data = _data; 1320 struct select_data *data = _data;
1311 1321
1312 if (d_mountpoint(data->start)) 1322 if (d_mountpoint(data->start))
1313 data->found = -EBUSY; 1323 data->found = -EBUSY;
1314 if (!data->found) 1324 if (!data->found)
1315 __d_drop(data->start); 1325 __d_drop(data->start);
1316 } 1326 }
1317 1327
1318 /** 1328 /**
1319 * check_submounts_and_drop - prune dcache, check for submounts and drop 1329 * check_submounts_and_drop - prune dcache, check for submounts and drop
1320 * 1330 *
1321 * All done as a single atomic operation relative to has_unlinked_ancestor(). 1331 * All done as a single atomic operation relative to has_unlinked_ancestor().
1322 * Returns 0 if successfully unhashed @parent. If there were submounts then 1332 * Returns 0 if successfully unhashed @parent. If there were submounts then
1323 * return -EBUSY. 1333 * return -EBUSY.
1324 * 1334 *
1325 * @dentry: dentry to prune and drop 1335 * @dentry: dentry to prune and drop
1326 */ 1336 */
1327 int check_submounts_and_drop(struct dentry *dentry) 1337 int check_submounts_and_drop(struct dentry *dentry)
1328 { 1338 {
1329 int ret = 0; 1339 int ret = 0;
1330 1340
1331 /* Negative dentries can be dropped without further checks */ 1341 /* Negative dentries can be dropped without further checks */
1332 if (!dentry->d_inode) { 1342 if (!dentry->d_inode) {
1333 d_drop(dentry); 1343 d_drop(dentry);
1334 goto out; 1344 goto out;
1335 } 1345 }
1336 1346
1337 for (;;) { 1347 for (;;) {
1338 struct select_data data; 1348 struct select_data data;
1339 1349
1340 INIT_LIST_HEAD(&data.dispose); 1350 INIT_LIST_HEAD(&data.dispose);
1341 data.start = dentry; 1351 data.start = dentry;
1342 data.found = 0; 1352 data.found = 0;
1343 1353
1344 d_walk(dentry, &data, check_and_collect, check_and_drop); 1354 d_walk(dentry, &data, check_and_collect, check_and_drop);
1345 ret = data.found; 1355 ret = data.found;
1346 1356
1347 if (!list_empty(&data.dispose)) 1357 if (!list_empty(&data.dispose))
1348 shrink_dentry_list(&data.dispose); 1358 shrink_dentry_list(&data.dispose);
1349 1359
1350 if (ret <= 0) 1360 if (ret <= 0)
1351 break; 1361 break;
1352 1362
1353 cond_resched(); 1363 cond_resched();
1354 } 1364 }
1355 1365
1356 out: 1366 out:
1357 return ret; 1367 return ret;
1358 } 1368 }
1359 EXPORT_SYMBOL(check_submounts_and_drop); 1369 EXPORT_SYMBOL(check_submounts_and_drop);
1360 1370
1361 /** 1371 /**
1362 * __d_alloc - allocate a dcache entry 1372 * __d_alloc - allocate a dcache entry
1363 * @sb: filesystem it will belong to 1373 * @sb: filesystem it will belong to
1364 * @name: qstr of the name 1374 * @name: qstr of the name
1365 * 1375 *
1366 * Allocates a dentry. It returns %NULL if there is insufficient memory 1376 * Allocates a dentry. It returns %NULL if there is insufficient memory
1367 * available. On a success the dentry is returned. The name passed in is 1377 * available. On a success the dentry is returned. The name passed in is
1368 * copied and the copy passed in may be reused after this call. 1378 * copied and the copy passed in may be reused after this call.
1369 */ 1379 */
1370 1380
1371 struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) 1381 struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1372 { 1382 {
1373 struct dentry *dentry; 1383 struct dentry *dentry;
1374 char *dname; 1384 char *dname;
1375 1385
1376 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL); 1386 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL);
1377 if (!dentry) 1387 if (!dentry)
1378 return NULL; 1388 return NULL;
1379 1389
1380 /* 1390 /*
1381 * We guarantee that the inline name is always NUL-terminated. 1391 * We guarantee that the inline name is always NUL-terminated.
1382 * This way the memcpy() done by the name switching in rename 1392 * This way the memcpy() done by the name switching in rename
1383 * will still always have a NUL at the end, even if we might 1393 * will still always have a NUL at the end, even if we might
1384 * be overwriting an internal NUL character 1394 * be overwriting an internal NUL character
1385 */ 1395 */
1386 dentry->d_iname[DNAME_INLINE_LEN-1] = 0; 1396 dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
1387 if (name->len > DNAME_INLINE_LEN-1) { 1397 if (name->len > DNAME_INLINE_LEN-1) {
1388 dname = kmalloc(name->len + 1, GFP_KERNEL); 1398 dname = kmalloc(name->len + 1, GFP_KERNEL);
1389 if (!dname) { 1399 if (!dname) {
1390 kmem_cache_free(dentry_cache, dentry); 1400 kmem_cache_free(dentry_cache, dentry);
1391 return NULL; 1401 return NULL;
1392 } 1402 }
1393 } else { 1403 } else {
1394 dname = dentry->d_iname; 1404 dname = dentry->d_iname;
1395 } 1405 }
1396 1406
1397 dentry->d_name.len = name->len; 1407 dentry->d_name.len = name->len;
1398 dentry->d_name.hash = name->hash; 1408 dentry->d_name.hash = name->hash;
1399 memcpy(dname, name->name, name->len); 1409 memcpy(dname, name->name, name->len);
1400 dname[name->len] = 0; 1410 dname[name->len] = 0;
1401 1411
1402 /* Make sure we always see the terminating NUL character */ 1412 /* Make sure we always see the terminating NUL character */
1403 smp_wmb(); 1413 smp_wmb();
1404 dentry->d_name.name = dname; 1414 dentry->d_name.name = dname;
1405 1415
1406 dentry->d_lockref.count = 1; 1416 dentry->d_lockref.count = 1;
1407 dentry->d_flags = 0; 1417 dentry->d_flags = 0;
1408 spin_lock_init(&dentry->d_lock); 1418 spin_lock_init(&dentry->d_lock);
1409 seqcount_init(&dentry->d_seq); 1419 seqcount_init(&dentry->d_seq);
1410 dentry->d_inode = NULL; 1420 dentry->d_inode = NULL;
1411 dentry->d_parent = dentry; 1421 dentry->d_parent = dentry;
1412 dentry->d_sb = sb; 1422 dentry->d_sb = sb;
1413 dentry->d_op = NULL; 1423 dentry->d_op = NULL;
1414 dentry->d_fsdata = NULL; 1424 dentry->d_fsdata = NULL;
1415 INIT_HLIST_BL_NODE(&dentry->d_hash); 1425 INIT_HLIST_BL_NODE(&dentry->d_hash);
1416 INIT_LIST_HEAD(&dentry->d_lru); 1426 INIT_LIST_HEAD(&dentry->d_lru);
1417 INIT_LIST_HEAD(&dentry->d_subdirs); 1427 INIT_LIST_HEAD(&dentry->d_subdirs);
1418 INIT_HLIST_NODE(&dentry->d_alias); 1428 INIT_HLIST_NODE(&dentry->d_alias);
1419 INIT_LIST_HEAD(&dentry->d_u.d_child); 1429 INIT_LIST_HEAD(&dentry->d_u.d_child);
1420 d_set_d_op(dentry, dentry->d_sb->s_d_op); 1430 d_set_d_op(dentry, dentry->d_sb->s_d_op);
1421 1431
1422 this_cpu_inc(nr_dentry); 1432 this_cpu_inc(nr_dentry);
1423 1433
1424 return dentry; 1434 return dentry;
1425 } 1435 }
1426 1436
1427 /** 1437 /**
1428 * d_alloc - allocate a dcache entry 1438 * d_alloc - allocate a dcache entry
1429 * @parent: parent of entry to allocate 1439 * @parent: parent of entry to allocate
1430 * @name: qstr of the name 1440 * @name: qstr of the name
1431 * 1441 *
1432 * Allocates a dentry. It returns %NULL if there is insufficient memory 1442 * Allocates a dentry. It returns %NULL if there is insufficient memory
1433 * available. On a success the dentry is returned. The name passed in is 1443 * available. On a success the dentry is returned. The name passed in is
1434 * copied and the copy passed in may be reused after this call. 1444 * copied and the copy passed in may be reused after this call.
1435 */ 1445 */
1436 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) 1446 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1437 { 1447 {
1438 struct dentry *dentry = __d_alloc(parent->d_sb, name); 1448 struct dentry *dentry = __d_alloc(parent->d_sb, name);
1439 if (!dentry) 1449 if (!dentry)
1440 return NULL; 1450 return NULL;
1441 1451
1442 spin_lock(&parent->d_lock); 1452 spin_lock(&parent->d_lock);
1443 /* 1453 /*
1444 * don't need child lock because it is not subject 1454 * don't need child lock because it is not subject
1445 * to concurrency here 1455 * to concurrency here
1446 */ 1456 */
1447 __dget_dlock(parent); 1457 __dget_dlock(parent);
1448 dentry->d_parent = parent; 1458 dentry->d_parent = parent;
1449 list_add(&dentry->d_u.d_child, &parent->d_subdirs); 1459 list_add(&dentry->d_u.d_child, &parent->d_subdirs);
1450 spin_unlock(&parent->d_lock); 1460 spin_unlock(&parent->d_lock);
1451 1461
1452 return dentry; 1462 return dentry;
1453 } 1463 }
1454 EXPORT_SYMBOL(d_alloc); 1464 EXPORT_SYMBOL(d_alloc);
1455 1465
1456 /** 1466 /**
1457 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems) 1467 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
1458 * @sb: the superblock 1468 * @sb: the superblock
1459 * @name: qstr of the name 1469 * @name: qstr of the name
1460 * 1470 *
1461 * For a filesystem that just pins its dentries in memory and never 1471 * For a filesystem that just pins its dentries in memory and never
1462 * performs lookups at all, return an unhashed IS_ROOT dentry. 1472 * performs lookups at all, return an unhashed IS_ROOT dentry.
1463 */ 1473 */
1464 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name) 1474 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1465 { 1475 {
1466 return __d_alloc(sb, name); 1476 return __d_alloc(sb, name);
1467 } 1477 }
1468 EXPORT_SYMBOL(d_alloc_pseudo); 1478 EXPORT_SYMBOL(d_alloc_pseudo);
1469 1479
1470 struct dentry *d_alloc_name(struct dentry *parent, const char *name) 1480 struct dentry *d_alloc_name(struct dentry *parent, const char *name)
1471 { 1481 {
1472 struct qstr q; 1482 struct qstr q;
1473 1483
1474 q.name = name; 1484 q.name = name;
1475 q.len = strlen(name); 1485 q.len = strlen(name);
1476 q.hash = full_name_hash(q.name, q.len); 1486 q.hash = full_name_hash(q.name, q.len);
1477 return d_alloc(parent, &q); 1487 return d_alloc(parent, &q);
1478 } 1488 }
1479 EXPORT_SYMBOL(d_alloc_name); 1489 EXPORT_SYMBOL(d_alloc_name);
1480 1490
1481 void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) 1491 void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1482 { 1492 {
1483 WARN_ON_ONCE(dentry->d_op); 1493 WARN_ON_ONCE(dentry->d_op);
1484 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH | 1494 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1485 DCACHE_OP_COMPARE | 1495 DCACHE_OP_COMPARE |
1486 DCACHE_OP_REVALIDATE | 1496 DCACHE_OP_REVALIDATE |
1487 DCACHE_OP_WEAK_REVALIDATE | 1497 DCACHE_OP_WEAK_REVALIDATE |
1488 DCACHE_OP_DELETE )); 1498 DCACHE_OP_DELETE ));
1489 dentry->d_op = op; 1499 dentry->d_op = op;
1490 if (!op) 1500 if (!op)
1491 return; 1501 return;
1492 if (op->d_hash) 1502 if (op->d_hash)
1493 dentry->d_flags |= DCACHE_OP_HASH; 1503 dentry->d_flags |= DCACHE_OP_HASH;
1494 if (op->d_compare) 1504 if (op->d_compare)
1495 dentry->d_flags |= DCACHE_OP_COMPARE; 1505 dentry->d_flags |= DCACHE_OP_COMPARE;
1496 if (op->d_revalidate) 1506 if (op->d_revalidate)
1497 dentry->d_flags |= DCACHE_OP_REVALIDATE; 1507 dentry->d_flags |= DCACHE_OP_REVALIDATE;
1498 if (op->d_weak_revalidate) 1508 if (op->d_weak_revalidate)
1499 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE; 1509 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1500 if (op->d_delete) 1510 if (op->d_delete)
1501 dentry->d_flags |= DCACHE_OP_DELETE; 1511 dentry->d_flags |= DCACHE_OP_DELETE;
1502 if (op->d_prune) 1512 if (op->d_prune)
1503 dentry->d_flags |= DCACHE_OP_PRUNE; 1513 dentry->d_flags |= DCACHE_OP_PRUNE;
1504 1514
1505 } 1515 }
1506 EXPORT_SYMBOL(d_set_d_op); 1516 EXPORT_SYMBOL(d_set_d_op);
1507 1517
1508 static unsigned d_flags_for_inode(struct inode *inode) 1518 static unsigned d_flags_for_inode(struct inode *inode)
1509 { 1519 {
1510 unsigned add_flags = DCACHE_FILE_TYPE; 1520 unsigned add_flags = DCACHE_FILE_TYPE;
1511 1521
1512 if (!inode) 1522 if (!inode)
1513 return DCACHE_MISS_TYPE; 1523 return DCACHE_MISS_TYPE;
1514 1524
1515 if (S_ISDIR(inode->i_mode)) { 1525 if (S_ISDIR(inode->i_mode)) {
1516 add_flags = DCACHE_DIRECTORY_TYPE; 1526 add_flags = DCACHE_DIRECTORY_TYPE;
1517 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) { 1527 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1518 if (unlikely(!inode->i_op->lookup)) 1528 if (unlikely(!inode->i_op->lookup))
1519 add_flags = DCACHE_AUTODIR_TYPE; 1529 add_flags = DCACHE_AUTODIR_TYPE;
1520 else 1530 else
1521 inode->i_opflags |= IOP_LOOKUP; 1531 inode->i_opflags |= IOP_LOOKUP;
1522 } 1532 }
1523 } else if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) { 1533 } else if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
1524 if (unlikely(inode->i_op->follow_link)) 1534 if (unlikely(inode->i_op->follow_link))
1525 add_flags = DCACHE_SYMLINK_TYPE; 1535 add_flags = DCACHE_SYMLINK_TYPE;
1526 else 1536 else
1527 inode->i_opflags |= IOP_NOFOLLOW; 1537 inode->i_opflags |= IOP_NOFOLLOW;
1528 } 1538 }
1529 1539
1530 if (unlikely(IS_AUTOMOUNT(inode))) 1540 if (unlikely(IS_AUTOMOUNT(inode)))
1531 add_flags |= DCACHE_NEED_AUTOMOUNT; 1541 add_flags |= DCACHE_NEED_AUTOMOUNT;
1532 return add_flags; 1542 return add_flags;
1533 } 1543 }
1534 1544
1535 static void __d_instantiate(struct dentry *dentry, struct inode *inode) 1545 static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1536 { 1546 {
1537 unsigned add_flags = d_flags_for_inode(inode); 1547 unsigned add_flags = d_flags_for_inode(inode);
1538 1548
1539 spin_lock(&dentry->d_lock); 1549 spin_lock(&dentry->d_lock);
1540 __d_set_type(dentry, add_flags); 1550 __d_set_type(dentry, add_flags);
1541 if (inode) 1551 if (inode)
1542 hlist_add_head(&dentry->d_alias, &inode->i_dentry); 1552 hlist_add_head(&dentry->d_alias, &inode->i_dentry);
1543 dentry->d_inode = inode; 1553 dentry->d_inode = inode;
1544 dentry_rcuwalk_barrier(dentry); 1554 dentry_rcuwalk_barrier(dentry);
1545 spin_unlock(&dentry->d_lock); 1555 spin_unlock(&dentry->d_lock);
1546 fsnotify_d_instantiate(dentry, inode); 1556 fsnotify_d_instantiate(dentry, inode);
1547 } 1557 }
1548 1558
1549 /** 1559 /**
1550 * d_instantiate - fill in inode information for a dentry 1560 * d_instantiate - fill in inode information for a dentry
1551 * @entry: dentry to complete 1561 * @entry: dentry to complete
1552 * @inode: inode to attach to this dentry 1562 * @inode: inode to attach to this dentry
1553 * 1563 *
1554 * Fill in inode information in the entry. 1564 * Fill in inode information in the entry.
1555 * 1565 *
1556 * This turns negative dentries into productive full members 1566 * This turns negative dentries into productive full members
1557 * of society. 1567 * of society.
1558 * 1568 *
1559 * NOTE! This assumes that the inode count has been incremented 1569 * NOTE! This assumes that the inode count has been incremented
1560 * (or otherwise set) by the caller to indicate that it is now 1570 * (or otherwise set) by the caller to indicate that it is now
1561 * in use by the dcache. 1571 * in use by the dcache.
1562 */ 1572 */
1563 1573
1564 void d_instantiate(struct dentry *entry, struct inode * inode) 1574 void d_instantiate(struct dentry *entry, struct inode * inode)
1565 { 1575 {
1566 BUG_ON(!hlist_unhashed(&entry->d_alias)); 1576 BUG_ON(!hlist_unhashed(&entry->d_alias));
1567 if (inode) 1577 if (inode)
1568 spin_lock(&inode->i_lock); 1578 spin_lock(&inode->i_lock);
1569 __d_instantiate(entry, inode); 1579 __d_instantiate(entry, inode);
1570 if (inode) 1580 if (inode)
1571 spin_unlock(&inode->i_lock); 1581 spin_unlock(&inode->i_lock);
1572 security_d_instantiate(entry, inode); 1582 security_d_instantiate(entry, inode);
1573 } 1583 }
1574 EXPORT_SYMBOL(d_instantiate); 1584 EXPORT_SYMBOL(d_instantiate);
1575 1585
1576 /** 1586 /**
1577 * d_instantiate_unique - instantiate a non-aliased dentry 1587 * d_instantiate_unique - instantiate a non-aliased dentry
1578 * @entry: dentry to instantiate 1588 * @entry: dentry to instantiate
1579 * @inode: inode to attach to this dentry 1589 * @inode: inode to attach to this dentry
1580 * 1590 *
1581 * Fill in inode information in the entry. On success, it returns NULL. 1591 * Fill in inode information in the entry. On success, it returns NULL.
1582 * If an unhashed alias of "entry" already exists, then we return the 1592 * If an unhashed alias of "entry" already exists, then we return the
1583 * aliased dentry instead and drop one reference to inode. 1593 * aliased dentry instead and drop one reference to inode.
1584 * 1594 *
1585 * Note that in order to avoid conflicts with rename() etc, the caller 1595 * Note that in order to avoid conflicts with rename() etc, the caller
1586 * had better be holding the parent directory semaphore. 1596 * had better be holding the parent directory semaphore.
1587 * 1597 *
1588 * This also assumes that the inode count has been incremented 1598 * This also assumes that the inode count has been incremented
1589 * (or otherwise set) by the caller to indicate that it is now 1599 * (or otherwise set) by the caller to indicate that it is now
1590 * in use by the dcache. 1600 * in use by the dcache.
1591 */ 1601 */
1592 static struct dentry *__d_instantiate_unique(struct dentry *entry, 1602 static struct dentry *__d_instantiate_unique(struct dentry *entry,
1593 struct inode *inode) 1603 struct inode *inode)
1594 { 1604 {
1595 struct dentry *alias; 1605 struct dentry *alias;
1596 int len = entry->d_name.len; 1606 int len = entry->d_name.len;
1597 const char *name = entry->d_name.name; 1607 const char *name = entry->d_name.name;
1598 unsigned int hash = entry->d_name.hash; 1608 unsigned int hash = entry->d_name.hash;
1599 1609
1600 if (!inode) { 1610 if (!inode) {
1601 __d_instantiate(entry, NULL); 1611 __d_instantiate(entry, NULL);
1602 return NULL; 1612 return NULL;
1603 } 1613 }
1604 1614
1605 hlist_for_each_entry(alias, &inode->i_dentry, d_alias) { 1615 hlist_for_each_entry(alias, &inode->i_dentry, d_alias) {
1606 /* 1616 /*
1607 * Don't need alias->d_lock here, because aliases with 1617 * Don't need alias->d_lock here, because aliases with
1608 * d_parent == entry->d_parent are not subject to name or 1618 * d_parent == entry->d_parent are not subject to name or
1609 * parent changes, because the parent inode i_mutex is held. 1619 * parent changes, because the parent inode i_mutex is held.
1610 */ 1620 */
1611 if (alias->d_name.hash != hash) 1621 if (alias->d_name.hash != hash)
1612 continue; 1622 continue;
1613 if (alias->d_parent != entry->d_parent) 1623 if (alias->d_parent != entry->d_parent)
1614 continue; 1624 continue;
1615 if (alias->d_name.len != len) 1625 if (alias->d_name.len != len)
1616 continue; 1626 continue;
1617 if (dentry_cmp(alias, name, len)) 1627 if (dentry_cmp(alias, name, len))
1618 continue; 1628 continue;
1619 __dget(alias); 1629 __dget(alias);
1620 return alias; 1630 return alias;
1621 } 1631 }
1622 1632
1623 __d_instantiate(entry, inode); 1633 __d_instantiate(entry, inode);
1624 return NULL; 1634 return NULL;
1625 } 1635 }
1626 1636
1627 struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode) 1637 struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
1628 { 1638 {
1629 struct dentry *result; 1639 struct dentry *result;
1630 1640
1631 BUG_ON(!hlist_unhashed(&entry->d_alias)); 1641 BUG_ON(!hlist_unhashed(&entry->d_alias));
1632 1642
1633 if (inode) 1643 if (inode)
1634 spin_lock(&inode->i_lock); 1644 spin_lock(&inode->i_lock);
1635 result = __d_instantiate_unique(entry, inode); 1645 result = __d_instantiate_unique(entry, inode);
1636 if (inode) 1646 if (inode)
1637 spin_unlock(&inode->i_lock); 1647 spin_unlock(&inode->i_lock);
1638 1648
1639 if (!result) { 1649 if (!result) {
1640 security_d_instantiate(entry, inode); 1650 security_d_instantiate(entry, inode);
1641 return NULL; 1651 return NULL;
1642 } 1652 }
1643 1653
1644 BUG_ON(!d_unhashed(result)); 1654 BUG_ON(!d_unhashed(result));
1645 iput(inode); 1655 iput(inode);
1646 return result; 1656 return result;
1647 } 1657 }
1648 1658
1649 EXPORT_SYMBOL(d_instantiate_unique); 1659 EXPORT_SYMBOL(d_instantiate_unique);
1650 1660
1651 /** 1661 /**
1652 * d_instantiate_no_diralias - instantiate a non-aliased dentry 1662 * d_instantiate_no_diralias - instantiate a non-aliased dentry
1653 * @entry: dentry to complete 1663 * @entry: dentry to complete
1654 * @inode: inode to attach to this dentry 1664 * @inode: inode to attach to this dentry
1655 * 1665 *
1656 * Fill in inode information in the entry. If a directory alias is found, then 1666 * Fill in inode information in the entry. If a directory alias is found, then
1657 * return an error (and drop inode). Together with d_materialise_unique() this 1667 * return an error (and drop inode). Together with d_materialise_unique() this
1658 * guarantees that a directory inode may never have more than one alias. 1668 * guarantees that a directory inode may never have more than one alias.
1659 */ 1669 */
1660 int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode) 1670 int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode)
1661 { 1671 {
1662 BUG_ON(!hlist_unhashed(&entry->d_alias)); 1672 BUG_ON(!hlist_unhashed(&entry->d_alias));
1663 1673
1664 spin_lock(&inode->i_lock); 1674 spin_lock(&inode->i_lock);
1665 if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) { 1675 if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) {
1666 spin_unlock(&inode->i_lock); 1676 spin_unlock(&inode->i_lock);
1667 iput(inode); 1677 iput(inode);
1668 return -EBUSY; 1678 return -EBUSY;
1669 } 1679 }
1670 __d_instantiate(entry, inode); 1680 __d_instantiate(entry, inode);
1671 spin_unlock(&inode->i_lock); 1681 spin_unlock(&inode->i_lock);
1672 security_d_instantiate(entry, inode); 1682 security_d_instantiate(entry, inode);
1673 1683
1674 return 0; 1684 return 0;
1675 } 1685 }
1676 EXPORT_SYMBOL(d_instantiate_no_diralias); 1686 EXPORT_SYMBOL(d_instantiate_no_diralias);
1677 1687
1678 struct dentry *d_make_root(struct inode *root_inode) 1688 struct dentry *d_make_root(struct inode *root_inode)
1679 { 1689 {
1680 struct dentry *res = NULL; 1690 struct dentry *res = NULL;
1681 1691
1682 if (root_inode) { 1692 if (root_inode) {
1683 static const struct qstr name = QSTR_INIT("/", 1); 1693 static const struct qstr name = QSTR_INIT("/", 1);
1684 1694
1685 res = __d_alloc(root_inode->i_sb, &name); 1695 res = __d_alloc(root_inode->i_sb, &name);
1686 if (res) 1696 if (res)
1687 d_instantiate(res, root_inode); 1697 d_instantiate(res, root_inode);
1688 else 1698 else
1689 iput(root_inode); 1699 iput(root_inode);
1690 } 1700 }
1691 return res; 1701 return res;
1692 } 1702 }
1693 EXPORT_SYMBOL(d_make_root); 1703 EXPORT_SYMBOL(d_make_root);
1694 1704
1695 static struct dentry * __d_find_any_alias(struct inode *inode) 1705 static struct dentry * __d_find_any_alias(struct inode *inode)
1696 { 1706 {
1697 struct dentry *alias; 1707 struct dentry *alias;
1698 1708
1699 if (hlist_empty(&inode->i_dentry)) 1709 if (hlist_empty(&inode->i_dentry))
1700 return NULL; 1710 return NULL;
1701 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias); 1711 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
1702 __dget(alias); 1712 __dget(alias);
1703 return alias; 1713 return alias;
1704 } 1714 }
1705 1715
1706 /** 1716 /**
1707 * d_find_any_alias - find any alias for a given inode 1717 * d_find_any_alias - find any alias for a given inode
1708 * @inode: inode to find an alias for 1718 * @inode: inode to find an alias for
1709 * 1719 *
1710 * If any aliases exist for the given inode, take and return a 1720 * If any aliases exist for the given inode, take and return a
1711 * reference for one of them. If no aliases exist, return %NULL. 1721 * reference for one of them. If no aliases exist, return %NULL.
1712 */ 1722 */
1713 struct dentry *d_find_any_alias(struct inode *inode) 1723 struct dentry *d_find_any_alias(struct inode *inode)
1714 { 1724 {
1715 struct dentry *de; 1725 struct dentry *de;
1716 1726
1717 spin_lock(&inode->i_lock); 1727 spin_lock(&inode->i_lock);
1718 de = __d_find_any_alias(inode); 1728 de = __d_find_any_alias(inode);
1719 spin_unlock(&inode->i_lock); 1729 spin_unlock(&inode->i_lock);
1720 return de; 1730 return de;
1721 } 1731 }
1722 EXPORT_SYMBOL(d_find_any_alias); 1732 EXPORT_SYMBOL(d_find_any_alias);
1723 1733
1724 /** 1734 /**
1725 * d_obtain_alias - find or allocate a dentry for a given inode 1735 * d_obtain_alias - find or allocate a dentry for a given inode
1726 * @inode: inode to allocate the dentry for 1736 * @inode: inode to allocate the dentry for
1727 * 1737 *
1728 * Obtain a dentry for an inode resulting from NFS filehandle conversion or 1738 * Obtain a dentry for an inode resulting from NFS filehandle conversion or
1729 * similar open by handle operations. The returned dentry may be anonymous, 1739 * similar open by handle operations. The returned dentry may be anonymous,
1730 * or may have a full name (if the inode was already in the cache). 1740 * or may have a full name (if the inode was already in the cache).
1731 * 1741 *
1732 * When called on a directory inode, we must ensure that the inode only ever 1742 * When called on a directory inode, we must ensure that the inode only ever
1733 * has one dentry. If a dentry is found, that is returned instead of 1743 * has one dentry. If a dentry is found, that is returned instead of
1734 * allocating a new one. 1744 * allocating a new one.
1735 * 1745 *
1736 * On successful return, the reference to the inode has been transferred 1746 * On successful return, the reference to the inode has been transferred
1737 * to the dentry. In case of an error the reference on the inode is released. 1747 * to the dentry. In case of an error the reference on the inode is released.
1738 * To make it easier to use in export operations a %NULL or IS_ERR inode may 1748 * To make it easier to use in export operations a %NULL or IS_ERR inode may
1739 * be passed in and will be the error will be propagate to the return value, 1749 * be passed in and will be the error will be propagate to the return value,
1740 * with a %NULL @inode replaced by ERR_PTR(-ESTALE). 1750 * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
1741 */ 1751 */
1742 struct dentry *d_obtain_alias(struct inode *inode) 1752 struct dentry *d_obtain_alias(struct inode *inode)
1743 { 1753 {
1744 static const struct qstr anonstring = QSTR_INIT("/", 1); 1754 static const struct qstr anonstring = QSTR_INIT("/", 1);
1745 struct dentry *tmp; 1755 struct dentry *tmp;
1746 struct dentry *res; 1756 struct dentry *res;
1747 unsigned add_flags; 1757 unsigned add_flags;
1748 1758
1749 if (!inode) 1759 if (!inode)
1750 return ERR_PTR(-ESTALE); 1760 return ERR_PTR(-ESTALE);
1751 if (IS_ERR(inode)) 1761 if (IS_ERR(inode))
1752 return ERR_CAST(inode); 1762 return ERR_CAST(inode);
1753 1763
1754 res = d_find_any_alias(inode); 1764 res = d_find_any_alias(inode);
1755 if (res) 1765 if (res)
1756 goto out_iput; 1766 goto out_iput;
1757 1767
1758 tmp = __d_alloc(inode->i_sb, &anonstring); 1768 tmp = __d_alloc(inode->i_sb, &anonstring);
1759 if (!tmp) { 1769 if (!tmp) {
1760 res = ERR_PTR(-ENOMEM); 1770 res = ERR_PTR(-ENOMEM);
1761 goto out_iput; 1771 goto out_iput;
1762 } 1772 }
1763 1773
1764 spin_lock(&inode->i_lock); 1774 spin_lock(&inode->i_lock);
1765 res = __d_find_any_alias(inode); 1775 res = __d_find_any_alias(inode);
1766 if (res) { 1776 if (res) {
1767 spin_unlock(&inode->i_lock); 1777 spin_unlock(&inode->i_lock);
1768 dput(tmp); 1778 dput(tmp);
1769 goto out_iput; 1779 goto out_iput;
1770 } 1780 }
1771 1781
1772 /* attach a disconnected dentry */ 1782 /* attach a disconnected dentry */
1773 add_flags = d_flags_for_inode(inode) | DCACHE_DISCONNECTED; 1783 add_flags = d_flags_for_inode(inode) | DCACHE_DISCONNECTED;
1774 1784
1775 spin_lock(&tmp->d_lock); 1785 spin_lock(&tmp->d_lock);
1776 tmp->d_inode = inode; 1786 tmp->d_inode = inode;
1777 tmp->d_flags |= add_flags; 1787 tmp->d_flags |= add_flags;
1778 hlist_add_head(&tmp->d_alias, &inode->i_dentry); 1788 hlist_add_head(&tmp->d_alias, &inode->i_dentry);
1779 hlist_bl_lock(&tmp->d_sb->s_anon); 1789 hlist_bl_lock(&tmp->d_sb->s_anon);
1780 hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon); 1790 hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
1781 hlist_bl_unlock(&tmp->d_sb->s_anon); 1791 hlist_bl_unlock(&tmp->d_sb->s_anon);
1782 spin_unlock(&tmp->d_lock); 1792 spin_unlock(&tmp->d_lock);
1783 spin_unlock(&inode->i_lock); 1793 spin_unlock(&inode->i_lock);
1784 security_d_instantiate(tmp, inode); 1794 security_d_instantiate(tmp, inode);
1785 1795
1786 return tmp; 1796 return tmp;
1787 1797
1788 out_iput: 1798 out_iput:
1789 if (res && !IS_ERR(res)) 1799 if (res && !IS_ERR(res))
1790 security_d_instantiate(res, inode); 1800 security_d_instantiate(res, inode);
1791 iput(inode); 1801 iput(inode);
1792 return res; 1802 return res;
1793 } 1803 }
1794 EXPORT_SYMBOL(d_obtain_alias); 1804 EXPORT_SYMBOL(d_obtain_alias);
1795 1805
1796 /** 1806 /**
1797 * d_splice_alias - splice a disconnected dentry into the tree if one exists 1807 * d_splice_alias - splice a disconnected dentry into the tree if one exists
1798 * @inode: the inode which may have a disconnected dentry 1808 * @inode: the inode which may have a disconnected dentry
1799 * @dentry: a negative dentry which we want to point to the inode. 1809 * @dentry: a negative dentry which we want to point to the inode.
1800 * 1810 *
1801 * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and 1811 * If inode is a directory and has a 'disconnected' dentry (i.e. IS_ROOT and
1802 * DCACHE_DISCONNECTED), then d_move that in place of the given dentry 1812 * DCACHE_DISCONNECTED), then d_move that in place of the given dentry
1803 * and return it, else simply d_add the inode to the dentry and return NULL. 1813 * and return it, else simply d_add the inode to the dentry and return NULL.
1804 * 1814 *
1805 * This is needed in the lookup routine of any filesystem that is exportable 1815 * This is needed in the lookup routine of any filesystem that is exportable
1806 * (via knfsd) so that we can build dcache paths to directories effectively. 1816 * (via knfsd) so that we can build dcache paths to directories effectively.
1807 * 1817 *
1808 * If a dentry was found and moved, then it is returned. Otherwise NULL 1818 * If a dentry was found and moved, then it is returned. Otherwise NULL
1809 * is returned. This matches the expected return value of ->lookup. 1819 * is returned. This matches the expected return value of ->lookup.
1810 * 1820 *
1811 * Cluster filesystems may call this function with a negative, hashed dentry. 1821 * Cluster filesystems may call this function with a negative, hashed dentry.
1812 * In that case, we know that the inode will be a regular file, and also this 1822 * In that case, we know that the inode will be a regular file, and also this
1813 * will only occur during atomic_open. So we need to check for the dentry 1823 * will only occur during atomic_open. So we need to check for the dentry
1814 * being already hashed only in the final case. 1824 * being already hashed only in the final case.
1815 */ 1825 */
1816 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) 1826 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
1817 { 1827 {
1818 struct dentry *new = NULL; 1828 struct dentry *new = NULL;
1819 1829
1820 if (IS_ERR(inode)) 1830 if (IS_ERR(inode))
1821 return ERR_CAST(inode); 1831 return ERR_CAST(inode);
1822 1832
1823 if (inode && S_ISDIR(inode->i_mode)) { 1833 if (inode && S_ISDIR(inode->i_mode)) {
1824 spin_lock(&inode->i_lock); 1834 spin_lock(&inode->i_lock);
1825 new = __d_find_alias(inode, 1); 1835 new = __d_find_alias(inode, 1);
1826 if (new) { 1836 if (new) {
1827 BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED)); 1837 BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
1828 spin_unlock(&inode->i_lock); 1838 spin_unlock(&inode->i_lock);
1829 security_d_instantiate(new, inode); 1839 security_d_instantiate(new, inode);
1830 d_move(new, dentry); 1840 d_move(new, dentry);
1831 iput(inode); 1841 iput(inode);
1832 } else { 1842 } else {
1833 /* already taking inode->i_lock, so d_add() by hand */ 1843 /* already taking inode->i_lock, so d_add() by hand */
1834 __d_instantiate(dentry, inode); 1844 __d_instantiate(dentry, inode);
1835 spin_unlock(&inode->i_lock); 1845 spin_unlock(&inode->i_lock);
1836 security_d_instantiate(dentry, inode); 1846 security_d_instantiate(dentry, inode);
1837 d_rehash(dentry); 1847 d_rehash(dentry);
1838 } 1848 }
1839 } else { 1849 } else {
1840 d_instantiate(dentry, inode); 1850 d_instantiate(dentry, inode);
1841 if (d_unhashed(dentry)) 1851 if (d_unhashed(dentry))
1842 d_rehash(dentry); 1852 d_rehash(dentry);
1843 } 1853 }
1844 return new; 1854 return new;
1845 } 1855 }
1846 EXPORT_SYMBOL(d_splice_alias); 1856 EXPORT_SYMBOL(d_splice_alias);
1847 1857
1848 /** 1858 /**
1849 * d_add_ci - lookup or allocate new dentry with case-exact name 1859 * d_add_ci - lookup or allocate new dentry with case-exact name
1850 * @inode: the inode case-insensitive lookup has found 1860 * @inode: the inode case-insensitive lookup has found
1851 * @dentry: the negative dentry that was passed to the parent's lookup func 1861 * @dentry: the negative dentry that was passed to the parent's lookup func
1852 * @name: the case-exact name to be associated with the returned dentry 1862 * @name: the case-exact name to be associated with the returned dentry
1853 * 1863 *
1854 * This is to avoid filling the dcache with case-insensitive names to the 1864 * This is to avoid filling the dcache with case-insensitive names to the
1855 * same inode, only the actual correct case is stored in the dcache for 1865 * same inode, only the actual correct case is stored in the dcache for
1856 * case-insensitive filesystems. 1866 * case-insensitive filesystems.
1857 * 1867 *
1858 * For a case-insensitive lookup match and if the the case-exact dentry 1868 * For a case-insensitive lookup match and if the the case-exact dentry
1859 * already exists in in the dcache, use it and return it. 1869 * already exists in in the dcache, use it and return it.
1860 * 1870 *
1861 * If no entry exists with the exact case name, allocate new dentry with 1871 * If no entry exists with the exact case name, allocate new dentry with
1862 * the exact case, and return the spliced entry. 1872 * the exact case, and return the spliced entry.
1863 */ 1873 */
1864 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, 1874 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
1865 struct qstr *name) 1875 struct qstr *name)
1866 { 1876 {
1867 struct dentry *found; 1877 struct dentry *found;
1868 struct dentry *new; 1878 struct dentry *new;
1869 1879
1870 /* 1880 /*
1871 * First check if a dentry matching the name already exists, 1881 * First check if a dentry matching the name already exists,
1872 * if not go ahead and create it now. 1882 * if not go ahead and create it now.
1873 */ 1883 */
1874 found = d_hash_and_lookup(dentry->d_parent, name); 1884 found = d_hash_and_lookup(dentry->d_parent, name);
1875 if (unlikely(IS_ERR(found))) 1885 if (unlikely(IS_ERR(found)))
1876 goto err_out; 1886 goto err_out;
1877 if (!found) { 1887 if (!found) {
1878 new = d_alloc(dentry->d_parent, name); 1888 new = d_alloc(dentry->d_parent, name);
1879 if (!new) { 1889 if (!new) {
1880 found = ERR_PTR(-ENOMEM); 1890 found = ERR_PTR(-ENOMEM);
1881 goto err_out; 1891 goto err_out;
1882 } 1892 }
1883 1893
1884 found = d_splice_alias(inode, new); 1894 found = d_splice_alias(inode, new);
1885 if (found) { 1895 if (found) {
1886 dput(new); 1896 dput(new);
1887 return found; 1897 return found;
1888 } 1898 }
1889 return new; 1899 return new;
1890 } 1900 }
1891 1901
1892 /* 1902 /*
1893 * If a matching dentry exists, and it's not negative use it. 1903 * If a matching dentry exists, and it's not negative use it.
1894 * 1904 *
1895 * Decrement the reference count to balance the iget() done 1905 * Decrement the reference count to balance the iget() done
1896 * earlier on. 1906 * earlier on.
1897 */ 1907 */
1898 if (found->d_inode) { 1908 if (found->d_inode) {
1899 if (unlikely(found->d_inode != inode)) { 1909 if (unlikely(found->d_inode != inode)) {
1900 /* This can't happen because bad inodes are unhashed. */ 1910 /* This can't happen because bad inodes are unhashed. */
1901 BUG_ON(!is_bad_inode(inode)); 1911 BUG_ON(!is_bad_inode(inode));
1902 BUG_ON(!is_bad_inode(found->d_inode)); 1912 BUG_ON(!is_bad_inode(found->d_inode));
1903 } 1913 }
1904 iput(inode); 1914 iput(inode);
1905 return found; 1915 return found;
1906 } 1916 }
1907 1917
1908 /* 1918 /*
1909 * Negative dentry: instantiate it unless the inode is a directory and 1919 * Negative dentry: instantiate it unless the inode is a directory and
1910 * already has a dentry. 1920 * already has a dentry.
1911 */ 1921 */
1912 new = d_splice_alias(inode, found); 1922 new = d_splice_alias(inode, found);
1913 if (new) { 1923 if (new) {
1914 dput(found); 1924 dput(found);
1915 found = new; 1925 found = new;
1916 } 1926 }
1917 return found; 1927 return found;
1918 1928
1919 err_out: 1929 err_out:
1920 iput(inode); 1930 iput(inode);
1921 return found; 1931 return found;
1922 } 1932 }
1923 EXPORT_SYMBOL(d_add_ci); 1933 EXPORT_SYMBOL(d_add_ci);
1924 1934
1925 /* 1935 /*
1926 * Do the slow-case of the dentry name compare. 1936 * Do the slow-case of the dentry name compare.
1927 * 1937 *
1928 * Unlike the dentry_cmp() function, we need to atomically 1938 * Unlike the dentry_cmp() function, we need to atomically
1929 * load the name and length information, so that the 1939 * load the name and length information, so that the
1930 * filesystem can rely on them, and can use the 'name' and 1940 * filesystem can rely on them, and can use the 'name' and
1931 * 'len' information without worrying about walking off the 1941 * 'len' information without worrying about walking off the
1932 * end of memory etc. 1942 * end of memory etc.
1933 * 1943 *
1934 * Thus the read_seqcount_retry() and the "duplicate" info 1944 * Thus the read_seqcount_retry() and the "duplicate" info
1935 * in arguments (the low-level filesystem should not look 1945 * in arguments (the low-level filesystem should not look
1936 * at the dentry inode or name contents directly, since 1946 * at the dentry inode or name contents directly, since
1937 * rename can change them while we're in RCU mode). 1947 * rename can change them while we're in RCU mode).
1938 */ 1948 */
1939 enum slow_d_compare { 1949 enum slow_d_compare {
1940 D_COMP_OK, 1950 D_COMP_OK,
1941 D_COMP_NOMATCH, 1951 D_COMP_NOMATCH,
1942 D_COMP_SEQRETRY, 1952 D_COMP_SEQRETRY,
1943 }; 1953 };
1944 1954
1945 static noinline enum slow_d_compare slow_dentry_cmp( 1955 static noinline enum slow_d_compare slow_dentry_cmp(
1946 const struct dentry *parent, 1956 const struct dentry *parent,
1947 struct dentry *dentry, 1957 struct dentry *dentry,
1948 unsigned int seq, 1958 unsigned int seq,
1949 const struct qstr *name) 1959 const struct qstr *name)
1950 { 1960 {
1951 int tlen = dentry->d_name.len; 1961 int tlen = dentry->d_name.len;
1952 const char *tname = dentry->d_name.name; 1962 const char *tname = dentry->d_name.name;
1953 1963
1954 if (read_seqcount_retry(&dentry->d_seq, seq)) { 1964 if (read_seqcount_retry(&dentry->d_seq, seq)) {
1955 cpu_relax(); 1965 cpu_relax();
1956 return D_COMP_SEQRETRY; 1966 return D_COMP_SEQRETRY;
1957 } 1967 }
1958 if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) 1968 if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
1959 return D_COMP_NOMATCH; 1969 return D_COMP_NOMATCH;
1960 return D_COMP_OK; 1970 return D_COMP_OK;
1961 } 1971 }
1962 1972
1963 /** 1973 /**
1964 * __d_lookup_rcu - search for a dentry (racy, store-free) 1974 * __d_lookup_rcu - search for a dentry (racy, store-free)
1965 * @parent: parent dentry 1975 * @parent: parent dentry
1966 * @name: qstr of name we wish to find 1976 * @name: qstr of name we wish to find
1967 * @seqp: returns d_seq value at the point where the dentry was found 1977 * @seqp: returns d_seq value at the point where the dentry was found
1968 * Returns: dentry, or NULL 1978 * Returns: dentry, or NULL
1969 * 1979 *
1970 * __d_lookup_rcu is the dcache lookup function for rcu-walk name 1980 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
1971 * resolution (store-free path walking) design described in 1981 * resolution (store-free path walking) design described in
1972 * Documentation/filesystems/path-lookup.txt. 1982 * Documentation/filesystems/path-lookup.txt.
1973 * 1983 *
1974 * This is not to be used outside core vfs. 1984 * This is not to be used outside core vfs.
1975 * 1985 *
1976 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock 1986 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
1977 * held, and rcu_read_lock held. The returned dentry must not be stored into 1987 * held, and rcu_read_lock held. The returned dentry must not be stored into
1978 * without taking d_lock and checking d_seq sequence count against @seq 1988 * without taking d_lock and checking d_seq sequence count against @seq
1979 * returned here. 1989 * returned here.
1980 * 1990 *
1981 * A refcount may be taken on the found dentry with the d_rcu_to_refcount 1991 * A refcount may be taken on the found dentry with the d_rcu_to_refcount
1982 * function. 1992 * function.
1983 * 1993 *
1984 * Alternatively, __d_lookup_rcu may be called again to look up the child of 1994 * Alternatively, __d_lookup_rcu may be called again to look up the child of
1985 * the returned dentry, so long as its parent's seqlock is checked after the 1995 * the returned dentry, so long as its parent's seqlock is checked after the
1986 * child is looked up. Thus, an interlocking stepping of sequence lock checks 1996 * child is looked up. Thus, an interlocking stepping of sequence lock checks
1987 * is formed, giving integrity down the path walk. 1997 * is formed, giving integrity down the path walk.
1988 * 1998 *
1989 * NOTE! The caller *has* to check the resulting dentry against the sequence 1999 * NOTE! The caller *has* to check the resulting dentry against the sequence
1990 * number we've returned before using any of the resulting dentry state! 2000 * number we've returned before using any of the resulting dentry state!
1991 */ 2001 */
1992 struct dentry *__d_lookup_rcu(const struct dentry *parent, 2002 struct dentry *__d_lookup_rcu(const struct dentry *parent,
1993 const struct qstr *name, 2003 const struct qstr *name,
1994 unsigned *seqp) 2004 unsigned *seqp)
1995 { 2005 {
1996 u64 hashlen = name->hash_len; 2006 u64 hashlen = name->hash_len;
1997 const unsigned char *str = name->name; 2007 const unsigned char *str = name->name;
1998 struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen)); 2008 struct hlist_bl_head *b = d_hash(parent, hashlen_hash(hashlen));
1999 struct hlist_bl_node *node; 2009 struct hlist_bl_node *node;
2000 struct dentry *dentry; 2010 struct dentry *dentry;
2001 2011
2002 /* 2012 /*
2003 * Note: There is significant duplication with __d_lookup_rcu which is 2013 * Note: There is significant duplication with __d_lookup_rcu which is
2004 * required to prevent single threaded performance regressions 2014 * required to prevent single threaded performance regressions
2005 * especially on architectures where smp_rmb (in seqcounts) are costly. 2015 * especially on architectures where smp_rmb (in seqcounts) are costly.
2006 * Keep the two functions in sync. 2016 * Keep the two functions in sync.
2007 */ 2017 */
2008 2018
2009 /* 2019 /*
2010 * The hash list is protected using RCU. 2020 * The hash list is protected using RCU.
2011 * 2021 *
2012 * Carefully use d_seq when comparing a candidate dentry, to avoid 2022 * Carefully use d_seq when comparing a candidate dentry, to avoid
2013 * races with d_move(). 2023 * races with d_move().
2014 * 2024 *
2015 * It is possible that concurrent renames can mess up our list 2025 * It is possible that concurrent renames can mess up our list
2016 * walk here and result in missing our dentry, resulting in the 2026 * walk here and result in missing our dentry, resulting in the
2017 * false-negative result. d_lookup() protects against concurrent 2027 * false-negative result. d_lookup() protects against concurrent
2018 * renames using rename_lock seqlock. 2028 * renames using rename_lock seqlock.
2019 * 2029 *
2020 * See Documentation/filesystems/path-lookup.txt for more details. 2030 * See Documentation/filesystems/path-lookup.txt for more details.
2021 */ 2031 */
2022 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2032 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2023 unsigned seq; 2033 unsigned seq;
2024 2034
2025 seqretry: 2035 seqretry:
2026 /* 2036 /*
2027 * The dentry sequence count protects us from concurrent 2037 * The dentry sequence count protects us from concurrent
2028 * renames, and thus protects parent and name fields. 2038 * renames, and thus protects parent and name fields.
2029 * 2039 *
2030 * The caller must perform a seqcount check in order 2040 * The caller must perform a seqcount check in order
2031 * to do anything useful with the returned dentry. 2041 * to do anything useful with the returned dentry.
2032 * 2042 *
2033 * NOTE! We do a "raw" seqcount_begin here. That means that 2043 * NOTE! We do a "raw" seqcount_begin here. That means that
2034 * we don't wait for the sequence count to stabilize if it 2044 * we don't wait for the sequence count to stabilize if it
2035 * is in the middle of a sequence change. If we do the slow 2045 * is in the middle of a sequence change. If we do the slow
2036 * dentry compare, we will do seqretries until it is stable, 2046 * dentry compare, we will do seqretries until it is stable,
2037 * and if we end up with a successful lookup, we actually 2047 * and if we end up with a successful lookup, we actually
2038 * want to exit RCU lookup anyway. 2048 * want to exit RCU lookup anyway.
2039 */ 2049 */
2040 seq = raw_seqcount_begin(&dentry->d_seq); 2050 seq = raw_seqcount_begin(&dentry->d_seq);
2041 if (dentry->d_parent != parent) 2051 if (dentry->d_parent != parent)
2042 continue; 2052 continue;
2043 if (d_unhashed(dentry)) 2053 if (d_unhashed(dentry))
2044 continue; 2054 continue;
2045 2055
2046 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { 2056 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) {
2047 if (dentry->d_name.hash != hashlen_hash(hashlen)) 2057 if (dentry->d_name.hash != hashlen_hash(hashlen))
2048 continue; 2058 continue;
2049 *seqp = seq; 2059 *seqp = seq;
2050 switch (slow_dentry_cmp(parent, dentry, seq, name)) { 2060 switch (slow_dentry_cmp(parent, dentry, seq, name)) {
2051 case D_COMP_OK: 2061 case D_COMP_OK:
2052 return dentry; 2062 return dentry;
2053 case D_COMP_NOMATCH: 2063 case D_COMP_NOMATCH:
2054 continue; 2064 continue;
2055 default: 2065 default:
2056 goto seqretry; 2066 goto seqretry;
2057 } 2067 }
2058 } 2068 }
2059 2069
2060 if (dentry->d_name.hash_len != hashlen) 2070 if (dentry->d_name.hash_len != hashlen)
2061 continue; 2071 continue;
2062 *seqp = seq; 2072 *seqp = seq;
2063 if (!dentry_cmp(dentry, str, hashlen_len(hashlen))) 2073 if (!dentry_cmp(dentry, str, hashlen_len(hashlen)))
2064 return dentry; 2074 return dentry;
2065 } 2075 }
2066 return NULL; 2076 return NULL;
2067 } 2077 }
2068 2078
2069 /** 2079 /**
2070 * d_lookup - search for a dentry 2080 * d_lookup - search for a dentry
2071 * @parent: parent dentry 2081 * @parent: parent dentry
2072 * @name: qstr of name we wish to find 2082 * @name: qstr of name we wish to find
2073 * Returns: dentry, or NULL 2083 * Returns: dentry, or NULL
2074 * 2084 *
2075 * d_lookup searches the children of the parent dentry for the name in 2085 * d_lookup searches the children of the parent dentry for the name in
2076 * question. If the dentry is found its reference count is incremented and the 2086 * question. If the dentry is found its reference count is incremented and the
2077 * dentry is returned. The caller must use dput to free the entry when it has 2087 * dentry is returned. The caller must use dput to free the entry when it has
2078 * finished using it. %NULL is returned if the dentry does not exist. 2088 * finished using it. %NULL is returned if the dentry does not exist.
2079 */ 2089 */
2080 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name) 2090 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
2081 { 2091 {
2082 struct dentry *dentry; 2092 struct dentry *dentry;
2083 unsigned seq; 2093 unsigned seq;
2084 2094
2085 do { 2095 do {
2086 seq = read_seqbegin(&rename_lock); 2096 seq = read_seqbegin(&rename_lock);
2087 dentry = __d_lookup(parent, name); 2097 dentry = __d_lookup(parent, name);
2088 if (dentry) 2098 if (dentry)
2089 break; 2099 break;
2090 } while (read_seqretry(&rename_lock, seq)); 2100 } while (read_seqretry(&rename_lock, seq));
2091 return dentry; 2101 return dentry;
2092 } 2102 }
2093 EXPORT_SYMBOL(d_lookup); 2103 EXPORT_SYMBOL(d_lookup);
2094 2104
2095 /** 2105 /**
2096 * __d_lookup - search for a dentry (racy) 2106 * __d_lookup - search for a dentry (racy)
2097 * @parent: parent dentry 2107 * @parent: parent dentry
2098 * @name: qstr of name we wish to find 2108 * @name: qstr of name we wish to find
2099 * Returns: dentry, or NULL 2109 * Returns: dentry, or NULL
2100 * 2110 *
2101 * __d_lookup is like d_lookup, however it may (rarely) return a 2111 * __d_lookup is like d_lookup, however it may (rarely) return a
2102 * false-negative result due to unrelated rename activity. 2112 * false-negative result due to unrelated rename activity.
2103 * 2113 *
2104 * __d_lookup is slightly faster by avoiding rename_lock read seqlock, 2114 * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
2105 * however it must be used carefully, eg. with a following d_lookup in 2115 * however it must be used carefully, eg. with a following d_lookup in
2106 * the case of failure. 2116 * the case of failure.
2107 * 2117 *
2108 * __d_lookup callers must be commented. 2118 * __d_lookup callers must be commented.
2109 */ 2119 */
2110 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) 2120 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
2111 { 2121 {
2112 unsigned int len = name->len; 2122 unsigned int len = name->len;
2113 unsigned int hash = name->hash; 2123 unsigned int hash = name->hash;
2114 const unsigned char *str = name->name; 2124 const unsigned char *str = name->name;
2115 struct hlist_bl_head *b = d_hash(parent, hash); 2125 struct hlist_bl_head *b = d_hash(parent, hash);
2116 struct hlist_bl_node *node; 2126 struct hlist_bl_node *node;
2117 struct dentry *found = NULL; 2127 struct dentry *found = NULL;
2118 struct dentry *dentry; 2128 struct dentry *dentry;
2119 2129
2120 /* 2130 /*
2121 * Note: There is significant duplication with __d_lookup_rcu which is 2131 * Note: There is significant duplication with __d_lookup_rcu which is
2122 * required to prevent single threaded performance regressions 2132 * required to prevent single threaded performance regressions
2123 * especially on architectures where smp_rmb (in seqcounts) are costly. 2133 * especially on architectures where smp_rmb (in seqcounts) are costly.
2124 * Keep the two functions in sync. 2134 * Keep the two functions in sync.
2125 */ 2135 */
2126 2136
2127 /* 2137 /*
2128 * The hash list is protected using RCU. 2138 * The hash list is protected using RCU.
2129 * 2139 *
2130 * Take d_lock when comparing a candidate dentry, to avoid races 2140 * Take d_lock when comparing a candidate dentry, to avoid races
2131 * with d_move(). 2141 * with d_move().
2132 * 2142 *
2133 * It is possible that concurrent renames can mess up our list 2143 * It is possible that concurrent renames can mess up our list
2134 * walk here and result in missing our dentry, resulting in the 2144 * walk here and result in missing our dentry, resulting in the
2135 * false-negative result. d_lookup() protects against concurrent 2145 * false-negative result. d_lookup() protects against concurrent
2136 * renames using rename_lock seqlock. 2146 * renames using rename_lock seqlock.
2137 * 2147 *
2138 * See Documentation/filesystems/path-lookup.txt for more details. 2148 * See Documentation/filesystems/path-lookup.txt for more details.
2139 */ 2149 */
2140 rcu_read_lock(); 2150 rcu_read_lock();
2141 2151
2142 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2152 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2143 2153
2144 if (dentry->d_name.hash != hash) 2154 if (dentry->d_name.hash != hash)
2145 continue; 2155 continue;
2146 2156
2147 spin_lock(&dentry->d_lock); 2157 spin_lock(&dentry->d_lock);
2148 if (dentry->d_parent != parent) 2158 if (dentry->d_parent != parent)
2149 goto next; 2159 goto next;
2150 if (d_unhashed(dentry)) 2160 if (d_unhashed(dentry))
2151 goto next; 2161 goto next;
2152 2162
2153 /* 2163 /*
2154 * It is safe to compare names since d_move() cannot 2164 * It is safe to compare names since d_move() cannot
2155 * change the qstr (protected by d_lock). 2165 * change the qstr (protected by d_lock).
2156 */ 2166 */
2157 if (parent->d_flags & DCACHE_OP_COMPARE) { 2167 if (parent->d_flags & DCACHE_OP_COMPARE) {
2158 int tlen = dentry->d_name.len; 2168 int tlen = dentry->d_name.len;
2159 const char *tname = dentry->d_name.name; 2169 const char *tname = dentry->d_name.name;
2160 if (parent->d_op->d_compare(parent, dentry, tlen, tname, name)) 2170 if (parent->d_op->d_compare(parent, dentry, tlen, tname, name))
2161 goto next; 2171 goto next;
2162 } else { 2172 } else {
2163 if (dentry->d_name.len != len) 2173 if (dentry->d_name.len != len)
2164 goto next; 2174 goto next;
2165 if (dentry_cmp(dentry, str, len)) 2175 if (dentry_cmp(dentry, str, len))
2166 goto next; 2176 goto next;
2167 } 2177 }
2168 2178
2169 dentry->d_lockref.count++; 2179 dentry->d_lockref.count++;
2170 found = dentry; 2180 found = dentry;
2171 spin_unlock(&dentry->d_lock); 2181 spin_unlock(&dentry->d_lock);
2172 break; 2182 break;
2173 next: 2183 next:
2174 spin_unlock(&dentry->d_lock); 2184 spin_unlock(&dentry->d_lock);
2175 } 2185 }
2176 rcu_read_unlock(); 2186 rcu_read_unlock();
2177 2187
2178 return found; 2188 return found;
2179 } 2189 }
2180 2190
2181 /** 2191 /**
2182 * d_hash_and_lookup - hash the qstr then search for a dentry 2192 * d_hash_and_lookup - hash the qstr then search for a dentry
2183 * @dir: Directory to search in 2193 * @dir: Directory to search in
2184 * @name: qstr of name we wish to find 2194 * @name: qstr of name we wish to find
2185 * 2195 *
2186 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error) 2196 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
2187 */ 2197 */
2188 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name) 2198 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
2189 { 2199 {
2190 /* 2200 /*
2191 * Check for a fs-specific hash function. Note that we must 2201 * Check for a fs-specific hash function. Note that we must
2192 * calculate the standard hash first, as the d_op->d_hash() 2202 * calculate the standard hash first, as the d_op->d_hash()
2193 * routine may choose to leave the hash value unchanged. 2203 * routine may choose to leave the hash value unchanged.
2194 */ 2204 */
2195 name->hash = full_name_hash(name->name, name->len); 2205 name->hash = full_name_hash(name->name, name->len);
2196 if (dir->d_flags & DCACHE_OP_HASH) { 2206 if (dir->d_flags & DCACHE_OP_HASH) {
2197 int err = dir->d_op->d_hash(dir, name); 2207 int err = dir->d_op->d_hash(dir, name);
2198 if (unlikely(err < 0)) 2208 if (unlikely(err < 0))
2199 return ERR_PTR(err); 2209 return ERR_PTR(err);
2200 } 2210 }
2201 return d_lookup(dir, name); 2211 return d_lookup(dir, name);
2202 } 2212 }
2203 EXPORT_SYMBOL(d_hash_and_lookup); 2213 EXPORT_SYMBOL(d_hash_and_lookup);
2204 2214
2205 /** 2215 /**
2206 * d_validate - verify dentry provided from insecure source (deprecated) 2216 * d_validate - verify dentry provided from insecure source (deprecated)
2207 * @dentry: The dentry alleged to be valid child of @dparent 2217 * @dentry: The dentry alleged to be valid child of @dparent
2208 * @dparent: The parent dentry (known to be valid) 2218 * @dparent: The parent dentry (known to be valid)
2209 * 2219 *
2210 * An insecure source has sent us a dentry, here we verify it and dget() it. 2220 * An insecure source has sent us a dentry, here we verify it and dget() it.
2211 * This is used by ncpfs in its readdir implementation. 2221 * This is used by ncpfs in its readdir implementation.
2212 * Zero is returned in the dentry is invalid. 2222 * Zero is returned in the dentry is invalid.
2213 * 2223 *
2214 * This function is slow for big directories, and deprecated, do not use it. 2224 * This function is slow for big directories, and deprecated, do not use it.
2215 */ 2225 */
2216 int d_validate(struct dentry *dentry, struct dentry *dparent) 2226 int d_validate(struct dentry *dentry, struct dentry *dparent)
2217 { 2227 {
2218 struct dentry *child; 2228 struct dentry *child;
2219 2229
2220 spin_lock(&dparent->d_lock); 2230 spin_lock(&dparent->d_lock);
2221 list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) { 2231 list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
2222 if (dentry == child) { 2232 if (dentry == child) {
2223 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); 2233 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
2224 __dget_dlock(dentry); 2234 __dget_dlock(dentry);
2225 spin_unlock(&dentry->d_lock); 2235 spin_unlock(&dentry->d_lock);
2226 spin_unlock(&dparent->d_lock); 2236 spin_unlock(&dparent->d_lock);
2227 return 1; 2237 return 1;
2228 } 2238 }
2229 } 2239 }
2230 spin_unlock(&dparent->d_lock); 2240 spin_unlock(&dparent->d_lock);
2231 2241
2232 return 0; 2242 return 0;
2233 } 2243 }
2234 EXPORT_SYMBOL(d_validate); 2244 EXPORT_SYMBOL(d_validate);
2235 2245
2236 /* 2246 /*
2237 * When a file is deleted, we have two options: 2247 * When a file is deleted, we have two options:
2238 * - turn this dentry into a negative dentry 2248 * - turn this dentry into a negative dentry
2239 * - unhash this dentry and free it. 2249 * - unhash this dentry and free it.
2240 * 2250 *
2241 * Usually, we want to just turn this into 2251 * Usually, we want to just turn this into
2242 * a negative dentry, but if anybody else is 2252 * a negative dentry, but if anybody else is
2243 * currently using the dentry or the inode 2253 * currently using the dentry or the inode
2244 * we can't do that and we fall back on removing 2254 * we can't do that and we fall back on removing
2245 * it from the hash queues and waiting for 2255 * it from the hash queues and waiting for
2246 * it to be deleted later when it has no users 2256 * it to be deleted later when it has no users
2247 */ 2257 */
2248 2258
2249 /** 2259 /**
2250 * d_delete - delete a dentry 2260 * d_delete - delete a dentry
2251 * @dentry: The dentry to delete 2261 * @dentry: The dentry to delete
2252 * 2262 *
2253 * Turn the dentry into a negative dentry if possible, otherwise 2263 * Turn the dentry into a negative dentry if possible, otherwise
2254 * remove it from the hash queues so it can be deleted later 2264 * remove it from the hash queues so it can be deleted later
2255 */ 2265 */
2256 2266
2257 void d_delete(struct dentry * dentry) 2267 void d_delete(struct dentry * dentry)
2258 { 2268 {
2259 struct inode *inode; 2269 struct inode *inode;
2260 int isdir = 0; 2270 int isdir = 0;
2261 /* 2271 /*
2262 * Are we the only user? 2272 * Are we the only user?
2263 */ 2273 */
2264 again: 2274 again:
2265 spin_lock(&dentry->d_lock); 2275 spin_lock(&dentry->d_lock);
2266 inode = dentry->d_inode; 2276 inode = dentry->d_inode;
2267 isdir = S_ISDIR(inode->i_mode); 2277 isdir = S_ISDIR(inode->i_mode);
2268 if (dentry->d_lockref.count == 1) { 2278 if (dentry->d_lockref.count == 1) {
2269 if (!spin_trylock(&inode->i_lock)) { 2279 if (!spin_trylock(&inode->i_lock)) {
2270 spin_unlock(&dentry->d_lock); 2280 spin_unlock(&dentry->d_lock);
2271 cpu_relax(); 2281 cpu_relax();
2272 goto again; 2282 goto again;
2273 } 2283 }
2274 dentry->d_flags &= ~DCACHE_CANT_MOUNT; 2284 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
2275 dentry_unlink_inode(dentry); 2285 dentry_unlink_inode(dentry);
2276 fsnotify_nameremove(dentry, isdir); 2286 fsnotify_nameremove(dentry, isdir);
2277 return; 2287 return;
2278 } 2288 }
2279 2289
2280 if (!d_unhashed(dentry)) 2290 if (!d_unhashed(dentry))
2281 __d_drop(dentry); 2291 __d_drop(dentry);
2282 2292
2283 spin_unlock(&dentry->d_lock); 2293 spin_unlock(&dentry->d_lock);
2284 2294
2285 fsnotify_nameremove(dentry, isdir); 2295 fsnotify_nameremove(dentry, isdir);
2286 } 2296 }
2287 EXPORT_SYMBOL(d_delete); 2297 EXPORT_SYMBOL(d_delete);
2288 2298
2289 static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b) 2299 static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b)
2290 { 2300 {
2291 BUG_ON(!d_unhashed(entry)); 2301 BUG_ON(!d_unhashed(entry));
2292 hlist_bl_lock(b); 2302 hlist_bl_lock(b);
2293 entry->d_flags |= DCACHE_RCUACCESS; 2303 entry->d_flags |= DCACHE_RCUACCESS;
2294 hlist_bl_add_head_rcu(&entry->d_hash, b); 2304 hlist_bl_add_head_rcu(&entry->d_hash, b);
2295 hlist_bl_unlock(b); 2305 hlist_bl_unlock(b);
2296 } 2306 }
2297 2307
2298 static void _d_rehash(struct dentry * entry) 2308 static void _d_rehash(struct dentry * entry)
2299 { 2309 {
2300 __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash)); 2310 __d_rehash(entry, d_hash(entry->d_parent, entry->d_name.hash));
2301 } 2311 }
2302 2312
2303 /** 2313 /**
2304 * d_rehash - add an entry back to the hash 2314 * d_rehash - add an entry back to the hash
2305 * @entry: dentry to add to the hash 2315 * @entry: dentry to add to the hash
2306 * 2316 *
2307 * Adds a dentry to the hash according to its name. 2317 * Adds a dentry to the hash according to its name.
2308 */ 2318 */
2309 2319
2310 void d_rehash(struct dentry * entry) 2320 void d_rehash(struct dentry * entry)
2311 { 2321 {
2312 spin_lock(&entry->d_lock); 2322 spin_lock(&entry->d_lock);
2313 _d_rehash(entry); 2323 _d_rehash(entry);
2314 spin_unlock(&entry->d_lock); 2324 spin_unlock(&entry->d_lock);
2315 } 2325 }
2316 EXPORT_SYMBOL(d_rehash); 2326 EXPORT_SYMBOL(d_rehash);
2317 2327
2318 /** 2328 /**
2319 * dentry_update_name_case - update case insensitive dentry with a new name 2329 * dentry_update_name_case - update case insensitive dentry with a new name
2320 * @dentry: dentry to be updated 2330 * @dentry: dentry to be updated
2321 * @name: new name 2331 * @name: new name
2322 * 2332 *
2323 * Update a case insensitive dentry with new case of name. 2333 * Update a case insensitive dentry with new case of name.
2324 * 2334 *
2325 * dentry must have been returned by d_lookup with name @name. Old and new 2335 * dentry must have been returned by d_lookup with name @name. Old and new
2326 * name lengths must match (ie. no d_compare which allows mismatched name 2336 * name lengths must match (ie. no d_compare which allows mismatched name
2327 * lengths). 2337 * lengths).
2328 * 2338 *
2329 * Parent inode i_mutex must be held over d_lookup and into this call (to 2339 * Parent inode i_mutex must be held over d_lookup and into this call (to
2330 * keep renames and concurrent inserts, and readdir(2) away). 2340 * keep renames and concurrent inserts, and readdir(2) away).
2331 */ 2341 */
2332 void dentry_update_name_case(struct dentry *dentry, struct qstr *name) 2342 void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
2333 { 2343 {
2334 BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex)); 2344 BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex));
2335 BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */ 2345 BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
2336 2346
2337 spin_lock(&dentry->d_lock); 2347 spin_lock(&dentry->d_lock);
2338 write_seqcount_begin(&dentry->d_seq); 2348 write_seqcount_begin(&dentry->d_seq);
2339 memcpy((unsigned char *)dentry->d_name.name, name->name, name->len); 2349 memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
2340 write_seqcount_end(&dentry->d_seq); 2350 write_seqcount_end(&dentry->d_seq);
2341 spin_unlock(&dentry->d_lock); 2351 spin_unlock(&dentry->d_lock);
2342 } 2352 }
2343 EXPORT_SYMBOL(dentry_update_name_case); 2353 EXPORT_SYMBOL(dentry_update_name_case);
2344 2354
2345 static void switch_names(struct dentry *dentry, struct dentry *target) 2355 static void switch_names(struct dentry *dentry, struct dentry *target)
2346 { 2356 {
2347 if (dname_external(target)) { 2357 if (dname_external(target)) {
2348 if (dname_external(dentry)) { 2358 if (dname_external(dentry)) {
2349 /* 2359 /*
2350 * Both external: swap the pointers 2360 * Both external: swap the pointers
2351 */ 2361 */
2352 swap(target->d_name.name, dentry->d_name.name); 2362 swap(target->d_name.name, dentry->d_name.name);
2353 } else { 2363 } else {
2354 /* 2364 /*
2355 * dentry:internal, target:external. Steal target's 2365 * dentry:internal, target:external. Steal target's
2356 * storage and make target internal. 2366 * storage and make target internal.
2357 */ 2367 */
2358 memcpy(target->d_iname, dentry->d_name.name, 2368 memcpy(target->d_iname, dentry->d_name.name,
2359 dentry->d_name.len + 1); 2369 dentry->d_name.len + 1);
2360 dentry->d_name.name = target->d_name.name; 2370 dentry->d_name.name = target->d_name.name;
2361 target->d_name.name = target->d_iname; 2371 target->d_name.name = target->d_iname;
2362 } 2372 }
2363 } else { 2373 } else {
2364 if (dname_external(dentry)) { 2374 if (dname_external(dentry)) {
2365 /* 2375 /*
2366 * dentry:external, target:internal. Give dentry's 2376 * dentry:external, target:internal. Give dentry's
2367 * storage to target and make dentry internal 2377 * storage to target and make dentry internal
2368 */ 2378 */
2369 memcpy(dentry->d_iname, target->d_name.name, 2379 memcpy(dentry->d_iname, target->d_name.name,
2370 target->d_name.len + 1); 2380 target->d_name.len + 1);
2371 target->d_name.name = dentry->d_name.name; 2381 target->d_name.name = dentry->d_name.name;
2372 dentry->d_name.name = dentry->d_iname; 2382 dentry->d_name.name = dentry->d_iname;
2373 } else { 2383 } else {
2374 /* 2384 /*
2375 * Both are internal. 2385 * Both are internal.
2376 */ 2386 */
2377 unsigned int i; 2387 unsigned int i;
2378 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long))); 2388 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2379 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) { 2389 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2380 swap(((long *) &dentry->d_iname)[i], 2390 swap(((long *) &dentry->d_iname)[i],
2381 ((long *) &target->d_iname)[i]); 2391 ((long *) &target->d_iname)[i]);
2382 } 2392 }
2383 } 2393 }
2384 } 2394 }
2385 swap(dentry->d_name.len, target->d_name.len); 2395 swap(dentry->d_name.len, target->d_name.len);
2386 } 2396 }
2387 2397
2388 static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target) 2398 static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
2389 { 2399 {
2390 /* 2400 /*
2391 * XXXX: do we really need to take target->d_lock? 2401 * XXXX: do we really need to take target->d_lock?
2392 */ 2402 */
2393 if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent) 2403 if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
2394 spin_lock(&target->d_parent->d_lock); 2404 spin_lock(&target->d_parent->d_lock);
2395 else { 2405 else {
2396 if (d_ancestor(dentry->d_parent, target->d_parent)) { 2406 if (d_ancestor(dentry->d_parent, target->d_parent)) {
2397 spin_lock(&dentry->d_parent->d_lock); 2407 spin_lock(&dentry->d_parent->d_lock);
2398 spin_lock_nested(&target->d_parent->d_lock, 2408 spin_lock_nested(&target->d_parent->d_lock,
2399 DENTRY_D_LOCK_NESTED); 2409 DENTRY_D_LOCK_NESTED);
2400 } else { 2410 } else {
2401 spin_lock(&target->d_parent->d_lock); 2411 spin_lock(&target->d_parent->d_lock);
2402 spin_lock_nested(&dentry->d_parent->d_lock, 2412 spin_lock_nested(&dentry->d_parent->d_lock,
2403 DENTRY_D_LOCK_NESTED); 2413 DENTRY_D_LOCK_NESTED);
2404 } 2414 }
2405 } 2415 }
2406 if (target < dentry) { 2416 if (target < dentry) {
2407 spin_lock_nested(&target->d_lock, 2); 2417 spin_lock_nested(&target->d_lock, 2);
2408 spin_lock_nested(&dentry->d_lock, 3); 2418 spin_lock_nested(&dentry->d_lock, 3);
2409 } else { 2419 } else {
2410 spin_lock_nested(&dentry->d_lock, 2); 2420 spin_lock_nested(&dentry->d_lock, 2);
2411 spin_lock_nested(&target->d_lock, 3); 2421 spin_lock_nested(&target->d_lock, 3);
2412 } 2422 }
2413 } 2423 }
2414 2424
2415 static void dentry_unlock_parents_for_move(struct dentry *dentry, 2425 static void dentry_unlock_parents_for_move(struct dentry *dentry,
2416 struct dentry *target) 2426 struct dentry *target)
2417 { 2427 {
2418 if (target->d_parent != dentry->d_parent) 2428 if (target->d_parent != dentry->d_parent)
2419 spin_unlock(&dentry->d_parent->d_lock); 2429 spin_unlock(&dentry->d_parent->d_lock);
2420 if (target->d_parent != target) 2430 if (target->d_parent != target)
2421 spin_unlock(&target->d_parent->d_lock); 2431 spin_unlock(&target->d_parent->d_lock);
2422 } 2432 }
2423 2433
2424 /* 2434 /*
2425 * When switching names, the actual string doesn't strictly have to 2435 * When switching names, the actual string doesn't strictly have to
2426 * be preserved in the target - because we're dropping the target 2436 * be preserved in the target - because we're dropping the target
2427 * anyway. As such, we can just do a simple memcpy() to copy over 2437 * anyway. As such, we can just do a simple memcpy() to copy over
2428 * the new name before we switch. 2438 * the new name before we switch.
2429 * 2439 *
2430 * Note that we have to be a lot more careful about getting the hash 2440 * Note that we have to be a lot more careful about getting the hash
2431 * switched - we have to switch the hash value properly even if it 2441 * switched - we have to switch the hash value properly even if it
2432 * then no longer matches the actual (corrupted) string of the target. 2442 * then no longer matches the actual (corrupted) string of the target.
2433 * The hash value has to match the hash queue that the dentry is on.. 2443 * The hash value has to match the hash queue that the dentry is on..
2434 */ 2444 */
2435 /* 2445 /*
2436 * __d_move - move a dentry 2446 * __d_move - move a dentry
2437 * @dentry: entry to move 2447 * @dentry: entry to move
2438 * @target: new dentry 2448 * @target: new dentry
2439 * @exchange: exchange the two dentries 2449 * @exchange: exchange the two dentries
2440 * 2450 *
2441 * Update the dcache to reflect the move of a file name. Negative 2451 * Update the dcache to reflect the move of a file name. Negative
2442 * dcache entries should not be moved in this way. Caller must hold 2452 * dcache entries should not be moved in this way. Caller must hold
2443 * rename_lock, the i_mutex of the source and target directories, 2453 * rename_lock, the i_mutex of the source and target directories,
2444 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename(). 2454 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
2445 */ 2455 */
2446 static void __d_move(struct dentry *dentry, struct dentry *target, 2456 static void __d_move(struct dentry *dentry, struct dentry *target,
2447 bool exchange) 2457 bool exchange)
2448 { 2458 {
2449 if (!dentry->d_inode) 2459 if (!dentry->d_inode)
2450 printk(KERN_WARNING "VFS: moving negative dcache entry\n"); 2460 printk(KERN_WARNING "VFS: moving negative dcache entry\n");
2451 2461
2452 BUG_ON(d_ancestor(dentry, target)); 2462 BUG_ON(d_ancestor(dentry, target));
2453 BUG_ON(d_ancestor(target, dentry)); 2463 BUG_ON(d_ancestor(target, dentry));
2454 2464
2455 dentry_lock_for_move(dentry, target); 2465 dentry_lock_for_move(dentry, target);
2456 2466
2457 write_seqcount_begin(&dentry->d_seq); 2467 write_seqcount_begin(&dentry->d_seq);
2458 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED); 2468 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2459 2469
2460 /* __d_drop does write_seqcount_barrier, but they're OK to nest. */ 2470 /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
2461 2471
2462 /* 2472 /*
2463 * Move the dentry to the target hash queue. Don't bother checking 2473 * Move the dentry to the target hash queue. Don't bother checking
2464 * for the same hash queue because of how unlikely it is. 2474 * for the same hash queue because of how unlikely it is.
2465 */ 2475 */
2466 __d_drop(dentry); 2476 __d_drop(dentry);
2467 __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash)); 2477 __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
2468 2478
2469 /* 2479 /*
2470 * Unhash the target (d_delete() is not usable here). If exchanging 2480 * Unhash the target (d_delete() is not usable here). If exchanging
2471 * the two dentries, then rehash onto the other's hash queue. 2481 * the two dentries, then rehash onto the other's hash queue.
2472 */ 2482 */
2473 __d_drop(target); 2483 __d_drop(target);
2474 if (exchange) { 2484 if (exchange) {
2475 __d_rehash(target, 2485 __d_rehash(target,
2476 d_hash(dentry->d_parent, dentry->d_name.hash)); 2486 d_hash(dentry->d_parent, dentry->d_name.hash));
2477 } 2487 }
2478 2488
2479 list_del(&dentry->d_u.d_child); 2489 list_del(&dentry->d_u.d_child);
2480 list_del(&target->d_u.d_child); 2490 list_del(&target->d_u.d_child);
2481 2491
2482 /* Switch the names.. */ 2492 /* Switch the names.. */
2483 switch_names(dentry, target); 2493 switch_names(dentry, target);
2484 swap(dentry->d_name.hash, target->d_name.hash); 2494 swap(dentry->d_name.hash, target->d_name.hash);
2485 2495
2486 /* ... and switch the parents */ 2496 /* ... and switch the parents */
2487 if (IS_ROOT(dentry)) { 2497 if (IS_ROOT(dentry)) {
2488 dentry->d_parent = target->d_parent; 2498 dentry->d_parent = target->d_parent;
2489 target->d_parent = target; 2499 target->d_parent = target;
2490 INIT_LIST_HEAD(&target->d_u.d_child); 2500 INIT_LIST_HEAD(&target->d_u.d_child);
2491 } else { 2501 } else {
2492 swap(dentry->d_parent, target->d_parent); 2502 swap(dentry->d_parent, target->d_parent);
2493 2503
2494 /* And add them back to the (new) parent lists */ 2504 /* And add them back to the (new) parent lists */
2495 list_add(&target->d_u.d_child, &target->d_parent->d_subdirs); 2505 list_add(&target->d_u.d_child, &target->d_parent->d_subdirs);
2496 } 2506 }
2497 2507
2498 list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs); 2508 list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
2499 2509
2500 write_seqcount_end(&target->d_seq); 2510 write_seqcount_end(&target->d_seq);
2501 write_seqcount_end(&dentry->d_seq); 2511 write_seqcount_end(&dentry->d_seq);
2502 2512
2503 dentry_unlock_parents_for_move(dentry, target); 2513 dentry_unlock_parents_for_move(dentry, target);
2504 if (exchange) 2514 if (exchange)
2505 fsnotify_d_move(target); 2515 fsnotify_d_move(target);
2506 spin_unlock(&target->d_lock); 2516 spin_unlock(&target->d_lock);
2507 fsnotify_d_move(dentry); 2517 fsnotify_d_move(dentry);
2508 spin_unlock(&dentry->d_lock); 2518 spin_unlock(&dentry->d_lock);
2509 } 2519 }
2510 2520
2511 /* 2521 /*
2512 * d_move - move a dentry 2522 * d_move - move a dentry
2513 * @dentry: entry to move 2523 * @dentry: entry to move
2514 * @target: new dentry 2524 * @target: new dentry
2515 * 2525 *
2516 * Update the dcache to reflect the move of a file name. Negative 2526 * Update the dcache to reflect the move of a file name. Negative
2517 * dcache entries should not be moved in this way. See the locking 2527 * dcache entries should not be moved in this way. See the locking
2518 * requirements for __d_move. 2528 * requirements for __d_move.
2519 */ 2529 */
2520 void d_move(struct dentry *dentry, struct dentry *target) 2530 void d_move(struct dentry *dentry, struct dentry *target)
2521 { 2531 {
2522 write_seqlock(&rename_lock); 2532 write_seqlock(&rename_lock);
2523 __d_move(dentry, target, false); 2533 __d_move(dentry, target, false);
2524 write_sequnlock(&rename_lock); 2534 write_sequnlock(&rename_lock);
2525 } 2535 }
2526 EXPORT_SYMBOL(d_move); 2536 EXPORT_SYMBOL(d_move);
2527 2537
2528 /* 2538 /*
2529 * d_exchange - exchange two dentries 2539 * d_exchange - exchange two dentries
2530 * @dentry1: first dentry 2540 * @dentry1: first dentry
2531 * @dentry2: second dentry 2541 * @dentry2: second dentry
2532 */ 2542 */
2533 void d_exchange(struct dentry *dentry1, struct dentry *dentry2) 2543 void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
2534 { 2544 {
2535 write_seqlock(&rename_lock); 2545 write_seqlock(&rename_lock);
2536 2546
2537 WARN_ON(!dentry1->d_inode); 2547 WARN_ON(!dentry1->d_inode);
2538 WARN_ON(!dentry2->d_inode); 2548 WARN_ON(!dentry2->d_inode);
2539 WARN_ON(IS_ROOT(dentry1)); 2549 WARN_ON(IS_ROOT(dentry1));
2540 WARN_ON(IS_ROOT(dentry2)); 2550 WARN_ON(IS_ROOT(dentry2));
2541 2551
2542 __d_move(dentry1, dentry2, true); 2552 __d_move(dentry1, dentry2, true);
2543 2553
2544 write_sequnlock(&rename_lock); 2554 write_sequnlock(&rename_lock);
2545 } 2555 }
2546 2556
2547 /** 2557 /**
2548 * d_ancestor - search for an ancestor 2558 * d_ancestor - search for an ancestor
2549 * @p1: ancestor dentry 2559 * @p1: ancestor dentry
2550 * @p2: child dentry 2560 * @p2: child dentry
2551 * 2561 *
2552 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is 2562 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
2553 * an ancestor of p2, else NULL. 2563 * an ancestor of p2, else NULL.
2554 */ 2564 */
2555 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) 2565 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
2556 { 2566 {
2557 struct dentry *p; 2567 struct dentry *p;
2558 2568
2559 for (p = p2; !IS_ROOT(p); p = p->d_parent) { 2569 for (p = p2; !IS_ROOT(p); p = p->d_parent) {
2560 if (p->d_parent == p1) 2570 if (p->d_parent == p1)
2561 return p; 2571 return p;
2562 } 2572 }
2563 return NULL; 2573 return NULL;
2564 } 2574 }
2565 2575
2566 /* 2576 /*
2567 * This helper attempts to cope with remotely renamed directories 2577 * This helper attempts to cope with remotely renamed directories
2568 * 2578 *
2569 * It assumes that the caller is already holding 2579 * It assumes that the caller is already holding
2570 * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock 2580 * dentry->d_parent->d_inode->i_mutex, inode->i_lock and rename_lock
2571 * 2581 *
2572 * Note: If ever the locking in lock_rename() changes, then please 2582 * Note: If ever the locking in lock_rename() changes, then please
2573 * remember to update this too... 2583 * remember to update this too...
2574 */ 2584 */
2575 static struct dentry *__d_unalias(struct inode *inode, 2585 static struct dentry *__d_unalias(struct inode *inode,
2576 struct dentry *dentry, struct dentry *alias) 2586 struct dentry *dentry, struct dentry *alias)
2577 { 2587 {
2578 struct mutex *m1 = NULL, *m2 = NULL; 2588 struct mutex *m1 = NULL, *m2 = NULL;
2579 struct dentry *ret = ERR_PTR(-EBUSY); 2589 struct dentry *ret = ERR_PTR(-EBUSY);
2580 2590
2581 /* If alias and dentry share a parent, then no extra locks required */ 2591 /* If alias and dentry share a parent, then no extra locks required */
2582 if (alias->d_parent == dentry->d_parent) 2592 if (alias->d_parent == dentry->d_parent)
2583 goto out_unalias; 2593 goto out_unalias;
2584 2594
2585 /* See lock_rename() */ 2595 /* See lock_rename() */
2586 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex)) 2596 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
2587 goto out_err; 2597 goto out_err;
2588 m1 = &dentry->d_sb->s_vfs_rename_mutex; 2598 m1 = &dentry->d_sb->s_vfs_rename_mutex;
2589 if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex)) 2599 if (!mutex_trylock(&alias->d_parent->d_inode->i_mutex))
2590 goto out_err; 2600 goto out_err;
2591 m2 = &alias->d_parent->d_inode->i_mutex; 2601 m2 = &alias->d_parent->d_inode->i_mutex;
2592 out_unalias: 2602 out_unalias:
2593 if (likely(!d_mountpoint(alias))) { 2603 if (likely(!d_mountpoint(alias))) {
2594 __d_move(alias, dentry, false); 2604 __d_move(alias, dentry, false);
2595 ret = alias; 2605 ret = alias;
2596 } 2606 }
2597 out_err: 2607 out_err:
2598 spin_unlock(&inode->i_lock); 2608 spin_unlock(&inode->i_lock);
2599 if (m2) 2609 if (m2)
2600 mutex_unlock(m2); 2610 mutex_unlock(m2);
2601 if (m1) 2611 if (m1)
2602 mutex_unlock(m1); 2612 mutex_unlock(m1);
2603 return ret; 2613 return ret;
2604 } 2614 }
2605 2615
2606 /* 2616 /*
2607 * Prepare an anonymous dentry for life in the superblock's dentry tree as a 2617 * Prepare an anonymous dentry for life in the superblock's dentry tree as a
2608 * named dentry in place of the dentry to be replaced. 2618 * named dentry in place of the dentry to be replaced.
2609 * returns with anon->d_lock held! 2619 * returns with anon->d_lock held!
2610 */ 2620 */
2611 static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon) 2621 static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
2612 { 2622 {
2613 struct dentry *dparent; 2623 struct dentry *dparent;
2614 2624
2615 dentry_lock_for_move(anon, dentry); 2625 dentry_lock_for_move(anon, dentry);
2616 2626
2617 write_seqcount_begin(&dentry->d_seq); 2627 write_seqcount_begin(&dentry->d_seq);
2618 write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED); 2628 write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED);
2619 2629
2620 dparent = dentry->d_parent; 2630 dparent = dentry->d_parent;
2621 2631
2622 switch_names(dentry, anon); 2632 switch_names(dentry, anon);
2623 swap(dentry->d_name.hash, anon->d_name.hash); 2633 swap(dentry->d_name.hash, anon->d_name.hash);
2624 2634
2625 dentry->d_parent = dentry; 2635 dentry->d_parent = dentry;
2626 list_del_init(&dentry->d_u.d_child); 2636 list_del_init(&dentry->d_u.d_child);
2627 anon->d_parent = dparent; 2637 anon->d_parent = dparent;
2628 list_move(&anon->d_u.d_child, &dparent->d_subdirs); 2638 list_move(&anon->d_u.d_child, &dparent->d_subdirs);
2629 2639
2630 write_seqcount_end(&dentry->d_seq); 2640 write_seqcount_end(&dentry->d_seq);
2631 write_seqcount_end(&anon->d_seq); 2641 write_seqcount_end(&anon->d_seq);
2632 2642
2633 dentry_unlock_parents_for_move(anon, dentry); 2643 dentry_unlock_parents_for_move(anon, dentry);
2634 spin_unlock(&dentry->d_lock); 2644 spin_unlock(&dentry->d_lock);
2635 2645
2636 /* anon->d_lock still locked, returns locked */ 2646 /* anon->d_lock still locked, returns locked */
2637 } 2647 }
2638 2648
2639 /** 2649 /**
2640 * d_materialise_unique - introduce an inode into the tree 2650 * d_materialise_unique - introduce an inode into the tree
2641 * @dentry: candidate dentry 2651 * @dentry: candidate dentry
2642 * @inode: inode to bind to the dentry, to which aliases may be attached 2652 * @inode: inode to bind to the dentry, to which aliases may be attached
2643 * 2653 *
2644 * Introduces an dentry into the tree, substituting an extant disconnected 2654 * Introduces an dentry into the tree, substituting an extant disconnected
2645 * root directory alias in its place if there is one. Caller must hold the 2655 * root directory alias in its place if there is one. Caller must hold the
2646 * i_mutex of the parent directory. 2656 * i_mutex of the parent directory.
2647 */ 2657 */
2648 struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode) 2658 struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
2649 { 2659 {
2650 struct dentry *actual; 2660 struct dentry *actual;
2651 2661
2652 BUG_ON(!d_unhashed(dentry)); 2662 BUG_ON(!d_unhashed(dentry));
2653 2663
2654 if (!inode) { 2664 if (!inode) {
2655 actual = dentry; 2665 actual = dentry;
2656 __d_instantiate(dentry, NULL); 2666 __d_instantiate(dentry, NULL);
2657 d_rehash(actual); 2667 d_rehash(actual);
2658 goto out_nolock; 2668 goto out_nolock;
2659 } 2669 }
2660 2670
2661 spin_lock(&inode->i_lock); 2671 spin_lock(&inode->i_lock);
2662 2672
2663 if (S_ISDIR(inode->i_mode)) { 2673 if (S_ISDIR(inode->i_mode)) {
2664 struct dentry *alias; 2674 struct dentry *alias;
2665 2675
2666 /* Does an aliased dentry already exist? */ 2676 /* Does an aliased dentry already exist? */
2667 alias = __d_find_alias(inode, 0); 2677 alias = __d_find_alias(inode, 0);
2668 if (alias) { 2678 if (alias) {
2669 actual = alias; 2679 actual = alias;
2670 write_seqlock(&rename_lock); 2680 write_seqlock(&rename_lock);
2671 2681
2672 if (d_ancestor(alias, dentry)) { 2682 if (d_ancestor(alias, dentry)) {
2673 /* Check for loops */ 2683 /* Check for loops */
2674 actual = ERR_PTR(-ELOOP); 2684 actual = ERR_PTR(-ELOOP);
2675 spin_unlock(&inode->i_lock); 2685 spin_unlock(&inode->i_lock);
2676 } else if (IS_ROOT(alias)) { 2686 } else if (IS_ROOT(alias)) {
2677 /* Is this an anonymous mountpoint that we 2687 /* Is this an anonymous mountpoint that we
2678 * could splice into our tree? */ 2688 * could splice into our tree? */
2679 __d_materialise_dentry(dentry, alias); 2689 __d_materialise_dentry(dentry, alias);
2680 write_sequnlock(&rename_lock); 2690 write_sequnlock(&rename_lock);
2681 __d_drop(alias); 2691 __d_drop(alias);
2682 goto found; 2692 goto found;
2683 } else { 2693 } else {
2684 /* Nope, but we must(!) avoid directory 2694 /* Nope, but we must(!) avoid directory
2685 * aliasing. This drops inode->i_lock */ 2695 * aliasing. This drops inode->i_lock */
2686 actual = __d_unalias(inode, dentry, alias); 2696 actual = __d_unalias(inode, dentry, alias);
2687 } 2697 }
2688 write_sequnlock(&rename_lock); 2698 write_sequnlock(&rename_lock);
2689 if (IS_ERR(actual)) { 2699 if (IS_ERR(actual)) {
2690 if (PTR_ERR(actual) == -ELOOP) 2700 if (PTR_ERR(actual) == -ELOOP)
2691 pr_warn_ratelimited( 2701 pr_warn_ratelimited(
2692 "VFS: Lookup of '%s' in %s %s" 2702 "VFS: Lookup of '%s' in %s %s"
2693 " would have caused loop\n", 2703 " would have caused loop\n",
2694 dentry->d_name.name, 2704 dentry->d_name.name,
2695 inode->i_sb->s_type->name, 2705 inode->i_sb->s_type->name,
2696 inode->i_sb->s_id); 2706 inode->i_sb->s_id);
2697 dput(alias); 2707 dput(alias);
2698 } 2708 }
2699 goto out_nolock; 2709 goto out_nolock;
2700 } 2710 }
2701 } 2711 }
2702 2712
2703 /* Add a unique reference */ 2713 /* Add a unique reference */
2704 actual = __d_instantiate_unique(dentry, inode); 2714 actual = __d_instantiate_unique(dentry, inode);
2705 if (!actual) 2715 if (!actual)
2706 actual = dentry; 2716 actual = dentry;
2707 else 2717 else
2708 BUG_ON(!d_unhashed(actual)); 2718 BUG_ON(!d_unhashed(actual));
2709 2719
2710 spin_lock(&actual->d_lock); 2720 spin_lock(&actual->d_lock);
2711 found: 2721 found:
2712 _d_rehash(actual); 2722 _d_rehash(actual);
2713 spin_unlock(&actual->d_lock); 2723 spin_unlock(&actual->d_lock);
2714 spin_unlock(&inode->i_lock); 2724 spin_unlock(&inode->i_lock);
2715 out_nolock: 2725 out_nolock:
2716 if (actual == dentry) { 2726 if (actual == dentry) {
2717 security_d_instantiate(dentry, inode); 2727 security_d_instantiate(dentry, inode);
2718 return NULL; 2728 return NULL;
2719 } 2729 }
2720 2730
2721 iput(inode); 2731 iput(inode);
2722 return actual; 2732 return actual;
2723 } 2733 }
2724 EXPORT_SYMBOL_GPL(d_materialise_unique); 2734 EXPORT_SYMBOL_GPL(d_materialise_unique);
2725 2735
2726 static int prepend(char **buffer, int *buflen, const char *str, int namelen) 2736 static int prepend(char **buffer, int *buflen, const char *str, int namelen)
2727 { 2737 {
2728 *buflen -= namelen; 2738 *buflen -= namelen;
2729 if (*buflen < 0) 2739 if (*buflen < 0)
2730 return -ENAMETOOLONG; 2740 return -ENAMETOOLONG;
2731 *buffer -= namelen; 2741 *buffer -= namelen;
2732 memcpy(*buffer, str, namelen); 2742 memcpy(*buffer, str, namelen);
2733 return 0; 2743 return 0;
2734 } 2744 }
2735 2745
2736 /** 2746 /**
2737 * prepend_name - prepend a pathname in front of current buffer pointer 2747 * prepend_name - prepend a pathname in front of current buffer pointer
2738 * @buffer: buffer pointer 2748 * @buffer: buffer pointer
2739 * @buflen: allocated length of the buffer 2749 * @buflen: allocated length of the buffer
2740 * @name: name string and length qstr structure 2750 * @name: name string and length qstr structure
2741 * 2751 *
2742 * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to 2752 * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
2743 * make sure that either the old or the new name pointer and length are 2753 * make sure that either the old or the new name pointer and length are
2744 * fetched. However, there may be mismatch between length and pointer. 2754 * fetched. However, there may be mismatch between length and pointer.
2745 * The length cannot be trusted, we need to copy it byte-by-byte until 2755 * The length cannot be trusted, we need to copy it byte-by-byte until
2746 * the length is reached or a null byte is found. It also prepends "/" at 2756 * the length is reached or a null byte is found. It also prepends "/" at
2747 * the beginning of the name. The sequence number check at the caller will 2757 * the beginning of the name. The sequence number check at the caller will
2748 * retry it again when a d_move() does happen. So any garbage in the buffer 2758 * retry it again when a d_move() does happen. So any garbage in the buffer
2749 * due to mismatched pointer and length will be discarded. 2759 * due to mismatched pointer and length will be discarded.
2750 */ 2760 */
2751 static int prepend_name(char **buffer, int *buflen, struct qstr *name) 2761 static int prepend_name(char **buffer, int *buflen, struct qstr *name)
2752 { 2762 {
2753 const char *dname = ACCESS_ONCE(name->name); 2763 const char *dname = ACCESS_ONCE(name->name);
2754 u32 dlen = ACCESS_ONCE(name->len); 2764 u32 dlen = ACCESS_ONCE(name->len);
2755 char *p; 2765 char *p;
2756 2766
2757 *buflen -= dlen + 1; 2767 *buflen -= dlen + 1;
2758 if (*buflen < 0) 2768 if (*buflen < 0)
2759 return -ENAMETOOLONG; 2769 return -ENAMETOOLONG;
2760 p = *buffer -= dlen + 1; 2770 p = *buffer -= dlen + 1;
2761 *p++ = '/'; 2771 *p++ = '/';
2762 while (dlen--) { 2772 while (dlen--) {
2763 char c = *dname++; 2773 char c = *dname++;
2764 if (!c) 2774 if (!c)
2765 break; 2775 break;
2766 *p++ = c; 2776 *p++ = c;
2767 } 2777 }
2768 return 0; 2778 return 0;
2769 } 2779 }
2770 2780
2771 /** 2781 /**
2772 * prepend_path - Prepend path string to a buffer 2782 * prepend_path - Prepend path string to a buffer
2773 * @path: the dentry/vfsmount to report 2783 * @path: the dentry/vfsmount to report
2774 * @root: root vfsmnt/dentry 2784 * @root: root vfsmnt/dentry
2775 * @buffer: pointer to the end of the buffer 2785 * @buffer: pointer to the end of the buffer
2776 * @buflen: pointer to buffer length 2786 * @buflen: pointer to buffer length
2777 * 2787 *
2778 * The function will first try to write out the pathname without taking any 2788 * The function will first try to write out the pathname without taking any
2779 * lock other than the RCU read lock to make sure that dentries won't go away. 2789 * lock other than the RCU read lock to make sure that dentries won't go away.
2780 * It only checks the sequence number of the global rename_lock as any change 2790 * It only checks the sequence number of the global rename_lock as any change
2781 * in the dentry's d_seq will be preceded by changes in the rename_lock 2791 * in the dentry's d_seq will be preceded by changes in the rename_lock
2782 * sequence number. If the sequence number had been changed, it will restart 2792 * sequence number. If the sequence number had been changed, it will restart
2783 * the whole pathname back-tracing sequence again by taking the rename_lock. 2793 * the whole pathname back-tracing sequence again by taking the rename_lock.
2784 * In this case, there is no need to take the RCU read lock as the recursive 2794 * In this case, there is no need to take the RCU read lock as the recursive
2785 * parent pointer references will keep the dentry chain alive as long as no 2795 * parent pointer references will keep the dentry chain alive as long as no
2786 * rename operation is performed. 2796 * rename operation is performed.
2787 */ 2797 */
2788 static int prepend_path(const struct path *path, 2798 static int prepend_path(const struct path *path,
2789 const struct path *root, 2799 const struct path *root,
2790 char **buffer, int *buflen) 2800 char **buffer, int *buflen)
2791 { 2801 {
2792 struct dentry *dentry; 2802 struct dentry *dentry;
2793 struct vfsmount *vfsmnt; 2803 struct vfsmount *vfsmnt;
2794 struct mount *mnt; 2804 struct mount *mnt;
2795 int error = 0; 2805 int error = 0;
2796 unsigned seq, m_seq = 0; 2806 unsigned seq, m_seq = 0;
2797 char *bptr; 2807 char *bptr;
2798 int blen; 2808 int blen;
2799 2809
2800 rcu_read_lock(); 2810 rcu_read_lock();
2801 restart_mnt: 2811 restart_mnt:
2802 read_seqbegin_or_lock(&mount_lock, &m_seq); 2812 read_seqbegin_or_lock(&mount_lock, &m_seq);
2803 seq = 0; 2813 seq = 0;
2804 rcu_read_lock(); 2814 rcu_read_lock();
2805 restart: 2815 restart:
2806 bptr = *buffer; 2816 bptr = *buffer;
2807 blen = *buflen; 2817 blen = *buflen;
2808 error = 0; 2818 error = 0;
2809 dentry = path->dentry; 2819 dentry = path->dentry;
2810 vfsmnt = path->mnt; 2820 vfsmnt = path->mnt;
2811 mnt = real_mount(vfsmnt); 2821 mnt = real_mount(vfsmnt);
2812 read_seqbegin_or_lock(&rename_lock, &seq); 2822 read_seqbegin_or_lock(&rename_lock, &seq);
2813 while (dentry != root->dentry || vfsmnt != root->mnt) { 2823 while (dentry != root->dentry || vfsmnt != root->mnt) {
2814 struct dentry * parent; 2824 struct dentry * parent;
2815 2825
2816 if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) { 2826 if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
2817 struct mount *parent = ACCESS_ONCE(mnt->mnt_parent); 2827 struct mount *parent = ACCESS_ONCE(mnt->mnt_parent);
2818 /* Global root? */ 2828 /* Global root? */
2819 if (mnt != parent) { 2829 if (mnt != parent) {
2820 dentry = ACCESS_ONCE(mnt->mnt_mountpoint); 2830 dentry = ACCESS_ONCE(mnt->mnt_mountpoint);
2821 mnt = parent; 2831 mnt = parent;
2822 vfsmnt = &mnt->mnt; 2832 vfsmnt = &mnt->mnt;
2823 continue; 2833 continue;
2824 } 2834 }
2825 /* 2835 /*
2826 * Filesystems needing to implement special "root names" 2836 * Filesystems needing to implement special "root names"
2827 * should do so with ->d_dname() 2837 * should do so with ->d_dname()
2828 */ 2838 */
2829 if (IS_ROOT(dentry) && 2839 if (IS_ROOT(dentry) &&
2830 (dentry->d_name.len != 1 || 2840 (dentry->d_name.len != 1 ||
2831 dentry->d_name.name[0] != '/')) { 2841 dentry->d_name.name[0] != '/')) {
2832 WARN(1, "Root dentry has weird name <%.*s>\n", 2842 WARN(1, "Root dentry has weird name <%.*s>\n",
2833 (int) dentry->d_name.len, 2843 (int) dentry->d_name.len,
2834 dentry->d_name.name); 2844 dentry->d_name.name);
2835 } 2845 }
2836 if (!error) 2846 if (!error)
2837 error = is_mounted(vfsmnt) ? 1 : 2; 2847 error = is_mounted(vfsmnt) ? 1 : 2;
2838 break; 2848 break;
2839 } 2849 }
2840 parent = dentry->d_parent; 2850 parent = dentry->d_parent;
2841 prefetch(parent); 2851 prefetch(parent);
2842 error = prepend_name(&bptr, &blen, &dentry->d_name); 2852 error = prepend_name(&bptr, &blen, &dentry->d_name);
2843 if (error) 2853 if (error)
2844 break; 2854 break;
2845 2855
2846 dentry = parent; 2856 dentry = parent;
2847 } 2857 }
2848 if (!(seq & 1)) 2858 if (!(seq & 1))
2849 rcu_read_unlock(); 2859 rcu_read_unlock();
2850 if (need_seqretry(&rename_lock, seq)) { 2860 if (need_seqretry(&rename_lock, seq)) {
2851 seq = 1; 2861 seq = 1;
2852 goto restart; 2862 goto restart;
2853 } 2863 }
2854 done_seqretry(&rename_lock, seq); 2864 done_seqretry(&rename_lock, seq);
2855 2865
2856 if (!(m_seq & 1)) 2866 if (!(m_seq & 1))
2857 rcu_read_unlock(); 2867 rcu_read_unlock();
2858 if (need_seqretry(&mount_lock, m_seq)) { 2868 if (need_seqretry(&mount_lock, m_seq)) {
2859 m_seq = 1; 2869 m_seq = 1;
2860 goto restart_mnt; 2870 goto restart_mnt;
2861 } 2871 }
2862 done_seqretry(&mount_lock, m_seq); 2872 done_seqretry(&mount_lock, m_seq);
2863 2873
2864 if (error >= 0 && bptr == *buffer) { 2874 if (error >= 0 && bptr == *buffer) {
2865 if (--blen < 0) 2875 if (--blen < 0)
2866 error = -ENAMETOOLONG; 2876 error = -ENAMETOOLONG;
2867 else 2877 else
2868 *--bptr = '/'; 2878 *--bptr = '/';
2869 } 2879 }
2870 *buffer = bptr; 2880 *buffer = bptr;
2871 *buflen = blen; 2881 *buflen = blen;
2872 return error; 2882 return error;
2873 } 2883 }
2874 2884
2875 /** 2885 /**
2876 * __d_path - return the path of a dentry 2886 * __d_path - return the path of a dentry
2877 * @path: the dentry/vfsmount to report 2887 * @path: the dentry/vfsmount to report
2878 * @root: root vfsmnt/dentry 2888 * @root: root vfsmnt/dentry
2879 * @buf: buffer to return value in 2889 * @buf: buffer to return value in
2880 * @buflen: buffer length 2890 * @buflen: buffer length
2881 * 2891 *
2882 * Convert a dentry into an ASCII path name. 2892 * Convert a dentry into an ASCII path name.
2883 * 2893 *
2884 * Returns a pointer into the buffer or an error code if the 2894 * Returns a pointer into the buffer or an error code if the
2885 * path was too long. 2895 * path was too long.
2886 * 2896 *
2887 * "buflen" should be positive. 2897 * "buflen" should be positive.
2888 * 2898 *
2889 * If the path is not reachable from the supplied root, return %NULL. 2899 * If the path is not reachable from the supplied root, return %NULL.
2890 */ 2900 */
2891 char *__d_path(const struct path *path, 2901 char *__d_path(const struct path *path,
2892 const struct path *root, 2902 const struct path *root,
2893 char *buf, int buflen) 2903 char *buf, int buflen)
2894 { 2904 {
2895 char *res = buf + buflen; 2905 char *res = buf + buflen;
2896 int error; 2906 int error;
2897 2907
2898 prepend(&res, &buflen, "\0", 1); 2908 prepend(&res, &buflen, "\0", 1);
2899 error = prepend_path(path, root, &res, &buflen); 2909 error = prepend_path(path, root, &res, &buflen);
2900 2910
2901 if (error < 0) 2911 if (error < 0)
2902 return ERR_PTR(error); 2912 return ERR_PTR(error);
2903 if (error > 0) 2913 if (error > 0)
2904 return NULL; 2914 return NULL;
2905 return res; 2915 return res;
2906 } 2916 }
2907 2917
2908 char *d_absolute_path(const struct path *path, 2918 char *d_absolute_path(const struct path *path,
2909 char *buf, int buflen) 2919 char *buf, int buflen)
2910 { 2920 {
2911 struct path root = {}; 2921 struct path root = {};
2912 char *res = buf + buflen; 2922 char *res = buf + buflen;
2913 int error; 2923 int error;
2914 2924
2915 prepend(&res, &buflen, "\0", 1); 2925 prepend(&res, &buflen, "\0", 1);
2916 error = prepend_path(path, &root, &res, &buflen); 2926 error = prepend_path(path, &root, &res, &buflen);
2917 2927
2918 if (error > 1) 2928 if (error > 1)
2919 error = -EINVAL; 2929 error = -EINVAL;
2920 if (error < 0) 2930 if (error < 0)
2921 return ERR_PTR(error); 2931 return ERR_PTR(error);
2922 return res; 2932 return res;
2923 } 2933 }
2924 2934
2925 /* 2935 /*
2926 * same as __d_path but appends "(deleted)" for unlinked files. 2936 * same as __d_path but appends "(deleted)" for unlinked files.
2927 */ 2937 */
2928 static int path_with_deleted(const struct path *path, 2938 static int path_with_deleted(const struct path *path,
2929 const struct path *root, 2939 const struct path *root,
2930 char **buf, int *buflen) 2940 char **buf, int *buflen)
2931 { 2941 {
2932 prepend(buf, buflen, "\0", 1); 2942 prepend(buf, buflen, "\0", 1);
2933 if (d_unlinked(path->dentry)) { 2943 if (d_unlinked(path->dentry)) {
2934 int error = prepend(buf, buflen, " (deleted)", 10); 2944 int error = prepend(buf, buflen, " (deleted)", 10);
2935 if (error) 2945 if (error)
2936 return error; 2946 return error;
2937 } 2947 }
2938 2948
2939 return prepend_path(path, root, buf, buflen); 2949 return prepend_path(path, root, buf, buflen);
2940 } 2950 }
2941 2951
2942 static int prepend_unreachable(char **buffer, int *buflen) 2952 static int prepend_unreachable(char **buffer, int *buflen)
2943 { 2953 {
2944 return prepend(buffer, buflen, "(unreachable)", 13); 2954 return prepend(buffer, buflen, "(unreachable)", 13);
2945 } 2955 }
2946 2956
2947 static void get_fs_root_rcu(struct fs_struct *fs, struct path *root) 2957 static void get_fs_root_rcu(struct fs_struct *fs, struct path *root)
2948 { 2958 {
2949 unsigned seq; 2959 unsigned seq;
2950 2960
2951 do { 2961 do {
2952 seq = read_seqcount_begin(&fs->seq); 2962 seq = read_seqcount_begin(&fs->seq);
2953 *root = fs->root; 2963 *root = fs->root;
2954 } while (read_seqcount_retry(&fs->seq, seq)); 2964 } while (read_seqcount_retry(&fs->seq, seq));
2955 } 2965 }
2956 2966
2957 /** 2967 /**
2958 * d_path - return the path of a dentry 2968 * d_path - return the path of a dentry
2959 * @path: path to report 2969 * @path: path to report
2960 * @buf: buffer to return value in 2970 * @buf: buffer to return value in
2961 * @buflen: buffer length 2971 * @buflen: buffer length
2962 * 2972 *
2963 * Convert a dentry into an ASCII path name. If the entry has been deleted 2973 * Convert a dentry into an ASCII path name. If the entry has been deleted
2964 * the string " (deleted)" is appended. Note that this is ambiguous. 2974 * the string " (deleted)" is appended. Note that this is ambiguous.
2965 * 2975 *
2966 * Returns a pointer into the buffer or an error code if the path was 2976 * Returns a pointer into the buffer or an error code if the path was
2967 * too long. Note: Callers should use the returned pointer, not the passed 2977 * too long. Note: Callers should use the returned pointer, not the passed
2968 * in buffer, to use the name! The implementation often starts at an offset 2978 * in buffer, to use the name! The implementation often starts at an offset
2969 * into the buffer, and may leave 0 bytes at the start. 2979 * into the buffer, and may leave 0 bytes at the start.
2970 * 2980 *
2971 * "buflen" should be positive. 2981 * "buflen" should be positive.
2972 */ 2982 */
2973 char *d_path(const struct path *path, char *buf, int buflen) 2983 char *d_path(const struct path *path, char *buf, int buflen)
2974 { 2984 {
2975 char *res = buf + buflen; 2985 char *res = buf + buflen;
2976 struct path root; 2986 struct path root;
2977 int error; 2987 int error;
2978 2988
2979 /* 2989 /*
2980 * We have various synthetic filesystems that never get mounted. On 2990 * We have various synthetic filesystems that never get mounted. On
2981 * these filesystems dentries are never used for lookup purposes, and 2991 * these filesystems dentries are never used for lookup purposes, and
2982 * thus don't need to be hashed. They also don't need a name until a 2992 * thus don't need to be hashed. They also don't need a name until a
2983 * user wants to identify the object in /proc/pid/fd/. The little hack 2993 * user wants to identify the object in /proc/pid/fd/. The little hack
2984 * below allows us to generate a name for these objects on demand: 2994 * below allows us to generate a name for these objects on demand:
2985 * 2995 *
2986 * Some pseudo inodes are mountable. When they are mounted 2996 * Some pseudo inodes are mountable. When they are mounted
2987 * path->dentry == path->mnt->mnt_root. In that case don't call d_dname 2997 * path->dentry == path->mnt->mnt_root. In that case don't call d_dname
2988 * and instead have d_path return the mounted path. 2998 * and instead have d_path return the mounted path.
2989 */ 2999 */
2990 if (path->dentry->d_op && path->dentry->d_op->d_dname && 3000 if (path->dentry->d_op && path->dentry->d_op->d_dname &&
2991 (!IS_ROOT(path->dentry) || path->dentry != path->mnt->mnt_root)) 3001 (!IS_ROOT(path->dentry) || path->dentry != path->mnt->mnt_root))
2992 return path->dentry->d_op->d_dname(path->dentry, buf, buflen); 3002 return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
2993 3003
2994 rcu_read_lock(); 3004 rcu_read_lock();
2995 get_fs_root_rcu(current->fs, &root); 3005 get_fs_root_rcu(current->fs, &root);
2996 error = path_with_deleted(path, &root, &res, &buflen); 3006 error = path_with_deleted(path, &root, &res, &buflen);
2997 rcu_read_unlock(); 3007 rcu_read_unlock();
2998 3008
2999 if (error < 0) 3009 if (error < 0)
3000 res = ERR_PTR(error); 3010 res = ERR_PTR(error);
3001 return res; 3011 return res;
3002 } 3012 }
3003 EXPORT_SYMBOL(d_path); 3013 EXPORT_SYMBOL(d_path);
3004 3014
3005 /* 3015 /*
3006 * Helper function for dentry_operations.d_dname() members 3016 * Helper function for dentry_operations.d_dname() members
3007 */ 3017 */
3008 char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen, 3018 char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
3009 const char *fmt, ...) 3019 const char *fmt, ...)
3010 { 3020 {
3011 va_list args; 3021 va_list args;
3012 char temp[64]; 3022 char temp[64];
3013 int sz; 3023 int sz;
3014 3024
3015 va_start(args, fmt); 3025 va_start(args, fmt);
3016 sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1; 3026 sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1;
3017 va_end(args); 3027 va_end(args);
3018 3028
3019 if (sz > sizeof(temp) || sz > buflen) 3029 if (sz > sizeof(temp) || sz > buflen)
3020 return ERR_PTR(-ENAMETOOLONG); 3030 return ERR_PTR(-ENAMETOOLONG);
3021 3031
3022 buffer += buflen - sz; 3032 buffer += buflen - sz;
3023 return memcpy(buffer, temp, sz); 3033 return memcpy(buffer, temp, sz);
3024 } 3034 }
3025 3035
3026 char *simple_dname(struct dentry *dentry, char *buffer, int buflen) 3036 char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
3027 { 3037 {
3028 char *end = buffer + buflen; 3038 char *end = buffer + buflen;
3029 /* these dentries are never renamed, so d_lock is not needed */ 3039 /* these dentries are never renamed, so d_lock is not needed */
3030 if (prepend(&end, &buflen, " (deleted)", 11) || 3040 if (prepend(&end, &buflen, " (deleted)", 11) ||
3031 prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) || 3041 prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
3032 prepend(&end, &buflen, "/", 1)) 3042 prepend(&end, &buflen, "/", 1))
3033 end = ERR_PTR(-ENAMETOOLONG); 3043 end = ERR_PTR(-ENAMETOOLONG);
3034 return end; 3044 return end;
3035 } 3045 }
3036 EXPORT_SYMBOL(simple_dname); 3046 EXPORT_SYMBOL(simple_dname);
3037 3047
3038 /* 3048 /*
3039 * Write full pathname from the root of the filesystem into the buffer. 3049 * Write full pathname from the root of the filesystem into the buffer.
3040 */ 3050 */
3041 static char *__dentry_path(struct dentry *d, char *buf, int buflen) 3051 static char *__dentry_path(struct dentry *d, char *buf, int buflen)
3042 { 3052 {
3043 struct dentry *dentry; 3053 struct dentry *dentry;
3044 char *end, *retval; 3054 char *end, *retval;
3045 int len, seq = 0; 3055 int len, seq = 0;
3046 int error = 0; 3056 int error = 0;
3047 3057
3048 if (buflen < 2) 3058 if (buflen < 2)
3049 goto Elong; 3059 goto Elong;
3050 3060
3051 rcu_read_lock(); 3061 rcu_read_lock();
3052 restart: 3062 restart:
3053 dentry = d; 3063 dentry = d;
3054 end = buf + buflen; 3064 end = buf + buflen;
3055 len = buflen; 3065 len = buflen;
3056 prepend(&end, &len, "\0", 1); 3066 prepend(&end, &len, "\0", 1);
3057 /* Get '/' right */ 3067 /* Get '/' right */
3058 retval = end-1; 3068 retval = end-1;
3059 *retval = '/'; 3069 *retval = '/';
3060 read_seqbegin_or_lock(&rename_lock, &seq); 3070 read_seqbegin_or_lock(&rename_lock, &seq);
3061 while (!IS_ROOT(dentry)) { 3071 while (!IS_ROOT(dentry)) {
3062 struct dentry *parent = dentry->d_parent; 3072 struct dentry *parent = dentry->d_parent;
3063 3073
3064 prefetch(parent); 3074 prefetch(parent);
3065 error = prepend_name(&end, &len, &dentry->d_name); 3075 error = prepend_name(&end, &len, &dentry->d_name);
3066 if (error) 3076 if (error)
3067 break; 3077 break;
3068 3078
3069 retval = end; 3079 retval = end;
3070 dentry = parent; 3080 dentry = parent;
3071 } 3081 }
3072 if (!(seq & 1)) 3082 if (!(seq & 1))
3073 rcu_read_unlock(); 3083 rcu_read_unlock();
3074 if (need_seqretry(&rename_lock, seq)) { 3084 if (need_seqretry(&rename_lock, seq)) {
3075 seq = 1; 3085 seq = 1;
3076 goto restart; 3086 goto restart;
3077 } 3087 }
3078 done_seqretry(&rename_lock, seq); 3088 done_seqretry(&rename_lock, seq);
3079 if (error) 3089 if (error)
3080 goto Elong; 3090 goto Elong;
3081 return retval; 3091 return retval;
3082 Elong: 3092 Elong:
3083 return ERR_PTR(-ENAMETOOLONG); 3093 return ERR_PTR(-ENAMETOOLONG);
3084 } 3094 }
3085 3095
3086 char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen) 3096 char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
3087 { 3097 {
3088 return __dentry_path(dentry, buf, buflen); 3098 return __dentry_path(dentry, buf, buflen);
3089 } 3099 }
3090 EXPORT_SYMBOL(dentry_path_raw); 3100 EXPORT_SYMBOL(dentry_path_raw);
3091 3101
3092 char *dentry_path(struct dentry *dentry, char *buf, int buflen) 3102 char *dentry_path(struct dentry *dentry, char *buf, int buflen)
3093 { 3103 {
3094 char *p = NULL; 3104 char *p = NULL;
3095 char *retval; 3105 char *retval;
3096 3106
3097 if (d_unlinked(dentry)) { 3107 if (d_unlinked(dentry)) {
3098 p = buf + buflen; 3108 p = buf + buflen;
3099 if (prepend(&p, &buflen, "//deleted", 10) != 0) 3109 if (prepend(&p, &buflen, "//deleted", 10) != 0)
3100 goto Elong; 3110 goto Elong;
3101 buflen++; 3111 buflen++;
3102 } 3112 }
3103 retval = __dentry_path(dentry, buf, buflen); 3113 retval = __dentry_path(dentry, buf, buflen);
3104 if (!IS_ERR(retval) && p) 3114 if (!IS_ERR(retval) && p)
3105 *p = '/'; /* restore '/' overriden with '\0' */ 3115 *p = '/'; /* restore '/' overriden with '\0' */
3106 return retval; 3116 return retval;
3107 Elong: 3117 Elong:
3108 return ERR_PTR(-ENAMETOOLONG); 3118 return ERR_PTR(-ENAMETOOLONG);
3109 } 3119 }
3110 3120
3111 static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root, 3121 static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
3112 struct path *pwd) 3122 struct path *pwd)
3113 { 3123 {
3114 unsigned seq; 3124 unsigned seq;
3115 3125
3116 do { 3126 do {
3117 seq = read_seqcount_begin(&fs->seq); 3127 seq = read_seqcount_begin(&fs->seq);
3118 *root = fs->root; 3128 *root = fs->root;
3119 *pwd = fs->pwd; 3129 *pwd = fs->pwd;
3120 } while (read_seqcount_retry(&fs->seq, seq)); 3130 } while (read_seqcount_retry(&fs->seq, seq));
3121 } 3131 }
3122 3132
3123 /* 3133 /*
3124 * NOTE! The user-level library version returns a 3134 * NOTE! The user-level library version returns a
3125 * character pointer. The kernel system call just 3135 * character pointer. The kernel system call just
3126 * returns the length of the buffer filled (which 3136 * returns the length of the buffer filled (which
3127 * includes the ending '\0' character), or a negative 3137 * includes the ending '\0' character), or a negative
3128 * error value. So libc would do something like 3138 * error value. So libc would do something like
3129 * 3139 *
3130 * char *getcwd(char * buf, size_t size) 3140 * char *getcwd(char * buf, size_t size)
3131 * { 3141 * {
3132 * int retval; 3142 * int retval;
3133 * 3143 *
3134 * retval = sys_getcwd(buf, size); 3144 * retval = sys_getcwd(buf, size);
3135 * if (retval >= 0) 3145 * if (retval >= 0)
3136 * return buf; 3146 * return buf;
3137 * errno = -retval; 3147 * errno = -retval;
3138 * return NULL; 3148 * return NULL;
3139 * } 3149 * }
3140 */ 3150 */
3141 SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size) 3151 SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
3142 { 3152 {
3143 int error; 3153 int error;
3144 struct path pwd, root; 3154 struct path pwd, root;
3145 char *page = __getname(); 3155 char *page = __getname();
3146 3156
3147 if (!page) 3157 if (!page)
3148 return -ENOMEM; 3158 return -ENOMEM;
3149 3159
3150 rcu_read_lock(); 3160 rcu_read_lock();
3151 get_fs_root_and_pwd_rcu(current->fs, &root, &pwd); 3161 get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
3152 3162
3153 error = -ENOENT; 3163 error = -ENOENT;
3154 if (!d_unlinked(pwd.dentry)) { 3164 if (!d_unlinked(pwd.dentry)) {
3155 unsigned long len; 3165 unsigned long len;
3156 char *cwd = page + PATH_MAX; 3166 char *cwd = page + PATH_MAX;
3157 int buflen = PATH_MAX; 3167 int buflen = PATH_MAX;
3158 3168
3159 prepend(&cwd, &buflen, "\0", 1); 3169 prepend(&cwd, &buflen, "\0", 1);
3160 error = prepend_path(&pwd, &root, &cwd, &buflen); 3170 error = prepend_path(&pwd, &root, &cwd, &buflen);
3161 rcu_read_unlock(); 3171 rcu_read_unlock();
3162 3172
3163 if (error < 0) 3173 if (error < 0)
3164 goto out; 3174 goto out;
3165 3175
3166 /* Unreachable from current root */ 3176 /* Unreachable from current root */
3167 if (error > 0) { 3177 if (error > 0) {
3168 error = prepend_unreachable(&cwd, &buflen); 3178 error = prepend_unreachable(&cwd, &buflen);
3169 if (error) 3179 if (error)
3170 goto out; 3180 goto out;
3171 } 3181 }
3172 3182
3173 error = -ERANGE; 3183 error = -ERANGE;
3174 len = PATH_MAX + page - cwd; 3184 len = PATH_MAX + page - cwd;
3175 if (len <= size) { 3185 if (len <= size) {
3176 error = len; 3186 error = len;
3177 if (copy_to_user(buf, cwd, len)) 3187 if (copy_to_user(buf, cwd, len))
3178 error = -EFAULT; 3188 error = -EFAULT;
3179 } 3189 }
3180 } else { 3190 } else {
3181 rcu_read_unlock(); 3191 rcu_read_unlock();
3182 } 3192 }
3183 3193
3184 out: 3194 out:
3185 __putname(page); 3195 __putname(page);
3186 return error; 3196 return error;
3187 } 3197 }
3188 3198
3189 /* 3199 /*
3190 * Test whether new_dentry is a subdirectory of old_dentry. 3200 * Test whether new_dentry is a subdirectory of old_dentry.
3191 * 3201 *
3192 * Trivially implemented using the dcache structure 3202 * Trivially implemented using the dcache structure
3193 */ 3203 */
3194 3204
3195 /** 3205 /**
3196 * is_subdir - is new dentry a subdirectory of old_dentry 3206 * is_subdir - is new dentry a subdirectory of old_dentry
3197 * @new_dentry: new dentry 3207 * @new_dentry: new dentry
3198 * @old_dentry: old dentry 3208 * @old_dentry: old dentry
3199 * 3209 *
3200 * Returns 1 if new_dentry is a subdirectory of the parent (at any depth). 3210 * Returns 1 if new_dentry is a subdirectory of the parent (at any depth).
3201 * Returns 0 otherwise. 3211 * Returns 0 otherwise.
3202 * Caller must ensure that "new_dentry" is pinned before calling is_subdir() 3212 * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
3203 */ 3213 */
3204 3214
3205 int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) 3215 int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
3206 { 3216 {
3207 int result; 3217 int result;
3208 unsigned seq; 3218 unsigned seq;
3209 3219
3210 if (new_dentry == old_dentry) 3220 if (new_dentry == old_dentry)
3211 return 1; 3221 return 1;
3212 3222
3213 do { 3223 do {
3214 /* for restarting inner loop in case of seq retry */ 3224 /* for restarting inner loop in case of seq retry */
3215 seq = read_seqbegin(&rename_lock); 3225 seq = read_seqbegin(&rename_lock);
3216 /* 3226 /*
3217 * Need rcu_readlock to protect against the d_parent trashing 3227 * Need rcu_readlock to protect against the d_parent trashing
3218 * due to d_move 3228 * due to d_move
3219 */ 3229 */
3220 rcu_read_lock(); 3230 rcu_read_lock();
3221 if (d_ancestor(old_dentry, new_dentry)) 3231 if (d_ancestor(old_dentry, new_dentry))
3222 result = 1; 3232 result = 1;
3223 else 3233 else
3224 result = 0; 3234 result = 0;
3225 rcu_read_unlock(); 3235 rcu_read_unlock();
3226 } while (read_seqretry(&rename_lock, seq)); 3236 } while (read_seqretry(&rename_lock, seq));
3227 3237
3228 return result; 3238 return result;
3229 } 3239 }
3230 3240
3231 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry) 3241 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
3232 { 3242 {
3233 struct dentry *root = data; 3243 struct dentry *root = data;
3234 if (dentry != root) { 3244 if (dentry != root) {
3235 if (d_unhashed(dentry) || !dentry->d_inode) 3245 if (d_unhashed(dentry) || !dentry->d_inode)
3236 return D_WALK_SKIP; 3246 return D_WALK_SKIP;
3237 3247
3238 if (!(dentry->d_flags & DCACHE_GENOCIDE)) { 3248 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
3239 dentry->d_flags |= DCACHE_GENOCIDE; 3249 dentry->d_flags |= DCACHE_GENOCIDE;
3240 dentry->d_lockref.count--; 3250 dentry->d_lockref.count--;
3241 } 3251 }
3242 } 3252 }
3243 return D_WALK_CONTINUE; 3253 return D_WALK_CONTINUE;
3244 } 3254 }
3245 3255
3246 void d_genocide(struct dentry *parent) 3256 void d_genocide(struct dentry *parent)
3247 { 3257 {
3248 d_walk(parent, parent, d_genocide_kill, NULL); 3258 d_walk(parent, parent, d_genocide_kill, NULL);
3249 } 3259 }
3250 3260
3251 void d_tmpfile(struct dentry *dentry, struct inode *inode) 3261 void d_tmpfile(struct dentry *dentry, struct inode *inode)
3252 { 3262 {
3253 inode_dec_link_count(inode); 3263 inode_dec_link_count(inode);
3254 BUG_ON(dentry->d_name.name != dentry->d_iname || 3264 BUG_ON(dentry->d_name.name != dentry->d_iname ||
3255 !hlist_unhashed(&dentry->d_alias) || 3265 !hlist_unhashed(&dentry->d_alias) ||
3256 !d_unlinked(dentry)); 3266 !d_unlinked(dentry));
3257 spin_lock(&dentry->d_parent->d_lock); 3267 spin_lock(&dentry->d_parent->d_lock);
3258 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); 3268 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
3259 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu", 3269 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
3260 (unsigned long long)inode->i_ino); 3270 (unsigned long long)inode->i_ino);
3261 spin_unlock(&dentry->d_lock); 3271 spin_unlock(&dentry->d_lock);
3262 spin_unlock(&dentry->d_parent->d_lock); 3272 spin_unlock(&dentry->d_parent->d_lock);
3263 d_instantiate(dentry, inode); 3273 d_instantiate(dentry, inode);
3264 } 3274 }
3265 EXPORT_SYMBOL(d_tmpfile); 3275 EXPORT_SYMBOL(d_tmpfile);
3266 3276
3267 static __initdata unsigned long dhash_entries; 3277 static __initdata unsigned long dhash_entries;
3268 static int __init set_dhash_entries(char *str) 3278 static int __init set_dhash_entries(char *str)
3269 { 3279 {
3270 if (!str) 3280 if (!str)
3271 return 0; 3281 return 0;
3272 dhash_entries = simple_strtoul(str, &str, 0); 3282 dhash_entries = simple_strtoul(str, &str, 0);
3273 return 1; 3283 return 1;
3274 } 3284 }
3275 __setup("dhash_entries=", set_dhash_entries); 3285 __setup("dhash_entries=", set_dhash_entries);
3276 3286
3277 static void __init dcache_init_early(void) 3287 static void __init dcache_init_early(void)
3278 { 3288 {
3279 unsigned int loop; 3289 unsigned int loop;
3280 3290
3281 /* If hashes are distributed across NUMA nodes, defer 3291 /* If hashes are distributed across NUMA nodes, defer
3282 * hash allocation until vmalloc space is available. 3292 * hash allocation until vmalloc space is available.
3283 */ 3293 */
3284 if (hashdist) 3294 if (hashdist)
3285 return; 3295 return;
3286 3296
3287 dentry_hashtable = 3297 dentry_hashtable =
3288 alloc_large_system_hash("Dentry cache", 3298 alloc_large_system_hash("Dentry cache",
3289 sizeof(struct hlist_bl_head), 3299 sizeof(struct hlist_bl_head),
3290 dhash_entries, 3300 dhash_entries,
3291 13, 3301 13,
3292 HASH_EARLY, 3302 HASH_EARLY,
3293 &d_hash_shift, 3303 &d_hash_shift,
3294 &d_hash_mask, 3304 &d_hash_mask,
3295 0, 3305 0,
3296 0); 3306 0);
3297 3307
3298 for (loop = 0; loop < (1U << d_hash_shift); loop++) 3308 for (loop = 0; loop < (1U << d_hash_shift); loop++)
3299 INIT_HLIST_BL_HEAD(dentry_hashtable + loop); 3309 INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
3300 } 3310 }
3301 3311
3302 static void __init dcache_init(void) 3312 static void __init dcache_init(void)
3303 { 3313 {
3304 unsigned int loop; 3314 unsigned int loop;
3305 3315
3306 /* 3316 /*
3307 * A constructor could be added for stable state like the lists, 3317 * A constructor could be added for stable state like the lists,
3308 * but it is probably not worth it because of the cache nature 3318 * but it is probably not worth it because of the cache nature
3309 * of the dcache. 3319 * of the dcache.
3310 */ 3320 */
3311 dentry_cache = KMEM_CACHE(dentry, 3321 dentry_cache = KMEM_CACHE(dentry,
3312 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD); 3322 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD);
3313 3323
3314 /* Hash may have been set up in dcache_init_early */ 3324 /* Hash may have been set up in dcache_init_early */
3315 if (!hashdist) 3325 if (!hashdist)
3316 return; 3326 return;
3317 3327
3318 dentry_hashtable = 3328 dentry_hashtable =
3319 alloc_large_system_hash("Dentry cache", 3329 alloc_large_system_hash("Dentry cache",
3320 sizeof(struct hlist_bl_head), 3330 sizeof(struct hlist_bl_head),
3321 dhash_entries, 3331 dhash_entries,
3322 13, 3332 13,
3323 0, 3333 0,
3324 &d_hash_shift, 3334 &d_hash_shift,
3325 &d_hash_mask, 3335 &d_hash_mask,
3326 0, 3336 0,
3327 0); 3337 0);
3328 3338
3329 for (loop = 0; loop < (1U << d_hash_shift); loop++) 3339 for (loop = 0; loop < (1U << d_hash_shift); loop++)
3330 INIT_HLIST_BL_HEAD(dentry_hashtable + loop); 3340 INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
3331 } 3341 }
3332 3342
3333 /* SLAB cache for __getname() consumers */ 3343 /* SLAB cache for __getname() consumers */
3334 struct kmem_cache *names_cachep __read_mostly; 3344 struct kmem_cache *names_cachep __read_mostly;
3335 EXPORT_SYMBOL(names_cachep); 3345 EXPORT_SYMBOL(names_cachep);
3336 3346
3337 EXPORT_SYMBOL(d_genocide); 3347 EXPORT_SYMBOL(d_genocide);
3338 3348
3339 void __init vfs_caches_init_early(void) 3349 void __init vfs_caches_init_early(void)
3340 { 3350 {