fs: cache optimise dentry and inode for rcu-walk

Put dentry and inode fields into top of data structure. This allows RCU path traversal to perform an RCU dentry lookup in a path walk by touching only the first 56 bytes of the dentry. We also fit in 8 bytes of inline name in the first 64 bytes, so for short names, only 64 bytes needs to be touched to perform the lookup. We should get rid of the hash->prev pointer from the first 64 bytes, and fit 16 bytes of name in there, which will take care of 81% rather than 32% of the kernel tree. inode is also rearranged so that RCU lookup will only touch a single cacheline in the inode, plus one in the i_ops structure. This is important for directory component lookups in RCU path walking. In the kernel source, directory names average is around 6 chars, so this works. When we reach the last element of the lookup, we need to lock it and take its refcount which requires another cacheline access. Align dentry and inode operations structs, so members will be at predictable offsets and we can group common operations into head of structure. Signed-off-by: Nick Piggin <npiggin@kernel.dk>

fs: cache optimise dentry and inode for rcu-walk
Put dentry and inode fields into top of data structure. This allows RCU path traversal to perform an RCU dentry lookup in a path walk by touching only the first 56 bytes of the dentry. We also fit in 8 bytes of inline name in the first 64 bytes, so for short names, only 64 bytes needs to be touched to perform the lookup. We should get rid of the hash->prev pointer from the first 64 bytes, and fit 16 bytes of name in there, which will take care of 81% rather than 32% of the kernel tree. inode is also rearranged so that RCU lookup will only touch a single cacheline in the inode, plus one in the i_ops structure. This is important for directory component lookups in RCU path walking. In the kernel source, directory names average is around 6 chars, so this works. When we reach the last element of the lookup, we need to lock it and take its refcount which requires another cacheline access. Align dentry and inode operations structs, so members will be at predictable offsets and we can group common operations into head of structure. Signed-off-by: Nick Piggin <npiggin@kernel.dk>
Nick Piggin
1 parent fb045adb99
Showing 3 changed files with 42 additions and 38 deletions Side-by-side Diff
fs/dcache.c
include/linux/dcache.h
include/linux/fs.h
@@ -83,8 +83,6 @@
  
 static struct kmem_cache *dentry_cache __read_mostly;
  
-#define DNAME_INLINE_LEN (sizeof(struct dentry)-offsetof(struct dentry,d_iname))
-
 /*
  * This is the single most critical data structure when it comes
  * to the dcache: the hashtable for lookups. Somebody should try
@@ -82,26 +82,34 @@
  * large memory footprint increase).
  */
 #ifdef CONFIG_64BIT
-#define DNAME_INLINE_LEN_MIN 32 /* 192 bytes */
+# define DNAME_INLINE_LEN 32 /* 192 bytes */
 #else
-#define DNAME_INLINE_LEN_MIN 40 /* 128 bytes */
+# ifdef CONFIG_SMP
+#  define DNAME_INLINE_LEN 36 /* 128 bytes */
+# else
+#  define DNAME_INLINE_LEN 40 /* 128 bytes */
+# endif
 #endif
  
 struct dentry {
-	unsigned int d_count;		/* protected by d_lock */
+	/* RCU lookup touched fields */
 	unsigned int d_flags;		/* protected by d_lock */
-	spinlock_t d_lock;		/* per dentry lock */
 	seqcount_t d_seq;		/* per dentry seqlock */
-	struct inode *d_inode;		/* Where the name belongs to - NULL is
-					 * negative */
-	/*
-	 * The next three fields are touched by __d_lookup.  Place them here
-	 * so they all fit in a cache line.
-	 */
 	struct hlist_node d_hash;	/* lookup hash list */
 	struct dentry *d_parent;	/* parent directory */
 	struct qstr d_name;
+	struct inode *d_inode;		/* Where the name belongs to - NULL is
+					 * negative */
+	unsigned char d_iname[DNAME_INLINE_LEN];	/* small names */
  
+	/* Ref lookup also touches following */
+	unsigned int d_count;		/* protected by d_lock */
+	spinlock_t d_lock;		/* per dentry lock */
+	const struct dentry_operations *d_op;
+	struct super_block *d_sb;	/* The root of the dentry tree */
+	unsigned long d_time;		/* used by d_revalidate */
+	void *d_fsdata;			/* fs-specific data */
+
 	struct list_head d_lru;		/* LRU list */
 	/*
 	 * d_child and d_rcu can share memory
@@ -112,12 +120,6 @@
 	} d_u;
 	struct list_head d_subdirs;	/* our children */
 	struct list_head d_alias;	/* inode alias list */
-	unsigned long d_time;		/* used by d_revalidate */
-	const struct dentry_operations *d_op;
-	struct super_block *d_sb;	/* The root of the dentry tree */
-	void *d_fsdata;			/* fs-specific data */
-
-	unsigned char d_iname[DNAME_INLINE_LEN_MIN];	/* small names */
 };
  
 /*
@@ -143,7 +145,7 @@
 	void (*d_release)(struct dentry *);
 	void (*d_iput)(struct dentry *, struct inode *);
 	char *(*d_dname)(struct dentry *, char *, int);
-};
+} ____cacheline_aligned;
  
 /*
  * Locking rules for dentry_operations callbacks are to be found in
@@ -733,6 +733,20 @@
 #define ACL_NOT_CACHED ((void *)(-1))
  
 struct inode {
+	/* RCU path lookup touches following: */
+	umode_t			i_mode;
+	uid_t			i_uid;
+	gid_t			i_gid;
+	const struct inode_operations	*i_op;
+	struct super_block	*i_sb;
+
+	spinlock_t		i_lock;	/* i_blocks, i_bytes, maybe i_size */
+	unsigned int		i_flags;
+	struct mutex		i_mutex;
+
+	unsigned long		i_state;
+	unsigned long		dirtied_when;	/* jiffies of first dirtying */
+
 	struct hlist_node	i_hash;
 	struct list_head	i_wb_list;	/* backing dev IO list */
 	struct list_head	i_lru;		/* inode LRU list */
@@ -744,8 +758,6 @@
 	unsigned long		i_ino;
 	atomic_t		i_count;
 	unsigned int		i_nlink;
-	uid_t			i_uid;
-	gid_t			i_gid;
 	dev_t			i_rdev;
 	unsigned int		i_blkbits;
 	u64			i_version;
  
  
@@ -758,13 +770,8 @@
 	struct timespec		i_ctime;
 	blkcnt_t		i_blocks;
 	unsigned short          i_bytes;
-	umode_t			i_mode;
-	spinlock_t		i_lock;	/* i_blocks, i_bytes, maybe i_size */
-	struct mutex		i_mutex;
 	struct rw_semaphore	i_alloc_sem;
-	const struct inode_operations	*i_op;
 	const struct file_operations	*i_fop;	/* former ->i_op->default_file_ops */
-	struct super_block	*i_sb;
 	struct file_lock	*i_flock;
 	struct address_space	*i_mapping;
 	struct address_space	i_data;
@@ -785,11 +792,6 @@
 	struct hlist_head	i_fsnotify_marks;
 #endif
  
-	unsigned long		i_state;
-	unsigned long		dirtied_when;	/* jiffies of first dirtying */
-
-	unsigned int		i_flags;
-
 #ifdef CONFIG_IMA
 	/* protected by i_lock */
 	unsigned int		i_readcount; /* struct files open RO */
  
@@ -1549,8 +1551,15 @@
 };
  
 struct inode_operations {
-	int (*create) (struct inode *,struct dentry *,int, struct nameidata *);
 	struct dentry * (*lookup) (struct inode *,struct dentry *, struct nameidata *);
+	void * (*follow_link) (struct dentry *, struct nameidata *);
+	int (*permission) (struct inode *, int);
+	int (*check_acl)(struct inode *, int);
+
+	int (*readlink) (struct dentry *, char __user *,int);
+	void (*put_link) (struct dentry *, struct nameidata *, void *);
+
+	int (*create) (struct inode *,struct dentry *,int, struct nameidata *);
 	int (*link) (struct dentry *,struct inode *,struct dentry *);
 	int (*unlink) (struct inode *,struct dentry *);
 	int (*symlink) (struct inode *,struct dentry *,const char *);
  
@@ -1559,12 +1568,7 @@
 	int (*mknod) (struct inode *,struct dentry *,int,dev_t);
 	int (*rename) (struct inode *, struct dentry *,
 			struct inode *, struct dentry *);
-	int (*readlink) (struct dentry *, char __user *,int);
-	void * (*follow_link) (struct dentry *, struct nameidata *);
-	void (*put_link) (struct dentry *, struct nameidata *, void *);
 	void (*truncate) (struct inode *);
-	int (*permission) (struct inode *, int);
-	int (*check_acl)(struct inode *, int);
 	int (*setattr) (struct dentry *, struct iattr *);
 	int (*getattr) (struct vfsmount *mnt, struct dentry *, struct kstat *);
 	int (*setxattr) (struct dentry *, const char *,const void *,size_t,int);
@@ -1576,7 +1580,7 @@
 			  loff_t len);
 	int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
 		      u64 len);
-};
+} ____cacheline_aligned;
  
 struct seq_file;
...	...	@@ -83,8 +83,6 @@
83	83
84	84	static struct kmem_cache *dentry_cache __read_mostly;
85	85
86		-#define DNAME_INLINE_LEN (sizeof(struct dentry)-offsetof(struct dentry,d_iname))
87		-
88	86	/*
89	87	* This is the single most critical data structure when it comes
90	88	* to the dcache: the hashtable for lookups. Somebody should try
...	...	@@ -82,26 +82,34 @@
82	82	* large memory footprint increase).
83	83	*/
84	84	#ifdef CONFIG_64BIT
85		-#define DNAME_INLINE_LEN_MIN 32 /* 192 bytes */
	85	+# define DNAME_INLINE_LEN 32 /* 192 bytes */
86	86	#else
87		-#define DNAME_INLINE_LEN_MIN 40 /* 128 bytes */
	87	+# ifdef CONFIG_SMP
	88	+# define DNAME_INLINE_LEN 36 /* 128 bytes */
	89	+# else
	90	+# define DNAME_INLINE_LEN 40 /* 128 bytes */
	91	+# endif
88	92	#endif
89	93
90	94	struct dentry {
91		- unsigned int d_count; /* protected by d_lock */
	95	+ /* RCU lookup touched fields */
92	96	unsigned int d_flags; /* protected by d_lock */
93		- spinlock_t d_lock; /* per dentry lock */
94	97	seqcount_t d_seq; /* per dentry seqlock */
95		- struct inode d_inode; / Where the name belongs to - NULL is
96		- * negative */
97		- /*
98		- * The next three fields are touched by __d_lookup. Place them here
99		- * so they all fit in a cache line.
100		- */
101	98	struct hlist_node d_hash; /* lookup hash list */
102	99	struct dentry d_parent; / parent directory */
103	100	struct qstr d_name;
	101	+ struct inode d_inode; / Where the name belongs to - NULL is
	102	+ * negative */
	103	+ unsigned char d_iname[DNAME_INLINE_LEN]; /* small names */
104	104
	105	+ /* Ref lookup also touches following */
	106	+ unsigned int d_count; /* protected by d_lock */
	107	+ spinlock_t d_lock; /* per dentry lock */
	108	+ const struct dentry_operations *d_op;
	109	+ struct super_block d_sb; / The root of the dentry tree */
	110	+ unsigned long d_time; /* used by d_revalidate */
	111	+ void d_fsdata; / fs-specific data */
	112	+
105	113	struct list_head d_lru; /* LRU list */
106	114	/*
107	115	* d_child and d_rcu can share memory
...	...	@@ -112,12 +120,6 @@
112	120	} d_u;
113	121	struct list_head d_subdirs; /* our children */
114	122	struct list_head d_alias; /* inode alias list */
115		- unsigned long d_time; /* used by d_revalidate */
116		- const struct dentry_operations *d_op;
117		- struct super_block d_sb; / The root of the dentry tree */
118		- void d_fsdata; / fs-specific data */
119		-
120		- unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */
121	123	};
122	124
123	125	/*
...	...	@@ -143,7 +145,7 @@
143	145	void (d_release)(struct dentry );
144	146	void (d_iput)(struct dentry , struct inode *);
145	147	char (d_dname)(struct dentry , char , int);
146		-};
	148	+} ____cacheline_aligned;
147	149
148	150	/*
149	151	* Locking rules for dentry_operations callbacks are to be found in
...	...	@@ -733,6 +733,20 @@
733	733	#define ACL_NOT_CACHED ((void *)(-1))
734	734
735	735	struct inode {
	736	+ /* RCU path lookup touches following: */
	737	+ umode_t i_mode;
	738	+ uid_t i_uid;
	739	+ gid_t i_gid;
	740	+ const struct inode_operations *i_op;
	741	+ struct super_block *i_sb;
	742	+
	743	+ spinlock_t i_lock; /* i_blocks, i_bytes, maybe i_size */
	744	+ unsigned int i_flags;
	745	+ struct mutex i_mutex;
	746	+
	747	+ unsigned long i_state;
	748	+ unsigned long dirtied_when; /* jiffies of first dirtying */
	749	+
736	750	struct hlist_node i_hash;
737	751	struct list_head i_wb_list; /* backing dev IO list */
738	752	struct list_head i_lru; /* inode LRU list */
...	...	@@ -744,8 +758,6 @@
744	758	unsigned long i_ino;
745	759	atomic_t i_count;
746	760	unsigned int i_nlink;
747		- uid_t i_uid;
748		- gid_t i_gid;
749	761	dev_t i_rdev;
750	762	unsigned int i_blkbits;
751	763	u64 i_version;
752	764
753	765
...	...	@@ -758,13 +770,8 @@
758	770	struct timespec i_ctime;
759	771	blkcnt_t i_blocks;
760	772	unsigned short i_bytes;
761		- umode_t i_mode;
762		- spinlock_t i_lock; /* i_blocks, i_bytes, maybe i_size */
763		- struct mutex i_mutex;
764	773	struct rw_semaphore i_alloc_sem;
765		- const struct inode_operations *i_op;
766	774	const struct file_operations i_fop; / former ->i_op->default_file_ops */
767		- struct super_block *i_sb;
768	775	struct file_lock *i_flock;
769	776	struct address_space *i_mapping;
770	777	struct address_space i_data;
...	...	@@ -785,11 +792,6 @@
785	792	struct hlist_head i_fsnotify_marks;
786	793	#endif
787	794
788		- unsigned long i_state;
789		- unsigned long dirtied_when; /* jiffies of first dirtying */
790		-
791		- unsigned int i_flags;
792		-
793	795	#ifdef CONFIG_IMA
794	796	/* protected by i_lock */
795	797	unsigned int i_readcount; /* struct files open RO */
796	798
...	...	@@ -1549,8 +1551,15 @@
1549	1551	};
1550	1552
1551	1553	struct inode_operations {
1552		- int (create) (struct inode ,struct dentry ,int, struct nameidata );
1553	1554	struct dentry * (lookup) (struct inode ,struct dentry , struct nameidata );
	1555	+ void * (follow_link) (struct dentry , struct nameidata *);
	1556	+ int (permission) (struct inode , int);
	1557	+ int (check_acl)(struct inode , int);
	1558	+
	1559	+ int (readlink) (struct dentry , char __user *,int);
	1560	+ void (put_link) (struct dentry , struct nameidata , void );
	1561	+
	1562	+ int (create) (struct inode ,struct dentry ,int, struct nameidata );
1554	1563	int (link) (struct dentry ,struct inode ,struct dentry );
1555	1564	int (unlink) (struct inode ,struct dentry *);
1556	1565	int (symlink) (struct inode ,struct dentry ,const char );
1557	1566
...	...	@@ -1559,12 +1568,7 @@
1559	1568	int (mknod) (struct inode ,struct dentry *,int,dev_t);
1560	1569	int (rename) (struct inode , struct dentry *,
1561	1570	struct inode , struct dentry );
1562		- int (readlink) (struct dentry , char __user *,int);
1563		- void * (follow_link) (struct dentry , struct nameidata *);
1564		- void (put_link) (struct dentry , struct nameidata , void );
1565	1571	void (truncate) (struct inode );
1566		- int (permission) (struct inode , int);
1567		- int (check_acl)(struct inode , int);
1568	1572	int (setattr) (struct dentry , struct iattr *);
1569	1573	int (getattr) (struct vfsmount mnt, struct dentry , struct kstat );
1570	1574	int (setxattr) (struct dentry , const char ,const void ,size_t,int);
...	...	@@ -1576,7 +1580,7 @@
1576	1580	loff_t len);
1577	1581	int (fiemap)(struct inode , struct fiemap_extent_info *, u64 start,
1578	1582	u64 len);
1579		-};
	1583	+} ____cacheline_aligned;
1580	1584
1581	1585	struct seq_file;
1582	1586