inetpeer: restore small inet_peer structures

Addition of rcu_head to struct inet_peer added 16bytes on 64bit arches. Thats a bit unfortunate, since old size was exactly 64 bytes. This can be solved, using an union between this rcu_head an four fields, that are normally used only when a refcount is taken on inet_peer. rcu_head is used only when refcnt=-1, right before structure freeing. Add a inet_peer_refcheck() function to check this assertion for a while. We can bring back SLAB_HWCACHE_ALIGN qualifier in kmem cache creation. Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>

inetpeer: restore small inet_peer structures
Addition of rcu_head to struct inet_peer added 16bytes on 64bit arches. Thats a bit unfortunate, since old size was exactly 64 bytes. This can be solved, using an union between this rcu_head an four fields, that are normally used only when a refcount is taken on inet_peer. rcu_head is used only when refcnt=-1, right before structure freeing. Add a inet_peer_refcheck() function to check this assertion for a while. We can bring back SLAB_HWCACHE_ALIGN qualifier in kmem cache creation. Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Eric Dumazet · David S. Miller
1 parent fdb93f8ac3
Showing 4 changed files with 36 additions and 11 deletions Side-by-side Diff
include/net/inetpeer.h
net/ipv4/inetpeer.c
net/ipv4/route.c
net/ipv4/tcp_ipv4.c
@@ -22,11 +22,21 @@
 	__u32			dtime;		/* the time of last use of not
 						 * referenced entries */
 	atomic_t		refcnt;
-	atomic_t		rid;		/* Frag reception counter */
-	atomic_t		ip_id_count;	/* IP ID for the next packet */
-	__u32			tcp_ts;
-	__u32			tcp_ts_stamp;
-	struct rcu_head		rcu;
+	/*
+	 * Once inet_peer is queued for deletion (refcnt == -1), following fields
+	 * are not available: rid, ip_id_count, tcp_ts, tcp_ts_stamp
+	 * We can share memory with rcu_head to keep inet_peer small
+	 * (less then 64 bytes)
+	 */
+	union {
+		struct {
+			atomic_t	rid;		/* Frag reception counter */
+			atomic_t	ip_id_count;	/* IP ID for the next packet */
+			__u32		tcp_ts;
+			__u32		tcp_ts_stamp;
+		};
+		struct rcu_head         rcu;
+	};
 };
  
 void			inet_initpeers(void) __init;
  
@@ -37,10 +47,21 @@
 /* can be called from BH context or outside */
 extern void inet_putpeer(struct inet_peer *p);
  
+/*
+ * temporary check to make sure we dont access rid, ip_id_count, tcp_ts,
+ * tcp_ts_stamp if no refcount is taken on inet_peer
+ */
+static inline void inet_peer_refcheck(const struct inet_peer *p)
+{
+	WARN_ON_ONCE(atomic_read(&p->refcnt) <= 0);
+}
+
+
 /* can be called with or without local BH being disabled */
 static inline __u16	inet_getid(struct inet_peer *p, int more)
 {
 	more++;
+	inet_peer_refcheck(p);
 	return atomic_add_return(more, &p->ip_id_count) - more;
 }
  
@@ -64,7 +64,7 @@
  *		   usually under some other lock to prevent node disappearing
  *		dtime: unused node list lock
  *		v4daddr: unchangeable
- *		ip_id_count: idlock
+ *		ip_id_count: atomic value (no lock needed)
  */
  
 static struct kmem_cache *peer_cachep __read_mostly;
@@ -129,7 +129,7 @@
  
 	peer_cachep = kmem_cache_create("inet_peer_cache",
 			sizeof(struct inet_peer),
-			0, SLAB_PANIC,
+			0, SLAB_HWCACHE_ALIGN | SLAB_PANIC,
 			NULL);
  
 	/* All the timers, started at system startup tend
@@ -2881,6 +2881,7 @@
 	error = rt->dst.error;
 	expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
 	if (rt->peer) {
+		inet_peer_refcheck(rt->peer);
 		id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
 		if (rt->peer->tcp_ts_stamp) {
 			ts = rt->peer->tcp_ts;
@@ -204,10 +204,12 @@
 		 * TIME-WAIT * and initialize rx_opt.ts_recent from it,
 		 * when trying new connection.
 		 */
-		if (peer != NULL &&
-		    (u32)get_seconds() - peer->tcp_ts_stamp <= TCP_PAWS_MSL) {
-			tp->rx_opt.ts_recent_stamp = peer->tcp_ts_stamp;
-			tp->rx_opt.ts_recent = peer->tcp_ts;
+		if (peer) {
+			inet_peer_refcheck(peer);
+			if ((u32)get_seconds() - peer->tcp_ts_stamp <= TCP_PAWS_MSL) {
+				tp->rx_opt.ts_recent_stamp = peer->tcp_ts_stamp;
+				tp->rx_opt.ts_recent = peer->tcp_ts;
+			}
 		}
 	}
  
@@ -1351,6 +1353,7 @@
 		    (dst = inet_csk_route_req(sk, req)) != NULL &&
 		    (peer = rt_get_peer((struct rtable *)dst)) != NULL &&
 		    peer->v4daddr == saddr) {
+			inet_peer_refcheck(peer);
 			if ((u32)get_seconds() - peer->tcp_ts_stamp < TCP_PAWS_MSL &&
 			    (s32)(peer->tcp_ts - req->ts_recent) >
 							TCP_PAWS_WINDOW) {
...	...	@@ -22,11 +22,21 @@
22	22	__u32 dtime; /* the time of last use of not
23	23	* referenced entries */
24	24	atomic_t refcnt;
25		- atomic_t rid; /* Frag reception counter */
26		- atomic_t ip_id_count; /* IP ID for the next packet */
27		- __u32 tcp_ts;
28		- __u32 tcp_ts_stamp;
29		- struct rcu_head rcu;
	25	+ /*
	26	+ * Once inet_peer is queued for deletion (refcnt == -1), following fields
	27	+ * are not available: rid, ip_id_count, tcp_ts, tcp_ts_stamp
	28	+ * We can share memory with rcu_head to keep inet_peer small
	29	+ * (less then 64 bytes)
	30	+ */
	31	+ union {
	32	+ struct {
	33	+ atomic_t rid; /* Frag reception counter */
	34	+ atomic_t ip_id_count; /* IP ID for the next packet */
	35	+ __u32 tcp_ts;
	36	+ __u32 tcp_ts_stamp;
	37	+ };
	38	+ struct rcu_head rcu;
	39	+ };
30	40	};
31	41
32	42	void inet_initpeers(void) __init;
33	43
...	...	@@ -37,10 +47,21 @@
37	47	/* can be called from BH context or outside */
38	48	extern void inet_putpeer(struct inet_peer *p);
39	49
	50	+/*
	51	+ * temporary check to make sure we dont access rid, ip_id_count, tcp_ts,
	52	+ * tcp_ts_stamp if no refcount is taken on inet_peer
	53	+ */
	54	+static inline void inet_peer_refcheck(const struct inet_peer *p)
	55	+{
	56	+ WARN_ON_ONCE(atomic_read(&p->refcnt) <= 0);
	57	+}
	58	+
	59	+
40	60	/* can be called with or without local BH being disabled */
41	61	static inline __u16 inet_getid(struct inet_peer *p, int more)
42	62	{
43	63	more++;
	64	+ inet_peer_refcheck(p);
44	65	return atomic_add_return(more, &p->ip_id_count) - more;
45	66	}
46	67
...	...	@@ -64,7 +64,7 @@
64	64	* usually under some other lock to prevent node disappearing
65	65	* dtime: unused node list lock
66	66	* v4daddr: unchangeable
67		- * ip_id_count: idlock
	67	+ * ip_id_count: atomic value (no lock needed)
68	68	*/
69	69
70	70	static struct kmem_cache *peer_cachep __read_mostly;
...	...	@@ -129,7 +129,7 @@
129	129
130	130	peer_cachep = kmem_cache_create("inet_peer_cache",
131	131	sizeof(struct inet_peer),
132		- 0, SLAB_PANIC,
	132	+ 0, SLAB_HWCACHE_ALIGN \| SLAB_PANIC,
133	133	NULL);
134	134
135	135	/* All the timers, started at system startup tend
...	...	@@ -2881,6 +2881,7 @@
2881	2881	error = rt->dst.error;
2882	2882	expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
2883	2883	if (rt->peer) {
	2884	+ inet_peer_refcheck(rt->peer);
2884	2885	id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
2885	2886	if (rt->peer->tcp_ts_stamp) {
2886	2887	ts = rt->peer->tcp_ts;
...	...	@@ -204,10 +204,12 @@
204	204	* TIME-WAIT * and initialize rx_opt.ts_recent from it,
205	205	* when trying new connection.
206	206	*/
207		- if (peer != NULL &&
208		- (u32)get_seconds() - peer->tcp_ts_stamp <= TCP_PAWS_MSL) {
209		- tp->rx_opt.ts_recent_stamp = peer->tcp_ts_stamp;
210		- tp->rx_opt.ts_recent = peer->tcp_ts;
	207	+ if (peer) {
	208	+ inet_peer_refcheck(peer);
	209	+ if ((u32)get_seconds() - peer->tcp_ts_stamp <= TCP_PAWS_MSL) {
	210	+ tp->rx_opt.ts_recent_stamp = peer->tcp_ts_stamp;
	211	+ tp->rx_opt.ts_recent = peer->tcp_ts;
	212	+ }
211	213	}
212	214	}
213	215
...	...	@@ -1351,6 +1353,7 @@
1351	1353	(dst = inet_csk_route_req(sk, req)) != NULL &&
1352	1354	(peer = rt_get_peer((struct rtable *)dst)) != NULL &&
1353	1355	peer->v4daddr == saddr) {
	1356	+ inet_peer_refcheck(peer);
1354	1357	if ((u32)get_seconds() - peer->tcp_ts_stamp < TCP_PAWS_MSL &&
1355	1358	(s32)(peer->tcp_ts - req->ts_recent) >
1356	1359	TCP_PAWS_WINDOW) {