veth: reduce stat overhead

veth stats are a bit bloated. There is no need to account transmit and receive stats, since they are absolutely symmetric. Also use a per device atomic64_t for the dropped counter, as it should never be used in fast path. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>

veth: reduce stat overhead
veth stats are a bit bloated. There is no need to account transmit and receive stats, since they are absolutely symmetric. Also use a per device atomic64_t for the dropped counter, as it should never be used in fast path. Signed-off-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Eric Dumazet · David S. Miller
1 parent 4cafe373d4
Showing 2 changed files with 48 additions and 68 deletions Side-by-side Diff
drivers/net/veth.c
include/linux/netdevice.h
@@ -25,18 +25,15 @@
 #define MIN_MTU 68		/* Min L3 MTU */
 #define MAX_MTU 65535		/* Max L3 MTU (arbitrary) */
  
-struct veth_net_stats {
-	u64			rx_packets;
-	u64			rx_bytes;
-	u64			tx_packets;
-	u64			tx_bytes;
-	u64			rx_dropped;
+struct pcpu_vstats {
+	u64			packets;
+	u64			bytes;
 	struct u64_stats_sync	syncp;
 };
  
 struct veth_priv {
-	struct net_device *peer;
-	struct veth_net_stats __percpu *stats;
+	struct net_device	*peer;
+	atomic64_t		dropped;
 };
  
 /*
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -107,84 +104,74 @@
 	.get_ethtool_stats	= veth_get_ethtool_stats,
 };
  
-/*
- * xmit
- */
-
 static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	struct net_device *rcv = NULL;
-	struct veth_priv *priv, *rcv_priv;
-	struct veth_net_stats *stats, *rcv_stats;
-	int length;
+	struct veth_priv *priv = netdev_priv(dev);
+	struct net_device *rcv = priv->peer;
+	int length = skb->len;
  
-	priv = netdev_priv(dev);
-	rcv = priv->peer;
-	rcv_priv = netdev_priv(rcv);
-
-	stats = this_cpu_ptr(priv->stats);
-	rcv_stats = this_cpu_ptr(rcv_priv->stats);
-
 	/* don't change ip_summed == CHECKSUM_PARTIAL, as that
-	   will cause bad checksum on forwarded packets */
+	 * will cause bad checksum on forwarded packets
+	 */
 	if (skb->ip_summed == CHECKSUM_NONE &&
 	    rcv->features & NETIF_F_RXCSUM)
 		skb->ip_summed = CHECKSUM_UNNECESSARY;
  
-	length = skb->len;
-	if (dev_forward_skb(rcv, skb) != NET_RX_SUCCESS)
-		goto rx_drop;
+	if (likely(dev_forward_skb(rcv, skb) == NET_RX_SUCCESS)) {
+		struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
  
-	u64_stats_update_begin(&stats->syncp);
-	stats->tx_bytes += length;
-	stats->tx_packets++;
-	u64_stats_update_end(&stats->syncp);
+		u64_stats_update_begin(&stats->syncp);
+		stats->bytes += length;
+		stats->packets++;
+		u64_stats_update_end(&stats->syncp);
+	} else {
+		atomic64_inc(&priv->dropped);
+	}
  
-	u64_stats_update_begin(&rcv_stats->syncp);
-	rcv_stats->rx_bytes += length;
-	rcv_stats->rx_packets++;
-	u64_stats_update_end(&rcv_stats->syncp);
-
 	return NETDEV_TX_OK;
-
-rx_drop:
-	u64_stats_update_begin(&rcv_stats->syncp);
-	rcv_stats->rx_dropped++;
-	u64_stats_update_end(&rcv_stats->syncp);
-	return NETDEV_TX_OK;
 }
  
 /*
  * general routines
  */
  
-static struct rtnl_link_stats64 *veth_get_stats64(struct net_device *dev,
-						  struct rtnl_link_stats64 *tot)
+static u64 veth_stats_one(struct pcpu_vstats *result, struct net_device *dev)
 {
 	struct veth_priv *priv = netdev_priv(dev);
 	int cpu;
  
+	result->packets = 0;
+	result->bytes = 0;
 	for_each_possible_cpu(cpu) {
-		struct veth_net_stats *stats = per_cpu_ptr(priv->stats, cpu);
-		u64 rx_packets, rx_bytes, rx_dropped;
-		u64 tx_packets, tx_bytes;
+		struct pcpu_vstats *stats = per_cpu_ptr(dev->vstats, cpu);
+		u64 packets, bytes;
 		unsigned int start;
  
 		do {
 			start = u64_stats_fetch_begin_bh(&stats->syncp);
-			rx_packets = stats->rx_packets;
-			tx_packets = stats->tx_packets;
-			rx_bytes = stats->rx_bytes;
-			tx_bytes = stats->tx_bytes;
-			rx_dropped = stats->rx_dropped;
+			packets = stats->packets;
+			bytes = stats->bytes;
 		} while (u64_stats_fetch_retry_bh(&stats->syncp, start));
-		tot->rx_packets += rx_packets;
-		tot->tx_packets += tx_packets;
-		tot->rx_bytes   += rx_bytes;
-		tot->tx_bytes   += tx_bytes;
-		tot->rx_dropped += rx_dropped;
+		result->packets += packets;
+		result->bytes += bytes;
 	}
+	return atomic64_read(&priv->dropped);
+}
  
+static struct rtnl_link_stats64 *veth_get_stats64(struct net_device *dev,
+						  struct rtnl_link_stats64 *tot)
+{
+	struct veth_priv *priv = netdev_priv(dev);
+	struct pcpu_vstats one;
+
+	tot->tx_dropped = veth_stats_one(&one, dev);
+	tot->tx_bytes = one.bytes;
+	tot->tx_packets = one.packets;
+
+	tot->rx_dropped = veth_stats_one(&one, priv->peer);
+	tot->rx_bytes = one.bytes;
+	tot->rx_packets = one.packets;
+
 	return tot;
 }
  
  
  
@@ -228,24 +215,16 @@
  
 static int veth_dev_init(struct net_device *dev)
 {
-	struct veth_net_stats __percpu *stats;
-	struct veth_priv *priv;
-
-	stats = alloc_percpu(struct veth_net_stats);
-	if (stats == NULL)
+	dev->vstats = alloc_percpu(struct pcpu_vstats);
+	if (!dev->vstats)
 		return -ENOMEM;
  
-	priv = netdev_priv(dev);
-	priv->stats = stats;
 	return 0;
 }
  
 static void veth_dev_free(struct net_device *dev)
 {
-	struct veth_priv *priv;
-
-	priv = netdev_priv(dev);
-	free_percpu(priv->stats);
+	free_percpu(dev->vstats);
 	free_netdev(dev);
 }
  
@@ -1284,6 +1284,7 @@
 		struct pcpu_lstats __percpu	*lstats; /* loopback stats */
 		struct pcpu_tstats __percpu	*tstats; /* tunnel stats */
 		struct pcpu_dstats __percpu	*dstats; /* dummy stats */
+		struct pcpu_vstats __percpu	*vstats; /* veth stats */
 	};
 	/* GARP */
 	struct garp_port __rcu	*garp_port;
...	...	@@ -25,18 +25,15 @@
25	25	#define MIN_MTU 68 /* Min L3 MTU */
26	26	#define MAX_MTU 65535 /* Max L3 MTU (arbitrary) */
27	27
28		-struct veth_net_stats {
29		- u64 rx_packets;
30		- u64 rx_bytes;
31		- u64 tx_packets;
32		- u64 tx_bytes;
33		- u64 rx_dropped;
	28	+struct pcpu_vstats {
	29	+ u64 packets;
	30	+ u64 bytes;
34	31	struct u64_stats_sync syncp;
35	32	};
36	33
37	34	struct veth_priv {
38		- struct net_device *peer;
39		- struct veth_net_stats __percpu *stats;
	35	+ struct net_device *peer;
	36	+ atomic64_t dropped;
40	37	};
41	38
42	39	/*
43	40
44	41
45	42
46	43
47	44
48	45
49	46
50	47
51	48
52	49
53	50
54	51
55	52
56	53
...	...	@@ -107,84 +104,74 @@
107	104	.get_ethtool_stats = veth_get_ethtool_stats,
108	105	};
109	106
110		-/*
111		- * xmit
112		- */
113		-
114	107	static netdev_tx_t veth_xmit(struct sk_buff skb, struct net_device dev)
115	108	{
116		- struct net_device *rcv = NULL;
117		- struct veth_priv priv, rcv_priv;
118		- struct veth_net_stats stats, rcv_stats;
119		- int length;
	109	+ struct veth_priv *priv = netdev_priv(dev);
	110	+ struct net_device *rcv = priv->peer;
	111	+ int length = skb->len;
120	112
121		- priv = netdev_priv(dev);
122		- rcv = priv->peer;
123		- rcv_priv = netdev_priv(rcv);
124		-
125		- stats = this_cpu_ptr(priv->stats);
126		- rcv_stats = this_cpu_ptr(rcv_priv->stats);
127		-
128	113	/* don't change ip_summed == CHECKSUM_PARTIAL, as that
129		- will cause bad checksum on forwarded packets */
	114	+ * will cause bad checksum on forwarded packets
	115	+ */
130	116	if (skb->ip_summed == CHECKSUM_NONE &&
131	117	rcv->features & NETIF_F_RXCSUM)
132	118	skb->ip_summed = CHECKSUM_UNNECESSARY;
133	119
134		- length = skb->len;
135		- if (dev_forward_skb(rcv, skb) != NET_RX_SUCCESS)
136		- goto rx_drop;
	120	+ if (likely(dev_forward_skb(rcv, skb) == NET_RX_SUCCESS)) {
	121	+ struct pcpu_vstats *stats = this_cpu_ptr(dev->vstats);
137	122
138		- u64_stats_update_begin(&stats->syncp);
139		- stats->tx_bytes += length;
140		- stats->tx_packets++;
141		- u64_stats_update_end(&stats->syncp);
	123	+ u64_stats_update_begin(&stats->syncp);
	124	+ stats->bytes += length;
	125	+ stats->packets++;
	126	+ u64_stats_update_end(&stats->syncp);
	127	+ } else {
	128	+ atomic64_inc(&priv->dropped);
	129	+ }
142	130
143		- u64_stats_update_begin(&rcv_stats->syncp);
144		- rcv_stats->rx_bytes += length;
145		- rcv_stats->rx_packets++;
146		- u64_stats_update_end(&rcv_stats->syncp);
147		-
148	131	return NETDEV_TX_OK;
149		-
150		-rx_drop:
151		- u64_stats_update_begin(&rcv_stats->syncp);
152		- rcv_stats->rx_dropped++;
153		- u64_stats_update_end(&rcv_stats->syncp);
154		- return NETDEV_TX_OK;
155	132	}
156	133
157	134	/*
158	135	* general routines
159	136	*/
160	137
161		-static struct rtnl_link_stats64 veth_get_stats64(struct net_device dev,
162		- struct rtnl_link_stats64 *tot)
	138	+static u64 veth_stats_one(struct pcpu_vstats result, struct net_device dev)
163	139	{
164	140	struct veth_priv *priv = netdev_priv(dev);
165	141	int cpu;
166	142
	143	+ result->packets = 0;
	144	+ result->bytes = 0;
167	145	for_each_possible_cpu(cpu) {
168		- struct veth_net_stats *stats = per_cpu_ptr(priv->stats, cpu);
169		- u64 rx_packets, rx_bytes, rx_dropped;
170		- u64 tx_packets, tx_bytes;
	146	+ struct pcpu_vstats *stats = per_cpu_ptr(dev->vstats, cpu);
	147	+ u64 packets, bytes;
171	148	unsigned int start;
172	149
173	150	do {
174	151	start = u64_stats_fetch_begin_bh(&stats->syncp);
175		- rx_packets = stats->rx_packets;
176		- tx_packets = stats->tx_packets;
177		- rx_bytes = stats->rx_bytes;
178		- tx_bytes = stats->tx_bytes;
179		- rx_dropped = stats->rx_dropped;
	152	+ packets = stats->packets;
	153	+ bytes = stats->bytes;
180	154	} while (u64_stats_fetch_retry_bh(&stats->syncp, start));
181		- tot->rx_packets += rx_packets;
182		- tot->tx_packets += tx_packets;
183		- tot->rx_bytes += rx_bytes;
184		- tot->tx_bytes += tx_bytes;
185		- tot->rx_dropped += rx_dropped;
	155	+ result->packets += packets;
	156	+ result->bytes += bytes;
186	157	}
	158	+ return atomic64_read(&priv->dropped);
	159	+}
187	160
	161	+static struct rtnl_link_stats64 veth_get_stats64(struct net_device dev,
	162	+ struct rtnl_link_stats64 *tot)
	163	+{
	164	+ struct veth_priv *priv = netdev_priv(dev);
	165	+ struct pcpu_vstats one;
	166	+
	167	+ tot->tx_dropped = veth_stats_one(&one, dev);
	168	+ tot->tx_bytes = one.bytes;
	169	+ tot->tx_packets = one.packets;
	170	+
	171	+ tot->rx_dropped = veth_stats_one(&one, priv->peer);
	172	+ tot->rx_bytes = one.bytes;
	173	+ tot->rx_packets = one.packets;
	174	+
188	175	return tot;
189	176	}
190	177
191	178
192	179
...	...	@@ -228,24 +215,16 @@
228	215
229	216	static int veth_dev_init(struct net_device *dev)
230	217	{
231		- struct veth_net_stats __percpu *stats;
232		- struct veth_priv *priv;
233		-
234		- stats = alloc_percpu(struct veth_net_stats);
235		- if (stats == NULL)
	218	+ dev->vstats = alloc_percpu(struct pcpu_vstats);
	219	+ if (!dev->vstats)
236	220	return -ENOMEM;
237	221
238		- priv = netdev_priv(dev);
239		- priv->stats = stats;
240	222	return 0;
241	223	}
242	224
243	225	static void veth_dev_free(struct net_device *dev)
244	226	{
245		- struct veth_priv *priv;
246		-
247		- priv = netdev_priv(dev);
248		- free_percpu(priv->stats);
	227	+ free_percpu(dev->vstats);
249	228	free_netdev(dev);
250	229	}
251	230
...	...	@@ -1284,6 +1284,7 @@
1284	1284	struct pcpu_lstats __percpu lstats; / loopback stats */
1285	1285	struct pcpu_tstats __percpu tstats; / tunnel stats */
1286	1286	struct pcpu_dstats __percpu dstats; / dummy stats */
	1287	+ struct pcpu_vstats __percpu vstats; / veth stats */
1287	1288	};
1288	1289	/* GARP */
1289	1290	struct garp_port __rcu *garp_port;