Blame view
drivers/net/veth.c
33.5 KB
09c434b8a treewide: Add SPD... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
e314dbdc1 [NET]: Virtual et... |
2 3 4 5 6 7 8 9 10 |
/* * drivers/net/veth.c * * Copyright (C) 2007 OpenVZ http://openvz.org, SWsoft Inc * * Author: Pavel Emelianov <xemul@openvz.org> * Ethtool interface from: Eric W. Biederman <ebiederm@xmission.com> * */ |
e314dbdc1 [NET]: Virtual et... |
11 |
#include <linux/netdevice.h> |
5a0e3ad6a include cleanup: ... |
12 |
#include <linux/slab.h> |
e314dbdc1 [NET]: Virtual et... |
13 14 |
#include <linux/ethtool.h> #include <linux/etherdevice.h> |
cf05c700c veth: fix 64bit s... |
15 |
#include <linux/u64_stats_sync.h> |
e314dbdc1 [NET]: Virtual et... |
16 |
|
f7b12606b rtnl: make ifla_p... |
17 |
#include <net/rtnetlink.h> |
e314dbdc1 [NET]: Virtual et... |
18 19 |
#include <net/dst.h> #include <net/xfrm.h> |
af87a3aa1 veth: Add ndo_xdp... |
20 |
#include <net/xdp.h> |
ecef969e5 [VETH]: move veth... |
21 |
#include <linux/veth.h> |
9d9779e72 drivers/net: Add ... |
22 |
#include <linux/module.h> |
948d4f214 veth: Add driver XDP |
23 24 25 |
#include <linux/bpf.h> #include <linux/filter.h> #include <linux/ptr_ring.h> |
948d4f214 veth: Add driver XDP |
26 |
#include <linux/bpf_trace.h> |
aa4e689ed veth: add softwar... |
27 |
#include <linux/net_tstamp.h> |
e314dbdc1 [NET]: Virtual et... |
28 29 30 |
#define DRV_NAME "veth" #define DRV_VERSION "1.0" |
9fc8d518d veth: Handle xdp_... |
31 |
#define VETH_XDP_FLAG BIT(0) |
948d4f214 veth: Add driver XDP |
32 33 |
#define VETH_RING_SIZE 256 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) |
9cda7807e veth: Support bul... |
34 |
#define VETH_XDP_TX_BULK_SIZE 16 |
65780c562 veth: move xdp st... |
35 |
struct veth_stats { |
1c5b82e55 veth: introduce m... |
36 37 |
u64 rx_drops; /* xdp */ |
65780c562 veth: move xdp st... |
38 39 |
u64 xdp_packets; u64 xdp_bytes; |
1c5b82e55 veth: introduce m... |
40 |
u64 xdp_redirect; |
65780c562 veth: move xdp st... |
41 |
u64 xdp_drops; |
1c5b82e55 veth: introduce m... |
42 |
u64 xdp_tx; |
9152cff0d veth: introduce m... |
43 |
u64 xdp_tx_err; |
5fe6e5677 veth: rely on pee... |
44 45 |
u64 peer_tq_xdp_xmit; u64 peer_tq_xdp_xmit_err; |
65780c562 veth: move xdp st... |
46 |
}; |
4195e54aa veth: Account for... |
47 |
struct veth_rq_stats { |
65780c562 veth: move xdp st... |
48 |
struct veth_stats vs; |
4195e54aa veth: Account for... |
49 50 |
struct u64_stats_sync syncp; }; |
638264dc9 veth: Support per... |
51 |
struct veth_rq { |
948d4f214 veth: Add driver XDP |
52 53 54 |
struct napi_struct xdp_napi; struct net_device *dev; struct bpf_prog __rcu *xdp_prog; |
d1396004d veth: Add XDP TX ... |
55 |
struct xdp_mem_info xdp_mem; |
4195e54aa veth: Account for... |
56 |
struct veth_rq_stats stats; |
948d4f214 veth: Add driver XDP |
57 58 59 |
bool rx_notify_masked; struct ptr_ring xdp_ring; struct xdp_rxq_info xdp_rxq; |
e314dbdc1 [NET]: Virtual et... |
60 |
}; |
638264dc9 veth: Support per... |
61 62 63 64 65 66 67 |
struct veth_priv { struct net_device __rcu *peer; atomic64_t dropped; struct bpf_prog *_xdp_prog; struct veth_rq *rq; unsigned int requested_headroom; }; |
9cda7807e veth: Support bul... |
68 69 70 71 |
struct veth_xdp_tx_bq { struct xdp_frame *q[VETH_XDP_TX_BULK_SIZE]; unsigned int count; }; |
e314dbdc1 [NET]: Virtual et... |
72 73 74 |
/* * ethtool interface */ |
d397b9682 veth: Add ethtool... |
75 76 77 78 |
struct veth_q_stat_desc { char desc[ETH_GSTRING_LEN]; size_t offset; }; |
65780c562 veth: move xdp st... |
79 |
#define VETH_RQ_STAT(m) offsetof(struct veth_stats, m) |
d397b9682 veth: Add ethtool... |
80 81 82 83 |
static const struct veth_q_stat_desc veth_rq_stats_desc[] = { { "xdp_packets", VETH_RQ_STAT(xdp_packets) }, { "xdp_bytes", VETH_RQ_STAT(xdp_bytes) }, |
5fe6e5677 veth: rely on pee... |
84 85 86 87 88 |
{ "drops", VETH_RQ_STAT(rx_drops) }, { "xdp_redirect", VETH_RQ_STAT(xdp_redirect) }, { "xdp_drops", VETH_RQ_STAT(xdp_drops) }, { "xdp_tx", VETH_RQ_STAT(xdp_tx) }, { "xdp_tx_errors", VETH_RQ_STAT(xdp_tx_err) }, |
d397b9682 veth: Add ethtool... |
89 90 91 |
}; #define VETH_RQ_STATS_LEN ARRAY_SIZE(veth_rq_stats_desc) |
5fe6e5677 veth: rely on pee... |
92 93 94 95 96 97 |
static const struct veth_q_stat_desc veth_tq_stats_desc[] = { { "xdp_xmit", VETH_RQ_STAT(peer_tq_xdp_xmit) }, { "xdp_xmit_errors", VETH_RQ_STAT(peer_tq_xdp_xmit_err) }, }; #define VETH_TQ_STATS_LEN ARRAY_SIZE(veth_tq_stats_desc) |
e314dbdc1 [NET]: Virtual et... |
98 99 100 101 102 |
static struct { const char string[ETH_GSTRING_LEN]; } ethtool_stats_keys[] = { { "peer_ifindex" }, }; |
56607b987 net: veth: use ne... |
103 104 |
static int veth_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *cmd) |
e314dbdc1 [NET]: Virtual et... |
105 |
{ |
56607b987 net: veth: use ne... |
106 107 108 109 |
cmd->base.speed = SPEED_10000; cmd->base.duplex = DUPLEX_FULL; cmd->base.port = PORT_TP; cmd->base.autoneg = AUTONEG_DISABLE; |
e314dbdc1 [NET]: Virtual et... |
110 111 112 113 114 |
return 0; } static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { |
33a5ba144 net: sweep-up som... |
115 116 |
strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); strlcpy(info->version, DRV_VERSION, sizeof(info->version)); |
e314dbdc1 [NET]: Virtual et... |
117 118 119 120 |
} static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) { |
d397b9682 veth: Add ethtool... |
121 122 |
char *p = (char *)buf; int i, j; |
e314dbdc1 [NET]: Virtual et... |
123 124 |
switch(stringset) { case ETH_SS_STATS: |
d397b9682 veth: Add ethtool... |
125 126 127 128 |
memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); p += sizeof(ethtool_stats_keys); for (i = 0; i < dev->real_num_rx_queues; i++) { for (j = 0; j < VETH_RQ_STATS_LEN; j++) { |
abdf47aab veth: Fix -Wforma... |
129 |
snprintf(p, ETH_GSTRING_LEN, |
9152cff0d veth: introduce m... |
130 |
"rx_queue_%u_%.18s", |
d397b9682 veth: Add ethtool... |
131 132 133 134 |
i, veth_rq_stats_desc[j].desc); p += ETH_GSTRING_LEN; } } |
5fe6e5677 veth: rely on pee... |
135 136 137 138 139 140 141 142 |
for (i = 0; i < dev->real_num_tx_queues; i++) { for (j = 0; j < VETH_TQ_STATS_LEN; j++) { snprintf(p, ETH_GSTRING_LEN, "tx_queue_%u_%.18s", i, veth_tq_stats_desc[j].desc); p += ETH_GSTRING_LEN; } } |
e314dbdc1 [NET]: Virtual et... |
143 144 145 |
break; } } |
b9f2c0440 [netdrvr] Stop us... |
146 |
static int veth_get_sset_count(struct net_device *dev, int sset) |
e314dbdc1 [NET]: Virtual et... |
147 |
{ |
b9f2c0440 [netdrvr] Stop us... |
148 149 |
switch (sset) { case ETH_SS_STATS: |
d397b9682 veth: Add ethtool... |
150 |
return ARRAY_SIZE(ethtool_stats_keys) + |
5fe6e5677 veth: rely on pee... |
151 152 |
VETH_RQ_STATS_LEN * dev->real_num_rx_queues + VETH_TQ_STATS_LEN * dev->real_num_tx_queues; |
b9f2c0440 [netdrvr] Stop us... |
153 154 155 |
default: return -EOPNOTSUPP; } |
e314dbdc1 [NET]: Virtual et... |
156 157 158 159 160 |
} static void veth_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *data) { |
5fe6e5677 veth: rely on pee... |
161 |
struct veth_priv *rcv_priv, *priv = netdev_priv(dev); |
d0e2c55e7 veth: avoid a NUL... |
162 |
struct net_device *peer = rtnl_dereference(priv->peer); |
d397b9682 veth: Add ethtool... |
163 |
int i, j, idx; |
e314dbdc1 [NET]: Virtual et... |
164 |
|
d0e2c55e7 veth: avoid a NUL... |
165 |
data[0] = peer ? peer->ifindex : 0; |
d397b9682 veth: Add ethtool... |
166 167 168 |
idx = 1; for (i = 0; i < dev->real_num_rx_queues; i++) { const struct veth_rq_stats *rq_stats = &priv->rq[i].stats; |
65780c562 veth: move xdp st... |
169 |
const void *stats_base = (void *)&rq_stats->vs; |
d397b9682 veth: Add ethtool... |
170 171 172 173 174 175 176 177 178 179 180 181 |
unsigned int start; size_t offset; do { start = u64_stats_fetch_begin_irq(&rq_stats->syncp); for (j = 0; j < VETH_RQ_STATS_LEN; j++) { offset = veth_rq_stats_desc[j].offset; data[idx + j] = *(u64 *)(stats_base + offset); } } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); idx += VETH_RQ_STATS_LEN; } |
5fe6e5677 veth: rely on pee... |
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 |
if (!peer) return; rcv_priv = netdev_priv(peer); for (i = 0; i < peer->real_num_rx_queues; i++) { const struct veth_rq_stats *rq_stats = &rcv_priv->rq[i].stats; const void *base = (void *)&rq_stats->vs; unsigned int start, tx_idx = idx; size_t offset; tx_idx += (i % dev->real_num_tx_queues) * VETH_TQ_STATS_LEN; do { start = u64_stats_fetch_begin_irq(&rq_stats->syncp); for (j = 0; j < VETH_TQ_STATS_LEN; j++) { offset = veth_tq_stats_desc[j].offset; data[tx_idx + j] += *(u64 *)(base + offset); } } while (u64_stats_fetch_retry_irq(&rq_stats->syncp, start)); } |
e314dbdc1 [NET]: Virtual et... |
202 |
} |
0fc0b732e netdev: drivers s... |
203 |
static const struct ethtool_ops veth_ethtool_ops = { |
e314dbdc1 [NET]: Virtual et... |
204 205 |
.get_drvinfo = veth_get_drvinfo, .get_link = ethtool_op_get_link, |
e314dbdc1 [NET]: Virtual et... |
206 |
.get_strings = veth_get_strings, |
b9f2c0440 [netdrvr] Stop us... |
207 |
.get_sset_count = veth_get_sset_count, |
e314dbdc1 [NET]: Virtual et... |
208 |
.get_ethtool_stats = veth_get_ethtool_stats, |
56607b987 net: veth: use ne... |
209 |
.get_link_ksettings = veth_get_link_ksettings, |
056b21fbe net: veth: use ge... |
210 |
.get_ts_info = ethtool_op_get_ts_info, |
e314dbdc1 [NET]: Virtual et... |
211 |
}; |
948d4f214 veth: Add driver XDP |
212 |
/* general routines */ |
9fc8d518d veth: Handle xdp_... |
213 214 215 216 |
static bool veth_is_xdp_frame(void *ptr) { return (unsigned long)ptr & VETH_XDP_FLAG; } |
defcffeb5 net-veth: Add typ... |
217 |
static struct xdp_frame *veth_ptr_to_xdp(void *ptr) |
9fc8d518d veth: Handle xdp_... |
218 219 220 |
{ return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); } |
defcffeb5 net-veth: Add typ... |
221 |
static void *veth_xdp_to_ptr(struct xdp_frame *xdp) |
af87a3aa1 veth: Add ndo_xdp... |
222 |
{ |
defcffeb5 net-veth: Add typ... |
223 |
return (void *)((unsigned long)xdp | VETH_XDP_FLAG); |
af87a3aa1 veth: Add ndo_xdp... |
224 |
} |
9fc8d518d veth: Handle xdp_... |
225 226 227 228 229 230 231 |
static void veth_ptr_free(void *ptr) { if (veth_is_xdp_frame(ptr)) xdp_return_frame(veth_ptr_to_xdp(ptr)); else kfree_skb(ptr); } |
638264dc9 veth: Support per... |
232 |
static void __veth_xdp_flush(struct veth_rq *rq) |
948d4f214 veth: Add driver XDP |
233 234 235 |
{ /* Write ptr_ring before reading rx_notify_masked */ smp_mb(); |
638264dc9 veth: Support per... |
236 237 238 |
if (!rq->rx_notify_masked) { rq->rx_notify_masked = true; napi_schedule(&rq->xdp_napi); |
948d4f214 veth: Add driver XDP |
239 240 |
} } |
638264dc9 veth: Support per... |
241 |
static int veth_xdp_rx(struct veth_rq *rq, struct sk_buff *skb) |
948d4f214 veth: Add driver XDP |
242 |
{ |
638264dc9 veth: Support per... |
243 |
if (unlikely(ptr_ring_produce(&rq->xdp_ring, skb))) { |
948d4f214 veth: Add driver XDP |
244 245 246 247 248 249 |
dev_kfree_skb_any(skb); return NET_RX_DROP; } return NET_RX_SUCCESS; } |
638264dc9 veth: Support per... |
250 251 |
static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb, struct veth_rq *rq, bool xdp) |
e314dbdc1 [NET]: Virtual et... |
252 |
{ |
948d4f214 veth: Add driver XDP |
253 |
return __dev_forward_skb(dev, skb) ?: xdp ? |
638264dc9 veth: Support per... |
254 |
veth_xdp_rx(rq, skb) : |
948d4f214 veth: Add driver XDP |
255 256 257 258 259 260 |
netif_rx(skb); } static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev) { struct veth_priv *rcv_priv, *priv = netdev_priv(dev); |
638264dc9 veth: Support per... |
261 |
struct veth_rq *rq = NULL; |
d0e2c55e7 veth: avoid a NUL... |
262 |
struct net_device *rcv; |
2681128f0 veth: reduce stat... |
263 |
int length = skb->len; |
948d4f214 veth: Add driver XDP |
264 |
bool rcv_xdp = false; |
638264dc9 veth: Support per... |
265 |
int rxq; |
e314dbdc1 [NET]: Virtual et... |
266 |
|
d0e2c55e7 veth: avoid a NUL... |
267 268 269 270 271 272 |
rcu_read_lock(); rcv = rcu_dereference(priv->peer); if (unlikely(!rcv)) { kfree_skb(skb); goto drop; } |
e314dbdc1 [NET]: Virtual et... |
273 |
|
948d4f214 veth: Add driver XDP |
274 |
rcv_priv = netdev_priv(rcv); |
638264dc9 veth: Support per... |
275 276 277 278 279 280 281 |
rxq = skb_get_queue_mapping(skb); if (rxq < rcv->real_num_rx_queues) { rq = &rcv_priv->rq[rxq]; rcv_xdp = rcu_access_pointer(rq->xdp_prog); if (rcv_xdp) skb_record_rx_queue(skb, rxq); } |
948d4f214 veth: Add driver XDP |
282 |
|
aa4e689ed veth: add softwar... |
283 |
skb_tx_timestamp(skb); |
638264dc9 veth: Support per... |
284 |
if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) { |
b4fba476d veth: use standar... |
285 286 |
if (!rcv_xdp) dev_lstats_add(dev, length); |
2681128f0 veth: reduce stat... |
287 |
} else { |
d0e2c55e7 veth: avoid a NUL... |
288 |
drop: |
2681128f0 veth: reduce stat... |
289 290 |
atomic64_inc(&priv->dropped); } |
948d4f214 veth: Add driver XDP |
291 292 |
if (rcv_xdp) |
638264dc9 veth: Support per... |
293 |
__veth_xdp_flush(rq); |
948d4f214 veth: Add driver XDP |
294 |
|
d0e2c55e7 veth: avoid a NUL... |
295 |
rcu_read_unlock(); |
948d4f214 veth: Add driver XDP |
296 |
|
6ed106549 net: use NETDEV_T... |
297 |
return NETDEV_TX_OK; |
e314dbdc1 [NET]: Virtual et... |
298 |
} |
b4fba476d veth: use standar... |
299 |
static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes) |
e314dbdc1 [NET]: Virtual et... |
300 |
{ |
cf05c700c veth: fix 64bit s... |
301 |
struct veth_priv *priv = netdev_priv(dev); |
cf05c700c veth: fix 64bit s... |
302 |
|
b4fba476d veth: use standar... |
303 |
dev_lstats_read(dev, packets, bytes); |
2681128f0 veth: reduce stat... |
304 305 |
return atomic64_read(&priv->dropped); } |
65780c562 veth: move xdp st... |
306 |
static void veth_stats_rx(struct veth_stats *result, struct net_device *dev) |
4195e54aa veth: Account for... |
307 308 309 |
{ struct veth_priv *priv = netdev_priv(dev); int i; |
5fe6e5677 veth: rely on pee... |
310 |
result->peer_tq_xdp_xmit_err = 0; |
4195e54aa veth: Account for... |
311 |
result->xdp_packets = 0; |
d99a7c2f3 veth: remove atom... |
312 |
result->xdp_tx_err = 0; |
4195e54aa veth: Account for... |
313 |
result->xdp_bytes = 0; |
66fe4a078 veth: distinguish... |
314 |
result->rx_drops = 0; |
4195e54aa veth: Account for... |
315 |
for (i = 0; i < dev->num_rx_queues; i++) { |
5fe6e5677 veth: rely on pee... |
316 |
u64 packets, bytes, drops, xdp_tx_err, peer_tq_xdp_xmit_err; |
4195e54aa veth: Account for... |
317 |
struct veth_rq_stats *stats = &priv->rq[i].stats; |
4195e54aa veth: Account for... |
318 319 320 321 |
unsigned int start; do { start = u64_stats_fetch_begin_irq(&stats->syncp); |
5fe6e5677 veth: rely on pee... |
322 |
peer_tq_xdp_xmit_err = stats->vs.peer_tq_xdp_xmit_err; |
d99a7c2f3 veth: remove atom... |
323 |
xdp_tx_err = stats->vs.xdp_tx_err; |
65780c562 veth: move xdp st... |
324 325 |
packets = stats->vs.xdp_packets; bytes = stats->vs.xdp_bytes; |
66fe4a078 veth: distinguish... |
326 |
drops = stats->vs.rx_drops; |
4195e54aa veth: Account for... |
327 |
} while (u64_stats_fetch_retry_irq(&stats->syncp, start)); |
5fe6e5677 veth: rely on pee... |
328 |
result->peer_tq_xdp_xmit_err += peer_tq_xdp_xmit_err; |
d99a7c2f3 veth: remove atom... |
329 |
result->xdp_tx_err += xdp_tx_err; |
4195e54aa veth: Account for... |
330 331 |
result->xdp_packets += packets; result->xdp_bytes += bytes; |
66fe4a078 veth: distinguish... |
332 |
result->rx_drops += drops; |
4195e54aa veth: Account for... |
333 334 |
} } |
bc1f44709 net: make ndo_get... |
335 336 |
static void veth_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *tot) |
2681128f0 veth: reduce stat... |
337 338 |
{ struct veth_priv *priv = netdev_priv(dev); |
d0e2c55e7 veth: avoid a NUL... |
339 |
struct net_device *peer; |
65780c562 veth: move xdp st... |
340 |
struct veth_stats rx; |
b4fba476d veth: use standar... |
341 |
u64 packets, bytes; |
4195e54aa veth: Account for... |
342 |
|
b4fba476d veth: use standar... |
343 344 345 |
tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes); tot->tx_bytes = bytes; tot->tx_packets = packets; |
2681128f0 veth: reduce stat... |
346 |
|
4195e54aa veth: Account for... |
347 |
veth_stats_rx(&rx, dev); |
5fe6e5677 veth: rely on pee... |
348 349 |
tot->tx_dropped += rx.xdp_tx_err; tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err; |
4195e54aa veth: Account for... |
350 351 |
tot->rx_bytes = rx.xdp_bytes; tot->rx_packets = rx.xdp_packets; |
2681128f0 veth: reduce stat... |
352 |
|
d0e2c55e7 veth: avoid a NUL... |
353 354 355 |
rcu_read_lock(); peer = rcu_dereference(priv->peer); if (peer) { |
e25d5dbcf veth: ignore peer... |
356 |
veth_stats_tx(peer, &packets, &bytes); |
b4fba476d veth: use standar... |
357 358 |
tot->rx_bytes += bytes; tot->rx_packets += packets; |
4195e54aa veth: Account for... |
359 360 |
veth_stats_rx(&rx, peer); |
5fe6e5677 veth: rely on pee... |
361 362 |
tot->tx_dropped += rx.peer_tq_xdp_xmit_err; tot->rx_dropped += rx.xdp_tx_err; |
4195e54aa veth: Account for... |
363 364 |
tot->tx_bytes += rx.xdp_bytes; tot->tx_packets += rx.xdp_packets; |
d0e2c55e7 veth: avoid a NUL... |
365 366 |
} rcu_read_unlock(); |
e314dbdc1 [NET]: Virtual et... |
367 |
} |
5c70ef85a veth: allow to se... |
368 369 370 371 |
/* fake multicast ability */ static void veth_set_multicast_list(struct net_device *dev) { } |
948d4f214 veth: Add driver XDP |
372 373 374 375 |
static struct sk_buff *veth_build_skb(void *head, int headroom, int len, int buflen) { struct sk_buff *skb; |
948d4f214 veth: Add driver XDP |
376 377 378 379 380 381 382 383 384 |
skb = build_skb(head, buflen); if (!skb) return NULL; skb_reserve(skb, headroom); skb_put(skb, len); return skb; } |
638264dc9 veth: Support per... |
385 386 387 388 |
static int veth_select_rxq(struct net_device *dev) { return smp_processor_id() % dev->real_num_rx_queues; } |
9aa1206e8 bpf: Add redirect... |
389 390 391 392 393 394 395 |
static struct net_device *veth_peer_dev(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); /* Callers must be under RCU read side. */ return rcu_dereference(priv->peer); } |
af87a3aa1 veth: Add ndo_xdp... |
396 |
static int veth_xdp_xmit(struct net_device *dev, int n, |
9152cff0d veth: introduce m... |
397 398 |
struct xdp_frame **frames, u32 flags, bool ndo_xmit) |
af87a3aa1 veth: Add ndo_xdp... |
399 400 |
{ struct veth_priv *rcv_priv, *priv = netdev_priv(dev); |
5fe6e5677 veth: rely on pee... |
401 |
int i, ret = -ENXIO, drops = 0; |
af87a3aa1 veth: Add ndo_xdp... |
402 |
struct net_device *rcv; |
5fe6e5677 veth: rely on pee... |
403 |
unsigned int max_len; |
638264dc9 veth: Support per... |
404 |
struct veth_rq *rq; |
af87a3aa1 veth: Add ndo_xdp... |
405 |
|
5fe6e5677 veth: rely on pee... |
406 |
if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) |
d99a7c2f3 veth: remove atom... |
407 |
return -EINVAL; |
af87a3aa1 veth: Add ndo_xdp... |
408 |
|
5fe6e5677 veth: rely on pee... |
409 |
rcu_read_lock(); |
af87a3aa1 veth: Add ndo_xdp... |
410 |
rcv = rcu_dereference(priv->peer); |
5fe6e5677 veth: rely on pee... |
411 412 |
if (unlikely(!rcv)) goto out; |
af87a3aa1 veth: Add ndo_xdp... |
413 414 |
rcv_priv = netdev_priv(rcv); |
5fe6e5677 veth: rely on pee... |
415 |
rq = &rcv_priv->rq[veth_select_rxq(rcv)]; |
af87a3aa1 veth: Add ndo_xdp... |
416 417 418 419 |
/* Non-NULL xdp_prog ensures that xdp_ring is initialized on receive * side. This means an XDP program is loaded on the peer and the peer * device is up. */ |
5fe6e5677 veth: rely on pee... |
420 421 |
if (!rcu_access_pointer(rq->xdp_prog)) goto out; |
af87a3aa1 veth: Add ndo_xdp... |
422 423 |
max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN; |
638264dc9 veth: Support per... |
424 |
spin_lock(&rq->xdp_ring.producer_lock); |
af87a3aa1 veth: Add ndo_xdp... |
425 426 427 428 429 |
for (i = 0; i < n; i++) { struct xdp_frame *frame = frames[i]; void *ptr = veth_xdp_to_ptr(frame); if (unlikely(frame->len > max_len || |
638264dc9 veth: Support per... |
430 |
__ptr_ring_produce(&rq->xdp_ring, ptr))) { |
af87a3aa1 veth: Add ndo_xdp... |
431 432 433 434 |
xdp_return_frame_rx_napi(frame); drops++; } } |
638264dc9 veth: Support per... |
435 |
spin_unlock(&rq->xdp_ring.producer_lock); |
af87a3aa1 veth: Add ndo_xdp... |
436 437 |
if (flags & XDP_XMIT_FLUSH) |
638264dc9 veth: Support per... |
438 |
__veth_xdp_flush(rq); |
af87a3aa1 veth: Add ndo_xdp... |
439 |
|
d99a7c2f3 veth: remove atom... |
440 |
ret = n - drops; |
9152cff0d veth: introduce m... |
441 |
if (ndo_xmit) { |
5fe6e5677 veth: rely on pee... |
442 443 444 445 |
u64_stats_update_begin(&rq->stats.syncp); rq->stats.vs.peer_tq_xdp_xmit += n - drops; rq->stats.vs.peer_tq_xdp_xmit_err += drops; u64_stats_update_end(&rq->stats.syncp); |
9152cff0d veth: introduce m... |
446 |
} |
9152cff0d veth: introduce m... |
447 |
|
5fe6e5677 veth: rely on pee... |
448 |
out: |
b23bfa563 bpf, xdp: Remove ... |
449 |
rcu_read_unlock(); |
2131479df veth: Account for... |
450 451 |
return ret; |
af87a3aa1 veth: Add ndo_xdp... |
452 |
} |
9152cff0d veth: introduce m... |
453 454 455 |
static int veth_ndo_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags) { |
5fe6e5677 veth: rely on pee... |
456 457 458 459 460 461 462 463 464 465 |
int err; err = veth_xdp_xmit(dev, n, frames, flags, true); if (err < 0) { struct veth_priv *priv = netdev_priv(dev); atomic64_add(n, &priv->dropped); } return err; |
9152cff0d veth: introduce m... |
466 |
} |
bd32aa1f5 veth: rely on vet... |
467 |
static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) |
9cda7807e veth: Support bul... |
468 469 |
{ int sent, i, err = 0; |
bd32aa1f5 veth: rely on vet... |
470 |
sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false); |
9cda7807e veth: Support bul... |
471 472 473 474 475 476 |
if (sent < 0) { err = sent; sent = 0; for (i = 0; i < bq->count; i++) xdp_return_frame(bq->q[i]); } |
bd32aa1f5 veth: rely on vet... |
477 |
trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err); |
9cda7807e veth: Support bul... |
478 |
|
5fe6e5677 veth: rely on pee... |
479 480 481 482 |
u64_stats_update_begin(&rq->stats.syncp); rq->stats.vs.xdp_tx += sent; rq->stats.vs.xdp_tx_err += bq->count - sent; u64_stats_update_end(&rq->stats.syncp); |
9cda7807e veth: Support bul... |
483 484 |
bq->count = 0; } |
bd32aa1f5 veth: rely on vet... |
485 |
static void veth_xdp_flush(struct veth_rq *rq, struct veth_xdp_tx_bq *bq) |
d1396004d veth: Add XDP TX ... |
486 |
{ |
bd32aa1f5 veth: rely on vet... |
487 |
struct veth_priv *rcv_priv, *priv = netdev_priv(rq->dev); |
d1396004d veth: Add XDP TX ... |
488 |
struct net_device *rcv; |
bd32aa1f5 veth: rely on vet... |
489 |
struct veth_rq *rcv_rq; |
d1396004d veth: Add XDP TX ... |
490 491 |
rcu_read_lock(); |
bd32aa1f5 veth: rely on vet... |
492 |
veth_xdp_flush_bq(rq, bq); |
d1396004d veth: Add XDP TX ... |
493 494 495 496 497 |
rcv = rcu_dereference(priv->peer); if (unlikely(!rcv)) goto out; rcv_priv = netdev_priv(rcv); |
bd32aa1f5 veth: rely on vet... |
498 |
rcv_rq = &rcv_priv->rq[veth_select_rxq(rcv)]; |
d1396004d veth: Add XDP TX ... |
499 |
/* xdp_ring is initialized on receive side? */ |
bd32aa1f5 veth: rely on vet... |
500 |
if (unlikely(!rcu_access_pointer(rcv_rq->xdp_prog))) |
d1396004d veth: Add XDP TX ... |
501 |
goto out; |
bd32aa1f5 veth: rely on vet... |
502 |
__veth_xdp_flush(rcv_rq); |
d1396004d veth: Add XDP TX ... |
503 504 505 |
out: rcu_read_unlock(); } |
bd32aa1f5 veth: rely on vet... |
506 |
static int veth_xdp_tx(struct veth_rq *rq, struct xdp_buff *xdp, |
9cda7807e veth: Support bul... |
507 |
struct veth_xdp_tx_bq *bq) |
d1396004d veth: Add XDP TX ... |
508 |
{ |
1b698fa5d xdp: Rename conve... |
509 |
struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); |
d1396004d veth: Add XDP TX ... |
510 511 512 |
if (unlikely(!frame)) return -EOVERFLOW; |
9cda7807e veth: Support bul... |
513 |
if (unlikely(bq->count == VETH_XDP_TX_BULK_SIZE)) |
bd32aa1f5 veth: rely on vet... |
514 |
veth_xdp_flush_bq(rq, bq); |
9cda7807e veth: Support bul... |
515 516 517 518 |
bq->q[bq->count++] = frame; return 0; |
d1396004d veth: Add XDP TX ... |
519 |
} |
638264dc9 veth: Support per... |
520 |
static struct sk_buff *veth_xdp_rcv_one(struct veth_rq *rq, |
d1396004d veth: Add XDP TX ... |
521 |
struct xdp_frame *frame, |
1c5b82e55 veth: introduce m... |
522 523 |
struct veth_xdp_tx_bq *bq, struct veth_stats *stats) |
9fc8d518d veth: Handle xdp_... |
524 525 |
{ void *hard_start = frame->data - frame->headroom; |
9fc8d518d veth: Handle xdp_... |
526 |
int len = frame->len, delta = 0; |
d1396004d veth: Add XDP TX ... |
527 |
struct xdp_frame orig_frame; |
9fc8d518d veth: Handle xdp_... |
528 529 530 |
struct bpf_prog *xdp_prog; unsigned int headroom; struct sk_buff *skb; |
5c8572251 veth: Adjust hard... |
531 532 |
/* bpf_xdp_adjust_head() assures BPF cannot access xdp_frame area */ hard_start -= sizeof(struct xdp_frame); |
9fc8d518d veth: Handle xdp_... |
533 |
rcu_read_lock(); |
638264dc9 veth: Support per... |
534 |
xdp_prog = rcu_dereference(rq->xdp_prog); |
9fc8d518d veth: Handle xdp_... |
535 536 537 |
if (likely(xdp_prog)) { struct xdp_buff xdp; u32 act; |
fc3798726 xdp: Introduce xd... |
538 |
xdp_convert_frame_to_buff(frame, &xdp); |
638264dc9 veth: Support per... |
539 |
xdp.rxq = &rq->xdp_rxq; |
9fc8d518d veth: Handle xdp_... |
540 541 542 543 544 545 546 547 |
act = bpf_prog_run_xdp(xdp_prog, &xdp); switch (act) { case XDP_PASS: delta = frame->data - xdp.data; len = xdp.data_end - xdp.data; break; |
d1396004d veth: Add XDP TX ... |
548 549 |
case XDP_TX: orig_frame = *frame; |
d1396004d veth: Add XDP TX ... |
550 |
xdp.rxq->mem = frame->mem; |
bd32aa1f5 veth: rely on vet... |
551 |
if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { |
638264dc9 veth: Support per... |
552 |
trace_xdp_exception(rq->dev, xdp_prog, act); |
d1396004d veth: Add XDP TX ... |
553 |
frame = &orig_frame; |
1c5b82e55 veth: introduce m... |
554 |
stats->rx_drops++; |
d1396004d veth: Add XDP TX ... |
555 556 |
goto err_xdp; } |
1c5b82e55 veth: introduce m... |
557 |
stats->xdp_tx++; |
d1396004d veth: Add XDP TX ... |
558 559 560 561 |
rcu_read_unlock(); goto xdp_xmit; case XDP_REDIRECT: orig_frame = *frame; |
d1396004d veth: Add XDP TX ... |
562 |
xdp.rxq->mem = frame->mem; |
638264dc9 veth: Support per... |
563 |
if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { |
d1396004d veth: Add XDP TX ... |
564 |
frame = &orig_frame; |
1c5b82e55 veth: introduce m... |
565 |
stats->rx_drops++; |
d1396004d veth: Add XDP TX ... |
566 567 |
goto err_xdp; } |
1c5b82e55 veth: introduce m... |
568 |
stats->xdp_redirect++; |
d1396004d veth: Add XDP TX ... |
569 570 |
rcu_read_unlock(); goto xdp_xmit; |
9fc8d518d veth: Handle xdp_... |
571 572 |
default: bpf_warn_invalid_xdp_action(act); |
df561f668 treewide: Use fal... |
573 |
fallthrough; |
9fc8d518d veth: Handle xdp_... |
574 |
case XDP_ABORTED: |
638264dc9 veth: Support per... |
575 |
trace_xdp_exception(rq->dev, xdp_prog, act); |
df561f668 treewide: Use fal... |
576 |
fallthrough; |
9fc8d518d veth: Handle xdp_... |
577 |
case XDP_DROP: |
1c5b82e55 veth: introduce m... |
578 |
stats->xdp_drops++; |
9fc8d518d veth: Handle xdp_... |
579 580 581 582 583 584 |
goto err_xdp; } } rcu_read_unlock(); headroom = sizeof(struct xdp_frame) + frame->headroom - delta; |
45a9e6d8a veth: Xdp using f... |
585 |
skb = veth_build_skb(hard_start, headroom, len, frame->frame_sz); |
9fc8d518d veth: Handle xdp_... |
586 587 |
if (!skb) { xdp_return_frame(frame); |
1c5b82e55 veth: introduce m... |
588 |
stats->rx_drops++; |
9fc8d518d veth: Handle xdp_... |
589 590 |
goto err; } |
cbf335106 veth: use xdp_rel... |
591 |
xdp_release_frame(frame); |
9fc8d518d veth: Handle xdp_... |
592 |
xdp_scrub_frame(frame); |
638264dc9 veth: Support per... |
593 |
skb->protocol = eth_type_trans(skb, rq->dev); |
9fc8d518d veth: Handle xdp_... |
594 595 596 597 598 |
err: return skb; err_xdp: rcu_read_unlock(); xdp_return_frame(frame); |
d1396004d veth: Add XDP TX ... |
599 |
xdp_xmit: |
9fc8d518d veth: Handle xdp_... |
600 601 |
return NULL; } |
1c5b82e55 veth: introduce m... |
602 603 604 605 |
static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, struct sk_buff *skb, struct veth_xdp_tx_bq *bq, struct veth_stats *stats) |
948d4f214 veth: Add driver XDP |
606 607 608 609 610 611 |
{ u32 pktlen, headroom, act, metalen; void *orig_data, *orig_data_end; struct bpf_prog *xdp_prog; int mac_len, delta, off; struct xdp_buff xdp; |
4bf9ffa0f veth: Orphan skb ... |
612 |
skb_orphan(skb); |
948d4f214 veth: Add driver XDP |
613 |
rcu_read_lock(); |
638264dc9 veth: Support per... |
614 |
xdp_prog = rcu_dereference(rq->xdp_prog); |
948d4f214 veth: Add driver XDP |
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 |
if (unlikely(!xdp_prog)) { rcu_read_unlock(); goto out; } mac_len = skb->data - skb_mac_header(skb); pktlen = skb->len + mac_len; headroom = skb_headroom(skb) - mac_len; if (skb_shared(skb) || skb_head_is_locked(skb) || skb_is_nonlinear(skb) || headroom < XDP_PACKET_HEADROOM) { struct sk_buff *nskb; int size, head_off; void *head, *start; struct page *page; size = SKB_DATA_ALIGN(VETH_XDP_HEADROOM + pktlen) + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); if (size > PAGE_SIZE) goto drop; page = alloc_page(GFP_ATOMIC | __GFP_NOWARN); if (!page) goto drop; head = page_address(page); start = head + VETH_XDP_HEADROOM; if (skb_copy_bits(skb, -mac_len, start, pktlen)) { page_frag_free(head); goto drop; } |
45a9e6d8a veth: Xdp using f... |
646 647 |
nskb = veth_build_skb(head, VETH_XDP_HEADROOM + mac_len, skb->len, PAGE_SIZE); |
948d4f214 veth: Add driver XDP |
648 649 650 651 652 653 654 655 |
if (!nskb) { page_frag_free(head); goto drop; } skb_copy_header(nskb, skb); head_off = skb_headroom(nskb) - skb_headroom(skb); skb_headers_offset_update(nskb, head_off); |
948d4f214 veth: Add driver XDP |
656 657 658 659 660 661 662 663 |
consume_skb(skb); skb = nskb; } xdp.data_hard_start = skb->head; xdp.data = skb_mac_header(skb); xdp.data_end = xdp.data + pktlen; xdp.data_meta = xdp.data; |
638264dc9 veth: Support per... |
664 |
xdp.rxq = &rq->xdp_rxq; |
45a9e6d8a veth: Xdp using f... |
665 666 667 668 |
/* SKB "head" area always have tailroom for skb_shared_info */ xdp.frame_sz = (void *)skb_end_pointer(skb) - xdp.data_hard_start; xdp.frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
948d4f214 veth: Add driver XDP |
669 670 671 672 673 674 675 676 |
orig_data = xdp.data; orig_data_end = xdp.data_end; act = bpf_prog_run_xdp(xdp_prog, &xdp); switch (act) { case XDP_PASS: break; |
d1396004d veth: Add XDP TX ... |
677 678 679 |
case XDP_TX: get_page(virt_to_page(xdp.data)); consume_skb(skb); |
638264dc9 veth: Support per... |
680 |
xdp.rxq->mem = rq->xdp_mem; |
bd32aa1f5 veth: rely on vet... |
681 |
if (unlikely(veth_xdp_tx(rq, &xdp, bq) < 0)) { |
638264dc9 veth: Support per... |
682 |
trace_xdp_exception(rq->dev, xdp_prog, act); |
1c5b82e55 veth: introduce m... |
683 |
stats->rx_drops++; |
d1396004d veth: Add XDP TX ... |
684 685 |
goto err_xdp; } |
1c5b82e55 veth: introduce m... |
686 |
stats->xdp_tx++; |
d1396004d veth: Add XDP TX ... |
687 688 689 690 691 |
rcu_read_unlock(); goto xdp_xmit; case XDP_REDIRECT: get_page(virt_to_page(xdp.data)); consume_skb(skb); |
638264dc9 veth: Support per... |
692 |
xdp.rxq->mem = rq->xdp_mem; |
1c5b82e55 veth: introduce m... |
693 694 |
if (xdp_do_redirect(rq->dev, &xdp, xdp_prog)) { stats->rx_drops++; |
d1396004d veth: Add XDP TX ... |
695 |
goto err_xdp; |
1c5b82e55 veth: introduce m... |
696 697 |
} stats->xdp_redirect++; |
d1396004d veth: Add XDP TX ... |
698 699 |
rcu_read_unlock(); goto xdp_xmit; |
948d4f214 veth: Add driver XDP |
700 701 |
default: bpf_warn_invalid_xdp_action(act); |
df561f668 treewide: Use fal... |
702 |
fallthrough; |
948d4f214 veth: Add driver XDP |
703 |
case XDP_ABORTED: |
638264dc9 veth: Support per... |
704 |
trace_xdp_exception(rq->dev, xdp_prog, act); |
df561f668 treewide: Use fal... |
705 |
fallthrough; |
948d4f214 veth: Add driver XDP |
706 |
case XDP_DROP: |
1c5b82e55 veth: introduce m... |
707 708 |
stats->xdp_drops++; goto xdp_drop; |
948d4f214 veth: Add driver XDP |
709 710 |
} rcu_read_unlock(); |
45a9e6d8a veth: Xdp using f... |
711 |
/* check if bpf_xdp_adjust_head was used */ |
948d4f214 veth: Add driver XDP |
712 713 714 715 716 717 718 |
delta = orig_data - xdp.data; off = mac_len + delta; if (off > 0) __skb_push(skb, off); else if (off < 0) __skb_pull(skb, -off); skb->mac_header -= delta; |
45a9e6d8a veth: Xdp using f... |
719 720 |
/* check if bpf_xdp_adjust_tail was used */ |
948d4f214 veth: Add driver XDP |
721 722 |
off = xdp.data_end - orig_data_end; if (off != 0) |
45a9e6d8a veth: Xdp using f... |
723 |
__skb_put(skb, off); /* positive on grow, negative on shrink */ |
638264dc9 veth: Support per... |
724 |
skb->protocol = eth_type_trans(skb, rq->dev); |
948d4f214 veth: Add driver XDP |
725 726 727 728 729 730 731 |
metalen = xdp.data - xdp.data_meta; if (metalen) skb_metadata_set(skb, metalen); out: return skb; drop: |
1c5b82e55 veth: introduce m... |
732 733 |
stats->rx_drops++; xdp_drop: |
948d4f214 veth: Add driver XDP |
734 735 736 |
rcu_read_unlock(); kfree_skb(skb); return NULL; |
d1396004d veth: Add XDP TX ... |
737 738 739 740 741 |
err_xdp: rcu_read_unlock(); page_frag_free(xdp.data); xdp_xmit: return NULL; |
948d4f214 veth: Add driver XDP |
742 |
} |
1c5b82e55 veth: introduce m... |
743 744 745 |
static int veth_xdp_rcv(struct veth_rq *rq, int budget, struct veth_xdp_tx_bq *bq, struct veth_stats *stats) |
948d4f214 veth: Add driver XDP |
746 |
{ |
1c5b82e55 veth: introduce m... |
747 |
int i, done = 0; |
948d4f214 veth: Add driver XDP |
748 749 |
for (i = 0; i < budget; i++) { |
638264dc9 veth: Support per... |
750 |
void *ptr = __ptr_ring_consume(&rq->xdp_ring); |
9fc8d518d veth: Handle xdp_... |
751 |
struct sk_buff *skb; |
948d4f214 veth: Add driver XDP |
752 |
|
9fc8d518d veth: Handle xdp_... |
753 |
if (!ptr) |
948d4f214 veth: Add driver XDP |
754 |
break; |
d1396004d veth: Add XDP TX ... |
755 |
if (veth_is_xdp_frame(ptr)) { |
4195e54aa veth: Account for... |
756 |
struct xdp_frame *frame = veth_ptr_to_xdp(ptr); |
1c5b82e55 veth: introduce m... |
757 758 |
stats->xdp_bytes += frame->len; skb = veth_xdp_rcv_one(rq, frame, bq, stats); |
d1396004d veth: Add XDP TX ... |
759 |
} else { |
4195e54aa veth: Account for... |
760 |
skb = ptr; |
1c5b82e55 veth: introduce m... |
761 762 |
stats->xdp_bytes += skb->len; skb = veth_xdp_rcv_skb(rq, skb, bq, stats); |
d1396004d veth: Add XDP TX ... |
763 |
} |
948d4f214 veth: Add driver XDP |
764 765 |
if (skb) |
638264dc9 veth: Support per... |
766 |
napi_gro_receive(&rq->xdp_napi, skb); |
948d4f214 veth: Add driver XDP |
767 768 769 |
done++; } |
4195e54aa veth: Account for... |
770 |
u64_stats_update_begin(&rq->stats.syncp); |
9152cff0d veth: introduce m... |
771 |
rq->stats.vs.xdp_redirect += stats->xdp_redirect; |
1c5b82e55 veth: introduce m... |
772 |
rq->stats.vs.xdp_bytes += stats->xdp_bytes; |
66fe4a078 veth: distinguish... |
773 774 |
rq->stats.vs.xdp_drops += stats->xdp_drops; rq->stats.vs.rx_drops += stats->rx_drops; |
65780c562 veth: move xdp st... |
775 |
rq->stats.vs.xdp_packets += done; |
4195e54aa veth: Account for... |
776 |
u64_stats_update_end(&rq->stats.syncp); |
948d4f214 veth: Add driver XDP |
777 778 779 780 781 |
return done; } static int veth_poll(struct napi_struct *napi, int budget) { |
638264dc9 veth: Support per... |
782 783 |
struct veth_rq *rq = container_of(napi, struct veth_rq, xdp_napi); |
1c5b82e55 veth: introduce m... |
784 |
struct veth_stats stats = {}; |
9cda7807e veth: Support bul... |
785 |
struct veth_xdp_tx_bq bq; |
948d4f214 veth: Add driver XDP |
786 |
int done; |
9cda7807e veth: Support bul... |
787 |
bq.count = 0; |
d1396004d veth: Add XDP TX ... |
788 |
xdp_set_return_frame_no_direct(); |
1c5b82e55 veth: introduce m... |
789 |
done = veth_xdp_rcv(rq, budget, &bq, &stats); |
948d4f214 veth: Add driver XDP |
790 791 792 |
if (done < budget && napi_complete_done(napi, done)) { /* Write rx_notify_masked before reading ptr_ring */ |
638264dc9 veth: Support per... |
793 794 795 796 |
smp_store_mb(rq->rx_notify_masked, false); if (unlikely(!__ptr_ring_empty(&rq->xdp_ring))) { rq->rx_notify_masked = true; napi_schedule(&rq->xdp_napi); |
948d4f214 veth: Add driver XDP |
797 798 |
} } |
1c5b82e55 veth: introduce m... |
799 |
if (stats.xdp_tx > 0) |
bd32aa1f5 veth: rely on vet... |
800 |
veth_xdp_flush(rq, &bq); |
1c5b82e55 veth: introduce m... |
801 |
if (stats.xdp_redirect > 0) |
1d233886d xdp: Use bulking ... |
802 |
xdp_do_flush(); |
d1396004d veth: Add XDP TX ... |
803 |
xdp_clear_return_frame_no_direct(); |
948d4f214 veth: Add driver XDP |
804 805 806 807 808 809 |
return done; } static int veth_napi_add(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); |
638264dc9 veth: Support per... |
810 |
int err, i; |
948d4f214 veth: Add driver XDP |
811 |
|
638264dc9 veth: Support per... |
812 813 814 815 816 817 818 |
for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL); if (err) goto err_xdp_ring; } |
948d4f214 veth: Add driver XDP |
819 |
|
638264dc9 veth: Support per... |
820 821 822 823 824 825 |
for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); napi_enable(&rq->xdp_napi); } |
948d4f214 veth: Add driver XDP |
826 827 |
return 0; |
638264dc9 veth: Support per... |
828 829 830 831 832 |
err_xdp_ring: for (i--; i >= 0; i--) ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free); return err; |
948d4f214 veth: Add driver XDP |
833 834 835 836 837 |
} static void veth_napi_del(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); |
638264dc9 veth: Support per... |
838 |
int i; |
948d4f214 veth: Add driver XDP |
839 |
|
638264dc9 veth: Support per... |
840 841 842 843 |
for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; napi_disable(&rq->xdp_napi); |
5198d545d net: remove napi_... |
844 |
__netif_napi_del(&rq->xdp_napi); |
638264dc9 veth: Support per... |
845 846 847 848 849 |
} synchronize_net(); for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; |
638264dc9 veth: Support per... |
850 851 852 |
rq->rx_notify_masked = false; ptr_ring_cleanup(&rq->xdp_ring, veth_ptr_free); } |
948d4f214 veth: Add driver XDP |
853 854 855 856 857 |
} static int veth_enable_xdp(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); |
638264dc9 veth: Support per... |
858 |
int err, i; |
948d4f214 veth: Add driver XDP |
859 |
|
638264dc9 veth: Support per... |
860 861 862 |
if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) { for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; |
948d4f214 veth: Add driver XDP |
863 |
|
638264dc9 veth: Support per... |
864 865 866 867 868 869 870 871 872 873 874 875 876 |
err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); if (err < 0) goto err_rxq_reg; err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); if (err < 0) goto err_reg_mem; /* Save original mem info as it can be overwritten */ rq->xdp_mem = rq->xdp_rxq.mem; } |
948d4f214 veth: Add driver XDP |
877 878 879 |
err = veth_napi_add(dev); if (err) |
638264dc9 veth: Support per... |
880 |
goto err_rxq_reg; |
948d4f214 veth: Add driver XDP |
881 |
} |
638264dc9 veth: Support per... |
882 883 |
for (i = 0; i < dev->real_num_rx_queues; i++) rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog); |
948d4f214 veth: Add driver XDP |
884 885 |
return 0; |
638264dc9 veth: Support per... |
886 887 888 889 890 |
err_reg_mem: xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); err_rxq_reg: for (i--; i >= 0; i--) xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); |
948d4f214 veth: Add driver XDP |
891 892 893 894 895 896 897 |
return err; } static void veth_disable_xdp(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); |
638264dc9 veth: Support per... |
898 |
int i; |
948d4f214 veth: Add driver XDP |
899 |
|
638264dc9 veth: Support per... |
900 901 |
for (i = 0; i < dev->real_num_rx_queues; i++) rcu_assign_pointer(priv->rq[i].xdp_prog, NULL); |
948d4f214 veth: Add driver XDP |
902 |
veth_napi_del(dev); |
638264dc9 veth: Support per... |
903 904 905 906 907 908 |
for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; rq->xdp_rxq.mem = rq->xdp_mem; xdp_rxq_info_unreg(&rq->xdp_rxq); } |
948d4f214 veth: Add driver XDP |
909 |
} |
e314dbdc1 [NET]: Virtual et... |
910 911 |
static int veth_open(struct net_device *dev) { |
d0e2c55e7 veth: avoid a NUL... |
912 913 |
struct veth_priv *priv = netdev_priv(dev); struct net_device *peer = rtnl_dereference(priv->peer); |
948d4f214 veth: Add driver XDP |
914 |
int err; |
e314dbdc1 [NET]: Virtual et... |
915 |
|
d0e2c55e7 veth: avoid a NUL... |
916 |
if (!peer) |
e314dbdc1 [NET]: Virtual et... |
917 |
return -ENOTCONN; |
948d4f214 veth: Add driver XDP |
918 919 920 921 922 |
if (priv->_xdp_prog) { err = veth_enable_xdp(dev); if (err) return err; } |
d0e2c55e7 veth: avoid a NUL... |
923 |
if (peer->flags & IFF_UP) { |
e314dbdc1 [NET]: Virtual et... |
924 |
netif_carrier_on(dev); |
d0e2c55e7 veth: avoid a NUL... |
925 |
netif_carrier_on(peer); |
e314dbdc1 [NET]: Virtual et... |
926 |
} |
948d4f214 veth: Add driver XDP |
927 |
|
e314dbdc1 [NET]: Virtual et... |
928 929 |
return 0; } |
2cf48a10a veth: Fix carrier... |
930 931 932 |
static int veth_close(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); |
2efd32ee1 veth: fix a NULL ... |
933 |
struct net_device *peer = rtnl_dereference(priv->peer); |
2cf48a10a veth: Fix carrier... |
934 935 |
netif_carrier_off(dev); |
2efd32ee1 veth: fix a NULL ... |
936 937 |
if (peer) netif_carrier_off(peer); |
2cf48a10a veth: Fix carrier... |
938 |
|
948d4f214 veth: Add driver XDP |
939 940 |
if (priv->_xdp_prog) veth_disable_xdp(dev); |
2cf48a10a veth: Fix carrier... |
941 942 |
return 0; } |
91572088e net: use core MTU... |
943 |
static int is_valid_veth_mtu(int mtu) |
38d408152 veth: Allow setti... |
944 |
{ |
91572088e net: use core MTU... |
945 |
return mtu >= ETH_MIN_MTU && mtu <= ETH_MAX_MTU; |
38d408152 veth: Allow setti... |
946 |
} |
7797b93b7 veth: Free queues... |
947 948 949 950 951 952 953 954 |
static int veth_alloc_queues(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); int i; priv->rq = kcalloc(dev->num_rx_queues, sizeof(*priv->rq), GFP_KERNEL); if (!priv->rq) return -ENOMEM; |
4195e54aa veth: Account for... |
955 |
for (i = 0; i < dev->num_rx_queues; i++) { |
7797b93b7 veth: Free queues... |
956 |
priv->rq[i].dev = dev; |
4195e54aa veth: Account for... |
957 958 |
u64_stats_init(&priv->rq[i].stats.syncp); } |
7797b93b7 veth: Free queues... |
959 960 961 962 963 964 965 966 967 968 |
return 0; } static void veth_free_queues(struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); kfree(priv->rq); } |
e314dbdc1 [NET]: Virtual et... |
969 970 |
static int veth_dev_init(struct net_device *dev) { |
7797b93b7 veth: Free queues... |
971 |
int err; |
14d734167 veth: rename pcpu... |
972 973 |
dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); if (!dev->lstats) |
e314dbdc1 [NET]: Virtual et... |
974 |
return -ENOMEM; |
7797b93b7 veth: Free queues... |
975 976 977 |
err = veth_alloc_queues(dev); if (err) { |
14d734167 veth: rename pcpu... |
978 |
free_percpu(dev->lstats); |
7797b93b7 veth: Free queues... |
979 980 |
return err; } |
e314dbdc1 [NET]: Virtual et... |
981 982 |
return 0; } |
11687a109 Revert "veth: pre... |
983 984 |
static void veth_dev_free(struct net_device *dev) { |
7797b93b7 veth: Free queues... |
985 |
veth_free_queues(dev); |
14d734167 veth: rename pcpu... |
986 |
free_percpu(dev->lstats); |
11687a109 Revert "veth: pre... |
987 |
} |
bb446c19f veth: add netpoll... |
988 989 990 991 |
#ifdef CONFIG_NET_POLL_CONTROLLER static void veth_poll_controller(struct net_device *dev) { /* veth only receives frames when its peer sends one |
948d4f214 veth: Add driver XDP |
992 |
* Since it has nothing to do with disabling irqs, we are guaranteed |
bb446c19f veth: add netpoll... |
993 994 995 996 997 998 999 1000 1001 |
* never to have pending data when we poll for it so * there is nothing to do here. * * We need this though so netpoll recognizes us as an interface that * supports polling, which enables bridge devices in virt setups to * still use netconsole */ } #endif /* CONFIG_NET_POLL_CONTROLLER */ |
a45253bf3 veth: set iflink ... |
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 |
static int veth_get_iflink(const struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); struct net_device *peer; int iflink; rcu_read_lock(); peer = rcu_dereference(priv->peer); iflink = peer ? peer->ifindex : 0; rcu_read_unlock(); return iflink; } |
dc2248220 veth: Avoid drops... |
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 |
static netdev_features_t veth_fix_features(struct net_device *dev, netdev_features_t features) { struct veth_priv *priv = netdev_priv(dev); struct net_device *peer; peer = rtnl_dereference(priv->peer); if (peer) { struct veth_priv *peer_priv = netdev_priv(peer); if (peer_priv->_xdp_prog) features &= ~NETIF_F_GSO_SOFTWARE; } return features; } |
163e52920 veth: implement n... |
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 |
static void veth_set_rx_headroom(struct net_device *dev, int new_hr) { struct veth_priv *peer_priv, *priv = netdev_priv(dev); struct net_device *peer; if (new_hr < 0) new_hr = 0; rcu_read_lock(); peer = rcu_dereference(priv->peer); if (unlikely(!peer)) goto out; peer_priv = netdev_priv(peer); priv->requested_headroom = new_hr; new_hr = max(priv->requested_headroom, peer_priv->requested_headroom); dev->needed_headroom = new_hr; peer->needed_headroom = new_hr; out: rcu_read_unlock(); } |
948d4f214 veth: Add driver XDP |
1053 1054 1055 1056 1057 1058 |
static int veth_xdp_set(struct net_device *dev, struct bpf_prog *prog, struct netlink_ext_ack *extack) { struct veth_priv *priv = netdev_priv(dev); struct bpf_prog *old_prog; struct net_device *peer; |
dc2248220 veth: Avoid drops... |
1059 |
unsigned int max_mtu; |
948d4f214 veth: Add driver XDP |
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 |
int err; old_prog = priv->_xdp_prog; priv->_xdp_prog = prog; peer = rtnl_dereference(priv->peer); if (prog) { if (!peer) { NL_SET_ERR_MSG_MOD(extack, "Cannot set XDP when peer is detached"); err = -ENOTCONN; goto err; } |
dc2248220 veth: Avoid drops... |
1072 1073 1074 1075 1076 1077 1078 1079 |
max_mtu = PAGE_SIZE - VETH_XDP_HEADROOM - peer->hard_header_len - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); if (peer->mtu > max_mtu) { NL_SET_ERR_MSG_MOD(extack, "Peer MTU is too large to set XDP"); err = -ERANGE; goto err; } |
638264dc9 veth: Support per... |
1080 1081 1082 1083 1084 |
if (dev->real_num_rx_queues < peer->real_num_tx_queues) { NL_SET_ERR_MSG_MOD(extack, "XDP expects number of rx queues not less than peer tx queues"); err = -ENOSPC; goto err; } |
948d4f214 veth: Add driver XDP |
1085 1086 1087 1088 1089 1090 1091 |
if (dev->flags & IFF_UP) { err = veth_enable_xdp(dev); if (err) { NL_SET_ERR_MSG_MOD(extack, "Setup for XDP failed"); goto err; } } |
dc2248220 veth: Avoid drops... |
1092 1093 1094 1095 1096 |
if (!old_prog) { peer->hw_features &= ~NETIF_F_GSO_SOFTWARE; peer->max_mtu = max_mtu; } |
948d4f214 veth: Add driver XDP |
1097 1098 1099 |
} if (old_prog) { |
dc2248220 veth: Avoid drops... |
1100 1101 1102 1103 1104 1105 1106 1107 1108 |
if (!prog) { if (dev->flags & IFF_UP) veth_disable_xdp(dev); if (peer) { peer->hw_features |= NETIF_F_GSO_SOFTWARE; peer->max_mtu = ETH_MAX_MTU; } } |
948d4f214 veth: Add driver XDP |
1109 1110 |
bpf_prog_put(old_prog); } |
dc2248220 veth: Avoid drops... |
1111 1112 |
if ((!!old_prog ^ !!prog) && peer) netdev_update_features(peer); |
948d4f214 veth: Add driver XDP |
1113 1114 1115 1116 1117 1118 |
return 0; err: priv->_xdp_prog = old_prog; return err; } |
948d4f214 veth: Add driver XDP |
1119 1120 1121 1122 1123 |
static int veth_xdp(struct net_device *dev, struct netdev_bpf *xdp) { switch (xdp->command) { case XDP_SETUP_PROG: return veth_xdp_set(dev, xdp->prog, xdp->extack); |
948d4f214 veth: Add driver XDP |
1124 1125 1126 1127 |
default: return -EINVAL; } } |
4456e7bdf veth: convert to ... |
1128 |
static const struct net_device_ops veth_netdev_ops = { |
ee9236231 veth : add the se... |
1129 1130 |
.ndo_init = veth_dev_init, .ndo_open = veth_open, |
2cf48a10a veth: Fix carrier... |
1131 |
.ndo_stop = veth_close, |
ee9236231 veth : add the se... |
1132 |
.ndo_start_xmit = veth_xmit, |
6311cc44a veth: convert to ... |
1133 |
.ndo_get_stats64 = veth_get_stats64, |
5c70ef85a veth: allow to se... |
1134 |
.ndo_set_rx_mode = veth_set_multicast_list, |
ee9236231 veth : add the se... |
1135 |
.ndo_set_mac_address = eth_mac_addr, |
bb446c19f veth: add netpoll... |
1136 1137 1138 |
#ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = veth_poll_controller, #endif |
a45253bf3 veth: set iflink ... |
1139 |
.ndo_get_iflink = veth_get_iflink, |
dc2248220 veth: Avoid drops... |
1140 |
.ndo_fix_features = veth_fix_features, |
1a04a8215 veth: Don't segme... |
1141 |
.ndo_features_check = passthru_features_check, |
163e52920 veth: implement n... |
1142 |
.ndo_set_rx_headroom = veth_set_rx_headroom, |
948d4f214 veth: Add driver XDP |
1143 |
.ndo_bpf = veth_xdp, |
9152cff0d veth: introduce m... |
1144 |
.ndo_xdp_xmit = veth_ndo_xdp_xmit, |
9aa1206e8 bpf: Add redirect... |
1145 |
.ndo_get_peer_dev = veth_peer_dev, |
4456e7bdf veth: convert to ... |
1146 |
}; |
732912d72 veth: Update feat... |
1147 |
#define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ |
c80fafbbb veth: sctp: add N... |
1148 |
NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | \ |
732912d72 veth: Update feat... |
1149 |
NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ |
28d2b136c net: vlan: announ... |
1150 1151 |
NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | \ NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX ) |
8093315a9 veth: extend devi... |
1152 |
|
e314dbdc1 [NET]: Virtual et... |
1153 1154 1155 |
static void veth_setup(struct net_device *dev) { ether_setup(dev); |
550fd08c2 net: Audit driver... |
1156 |
dev->priv_flags &= ~IFF_TX_SKB_SHARING; |
23ea5a963 veth: allow chang... |
1157 |
dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; |
02f01ec1c net: veth: enable... |
1158 |
dev->priv_flags |= IFF_NO_QUEUE; |
163e52920 veth: implement n... |
1159 |
dev->priv_flags |= IFF_PHONY_HEADROOM; |
550fd08c2 net: Audit driver... |
1160 |
|
4456e7bdf veth: convert to ... |
1161 |
dev->netdev_ops = &veth_netdev_ops; |
e314dbdc1 [NET]: Virtual et... |
1162 1163 |
dev->ethtool_ops = &veth_ethtool_ops; dev->features |= NETIF_F_LLTX; |
8093315a9 veth: extend devi... |
1164 |
dev->features |= VETH_FEATURES; |
8d0d21f40 veth: Fix vlan_fe... |
1165 |
dev->vlan_features = dev->features & |
3f8c707b9 veth: Turn off vl... |
1166 1167 1168 1169 |
~(NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX); |
cf124db56 net: Fix inconsis... |
1170 1171 |
dev->needs_free_netdev = true; dev->priv_destructor = veth_dev_free; |
91572088e net: use core MTU... |
1172 |
dev->max_mtu = ETH_MAX_MTU; |
a2c725fa3 veth: convert to ... |
1173 |
|
8093315a9 veth: extend devi... |
1174 |
dev->hw_features = VETH_FEATURES; |
82d818982 veth: extend feat... |
1175 |
dev->hw_enc_features = VETH_FEATURES; |
607fca9ac net: veth: Set fe... |
1176 |
dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; |
e314dbdc1 [NET]: Virtual et... |
1177 1178 1179 1180 1181 |
} /* * netlink interface */ |
a8b8a889e net: add netlink_... |
1182 1183 |
static int veth_validate(struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) |
e314dbdc1 [NET]: Virtual et... |
1184 1185 1186 1187 1188 1189 1190 |
{ if (tb[IFLA_ADDRESS]) { if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) return -EINVAL; if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) return -EADDRNOTAVAIL; } |
38d408152 veth: Allow setti... |
1191 1192 1193 1194 |
if (tb[IFLA_MTU]) { if (!is_valid_veth_mtu(nla_get_u32(tb[IFLA_MTU]))) return -EINVAL; } |
e314dbdc1 [NET]: Virtual et... |
1195 1196 1197 1198 |
return 0; } static struct rtnl_link_ops veth_link_ops; |
81adee47d net: Support spec... |
1199 |
static int veth_newlink(struct net *src_net, struct net_device *dev, |
7a3f4a185 net: add netlink_... |
1200 1201 |
struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) |
e314dbdc1 [NET]: Virtual et... |
1202 |
{ |
7797b93b7 veth: Free queues... |
1203 |
int err; |
e314dbdc1 [NET]: Virtual et... |
1204 1205 1206 1207 |
struct net_device *peer; struct veth_priv *priv; char ifname[IFNAMSIZ]; struct nlattr *peer_tb[IFLA_MAX + 1], **tbp; |
5517750f0 net: rtnetlink - ... |
1208 |
unsigned char name_assign_type; |
3729d5021 rtnetlink: suppor... |
1209 |
struct ifinfomsg *ifmp; |
81adee47d net: Support spec... |
1210 |
struct net *net; |
e314dbdc1 [NET]: Virtual et... |
1211 1212 1213 |
/* * create and register peer first |
e314dbdc1 [NET]: Virtual et... |
1214 |
*/ |
e314dbdc1 [NET]: Virtual et... |
1215 1216 1217 1218 |
if (data != NULL && data[VETH_INFO_PEER] != NULL) { struct nlattr *nla_peer; nla_peer = data[VETH_INFO_PEER]; |
3729d5021 rtnetlink: suppor... |
1219 |
ifmp = nla_data(nla_peer); |
f7b12606b rtnl: make ifla_p... |
1220 1221 |
err = rtnl_nla_parse_ifla(peer_tb, nla_data(nla_peer) + sizeof(struct ifinfomsg), |
fceb6435e netlink: pass ext... |
1222 1223 |
nla_len(nla_peer) - sizeof(struct ifinfomsg), NULL); |
e314dbdc1 [NET]: Virtual et... |
1224 1225 |
if (err < 0) return err; |
a8b8a889e net: add netlink_... |
1226 |
err = veth_validate(peer_tb, NULL, extack); |
e314dbdc1 [NET]: Virtual et... |
1227 1228 1229 1230 |
if (err < 0) return err; tbp = peer_tb; |
3729d5021 rtnetlink: suppor... |
1231 1232 |
} else { ifmp = NULL; |
e314dbdc1 [NET]: Virtual et... |
1233 |
tbp = tb; |
3729d5021 rtnetlink: suppor... |
1234 |
} |
e314dbdc1 [NET]: Virtual et... |
1235 |
|
191cdb382 veth: Be more rob... |
1236 |
if (ifmp && tbp[IFLA_IFNAME]) { |
e314dbdc1 [NET]: Virtual et... |
1237 |
nla_strlcpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); |
5517750f0 net: rtnetlink - ... |
1238 1239 |
name_assign_type = NET_NAME_USER; } else { |
e314dbdc1 [NET]: Virtual et... |
1240 |
snprintf(ifname, IFNAMSIZ, DRV_NAME "%%d"); |
5517750f0 net: rtnetlink - ... |
1241 1242 |
name_assign_type = NET_NAME_ENUM; } |
e314dbdc1 [NET]: Virtual et... |
1243 |
|
81adee47d net: Support spec... |
1244 1245 1246 |
net = rtnl_link_get_net(src_net, tbp); if (IS_ERR(net)) return PTR_ERR(net); |
5517750f0 net: rtnetlink - ... |
1247 |
peer = rtnl_create_link(net, ifname, name_assign_type, |
d0522f1cd net: Add extack a... |
1248 |
&veth_link_ops, tbp, extack); |
81adee47d net: Support spec... |
1249 1250 |
if (IS_ERR(peer)) { put_net(net); |
e314dbdc1 [NET]: Virtual et... |
1251 |
return PTR_ERR(peer); |
81adee47d net: Support spec... |
1252 |
} |
e314dbdc1 [NET]: Virtual et... |
1253 |
|
191cdb382 veth: Be more rob... |
1254 |
if (!ifmp || !tbp[IFLA_ADDRESS]) |
f2cedb63d net: replace rand... |
1255 |
eth_hw_addr_random(peer); |
e6f8f1a73 veth: Allow to cr... |
1256 1257 1258 |
if (ifmp && (dev->ifindex != 0)) peer->ifindex = ifmp->ifi_index; |
e314dbdc1 [NET]: Virtual et... |
1259 |
|
72d24955b veth: set peer GS... |
1260 1261 |
peer->gso_max_size = dev->gso_max_size; peer->gso_max_segs = dev->gso_max_segs; |
e314dbdc1 [NET]: Virtual et... |
1262 |
err = register_netdevice(peer); |
81adee47d net: Support spec... |
1263 1264 |
put_net(net); net = NULL; |
e314dbdc1 [NET]: Virtual et... |
1265 1266 1267 1268 |
if (err < 0) goto err_register_peer; netif_carrier_off(peer); |
3729d5021 rtnetlink: suppor... |
1269 1270 1271 |
err = rtnl_configure_link(peer, ifmp); if (err < 0) goto err_configure_peer; |
e314dbdc1 [NET]: Virtual et... |
1272 1273 1274 1275 1276 1277 1278 1279 |
/* * register dev last * * note, that since we've registered new device the dev's name * should be re-allocated */ if (tb[IFLA_ADDRESS] == NULL) |
f2cedb63d net: replace rand... |
1280 |
eth_hw_addr_random(dev); |
e314dbdc1 [NET]: Virtual et... |
1281 |
|
6c8c44462 Revert: veth: rem... |
1282 1283 1284 1285 |
if (tb[IFLA_IFNAME]) nla_strlcpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); else snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d"); |
e314dbdc1 [NET]: Virtual et... |
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 |
err = register_netdevice(dev); if (err < 0) goto err_register_dev; netif_carrier_off(dev); /* * tie the deviced together */ priv = netdev_priv(dev); |
d0e2c55e7 veth: avoid a NUL... |
1297 |
rcu_assign_pointer(priv->peer, peer); |
e314dbdc1 [NET]: Virtual et... |
1298 1299 |
priv = netdev_priv(peer); |
d0e2c55e7 veth: avoid a NUL... |
1300 |
rcu_assign_pointer(priv->peer, dev); |
948d4f214 veth: Add driver XDP |
1301 |
|
e314dbdc1 [NET]: Virtual et... |
1302 1303 1304 1305 |
return 0; err_register_dev: /* nothing to do */ |
3729d5021 rtnetlink: suppor... |
1306 |
err_configure_peer: |
e314dbdc1 [NET]: Virtual et... |
1307 1308 1309 1310 1311 1312 1313 |
unregister_netdevice(peer); return err; err_register_peer: free_netdev(peer); return err; } |
23289a37e net: add a list_h... |
1314 |
static void veth_dellink(struct net_device *dev, struct list_head *head) |
e314dbdc1 [NET]: Virtual et... |
1315 1316 1317 1318 1319 |
{ struct veth_priv *priv; struct net_device *peer; priv = netdev_priv(dev); |
d0e2c55e7 veth: avoid a NUL... |
1320 1321 1322 1323 1324 1325 1326 |
peer = rtnl_dereference(priv->peer); /* Note : dellink() is called from default_device_exit_batch(), * before a rcu_synchronize() point. The devices are guaranteed * not being freed before one RCU grace period. */ RCU_INIT_POINTER(priv->peer, NULL); |
24540535d veth: Fix veth_de... |
1327 |
unregister_netdevice_queue(dev, head); |
f45a5c267 veth: fix NULL de... |
1328 1329 1330 1331 1332 1333 |
if (peer) { priv = netdev_priv(peer); RCU_INIT_POINTER(priv->peer, NULL); unregister_netdevice_queue(peer, head); } |
e314dbdc1 [NET]: Virtual et... |
1334 |
} |
237114384 veth: Enforce min... |
1335 1336 1337 |
static const struct nla_policy veth_policy[VETH_INFO_MAX + 1] = { [VETH_INFO_PEER] = { .len = sizeof(struct ifinfomsg) }, }; |
e314dbdc1 [NET]: Virtual et... |
1338 |
|
e5f4e7b9f veth: advertise l... |
1339 1340 1341 1342 1343 1344 1345 |
static struct net *veth_get_link_net(const struct net_device *dev) { struct veth_priv *priv = netdev_priv(dev); struct net_device *peer = rtnl_dereference(priv->peer); return peer ? dev_net(peer) : dev_net(dev); } |
e314dbdc1 [NET]: Virtual et... |
1346 1347 1348 1349 1350 1351 1352 1353 1354 |
static struct rtnl_link_ops veth_link_ops = { .kind = DRV_NAME, .priv_size = sizeof(struct veth_priv), .setup = veth_setup, .validate = veth_validate, .newlink = veth_newlink, .dellink = veth_dellink, .policy = veth_policy, .maxtype = VETH_INFO_MAX, |
e5f4e7b9f veth: advertise l... |
1355 |
.get_link_net = veth_get_link_net, |
e314dbdc1 [NET]: Virtual et... |
1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 |
}; /* * init/fini */ static __init int veth_init(void) { return rtnl_link_register(&veth_link_ops); } static __exit void veth_exit(void) { |
68365458a [NET]: rtnl_link:... |
1369 |
rtnl_link_unregister(&veth_link_ops); |
e314dbdc1 [NET]: Virtual et... |
1370 1371 1372 1373 1374 1375 1376 1377 |
} module_init(veth_init); module_exit(veth_exit); MODULE_DESCRIPTION("Virtual Ethernet Tunnel"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_RTNL_LINK(DRV_NAME); |