Blame view
net/ipv4/udp_offload.c
10.2 KB
da5bab079 net: udp4: move G... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
/* * IPV4 GSO/GRO offload support * Linux INET implementation * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. * * UDPv4 GSO support */ #include <linux/skbuff.h> #include <net/udp.h> #include <net/protocol.h> |
8bce6d7d0 udp: Generalize s... |
16 17 18 19 |
static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, netdev_features_t features, struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, netdev_features_t features), |
4bcb877d2 udp: Offload oute... |
20 |
__be16 new_protocol, bool is_ipv6) |
155e010ed udp: Move udp_tun... |
21 |
{ |
dbef491eb udp: Use uh->len ... |
22 |
int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); |
07b26c945 gso: Support part... |
23 |
bool remcsum, need_csum, offload_csum, ufo, gso_partial; |
155e010ed udp: Move udp_tun... |
24 |
struct sk_buff *segs = ERR_PTR(-EINVAL); |
dbef491eb udp: Use uh->len ... |
25 |
struct udphdr *uh = udp_hdr(skb); |
155e010ed udp: Move udp_tun... |
26 |
u16 mac_offset = skb->mac_header; |
155e010ed udp: Move udp_tun... |
27 |
__be16 protocol = skb->protocol; |
dbef491eb udp: Use uh->len ... |
28 |
u16 mac_len = skb->mac_len; |
155e010ed udp: Move udp_tun... |
29 |
int udp_offset, outer_hlen; |
083348249 GSO/UDP: Use skb-... |
30 |
__wsum partial; |
155e010ed udp: Move udp_tun... |
31 32 33 |
if (unlikely(!pskb_may_pull(skb, tnl_hlen))) goto out; |
083348249 GSO/UDP: Use skb-... |
34 35 36 37 38 39 |
/* Adjust partial header checksum to negate old length. * We cannot rely on the value contained in uh->len as it is * possible that the actual value exceeds the boundaries of the * 16 bit length field due to the header being added outside of an * IP or IPv6 frame that was already limited to 64K - 1. */ |
802ab55ad GSO: Support part... |
40 41 42 43 44 |
if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) partial = (__force __wsum)uh->len; else partial = (__force __wsum)htonl(skb->len); partial = csum_sub(csum_unfold(uh->check), partial); |
dbef491eb udp: Use uh->len ... |
45 46 |
/* setup inner skb. */ |
155e010ed udp: Move udp_tun... |
47 |
skb->encapsulation = 0; |
5197f3499 net: Reset encap_... |
48 |
SKB_GSO_CB(skb)->encap_level = 0; |
155e010ed udp: Move udp_tun... |
49 50 51 52 |
__skb_pull(skb, tnl_hlen); skb_reset_mac_header(skb); skb_set_network_header(skb, skb_inner_network_offset(skb)); skb->mac_len = skb_inner_network_offset(skb); |
8bce6d7d0 udp: Generalize s... |
53 |
skb->protocol = new_protocol; |
fdaefd62f udp: Clean up the... |
54 55 |
need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM); |
4bcb877d2 udp: Offload oute... |
56 |
skb->encap_hdr_csum = need_csum; |
fdaefd62f udp: Clean up the... |
57 58 |
remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM); |
e585f2363 udp: Changes to u... |
59 |
skb->remcsum_offload = remcsum; |
155e010ed udp: Move udp_tun... |
60 |
|
224638766 GSO: Provide soft... |
61 |
ufo = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP); |
4bcb877d2 udp: Offload oute... |
62 63 |
/* Try to offload checksum if possible */ offload_csum = !!(need_csum && |
fdaefd62f udp: Clean up the... |
64 65 66 |
(skb->dev->features & (is_ipv6 ? (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM) : (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)))); |
155e010ed udp: Move udp_tun... |
67 |
|
bef3c6c93 net: Drop unecess... |
68 |
features &= skb->dev->hw_enc_features; |
7fbeffed7 net: Update remot... |
69 70 71 72 |
/* The only checksum offload we care about from here on out is the * outer one so strip the existing checksum feature flags and * instead set the flag based on our outer checksum offload value. */ |
224638766 GSO: Provide soft... |
73 |
if (remcsum || ufo) { |
7fbeffed7 net: Update remot... |
74 |
features &= ~NETIF_F_CSUM_MASK; |
224638766 GSO: Provide soft... |
75 |
if (!need_csum || offload_csum) |
7fbeffed7 net: Update remot... |
76 77 |
features |= NETIF_F_HW_CSUM; } |
155e010ed udp: Move udp_tun... |
78 |
/* segment inner packet. */ |
bef3c6c93 net: Drop unecess... |
79 |
segs = gso_inner_segment(skb, features); |
27446442a net/udp_offload: ... |
80 |
if (IS_ERR_OR_NULL(segs)) { |
155e010ed udp: Move udp_tun... |
81 82 83 84 |
skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, mac_len); goto out; } |
07b26c945 gso: Support part... |
85 |
gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL); |
155e010ed udp: Move udp_tun... |
86 87 88 89 |
outer_hlen = skb_tnl_header_len(skb); udp_offset = outer_hlen - tnl_hlen; skb = segs; do { |
802ab55ad GSO: Support part... |
90 |
unsigned int len; |
4bcb877d2 udp: Offload oute... |
91 |
|
fdaefd62f udp: Clean up the... |
92 |
if (remcsum) |
4bcb877d2 udp: Offload oute... |
93 |
skb->ip_summed = CHECKSUM_NONE; |
fdaefd62f udp: Clean up the... |
94 95 96 |
/* Set up inner headers if we are offloading inner checksum */ if (skb->ip_summed == CHECKSUM_PARTIAL) { |
4bcb877d2 udp: Offload oute... |
97 98 99 |
skb_reset_inner_headers(skb); skb->encapsulation = 1; } |
155e010ed udp: Move udp_tun... |
100 101 |
skb->mac_len = mac_len; |
4bcb877d2 udp: Offload oute... |
102 |
skb->protocol = protocol; |
155e010ed udp: Move udp_tun... |
103 |
|
dbef491eb udp: Use uh->len ... |
104 |
__skb_push(skb, outer_hlen); |
155e010ed udp: Move udp_tun... |
105 106 107 |
skb_reset_mac_header(skb); skb_set_network_header(skb, mac_len); skb_set_transport_header(skb, udp_offset); |
802ab55ad GSO: Support part... |
108 |
len = skb->len - udp_offset; |
155e010ed udp: Move udp_tun... |
109 |
uh = udp_hdr(skb); |
802ab55ad GSO: Support part... |
110 111 112 113 114 |
/* If we are only performing partial GSO the inner header * will be using a length value equal to only one MSS sized * segment instead of the entire frame. */ |
07b26c945 gso: Support part... |
115 |
if (gso_partial) { |
802ab55ad GSO: Support part... |
116 117 118 119 120 121 |
uh->len = htons(skb_shinfo(skb)->gso_size + SKB_GSO_CB(skb)->data_offset + skb->head - (unsigned char *)uh); } else { uh->len = htons(len); } |
155e010ed udp: Move udp_tun... |
122 |
|
4bcb877d2 udp: Offload oute... |
123 124 |
if (!need_csum) continue; |
802ab55ad GSO: Support part... |
125 126 |
uh->check = ~csum_fold(csum_add(partial, (__force __wsum)htonl(len))); |
155e010ed udp: Move udp_tun... |
127 |
|
fdaefd62f udp: Clean up the... |
128 129 |
if (skb->encapsulation || !offload_csum) { uh->check = gso_make_checksum(skb, ~uh->check); |
155e010ed udp: Move udp_tun... |
130 131 |
if (uh->check == 0) uh->check = CSUM_MANGLED_0; |
fdaefd62f udp: Clean up the... |
132 133 134 135 |
} else { skb->ip_summed = CHECKSUM_PARTIAL; skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct udphdr, check); |
155e010ed udp: Move udp_tun... |
136 |
} |
155e010ed udp: Move udp_tun... |
137 138 139 140 |
} while ((skb = skb->next)); out: return segs; } |
8bce6d7d0 udp: Generalize s... |
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 |
struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, netdev_features_t features, bool is_ipv6) { __be16 protocol = skb->protocol; const struct net_offload **offloads; const struct net_offload *ops; struct sk_buff *segs = ERR_PTR(-EINVAL); struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb, netdev_features_t features); rcu_read_lock(); switch (skb->inner_protocol_type) { case ENCAP_TYPE_ETHER: protocol = skb->inner_protocol; gso_inner_segment = skb_mac_gso_segment; break; case ENCAP_TYPE_IPPROTO: offloads = is_ipv6 ? inet6_offloads : inet_offloads; ops = rcu_dereference(offloads[skb->inner_ipproto]); if (!ops || !ops->callbacks.gso_segment) goto out_unlock; gso_inner_segment = ops->callbacks.gso_segment; break; default: goto out_unlock; } segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment, |
4bcb877d2 udp: Offload oute... |
171 |
protocol, is_ipv6); |
8bce6d7d0 udp: Generalize s... |
172 173 174 175 176 177 |
out_unlock: rcu_read_unlock(); return segs; } |
a6024562f udp: Add GRO func... |
178 |
EXPORT_SYMBOL(skb_udp_tunnel_segment); |
8bce6d7d0 udp: Generalize s... |
179 |
|
da5bab079 net: udp4: move G... |
180 181 182 183 184 |
static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, netdev_features_t features) { struct sk_buff *segs = ERR_PTR(-EINVAL); unsigned int mss; |
7a7ffbabf ipv4: fix tunnele... |
185 |
__wsum csum; |
f71470b37 udp: move logic o... |
186 187 |
struct udphdr *uh; struct iphdr *iph; |
7a7ffbabf ipv4: fix tunnele... |
188 189 |
if (skb->encapsulation && |
0f4f4ffa7 net: Add GSO supp... |
190 191 |
(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))) { |
8bce6d7d0 udp: Generalize s... |
192 |
segs = skb_udp_tunnel_segment(skb, features, false); |
7a7ffbabf ipv4: fix tunnele... |
193 194 |
goto out; } |
da5bab079 net: udp4: move G... |
195 |
|
f71470b37 udp: move logic o... |
196 197 |
if (!pskb_may_pull(skb, sizeof(struct udphdr))) goto out; |
da5bab079 net: udp4: move G... |
198 199 200 201 202 203 |
mss = skb_shinfo(skb)->gso_size; if (unlikely(skb->len <= mss)) goto out; if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) { /* Packet is from an untrusted source, reset gso_segs. */ |
da5bab079 net: udp4: move G... |
204 205 206 207 208 209 |
skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss); segs = NULL; goto out; } |
7a7ffbabf ipv4: fix tunnele... |
210 211 212 213 |
/* Do software UFO. Complete and fill in the UDP checksum as * HW cannot do checksum of UDP packets sent as multiple * IP fragments. */ |
f71470b37 udp: move logic o... |
214 215 216 217 218 219 220 221 222 |
uh = udp_hdr(skb); iph = ip_hdr(skb); uh->check = 0; csum = skb_checksum(skb, 0, skb->len, 0); uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum); if (uh->check == 0) uh->check = CSUM_MANGLED_0; |
7a7ffbabf ipv4: fix tunnele... |
223 |
skb->ip_summed = CHECKSUM_NONE; |
224638766 GSO: Provide soft... |
224 225 226 227 228 229 |
/* If there is no outer header we can fake a checksum offload * due to the fact that we have already done the checksum in * software prior to segmenting the frame. */ if (!skb->encap_hdr_csum) features |= NETIF_F_HW_CSUM; |
da5bab079 net: udp4: move G... |
230 231 232 |
/* Fragment the skb. IP headers of the fragments are updated in * inet_gso_segment() */ |
7a7ffbabf ipv4: fix tunnele... |
233 |
segs = skb_segment(skb, features); |
da5bab079 net: udp4: move G... |
234 235 236 |
out: return segs; } |
57c67ff4b udp: additional G... |
237 |
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, |
a6024562f udp: Add GRO func... |
238 |
struct udphdr *uh, udp_lookup_t lookup) |
b582ef099 net: Add GRO supp... |
239 |
{ |
b582ef099 net: Add GRO supp... |
240 |
struct sk_buff *p, **pp = NULL; |
57c67ff4b udp: additional G... |
241 242 |
struct udphdr *uh2; unsigned int off = skb_gro_offset(skb); |
b582ef099 net: Add GRO supp... |
243 |
int flush = 1; |
a6024562f udp: Add GRO func... |
244 |
struct sock *sk; |
b582ef099 net: Add GRO supp... |
245 |
|
fac8e0f57 tunnels: Don't ap... |
246 |
if (NAPI_GRO_CB(skb)->encap_mark || |
662880f44 net: Allow GRO to... |
247 248 249 |
(skb->ip_summed != CHECKSUM_PARTIAL && NAPI_GRO_CB(skb)->csum_cnt == 0 && !NAPI_GRO_CB(skb)->csum_valid)) |
b582ef099 net: Add GRO supp... |
250 |
goto out; |
fac8e0f57 tunnels: Don't ap... |
251 252 |
/* mark that this skb passed once through the tunnel gro layer */ NAPI_GRO_CB(skb)->encap_mark = 1; |
b582ef099 net: Add GRO supp... |
253 254 |
rcu_read_lock(); |
a6024562f udp: Add GRO func... |
255 256 257 258 |
sk = (*lookup)(skb, uh->source, uh->dest); if (sk && udp_sk(sk)->gro_receive) goto unflush; |
b582ef099 net: Add GRO supp... |
259 260 261 262 263 264 265 266 267 268 |
goto out_unlock; unflush: flush = 0; for (p = *head; p; p = p->next) { if (!NAPI_GRO_CB(p)->same_flow) continue; uh2 = (struct udphdr *)(p->data + off); |
57c67ff4b udp: additional G... |
269 270 271 272 273 274 |
/* Match ports and either checksums are either both zero * or nonzero. */ if ((*(u32 *)&uh->source != *(u32 *)&uh2->source) || (!uh->check ^ !uh2->check)) { |
b582ef099 net: Add GRO supp... |
275 276 277 278 279 280 |
NAPI_GRO_CB(p)->same_flow = 0; continue; } } skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ |
6bae1d4cc net: Add skb_gro_... |
281 |
skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); |
fcd91dd44 net: add recursio... |
282 |
pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); |
b582ef099 net: Add GRO supp... |
283 284 285 286 287 288 289 |
out_unlock: rcu_read_unlock(); out: NAPI_GRO_CB(skb)->flush |= flush; return pp; } |
a6024562f udp: Add GRO func... |
290 |
EXPORT_SYMBOL(udp_gro_receive); |
b582ef099 net: Add GRO supp... |
291 |
|
57c67ff4b udp: additional G... |
292 293 294 295 |
static struct sk_buff **udp4_gro_receive(struct sk_buff **head, struct sk_buff *skb) { struct udphdr *uh = udp_gro_udphdr(skb); |
2abb7cdc0 udp: Add support ... |
296 297 |
if (unlikely(!uh)) goto flush; |
57c67ff4b udp: additional G... |
298 |
|
2abb7cdc0 udp: Add support ... |
299 |
/* Don't bother verifying checksum if we're going to flush anyway. */ |
2d8f7e2c8 udp: Fix inverted... |
300 |
if (NAPI_GRO_CB(skb)->flush) |
2abb7cdc0 udp: Add support ... |
301 302 303 304 305 306 307 308 309 |
goto skip; if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check, inet_gro_compute_pseudo)) goto flush; else if (uh->check) skb_gro_checksum_try_convert(skb, IPPROTO_UDP, uh->check, inet_gro_compute_pseudo); skip: |
efc98d08e fou: eliminate IP... |
310 |
NAPI_GRO_CB(skb)->is_ipv6 = 0; |
a6024562f udp: Add GRO func... |
311 |
return udp_gro_receive(head, skb, uh, udp4_lib_lookup_skb); |
2abb7cdc0 udp: Add support ... |
312 313 314 315 |
flush: NAPI_GRO_CB(skb)->flush = 1; return NULL; |
57c67ff4b udp: additional G... |
316 |
} |
a6024562f udp: Add GRO func... |
317 318 |
int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup) |
b582ef099 net: Add GRO supp... |
319 |
{ |
b582ef099 net: Add GRO supp... |
320 321 322 |
__be16 newlen = htons(skb->len - nhoff); struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); int err = -ENOSYS; |
a6024562f udp: Add GRO func... |
323 |
struct sock *sk; |
b582ef099 net: Add GRO supp... |
324 325 |
uh->len = newlen; |
229740c63 udp_offload: Set ... |
326 327 328 329 |
/* Set encapsulation before calling into inner gro_complete() functions * to make them set up the inner offsets. */ skb->encapsulation = 1; |
b582ef099 net: Add GRO supp... |
330 |
rcu_read_lock(); |
a6024562f udp: Add GRO func... |
331 332 333 334 |
sk = (*lookup)(skb, uh->source, uh->dest); if (sk && udp_sk(sk)->gro_complete) err = udp_sk(sk)->gro_complete(sk, skb, nhoff + sizeof(struct udphdr)); |
b582ef099 net: Add GRO supp... |
335 |
rcu_read_unlock(); |
6db93ea13 udp: Set SKB_GSO_... |
336 337 338 |
if (skb->remcsum_offload) skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM; |
b582ef099 net: Add GRO supp... |
339 340 |
return err; } |
a6024562f udp: Add GRO func... |
341 |
EXPORT_SYMBOL(udp_gro_complete); |
b582ef099 net: Add GRO supp... |
342 |
|
72bb17b37 ipv4: udp4_gro_co... |
343 |
static int udp4_gro_complete(struct sk_buff *skb, int nhoff) |
57c67ff4b udp: additional G... |
344 345 346 |
{ const struct iphdr *iph = ip_hdr(skb); struct udphdr *uh = (struct udphdr *)(skb->data + nhoff); |
6db93ea13 udp: Set SKB_GSO_... |
347 348 |
if (uh->check) { skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; |
57c67ff4b udp: additional G... |
349 350 |
uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr, iph->daddr, 0); |
6db93ea13 udp: Set SKB_GSO_... |
351 352 353 |
} else { skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; } |
57c67ff4b udp: additional G... |
354 |
|
a6024562f udp: Add GRO func... |
355 |
return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb); |
57c67ff4b udp: additional G... |
356 |
} |
da5bab079 net: udp4: move G... |
357 358 |
static const struct net_offload udpv4_offload = { .callbacks = { |
da5bab079 net: udp4: move G... |
359 |
.gso_segment = udp4_ufo_fragment, |
57c67ff4b udp: additional G... |
360 361 |
.gro_receive = udp4_gro_receive, .gro_complete = udp4_gro_complete, |
da5bab079 net: udp4: move G... |
362 363 364 365 366 367 368 |
}, }; int __init udpv4_offload_init(void) { return inet_add_offload(&udpv4_offload, IPPROTO_UDP); } |