Blame view
net/ipv4/gre_offload.c
7.32 KB
c50cd3578 net: gre: move GS... |
1 2 3 4 5 6 7 8 9 10 11 12 13 |
/* * IPV4 GSO/GRO offload support * Linux INET implementation * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. * * GRE GSO support */ #include <linux/skbuff.h> |
cf1722837 net/ipv4: don't u... |
14 |
#include <linux/init.h> |
c50cd3578 net: gre: move GS... |
15 16 |
#include <net/protocol.h> #include <net/gre.h> |
c50cd3578 net: gre: move GS... |
17 18 19 |
static struct sk_buff *gre_gso_segment(struct sk_buff *skb, netdev_features_t features) { |
2e598af71 gre: Use GSO flag... |
20 |
int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); |
c50cd3578 net: gre: move GS... |
21 |
struct sk_buff *segs = ERR_PTR(-EINVAL); |
7a7ffbabf ipv4: fix tunnele... |
22 |
u16 mac_offset = skb->mac_header; |
c50cd3578 net: gre: move GS... |
23 |
__be16 protocol = skb->protocol; |
2e598af71 gre: Use GSO flag... |
24 25 |
u16 mac_len = skb->mac_len; int gre_offset, outer_hlen; |
07b26c945 gso: Support part... |
26 |
bool need_csum, ufo, gso_partial; |
c50cd3578 net: gre: move GS... |
27 |
|
53e503989 net: Remove gso_s... |
28 29 |
if (!skb->encapsulation) goto out; |
2e598af71 gre: Use GSO flag... |
30 |
if (unlikely(tnl_hlen < sizeof(struct gre_base_hdr))) |
c50cd3578 net: gre: move GS... |
31 |
goto out; |
2e598af71 gre: Use GSO flag... |
32 |
if (unlikely(!pskb_may_pull(skb, tnl_hlen))) |
b884b1a46 gre_offload: simp... |
33 |
goto out; |
c50cd3578 net: gre: move GS... |
34 |
/* setup inner skb. */ |
c50cd3578 net: gre: move GS... |
35 |
skb->encapsulation = 0; |
5197f3499 net: Reset encap_... |
36 |
SKB_GSO_CB(skb)->encap_level = 0; |
2e598af71 gre: Use GSO flag... |
37 |
__skb_pull(skb, tnl_hlen); |
c50cd3578 net: gre: move GS... |
38 39 40 |
skb_reset_mac_header(skb); skb_set_network_header(skb, skb_inner_network_offset(skb)); skb->mac_len = skb_inner_network_offset(skb); |
387203524 gre: Use inner_pr... |
41 |
skb->protocol = skb->inner_protocol; |
c50cd3578 net: gre: move GS... |
42 |
|
2e598af71 gre: Use GSO flag... |
43 44 |
need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM); skb->encap_hdr_csum = need_csum; |
224638766 GSO: Provide soft... |
45 |
ufo = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP); |
bef3c6c93 net: Drop unecess... |
46 |
features &= skb->dev->hw_enc_features; |
224638766 GSO: Provide soft... |
47 48 49 50 51 52 53 54 55 |
/* The only checksum offload we care about from here on out is the * outer one so strip the existing checksum feature flags based * on the fact that we will be computing our checksum in software. */ if (ufo) { features &= ~NETIF_F_CSUM_MASK; if (!need_csum) features |= NETIF_F_HW_CSUM; } |
c50cd3578 net: gre: move GS... |
56 |
/* segment inner packet. */ |
bef3c6c93 net: Drop unecess... |
57 |
segs = skb_mac_gso_segment(skb, features); |
5a8dbf03d net/ipv4: Use IS_... |
58 |
if (IS_ERR_OR_NULL(segs)) { |
2e598af71 gre: Use GSO flag... |
59 60 |
skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset, mac_len); |
c50cd3578 net: gre: move GS... |
61 |
goto out; |
7a7ffbabf ipv4: fix tunnele... |
62 |
} |
c50cd3578 net: gre: move GS... |
63 |
|
07b26c945 gso: Support part... |
64 |
gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL); |
2e598af71 gre: Use GSO flag... |
65 66 |
outer_hlen = skb_tnl_header_len(skb); gre_offset = outer_hlen - tnl_hlen; |
c50cd3578 net: gre: move GS... |
67 |
skb = segs; |
c50cd3578 net: gre: move GS... |
68 |
do { |
387203524 gre: Use inner_pr... |
69 |
struct gre_base_hdr *greh; |
802ab55ad GSO: Support part... |
70 |
__sum16 *pcsum; |
c50cd3578 net: gre: move GS... |
71 |
|
224638766 GSO: Provide soft... |
72 73 74 75 76 |
/* Set up inner headers if we are offloading inner checksum */ if (skb->ip_summed == CHECKSUM_PARTIAL) { skb_reset_inner_headers(skb); skb->encapsulation = 1; } |
cdbaa0bb2 gso: Update tunne... |
77 |
|
c50cd3578 net: gre: move GS... |
78 79 |
skb->mac_len = mac_len; skb->protocol = protocol; |
2e598af71 gre: Use GSO flag... |
80 81 82 83 84 85 86 87 88 89 |
__skb_push(skb, outer_hlen); skb_reset_mac_header(skb); skb_set_network_header(skb, mac_len); skb_set_transport_header(skb, gre_offset); if (!need_csum) continue; greh = (struct gre_base_hdr *)skb_transport_header(skb); |
802ab55ad GSO: Support part... |
90 |
pcsum = (__sum16 *)(greh + 1); |
07b26c945 gso: Support part... |
91 |
if (gso_partial) { |
802ab55ad GSO: Support part... |
92 93 94 95 96 97 98 99 100 101 102 103 104 |
unsigned int partial_adj; /* Adjust checksum to account for the fact that * the partial checksum is based on actual size * whereas headers should be based on MSS size. */ partial_adj = skb->len + skb_headroom(skb) - SKB_GSO_CB(skb)->data_offset - skb_shinfo(skb)->gso_size; *pcsum = ~csum_fold((__force __wsum)htonl(partial_adj)); } else { *pcsum = 0; } |
2e598af71 gre: Use GSO flag... |
105 |
|
802ab55ad GSO: Support part... |
106 107 |
*(pcsum + 1) = 0; *pcsum = gso_make_checksum(skb, 0); |
c50cd3578 net: gre: move GS... |
108 109 110 111 |
} while ((skb = skb->next)); out: return segs; } |
bf5a755f5 net-gre-gro: Add ... |
112 113 114 115 116 117 118 119 120 121 122 |
static struct sk_buff **gre_gro_receive(struct sk_buff **head, struct sk_buff *skb) { struct sk_buff **pp = NULL; struct sk_buff *p; const struct gre_base_hdr *greh; unsigned int hlen, grehlen; unsigned int off; int flush = 1; struct packet_offload *ptype; __be16 type; |
fac8e0f57 tunnels: Don't ap... |
123 124 125 126 |
if (NAPI_GRO_CB(skb)->encap_mark) goto out; NAPI_GRO_CB(skb)->encap_mark = 1; |
bf5a755f5 net-gre-gro: Add ... |
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 |
off = skb_gro_offset(skb); hlen = off + sizeof(*greh); greh = skb_gro_header_fast(skb, off); if (skb_gro_header_hard(skb, hlen)) { greh = skb_gro_header_slow(skb, hlen, off); if (unlikely(!greh)) goto out; } /* Only support version 0 and K (key), C (csum) flags. Note that * although the support for the S (seq#) flag can be added easily * for GRO, this is problematic for GSO hence can not be enabled * here because a GRO pkt may end up in the forwarding path, thus * requiring GSO support to break it up correctly. */ if ((greh->flags & ~(GRE_KEY|GRE_CSUM)) != 0) goto out; |
a0ca153f9 GRE: Disable segm... |
144 145 146 147 148 149 150 |
/* We can only support GRE_CSUM if we can track the location of * the GRE header. In the case of FOU/GUE we cannot because the * outer UDP header displaces the GRE header leaving us in a state * of limbo. */ if ((greh->flags & GRE_CSUM) && NAPI_GRO_CB(skb)->is_fou) goto out; |
bf5a755f5 net-gre-gro: Add ... |
151 152 153 154 |
type = greh->protocol; rcu_read_lock(); ptype = gro_find_receive_by_type(type); |
51456b291 ipv4: coding styl... |
155 |
if (!ptype) |
bf5a755f5 net-gre-gro: Add ... |
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 |
goto out_unlock; grehlen = GRE_HEADER_SECTION; if (greh->flags & GRE_KEY) grehlen += GRE_HEADER_SECTION; if (greh->flags & GRE_CSUM) grehlen += GRE_HEADER_SECTION; hlen = off + grehlen; if (skb_gro_header_hard(skb, hlen)) { greh = skb_gro_header_slow(skb, hlen, off); if (unlikely(!greh)) goto out_unlock; } |
758f75d1f gre: call skb_gro... |
172 173 |
/* Don't bother verifying checksum if we're going to flush anyway. */ |
884d338c0 gre: Add support ... |
174 175 |
if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) { if (skb_gro_checksum_simple_validate(skb)) |
bf5a755f5 net-gre-gro: Add ... |
176 |
goto out_unlock; |
758f75d1f gre: call skb_gro... |
177 |
|
884d338c0 gre: Add support ... |
178 179 180 |
skb_gro_checksum_try_convert(skb, IPPROTO_GRE, 0, null_compute_pseudo); } |
bf5a755f5 net-gre-gro: Add ... |
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 |
for (p = *head; p; p = p->next) { const struct gre_base_hdr *greh2; if (!NAPI_GRO_CB(p)->same_flow) continue; /* The following checks are needed to ensure only pkts * from the same tunnel are considered for aggregation. * The criteria for "the same tunnel" includes: * 1) same version (we only support version 0 here) * 2) same protocol (we only support ETH_P_IP for now) * 3) same set of flags * 4) same key if the key field is present. */ greh2 = (struct gre_base_hdr *)(p->data + off); if (greh2->flags != greh->flags || greh2->protocol != greh->protocol) { NAPI_GRO_CB(p)->same_flow = 0; continue; } if (greh->flags & GRE_KEY) { /* compare keys */ if (*(__be32 *)(greh2+1) != *(__be32 *)(greh+1)) { NAPI_GRO_CB(p)->same_flow = 0; continue; } } } skb_gro_pull(skb, grehlen); /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/ skb_gro_postpull_rcsum(skb, greh, grehlen); |
fcd91dd44 net: add recursio... |
215 |
pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); |
c194cf93c gro: Defer cleari... |
216 |
flush = 0; |
bf5a755f5 net-gre-gro: Add ... |
217 218 219 220 221 222 223 224 |
out_unlock: rcu_read_unlock(); out: NAPI_GRO_CB(skb)->flush |= flush; return pp; } |
d10dbad2a gre_offload: fix ... |
225 |
static int gre_gro_complete(struct sk_buff *skb, int nhoff) |
bf5a755f5 net-gre-gro: Add ... |
226 227 228 229 230 231 |
{ struct gre_base_hdr *greh = (struct gre_base_hdr *)(skb->data + nhoff); struct packet_offload *ptype; unsigned int grehlen = sizeof(*greh); int err = -ENOENT; __be16 type; |
c3caf1192 net-gre-gro: Fix ... |
232 233 |
skb->encapsulation = 1; skb_shinfo(skb)->gso_type = SKB_GSO_GRE; |
bf5a755f5 net-gre-gro: Add ... |
234 235 236 237 238 239 240 241 242 |
type = greh->protocol; if (greh->flags & GRE_KEY) grehlen += GRE_HEADER_SECTION; if (greh->flags & GRE_CSUM) grehlen += GRE_HEADER_SECTION; rcu_read_lock(); ptype = gro_find_complete_by_type(type); |
00db41243 ipv4: coding styl... |
243 |
if (ptype) |
bf5a755f5 net-gre-gro: Add ... |
244 245 246 |
err = ptype->callbacks.gro_complete(skb, nhoff + grehlen); rcu_read_unlock(); |
6fb2a7567 gre: Set inner ma... |
247 248 |
skb_set_inner_mac_header(skb, nhoff + grehlen); |
bf5a755f5 net-gre-gro: Add ... |
249 250 |
return err; } |
c50cd3578 net: gre: move GS... |
251 252 |
static const struct net_offload gre_offload = { .callbacks = { |
c50cd3578 net: gre: move GS... |
253 |
.gso_segment = gre_gso_segment, |
bf5a755f5 net-gre-gro: Add ... |
254 255 |
.gro_receive = gre_gro_receive, .gro_complete = gre_gro_complete, |
c50cd3578 net: gre: move GS... |
256 257 |
}, }; |
438e38fad gre_offload: stat... |
258 |
static int __init gre_offload_init(void) |
c50cd3578 net: gre: move GS... |
259 |
{ |
e0c20967c GRE: Add support ... |
260 261 262 263 264 265 266 267 268 269 270 271 272 |
int err; err = inet_add_offload(&gre_offload, IPPROTO_GRE); #if IS_ENABLED(CONFIG_IPV6) if (err) return err; err = inet6_add_offload(&gre_offload, IPPROTO_GRE); if (err) inet_del_offload(&gre_offload, IPPROTO_GRE); #endif return err; |
c50cd3578 net: gre: move GS... |
273 |
} |
cf1722837 net/ipv4: don't u... |
274 |
device_initcall(gre_offload_init); |