Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit a2da570d62fcb9e8816f6920e1ec02c706b289fa

Authored by Eric Dumazet
Committed by David S. Miller
1 parent fd245a4adb
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

net_sched: RCU conversion of stab 

This patch converts stab qdisc management to RCU, so that we can perform
the qdisc_calculate_pkt_len() call before getting qdisc lock.

This shortens the lock's held time in __dev_xmit_skb().

This permits more qdiscs to get TCQ_F_CAN_BYPASS status, avoiding lot of
cache misses and so reducing latencies.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
CC: Patrick McHardy <kaber@trash.net>
CC: Jesper Dangaard Brouer <hawk@diku.dk>
CC: Jarek Poplawski <jarkao2@gmail.com>
CC: Jamal Hadi Salim <hadi@cyberus.ca>
CC: Stephen Hemminger <shemminger@vyatta.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

Showing 4 changed files with 38 additions and 19 deletions Inline Diff

include/net/sch_generic.h
Diff comments   View file @ a2da570
1 #ifndef __NET_SCHED_GENERIC_H 1 #ifndef __NET_SCHED_GENERIC_H
2 #define __NET_SCHED_GENERIC_H 2 #define __NET_SCHED_GENERIC_H
3 3
4 #include <linux/netdevice.h> 4 #include <linux/netdevice.h>
5 #include <linux/types.h> 5 #include <linux/types.h>
6 #include <linux/rcupdate.h> 6 #include <linux/rcupdate.h>
7 #include <linux/module.h> 7 #include <linux/module.h>
8 #include <linux/pkt_sched.h> 8 #include <linux/pkt_sched.h>
9 #include <linux/pkt_cls.h> 9 #include <linux/pkt_cls.h>
10 #include <net/gen_stats.h> 10 #include <net/gen_stats.h>
11 #include <net/rtnetlink.h> 11 #include <net/rtnetlink.h>
12 12
13 struct Qdisc_ops; 13 struct Qdisc_ops;
14 struct qdisc_walker; 14 struct qdisc_walker;
15 struct tcf_walker; 15 struct tcf_walker;
16 struct module; 16 struct module;
17 17
18 struct qdisc_rate_table { 18 struct qdisc_rate_table {
19 struct tc_ratespec rate; 19 struct tc_ratespec rate;
20 u32 data[256]; 20 u32 data[256];
21 struct qdisc_rate_table *next; 21 struct qdisc_rate_table *next;
22 int refcnt; 22 int refcnt;
23 }; 23 };
24 24
25 enum qdisc_state_t { 25 enum qdisc_state_t {
26 __QDISC_STATE_SCHED, 26 __QDISC_STATE_SCHED,
27 __QDISC_STATE_DEACTIVATED, 27 __QDISC_STATE_DEACTIVATED,
28 }; 28 };
29 29
30 /* 30 /*
31 * following bits are only changed while qdisc lock is held 31 * following bits are only changed while qdisc lock is held
32 */ 32 */
33 enum qdisc___state_t { 33 enum qdisc___state_t {
34 __QDISC___STATE_RUNNING = 1, 34 __QDISC___STATE_RUNNING = 1,
35 __QDISC___STATE_THROTTLED = 2, 35 __QDISC___STATE_THROTTLED = 2,
36 }; 36 };
37 37
38 struct qdisc_size_table { 38 struct qdisc_size_table {
39 struct rcu_head rcu;
39 struct list_head list; 40 struct list_head list;
40 struct tc_sizespec szopts; 41 struct tc_sizespec szopts;
41 int refcnt; 42 int refcnt;
42 u16 data[]; 43 u16 data[];
43 }; 44 };
44 45
45 struct Qdisc { 46 struct Qdisc {
46 int (*enqueue)(struct sk_buff *skb, struct Qdisc *dev); 47 int (*enqueue)(struct sk_buff *skb, struct Qdisc *dev);
47 struct sk_buff * (*dequeue)(struct Qdisc *dev); 48 struct sk_buff * (*dequeue)(struct Qdisc *dev);
48 unsigned flags; 49 unsigned flags;
49 #define TCQ_F_BUILTIN 1 50 #define TCQ_F_BUILTIN 1
50 #define TCQ_F_INGRESS 2 51 #define TCQ_F_INGRESS 2
51 #define TCQ_F_CAN_BYPASS 4 52 #define TCQ_F_CAN_BYPASS 4
52 #define TCQ_F_MQROOT 8 53 #define TCQ_F_MQROOT 8
53 #define TCQ_F_WARN_NONWC (1 << 16) 54 #define TCQ_F_WARN_NONWC (1 << 16)
54 int padded; 55 int padded;
55 struct Qdisc_ops *ops; 56 struct Qdisc_ops *ops;
56 struct qdisc_size_table *stab; 57 struct qdisc_size_table __rcu *stab;
57 struct list_head list; 58 struct list_head list;
58 u32 handle; 59 u32 handle;
59 u32 parent; 60 u32 parent;
60 atomic_t refcnt; 61 atomic_t refcnt;
61 struct gnet_stats_rate_est rate_est; 62 struct gnet_stats_rate_est rate_est;
62 int (*reshape_fail)(struct sk_buff *skb, 63 int (*reshape_fail)(struct sk_buff *skb,
63 struct Qdisc *q); 64 struct Qdisc *q);
64 65
65 void *u32_node; 66 void *u32_node;
66 67
67 /* This field is deprecated, but it is still used by CBQ 68 /* This field is deprecated, but it is still used by CBQ
68 * and it will live until better solution will be invented. 69 * and it will live until better solution will be invented.
69 */ 70 */
70 struct Qdisc *__parent; 71 struct Qdisc *__parent;
71 struct netdev_queue *dev_queue; 72 struct netdev_queue *dev_queue;
72 struct Qdisc *next_sched; 73 struct Qdisc *next_sched;
73 74
74 struct sk_buff *gso_skb; 75 struct sk_buff *gso_skb;
75 /* 76 /*
76 * For performance sake on SMP, we put highly modified fields at the end 77 * For performance sake on SMP, we put highly modified fields at the end
77 */ 78 */
78 unsigned long state; 79 unsigned long state;
79 struct sk_buff_head q; 80 struct sk_buff_head q;
80 struct gnet_stats_basic_packed bstats; 81 struct gnet_stats_basic_packed bstats;
81 unsigned int __state; 82 unsigned int __state;
82 struct gnet_stats_queue qstats; 83 struct gnet_stats_queue qstats;
83 struct rcu_head rcu_head; 84 struct rcu_head rcu_head;
84 spinlock_t busylock; 85 spinlock_t busylock;
85 }; 86 };
86 87
87 static inline bool qdisc_is_running(const struct Qdisc *qdisc) 88 static inline bool qdisc_is_running(const struct Qdisc *qdisc)
88 { 89 {
89 return (qdisc->__state & __QDISC___STATE_RUNNING) ? true : false; 90 return (qdisc->__state & __QDISC___STATE_RUNNING) ? true : false;
90 } 91 }
91 92
92 static inline bool qdisc_run_begin(struct Qdisc *qdisc) 93 static inline bool qdisc_run_begin(struct Qdisc *qdisc)
93 { 94 {
94 if (qdisc_is_running(qdisc)) 95 if (qdisc_is_running(qdisc))
95 return false; 96 return false;
96 qdisc->__state |= __QDISC___STATE_RUNNING; 97 qdisc->__state |= __QDISC___STATE_RUNNING;
97 return true; 98 return true;
98 } 99 }
99 100
100 static inline void qdisc_run_end(struct Qdisc *qdisc) 101 static inline void qdisc_run_end(struct Qdisc *qdisc)
101 { 102 {
102 qdisc->__state &= ~__QDISC___STATE_RUNNING; 103 qdisc->__state &= ~__QDISC___STATE_RUNNING;
103 } 104 }
104 105
105 static inline bool qdisc_is_throttled(const struct Qdisc *qdisc) 106 static inline bool qdisc_is_throttled(const struct Qdisc *qdisc)
106 { 107 {
107 return (qdisc->__state & __QDISC___STATE_THROTTLED) ? true : false; 108 return (qdisc->__state & __QDISC___STATE_THROTTLED) ? true : false;
108 } 109 }
109 110
110 static inline void qdisc_throttled(struct Qdisc *qdisc) 111 static inline void qdisc_throttled(struct Qdisc *qdisc)
111 { 112 {
112 qdisc->__state |= __QDISC___STATE_THROTTLED; 113 qdisc->__state |= __QDISC___STATE_THROTTLED;
113 } 114 }
114 115
115 static inline void qdisc_unthrottled(struct Qdisc *qdisc) 116 static inline void qdisc_unthrottled(struct Qdisc *qdisc)
116 { 117 {
117 qdisc->__state &= ~__QDISC___STATE_THROTTLED; 118 qdisc->__state &= ~__QDISC___STATE_THROTTLED;
118 } 119 }
119 120
120 struct Qdisc_class_ops { 121 struct Qdisc_class_ops {
121 /* Child qdisc manipulation */ 122 /* Child qdisc manipulation */
122 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *); 123 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *);
123 int (*graft)(struct Qdisc *, unsigned long cl, 124 int (*graft)(struct Qdisc *, unsigned long cl,
124 struct Qdisc *, struct Qdisc **); 125 struct Qdisc *, struct Qdisc **);
125 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl); 126 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl);
126 void (*qlen_notify)(struct Qdisc *, unsigned long); 127 void (*qlen_notify)(struct Qdisc *, unsigned long);
127 128
128 /* Class manipulation routines */ 129 /* Class manipulation routines */
129 unsigned long (*get)(struct Qdisc *, u32 classid); 130 unsigned long (*get)(struct Qdisc *, u32 classid);
130 void (*put)(struct Qdisc *, unsigned long); 131 void (*put)(struct Qdisc *, unsigned long);
131 int (*change)(struct Qdisc *, u32, u32, 132 int (*change)(struct Qdisc *, u32, u32,
132 struct nlattr **, unsigned long *); 133 struct nlattr **, unsigned long *);
133 int (*delete)(struct Qdisc *, unsigned long); 134 int (*delete)(struct Qdisc *, unsigned long);
134 void (*walk)(struct Qdisc *, struct qdisc_walker * arg); 135 void (*walk)(struct Qdisc *, struct qdisc_walker * arg);
135 136
136 /* Filter manipulation */ 137 /* Filter manipulation */
137 struct tcf_proto ** (*tcf_chain)(struct Qdisc *, unsigned long); 138 struct tcf_proto ** (*tcf_chain)(struct Qdisc *, unsigned long);
138 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long, 139 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long,
139 u32 classid); 140 u32 classid);
140 void (*unbind_tcf)(struct Qdisc *, unsigned long); 141 void (*unbind_tcf)(struct Qdisc *, unsigned long);
141 142
142 /* rtnetlink specific */ 143 /* rtnetlink specific */
143 int (*dump)(struct Qdisc *, unsigned long, 144 int (*dump)(struct Qdisc *, unsigned long,
144 struct sk_buff *skb, struct tcmsg*); 145 struct sk_buff *skb, struct tcmsg*);
145 int (*dump_stats)(struct Qdisc *, unsigned long, 146 int (*dump_stats)(struct Qdisc *, unsigned long,
146 struct gnet_dump *); 147 struct gnet_dump *);
147 }; 148 };
148 149
149 struct Qdisc_ops { 150 struct Qdisc_ops {
150 struct Qdisc_ops *next; 151 struct Qdisc_ops *next;
151 const struct Qdisc_class_ops *cl_ops; 152 const struct Qdisc_class_ops *cl_ops;
152 char id[IFNAMSIZ]; 153 char id[IFNAMSIZ];
153 int priv_size; 154 int priv_size;
154 155
155 int (*enqueue)(struct sk_buff *, struct Qdisc *); 156 int (*enqueue)(struct sk_buff *, struct Qdisc *);
156 struct sk_buff * (*dequeue)(struct Qdisc *); 157 struct sk_buff * (*dequeue)(struct Qdisc *);
157 struct sk_buff * (*peek)(struct Qdisc *); 158 struct sk_buff * (*peek)(struct Qdisc *);
158 unsigned int (*drop)(struct Qdisc *); 159 unsigned int (*drop)(struct Qdisc *);
159 160
160 int (*init)(struct Qdisc *, struct nlattr *arg); 161 int (*init)(struct Qdisc *, struct nlattr *arg);
161 void (*reset)(struct Qdisc *); 162 void (*reset)(struct Qdisc *);
162 void (*destroy)(struct Qdisc *); 163 void (*destroy)(struct Qdisc *);
163 int (*change)(struct Qdisc *, struct nlattr *arg); 164 int (*change)(struct Qdisc *, struct nlattr *arg);
164 void (*attach)(struct Qdisc *); 165 void (*attach)(struct Qdisc *);
165 166
166 int (*dump)(struct Qdisc *, struct sk_buff *); 167 int (*dump)(struct Qdisc *, struct sk_buff *);
167 int (*dump_stats)(struct Qdisc *, struct gnet_dump *); 168 int (*dump_stats)(struct Qdisc *, struct gnet_dump *);
168 169
169 struct module *owner; 170 struct module *owner;
170 }; 171 };
171 172
172 173
173 struct tcf_result { 174 struct tcf_result {
174 unsigned long class; 175 unsigned long class;
175 u32 classid; 176 u32 classid;
176 }; 177 };
177 178
178 struct tcf_proto_ops { 179 struct tcf_proto_ops {
179 struct tcf_proto_ops *next; 180 struct tcf_proto_ops *next;
180 char kind[IFNAMSIZ]; 181 char kind[IFNAMSIZ];
181 182
182 int (*classify)(struct sk_buff*, struct tcf_proto*, 183 int (*classify)(struct sk_buff*, struct tcf_proto*,
183 struct tcf_result *); 184 struct tcf_result *);
184 int (*init)(struct tcf_proto*); 185 int (*init)(struct tcf_proto*);
185 void (*destroy)(struct tcf_proto*); 186 void (*destroy)(struct tcf_proto*);
186 187
187 unsigned long (*get)(struct tcf_proto*, u32 handle); 188 unsigned long (*get)(struct tcf_proto*, u32 handle);
188 void (*put)(struct tcf_proto*, unsigned long); 189 void (*put)(struct tcf_proto*, unsigned long);
189 int (*change)(struct tcf_proto*, unsigned long, 190 int (*change)(struct tcf_proto*, unsigned long,
190 u32 handle, struct nlattr **, 191 u32 handle, struct nlattr **,
191 unsigned long *); 192 unsigned long *);
192 int (*delete)(struct tcf_proto*, unsigned long); 193 int (*delete)(struct tcf_proto*, unsigned long);
193 void (*walk)(struct tcf_proto*, struct tcf_walker *arg); 194 void (*walk)(struct tcf_proto*, struct tcf_walker *arg);
194 195
195 /* rtnetlink specific */ 196 /* rtnetlink specific */
196 int (*dump)(struct tcf_proto*, unsigned long, 197 int (*dump)(struct tcf_proto*, unsigned long,
197 struct sk_buff *skb, struct tcmsg*); 198 struct sk_buff *skb, struct tcmsg*);
198 199
199 struct module *owner; 200 struct module *owner;
200 }; 201 };
201 202
202 struct tcf_proto { 203 struct tcf_proto {
203 /* Fast access part */ 204 /* Fast access part */
204 struct tcf_proto *next; 205 struct tcf_proto *next;
205 void *root; 206 void *root;
206 int (*classify)(struct sk_buff*, struct tcf_proto*, 207 int (*classify)(struct sk_buff*, struct tcf_proto*,
207 struct tcf_result *); 208 struct tcf_result *);
208 __be16 protocol; 209 __be16 protocol;
209 210
210 /* All the rest */ 211 /* All the rest */
211 u32 prio; 212 u32 prio;
212 u32 classid; 213 u32 classid;
213 struct Qdisc *q; 214 struct Qdisc *q;
214 void *data; 215 void *data;
215 struct tcf_proto_ops *ops; 216 struct tcf_proto_ops *ops;
216 }; 217 };
217 218
218 struct qdisc_skb_cb { 219 struct qdisc_skb_cb {
219 unsigned int pkt_len; 220 unsigned int pkt_len;
220 char data[]; 221 char data[];
221 }; 222 };
222 223
223 static inline int qdisc_qlen(struct Qdisc *q) 224 static inline int qdisc_qlen(struct Qdisc *q)
224 { 225 {
225 return q->q.qlen; 226 return q->q.qlen;
226 } 227 }
227 228
228 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb) 229 static inline struct qdisc_skb_cb *qdisc_skb_cb(const struct sk_buff *skb)
229 { 230 {
230 return (struct qdisc_skb_cb *)skb->cb; 231 return (struct qdisc_skb_cb *)skb->cb;
231 } 232 }
232 233
233 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) 234 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc)
234 { 235 {
235 return &qdisc->q.lock; 236 return &qdisc->q.lock;
236 } 237 }
237 238
238 static inline struct Qdisc *qdisc_root(struct Qdisc *qdisc) 239 static inline struct Qdisc *qdisc_root(struct Qdisc *qdisc)
239 { 240 {
240 return qdisc->dev_queue->qdisc; 241 return qdisc->dev_queue->qdisc;
241 } 242 }
242 243
243 static inline struct Qdisc *qdisc_root_sleeping(struct Qdisc *qdisc) 244 static inline struct Qdisc *qdisc_root_sleeping(struct Qdisc *qdisc)
244 { 245 {
245 return qdisc->dev_queue->qdisc_sleeping; 246 return qdisc->dev_queue->qdisc_sleeping;
246 } 247 }
247 248
248 /* The qdisc root lock is a mechanism by which to top level 249 /* The qdisc root lock is a mechanism by which to top level
249 * of a qdisc tree can be locked from any qdisc node in the 250 * of a qdisc tree can be locked from any qdisc node in the
250 * forest. This allows changing the configuration of some 251 * forest. This allows changing the configuration of some
251 * aspect of the qdisc tree while blocking out asynchronous 252 * aspect of the qdisc tree while blocking out asynchronous
252 * qdisc access in the packet processing paths. 253 * qdisc access in the packet processing paths.
253 * 254 *
254 * It is only legal to do this when the root will not change 255 * It is only legal to do this when the root will not change
255 * on us. Otherwise we'll potentially lock the wrong qdisc 256 * on us. Otherwise we'll potentially lock the wrong qdisc
256 * root. This is enforced by holding the RTNL semaphore, which 257 * root. This is enforced by holding the RTNL semaphore, which
257 * all users of this lock accessor must do. 258 * all users of this lock accessor must do.
258 */ 259 */
259 static inline spinlock_t *qdisc_root_lock(struct Qdisc *qdisc) 260 static inline spinlock_t *qdisc_root_lock(struct Qdisc *qdisc)
260 { 261 {
261 struct Qdisc *root = qdisc_root(qdisc); 262 struct Qdisc *root = qdisc_root(qdisc);
262 263
263 ASSERT_RTNL(); 264 ASSERT_RTNL();
264 return qdisc_lock(root); 265 return qdisc_lock(root);
265 } 266 }
266 267
267 static inline spinlock_t *qdisc_root_sleeping_lock(struct Qdisc *qdisc) 268 static inline spinlock_t *qdisc_root_sleeping_lock(struct Qdisc *qdisc)
268 { 269 {
269 struct Qdisc *root = qdisc_root_sleeping(qdisc); 270 struct Qdisc *root = qdisc_root_sleeping(qdisc);
270 271
271 ASSERT_RTNL(); 272 ASSERT_RTNL();
272 return qdisc_lock(root); 273 return qdisc_lock(root);
273 } 274 }
274 275
275 static inline struct net_device *qdisc_dev(struct Qdisc *qdisc) 276 static inline struct net_device *qdisc_dev(struct Qdisc *qdisc)
276 { 277 {
277 return qdisc->dev_queue->dev; 278 return qdisc->dev_queue->dev;
278 } 279 }
279 280
280 static inline void sch_tree_lock(struct Qdisc *q) 281 static inline void sch_tree_lock(struct Qdisc *q)
281 { 282 {
282 spin_lock_bh(qdisc_root_sleeping_lock(q)); 283 spin_lock_bh(qdisc_root_sleeping_lock(q));
283 } 284 }
284 285
285 static inline void sch_tree_unlock(struct Qdisc *q) 286 static inline void sch_tree_unlock(struct Qdisc *q)
286 { 287 {
287 spin_unlock_bh(qdisc_root_sleeping_lock(q)); 288 spin_unlock_bh(qdisc_root_sleeping_lock(q));
288 } 289 }
289 290
290 #define tcf_tree_lock(tp) sch_tree_lock((tp)->q) 291 #define tcf_tree_lock(tp) sch_tree_lock((tp)->q)
291 #define tcf_tree_unlock(tp) sch_tree_unlock((tp)->q) 292 #define tcf_tree_unlock(tp) sch_tree_unlock((tp)->q)
292 293
293 extern struct Qdisc noop_qdisc; 294 extern struct Qdisc noop_qdisc;
294 extern struct Qdisc_ops noop_qdisc_ops; 295 extern struct Qdisc_ops noop_qdisc_ops;
295 extern struct Qdisc_ops pfifo_fast_ops; 296 extern struct Qdisc_ops pfifo_fast_ops;
296 extern struct Qdisc_ops mq_qdisc_ops; 297 extern struct Qdisc_ops mq_qdisc_ops;
297 298
298 struct Qdisc_class_common { 299 struct Qdisc_class_common {
299 u32 classid; 300 u32 classid;
300 struct hlist_node hnode; 301 struct hlist_node hnode;
301 }; 302 };
302 303
303 struct Qdisc_class_hash { 304 struct Qdisc_class_hash {
304 struct hlist_head *hash; 305 struct hlist_head *hash;
305 unsigned int hashsize; 306 unsigned int hashsize;
306 unsigned int hashmask; 307 unsigned int hashmask;
307 unsigned int hashelems; 308 unsigned int hashelems;
308 }; 309 };
309 310
310 static inline unsigned int qdisc_class_hash(u32 id, u32 mask) 311 static inline unsigned int qdisc_class_hash(u32 id, u32 mask)
311 { 312 {
312 id ^= id >> 8; 313 id ^= id >> 8;
313 id ^= id >> 4; 314 id ^= id >> 4;
314 return id & mask; 315 return id & mask;
315 } 316 }
316 317
317 static inline struct Qdisc_class_common * 318 static inline struct Qdisc_class_common *
318 qdisc_class_find(struct Qdisc_class_hash *hash, u32 id) 319 qdisc_class_find(struct Qdisc_class_hash *hash, u32 id)
319 { 320 {
320 struct Qdisc_class_common *cl; 321 struct Qdisc_class_common *cl;
321 struct hlist_node *n; 322 struct hlist_node *n;
322 unsigned int h; 323 unsigned int h;
323 324
324 h = qdisc_class_hash(id, hash->hashmask); 325 h = qdisc_class_hash(id, hash->hashmask);
325 hlist_for_each_entry(cl, n, &hash->hash[h], hnode) { 326 hlist_for_each_entry(cl, n, &hash->hash[h], hnode) {
326 if (cl->classid == id) 327 if (cl->classid == id)
327 return cl; 328 return cl;
328 } 329 }
329 return NULL; 330 return NULL;
330 } 331 }
331 332
332 extern int qdisc_class_hash_init(struct Qdisc_class_hash *); 333 extern int qdisc_class_hash_init(struct Qdisc_class_hash *);
333 extern void qdisc_class_hash_insert(struct Qdisc_class_hash *, struct Qdisc_class_common *); 334 extern void qdisc_class_hash_insert(struct Qdisc_class_hash *, struct Qdisc_class_common *);
334 extern void qdisc_class_hash_remove(struct Qdisc_class_hash *, struct Qdisc_class_common *); 335 extern void qdisc_class_hash_remove(struct Qdisc_class_hash *, struct Qdisc_class_common *);
335 extern void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *); 336 extern void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);
336 extern void qdisc_class_hash_destroy(struct Qdisc_class_hash *); 337 extern void qdisc_class_hash_destroy(struct Qdisc_class_hash *);
337 338
338 extern void dev_init_scheduler(struct net_device *dev); 339 extern void dev_init_scheduler(struct net_device *dev);
339 extern void dev_shutdown(struct net_device *dev); 340 extern void dev_shutdown(struct net_device *dev);
340 extern void dev_activate(struct net_device *dev); 341 extern void dev_activate(struct net_device *dev);
341 extern void dev_deactivate(struct net_device *dev); 342 extern void dev_deactivate(struct net_device *dev);
342 extern void dev_deactivate_many(struct list_head *head); 343 extern void dev_deactivate_many(struct list_head *head);
343 extern struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 344 extern struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
344 struct Qdisc *qdisc); 345 struct Qdisc *qdisc);
345 extern void qdisc_reset(struct Qdisc *qdisc); 346 extern void qdisc_reset(struct Qdisc *qdisc);
346 extern void qdisc_destroy(struct Qdisc *qdisc); 347 extern void qdisc_destroy(struct Qdisc *qdisc);
347 extern void qdisc_tree_decrease_qlen(struct Qdisc *qdisc, unsigned int n); 348 extern void qdisc_tree_decrease_qlen(struct Qdisc *qdisc, unsigned int n);
348 extern struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 349 extern struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
349 struct Qdisc_ops *ops); 350 struct Qdisc_ops *ops);
350 extern struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 351 extern struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
351 struct Qdisc_ops *ops, u32 parentid); 352 struct Qdisc_ops *ops, u32 parentid);
352 extern void qdisc_calculate_pkt_len(struct sk_buff *skb, 353 extern void __qdisc_calculate_pkt_len(struct sk_buff *skb,
353 struct qdisc_size_table *stab); 354 const struct qdisc_size_table *stab);
354 extern void tcf_destroy(struct tcf_proto *tp); 355 extern void tcf_destroy(struct tcf_proto *tp);
355 extern void tcf_destroy_chain(struct tcf_proto **fl); 356 extern void tcf_destroy_chain(struct tcf_proto **fl);
356 357
357 /* Reset all TX qdiscs greater then index of a device. */ 358 /* Reset all TX qdiscs greater then index of a device. */
358 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) 359 static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i)
359 { 360 {
360 struct Qdisc *qdisc; 361 struct Qdisc *qdisc;
361 362
362 for (; i < dev->num_tx_queues; i++) { 363 for (; i < dev->num_tx_queues; i++) {
363 qdisc = netdev_get_tx_queue(dev, i)->qdisc; 364 qdisc = netdev_get_tx_queue(dev, i)->qdisc;
364 if (qdisc) { 365 if (qdisc) {
365 spin_lock_bh(qdisc_lock(qdisc)); 366 spin_lock_bh(qdisc_lock(qdisc));
366 qdisc_reset(qdisc); 367 qdisc_reset(qdisc);
367 spin_unlock_bh(qdisc_lock(qdisc)); 368 spin_unlock_bh(qdisc_lock(qdisc));
368 } 369 }
369 } 370 }
370 } 371 }
371 372
372 static inline void qdisc_reset_all_tx(struct net_device *dev) 373 static inline void qdisc_reset_all_tx(struct net_device *dev)
373 { 374 {
374 qdisc_reset_all_tx_gt(dev, 0); 375 qdisc_reset_all_tx_gt(dev, 0);
375 } 376 }
376 377
377 /* Are all TX queues of the device empty? */ 378 /* Are all TX queues of the device empty? */
378 static inline bool qdisc_all_tx_empty(const struct net_device *dev) 379 static inline bool qdisc_all_tx_empty(const struct net_device *dev)
379 { 380 {
380 unsigned int i; 381 unsigned int i;
381 for (i = 0; i < dev->num_tx_queues; i++) { 382 for (i = 0; i < dev->num_tx_queues; i++) {
382 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 383 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
383 const struct Qdisc *q = txq->qdisc; 384 const struct Qdisc *q = txq->qdisc;
384 385
385 if (q->q.qlen) 386 if (q->q.qlen)
386 return false; 387 return false;
387 } 388 }
388 return true; 389 return true;
389 } 390 }
390 391
391 /* Are any of the TX qdiscs changing? */ 392 /* Are any of the TX qdiscs changing? */
392 static inline bool qdisc_tx_changing(struct net_device *dev) 393 static inline bool qdisc_tx_changing(struct net_device *dev)
393 { 394 {
394 unsigned int i; 395 unsigned int i;
395 for (i = 0; i < dev->num_tx_queues; i++) { 396 for (i = 0; i < dev->num_tx_queues; i++) {
396 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 397 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
397 if (txq->qdisc != txq->qdisc_sleeping) 398 if (txq->qdisc != txq->qdisc_sleeping)
398 return true; 399 return true;
399 } 400 }
400 return false; 401 return false;
401 } 402 }
402 403
403 /* Is the device using the noop qdisc on all queues? */ 404 /* Is the device using the noop qdisc on all queues? */
404 static inline bool qdisc_tx_is_noop(const struct net_device *dev) 405 static inline bool qdisc_tx_is_noop(const struct net_device *dev)
405 { 406 {
406 unsigned int i; 407 unsigned int i;
407 for (i = 0; i < dev->num_tx_queues; i++) { 408 for (i = 0; i < dev->num_tx_queues; i++) {
408 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 409 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
409 if (txq->qdisc != &noop_qdisc) 410 if (txq->qdisc != &noop_qdisc)
410 return false; 411 return false;
411 } 412 }
412 return true; 413 return true;
413 } 414 }
414 415
415 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb) 416 static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb)
416 { 417 {
417 return qdisc_skb_cb(skb)->pkt_len; 418 return qdisc_skb_cb(skb)->pkt_len;
418 } 419 }
419 420
420 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */ 421 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
421 enum net_xmit_qdisc_t { 422 enum net_xmit_qdisc_t {
422 __NET_XMIT_STOLEN = 0x00010000, 423 __NET_XMIT_STOLEN = 0x00010000,
423 __NET_XMIT_BYPASS = 0x00020000, 424 __NET_XMIT_BYPASS = 0x00020000,
424 }; 425 };
425 426
426 #ifdef CONFIG_NET_CLS_ACT 427 #ifdef CONFIG_NET_CLS_ACT
427 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1) 428 #define net_xmit_drop_count(e) ((e) & __NET_XMIT_STOLEN ? 0 : 1)
428 #else 429 #else
429 #define net_xmit_drop_count(e) (1) 430 #define net_xmit_drop_count(e) (1)
430 #endif 431 #endif
431 432
432 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch) 433 static inline void qdisc_calculate_pkt_len(struct sk_buff *skb,
434 const struct Qdisc *sch)
433 { 435 {
434 #ifdef CONFIG_NET_SCHED 436 #ifdef CONFIG_NET_SCHED
435 if (sch->stab) 437 struct qdisc_size_table *stab = rcu_dereference_bh(sch->stab);
436 qdisc_calculate_pkt_len(skb, sch->stab); 438
439 if (stab)
440 __qdisc_calculate_pkt_len(skb, stab);
437 #endif 441 #endif
442 }
443
444 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch)
445 {
446 qdisc_calculate_pkt_len(skb, sch);
438 return sch->enqueue(skb, sch); 447 return sch->enqueue(skb, sch);
439 } 448 }
440 449
441 static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch) 450 static inline int qdisc_enqueue_root(struct sk_buff *skb, struct Qdisc *sch)
442 { 451 {
443 qdisc_skb_cb(skb)->pkt_len = skb->len; 452 qdisc_skb_cb(skb)->pkt_len = skb->len;
444 return qdisc_enqueue(skb, sch) & NET_XMIT_MASK; 453 return qdisc_enqueue(skb, sch) & NET_XMIT_MASK;
445 } 454 }
446 455
447 456
448 static inline void bstats_update(struct gnet_stats_basic_packed *bstats, 457 static inline void bstats_update(struct gnet_stats_basic_packed *bstats,
449 const struct sk_buff *skb) 458 const struct sk_buff *skb)
450 { 459 {
451 bstats->bytes += qdisc_pkt_len(skb); 460 bstats->bytes += qdisc_pkt_len(skb);
452 bstats->packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 461 bstats->packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
453 } 462 }
454 463
455 static inline void qdisc_bstats_update(struct Qdisc *sch, 464 static inline void qdisc_bstats_update(struct Qdisc *sch,
456 const struct sk_buff *skb) 465 const struct sk_buff *skb)
457 { 466 {
458 bstats_update(&sch->bstats, skb); 467 bstats_update(&sch->bstats, skb);
459 } 468 }
460 469
461 static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch, 470 static inline int __qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch,
462 struct sk_buff_head *list) 471 struct sk_buff_head *list)
463 { 472 {
464 __skb_queue_tail(list, skb); 473 __skb_queue_tail(list, skb);
465 sch->qstats.backlog += qdisc_pkt_len(skb); 474 sch->qstats.backlog += qdisc_pkt_len(skb);
466 qdisc_bstats_update(sch, skb); 475 qdisc_bstats_update(sch, skb);
467 476
468 return NET_XMIT_SUCCESS; 477 return NET_XMIT_SUCCESS;
469 } 478 }
470 479
471 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) 480 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
472 { 481 {
473 return __qdisc_enqueue_tail(skb, sch, &sch->q); 482 return __qdisc_enqueue_tail(skb, sch, &sch->q);
474 } 483 }
475 484
476 static inline struct sk_buff *__qdisc_dequeue_head(struct Qdisc *sch, 485 static inline struct sk_buff *__qdisc_dequeue_head(struct Qdisc *sch,
477 struct sk_buff_head *list) 486 struct sk_buff_head *list)
478 { 487 {
479 struct sk_buff *skb = __skb_dequeue(list); 488 struct sk_buff *skb = __skb_dequeue(list);
480 489
481 if (likely(skb != NULL)) 490 if (likely(skb != NULL))
482 sch->qstats.backlog -= qdisc_pkt_len(skb); 491 sch->qstats.backlog -= qdisc_pkt_len(skb);
483 492
484 return skb; 493 return skb;
485 } 494 }
486 495
487 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) 496 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch)
488 { 497 {
489 return __qdisc_dequeue_head(sch, &sch->q); 498 return __qdisc_dequeue_head(sch, &sch->q);
490 } 499 }
491 500
492 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, 501 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch,
493 struct sk_buff_head *list) 502 struct sk_buff_head *list)
494 { 503 {
495 struct sk_buff *skb = __qdisc_dequeue_head(sch, list); 504 struct sk_buff *skb = __qdisc_dequeue_head(sch, list);
496 505
497 if (likely(skb != NULL)) { 506 if (likely(skb != NULL)) {
498 unsigned int len = qdisc_pkt_len(skb); 507 unsigned int len = qdisc_pkt_len(skb);
499 kfree_skb(skb); 508 kfree_skb(skb);
500 return len; 509 return len;
501 } 510 }
502 511
503 return 0; 512 return 0;
504 } 513 }
505 514
506 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch) 515 static inline unsigned int qdisc_queue_drop_head(struct Qdisc *sch)
507 { 516 {
508 return __qdisc_queue_drop_head(sch, &sch->q); 517 return __qdisc_queue_drop_head(sch, &sch->q);
509 } 518 }
510 519
511 static inline struct sk_buff *__qdisc_dequeue_tail(struct Qdisc *sch, 520 static inline struct sk_buff *__qdisc_dequeue_tail(struct Qdisc *sch,
512 struct sk_buff_head *list) 521 struct sk_buff_head *list)
513 { 522 {
514 struct sk_buff *skb = __skb_dequeue_tail(list); 523 struct sk_buff *skb = __skb_dequeue_tail(list);
515 524
516 if (likely(skb != NULL)) 525 if (likely(skb != NULL))
517 sch->qstats.backlog -= qdisc_pkt_len(skb); 526 sch->qstats.backlog -= qdisc_pkt_len(skb);
518 527
519 return skb; 528 return skb;
520 } 529 }
521 530
522 static inline struct sk_buff *qdisc_dequeue_tail(struct Qdisc *sch) 531 static inline struct sk_buff *qdisc_dequeue_tail(struct Qdisc *sch)
523 { 532 {
524 return __qdisc_dequeue_tail(sch, &sch->q); 533 return __qdisc_dequeue_tail(sch, &sch->q);
525 } 534 }
526 535
527 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) 536 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch)
528 { 537 {
529 return skb_peek(&sch->q); 538 return skb_peek(&sch->q);
530 } 539 }
531 540
532 /* generic pseudo peek method for non-work-conserving qdisc */ 541 /* generic pseudo peek method for non-work-conserving qdisc */
533 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) 542 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)
534 { 543 {
535 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */ 544 /* we can reuse ->gso_skb because peek isn't called for root qdiscs */
536 if (!sch->gso_skb) { 545 if (!sch->gso_skb) {
537 sch->gso_skb = sch->dequeue(sch); 546 sch->gso_skb = sch->dequeue(sch);
538 if (sch->gso_skb) 547 if (sch->gso_skb)
539 /* it's still part of the queue */ 548 /* it's still part of the queue */
540 sch->q.qlen++; 549 sch->q.qlen++;
541 } 550 }
542 551
543 return sch->gso_skb; 552 return sch->gso_skb;
544 } 553 }
545 554
546 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */ 555 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */
547 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) 556 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
548 { 557 {
549 struct sk_buff *skb = sch->gso_skb; 558 struct sk_buff *skb = sch->gso_skb;
550 559
551 if (skb) { 560 if (skb) {
552 sch->gso_skb = NULL; 561 sch->gso_skb = NULL;
553 sch->q.qlen--; 562 sch->q.qlen--;
554 } else { 563 } else {
555 skb = sch->dequeue(sch); 564 skb = sch->dequeue(sch);
556 } 565 }
557 566
558 return skb; 567 return skb;
559 } 568 }
560 569
561 static inline void __qdisc_reset_queue(struct Qdisc *sch, 570 static inline void __qdisc_reset_queue(struct Qdisc *sch,
562 struct sk_buff_head *list) 571 struct sk_buff_head *list)
563 { 572 {
564 /* 573 /*
565 * We do not know the backlog in bytes of this list, it 574 * We do not know the backlog in bytes of this list, it
566 * is up to the caller to correct it 575 * is up to the caller to correct it
567 */ 576 */
568 __skb_queue_purge(list); 577 __skb_queue_purge(list);
569 } 578 }
570 579
571 static inline void qdisc_reset_queue(struct Qdisc *sch) 580 static inline void qdisc_reset_queue(struct Qdisc *sch)
572 { 581 {
573 __qdisc_reset_queue(sch, &sch->q); 582 __qdisc_reset_queue(sch, &sch->q);
574 sch->qstats.backlog = 0; 583 sch->qstats.backlog = 0;
575 } 584 }
576 585
577 static inline unsigned int __qdisc_queue_drop(struct Qdisc *sch, 586 static inline unsigned int __qdisc_queue_drop(struct Qdisc *sch,
578 struct sk_buff_head *list) 587 struct sk_buff_head *list)
579 { 588 {
580 struct sk_buff *skb = __qdisc_dequeue_tail(sch, list); 589 struct sk_buff *skb = __qdisc_dequeue_tail(sch, list);
581 590
582 if (likely(skb != NULL)) { 591 if (likely(skb != NULL)) {
583 unsigned int len = qdisc_pkt_len(skb); 592 unsigned int len = qdisc_pkt_len(skb);
584 kfree_skb(skb); 593 kfree_skb(skb);
585 return len; 594 return len;
586 } 595 }
587 596
588 return 0; 597 return 0;
589 } 598 }
590 599
591 static inline unsigned int qdisc_queue_drop(struct Qdisc *sch) 600 static inline unsigned int qdisc_queue_drop(struct Qdisc *sch)
592 { 601 {
593 return __qdisc_queue_drop(sch, &sch->q); 602 return __qdisc_queue_drop(sch, &sch->q);
594 } 603 }
595 604
596 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) 605 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch)
597 { 606 {
598 kfree_skb(skb); 607 kfree_skb(skb);
599 sch->qstats.drops++; 608 sch->qstats.drops++;
600 609
601 return NET_XMIT_DROP; 610 return NET_XMIT_DROP;
602 } 611 }
603 612
604 static inline int qdisc_reshape_fail(struct sk_buff *skb, struct Qdisc *sch) 613 static inline int qdisc_reshape_fail(struct sk_buff *skb, struct Qdisc *sch)
605 { 614 {
606 sch->qstats.drops++; 615 sch->qstats.drops++;
607 616
608 #ifdef CONFIG_NET_CLS_ACT 617 #ifdef CONFIG_NET_CLS_ACT
609 if (sch->reshape_fail == NULL || sch->reshape_fail(skb, sch)) 618 if (sch->reshape_fail == NULL || sch->reshape_fail(skb, sch))
610 goto drop; 619 goto drop;
611 620
612 return NET_XMIT_SUCCESS; 621 return NET_XMIT_SUCCESS;
613 622
614 drop: 623 drop:
615 #endif 624 #endif
616 kfree_skb(skb); 625 kfree_skb(skb);
617 return NET_XMIT_DROP; 626 return NET_XMIT_DROP;
618 } 627 }
619 628
620 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how 629 /* Length to Time (L2T) lookup in a qdisc_rate_table, to determine how
621 long it will take to send a packet given its size. 630 long it will take to send a packet given its size.
622 */ 631 */
623 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen) 632 static inline u32 qdisc_l2t(struct qdisc_rate_table* rtab, unsigned int pktlen)
624 { 633 {
625 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead; 634 int slot = pktlen + rtab->rate.cell_align + rtab->rate.overhead;
626 if (slot < 0) 635 if (slot < 0)
627 slot = 0; 636 slot = 0;
628 slot >>= rtab->rate.cell_log; 637 slot >>= rtab->rate.cell_log;
629 if (slot > 255) 638 if (slot > 255)
630 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF]; 639 return rtab->data[255]*(slot >> 8) + rtab->data[slot & 0xFF];
631 return rtab->data[slot]; 640 return rtab->data[slot];
632 } 641 }
633 642
634 #ifdef CONFIG_NET_CLS_ACT 643 #ifdef CONFIG_NET_CLS_ACT
635 static inline struct sk_buff *skb_act_clone(struct sk_buff *skb, gfp_t gfp_mask, 644 static inline struct sk_buff *skb_act_clone(struct sk_buff *skb, gfp_t gfp_mask,
636 int action) 645 int action)
637 { 646 {
638 struct sk_buff *n; 647 struct sk_buff *n;
639 648
640 n = skb_clone(skb, gfp_mask); 649 n = skb_clone(skb, gfp_mask);
641 650
642 if (n) { 651 if (n) {
643 n->tc_verd = SET_TC_VERD(n->tc_verd, 0); 652 n->tc_verd = SET_TC_VERD(n->tc_verd, 0);
644 n->tc_verd = CLR_TC_OK2MUNGE(n->tc_verd); 653 n->tc_verd = CLR_TC_OK2MUNGE(n->tc_verd);
645 n->tc_verd = CLR_TC_MUNGED(n->tc_verd); 654 n->tc_verd = CLR_TC_MUNGED(n->tc_verd);
646 } 655 }
647 return n; 656 return n;
648 } 657 }
649 #endif 658 #endif
650 659
651 #endif 660 #endif
652 661
1 /* 1 /*
2 * NET3 Protocol independent device support routines. 2 * NET3 Protocol independent device support routines.
3 * 3 *
4 * This program is free software; you can redistribute it and/or 4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License 5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version. 7 * 2 of the License, or (at your option) any later version.
8 * 8 *
9 * Derived from the non IP parts of dev.c 1.0.19 9 * Derived from the non IP parts of dev.c 1.0.19
10 * Authors: Ross Biro 10 * Authors: Ross Biro
11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk>
13 * 13 *
14 * Additional Authors: 14 * Additional Authors:
15 * Florian la Roche <rzsfl@rz.uni-sb.de> 15 * Florian la Roche <rzsfl@rz.uni-sb.de>
16 * Alan Cox <gw4pts@gw4pts.ampr.org> 16 * Alan Cox <gw4pts@gw4pts.ampr.org>
17 * David Hinds <dahinds@users.sourceforge.net> 17 * David Hinds <dahinds@users.sourceforge.net>
18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 18 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19 * Adam Sulmicki <adam@cfar.umd.edu> 19 * Adam Sulmicki <adam@cfar.umd.edu>
20 * Pekka Riikonen <priikone@poesidon.pspt.fi> 20 * Pekka Riikonen <priikone@poesidon.pspt.fi>
21 * 21 *
22 * Changes: 22 * Changes:
23 * D.J. Barrow : Fixed bug where dev->refcnt gets set 23 * D.J. Barrow : Fixed bug where dev->refcnt gets set
24 * to 2 if register_netdev gets called 24 * to 2 if register_netdev gets called
25 * before net_dev_init & also removed a 25 * before net_dev_init & also removed a
26 * few lines of code in the process. 26 * few lines of code in the process.
27 * Alan Cox : device private ioctl copies fields back. 27 * Alan Cox : device private ioctl copies fields back.
28 * Alan Cox : Transmit queue code does relevant 28 * Alan Cox : Transmit queue code does relevant
29 * stunts to keep the queue safe. 29 * stunts to keep the queue safe.
30 * Alan Cox : Fixed double lock. 30 * Alan Cox : Fixed double lock.
31 * Alan Cox : Fixed promisc NULL pointer trap 31 * Alan Cox : Fixed promisc NULL pointer trap
32 * ???????? : Support the full private ioctl range 32 * ???????? : Support the full private ioctl range
33 * Alan Cox : Moved ioctl permission check into 33 * Alan Cox : Moved ioctl permission check into
34 * drivers 34 * drivers
35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 35 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
36 * Alan Cox : 100 backlog just doesn't cut it when 36 * Alan Cox : 100 backlog just doesn't cut it when
37 * you start doing multicast video 8) 37 * you start doing multicast video 8)
38 * Alan Cox : Rewrote net_bh and list manager. 38 * Alan Cox : Rewrote net_bh and list manager.
39 * Alan Cox : Fix ETH_P_ALL echoback lengths. 39 * Alan Cox : Fix ETH_P_ALL echoback lengths.
40 * Alan Cox : Took out transmit every packet pass 40 * Alan Cox : Took out transmit every packet pass
41 * Saved a few bytes in the ioctl handler 41 * Saved a few bytes in the ioctl handler
42 * Alan Cox : Network driver sets packet type before 42 * Alan Cox : Network driver sets packet type before
43 * calling netif_rx. Saves a function 43 * calling netif_rx. Saves a function
44 * call a packet. 44 * call a packet.
45 * Alan Cox : Hashed net_bh() 45 * Alan Cox : Hashed net_bh()
46 * Richard Kooijman: Timestamp fixes. 46 * Richard Kooijman: Timestamp fixes.
47 * Alan Cox : Wrong field in SIOCGIFDSTADDR 47 * Alan Cox : Wrong field in SIOCGIFDSTADDR
48 * Alan Cox : Device lock protection. 48 * Alan Cox : Device lock protection.
49 * Alan Cox : Fixed nasty side effect of device close 49 * Alan Cox : Fixed nasty side effect of device close
50 * changes. 50 * changes.
51 * Rudi Cilibrasi : Pass the right thing to 51 * Rudi Cilibrasi : Pass the right thing to
52 * set_mac_address() 52 * set_mac_address()
53 * Dave Miller : 32bit quantity for the device lock to 53 * Dave Miller : 32bit quantity for the device lock to
54 * make it work out on a Sparc. 54 * make it work out on a Sparc.
55 * Bjorn Ekwall : Added KERNELD hack. 55 * Bjorn Ekwall : Added KERNELD hack.
56 * Alan Cox : Cleaned up the backlog initialise. 56 * Alan Cox : Cleaned up the backlog initialise.
57 * Craig Metz : SIOCGIFCONF fix if space for under 57 * Craig Metz : SIOCGIFCONF fix if space for under
58 * 1 device. 58 * 1 device.
59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 59 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
60 * is no device open function. 60 * is no device open function.
61 * Andi Kleen : Fix error reporting for SIOCGIFCONF 61 * Andi Kleen : Fix error reporting for SIOCGIFCONF
62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 62 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
63 * Cyrus Durgin : Cleaned for KMOD 63 * Cyrus Durgin : Cleaned for KMOD
64 * Adam Sulmicki : Bug Fix : Network Device Unload 64 * Adam Sulmicki : Bug Fix : Network Device Unload
65 * A network device unload needs to purge 65 * A network device unload needs to purge
66 * the backlog queue. 66 * the backlog queue.
67 * Paul Rusty Russell : SIOCSIFNAME 67 * Paul Rusty Russell : SIOCSIFNAME
68 * Pekka Riikonen : Netdev boot-time settings code 68 * Pekka Riikonen : Netdev boot-time settings code
69 * Andrew Morton : Make unregister_netdevice wait 69 * Andrew Morton : Make unregister_netdevice wait
70 * indefinitely on dev->refcnt 70 * indefinitely on dev->refcnt
71 * J Hadi Salim : - Backlog queue sampling 71 * J Hadi Salim : - Backlog queue sampling
72 * - netif_rx() feedback 72 * - netif_rx() feedback
73 */ 73 */
74 74
75 #include <asm/uaccess.h> 75 #include <asm/uaccess.h>
76 #include <asm/system.h> 76 #include <asm/system.h>
77 #include <linux/bitops.h> 77 #include <linux/bitops.h>
78 #include <linux/capability.h> 78 #include <linux/capability.h>
79 #include <linux/cpu.h> 79 #include <linux/cpu.h>
80 #include <linux/types.h> 80 #include <linux/types.h>
81 #include <linux/kernel.h> 81 #include <linux/kernel.h>
82 #include <linux/hash.h> 82 #include <linux/hash.h>
83 #include <linux/slab.h> 83 #include <linux/slab.h>
84 #include <linux/sched.h> 84 #include <linux/sched.h>
85 #include <linux/mutex.h> 85 #include <linux/mutex.h>
86 #include <linux/string.h> 86 #include <linux/string.h>
87 #include <linux/mm.h> 87 #include <linux/mm.h>
88 #include <linux/socket.h> 88 #include <linux/socket.h>
89 #include <linux/sockios.h> 89 #include <linux/sockios.h>
90 #include <linux/errno.h> 90 #include <linux/errno.h>
91 #include <linux/interrupt.h> 91 #include <linux/interrupt.h>
92 #include <linux/if_ether.h> 92 #include <linux/if_ether.h>
93 #include <linux/netdevice.h> 93 #include <linux/netdevice.h>
94 #include <linux/etherdevice.h> 94 #include <linux/etherdevice.h>
95 #include <linux/ethtool.h> 95 #include <linux/ethtool.h>
96 #include <linux/notifier.h> 96 #include <linux/notifier.h>
97 #include <linux/skbuff.h> 97 #include <linux/skbuff.h>
98 #include <net/net_namespace.h> 98 #include <net/net_namespace.h>
99 #include <net/sock.h> 99 #include <net/sock.h>
100 #include <linux/rtnetlink.h> 100 #include <linux/rtnetlink.h>
101 #include <linux/proc_fs.h> 101 #include <linux/proc_fs.h>
102 #include <linux/seq_file.h> 102 #include <linux/seq_file.h>
103 #include <linux/stat.h> 103 #include <linux/stat.h>
104 #include <net/dst.h> 104 #include <net/dst.h>
105 #include <net/pkt_sched.h> 105 #include <net/pkt_sched.h>
106 #include <net/checksum.h> 106 #include <net/checksum.h>
107 #include <net/xfrm.h> 107 #include <net/xfrm.h>
108 #include <linux/highmem.h> 108 #include <linux/highmem.h>
109 #include <linux/init.h> 109 #include <linux/init.h>
110 #include <linux/kmod.h> 110 #include <linux/kmod.h>
111 #include <linux/module.h> 111 #include <linux/module.h>
112 #include <linux/netpoll.h> 112 #include <linux/netpoll.h>
113 #include <linux/rcupdate.h> 113 #include <linux/rcupdate.h>
114 #include <linux/delay.h> 114 #include <linux/delay.h>
115 #include <net/wext.h> 115 #include <net/wext.h>
116 #include <net/iw_handler.h> 116 #include <net/iw_handler.h>
117 #include <asm/current.h> 117 #include <asm/current.h>
118 #include <linux/audit.h> 118 #include <linux/audit.h>
119 #include <linux/dmaengine.h> 119 #include <linux/dmaengine.h>
120 #include <linux/err.h> 120 #include <linux/err.h>
121 #include <linux/ctype.h> 121 #include <linux/ctype.h>
122 #include <linux/if_arp.h> 122 #include <linux/if_arp.h>
123 #include <linux/if_vlan.h> 123 #include <linux/if_vlan.h>
124 #include <linux/ip.h> 124 #include <linux/ip.h>
125 #include <net/ip.h> 125 #include <net/ip.h>
126 #include <linux/ipv6.h> 126 #include <linux/ipv6.h>
127 #include <linux/in.h> 127 #include <linux/in.h>
128 #include <linux/jhash.h> 128 #include <linux/jhash.h>
129 #include <linux/random.h> 129 #include <linux/random.h>
130 #include <trace/events/napi.h> 130 #include <trace/events/napi.h>
131 #include <trace/events/net.h> 131 #include <trace/events/net.h>
132 #include <trace/events/skb.h> 132 #include <trace/events/skb.h>
133 #include <linux/pci.h> 133 #include <linux/pci.h>
134 #include <linux/inetdevice.h> 134 #include <linux/inetdevice.h>
135 135
136 #include "net-sysfs.h" 136 #include "net-sysfs.h"
137 137
138 /* Instead of increasing this, you should create a hash table. */ 138 /* Instead of increasing this, you should create a hash table. */
139 #define MAX_GRO_SKBS 8 139 #define MAX_GRO_SKBS 8
140 140
141 /* This should be increased if a protocol with a bigger head is added. */ 141 /* This should be increased if a protocol with a bigger head is added. */
142 #define GRO_MAX_HEAD (MAX_HEADER + 128) 142 #define GRO_MAX_HEAD (MAX_HEADER + 128)
143 143
144 /* 144 /*
145 * The list of packet types we will receive (as opposed to discard) 145 * The list of packet types we will receive (as opposed to discard)
146 * and the routines to invoke. 146 * and the routines to invoke.
147 * 147 *
148 * Why 16. Because with 16 the only overlap we get on a hash of the 148 * Why 16. Because with 16 the only overlap we get on a hash of the
149 * low nibble of the protocol value is RARP/SNAP/X.25. 149 * low nibble of the protocol value is RARP/SNAP/X.25.
150 * 150 *
151 * NOTE: That is no longer true with the addition of VLAN tags. Not 151 * NOTE: That is no longer true with the addition of VLAN tags. Not
152 * sure which should go first, but I bet it won't make much 152 * sure which should go first, but I bet it won't make much
153 * difference if we are running VLANs. The good news is that 153 * difference if we are running VLANs. The good news is that
154 * this protocol won't be in the list unless compiled in, so 154 * this protocol won't be in the list unless compiled in, so
155 * the average user (w/out VLANs) will not be adversely affected. 155 * the average user (w/out VLANs) will not be adversely affected.
156 * --BLG 156 * --BLG
157 * 157 *
158 * 0800 IP 158 * 0800 IP
159 * 8100 802.1Q VLAN 159 * 8100 802.1Q VLAN
160 * 0001 802.3 160 * 0001 802.3
161 * 0002 AX.25 161 * 0002 AX.25
162 * 0004 802.2 162 * 0004 802.2
163 * 8035 RARP 163 * 8035 RARP
164 * 0005 SNAP 164 * 0005 SNAP
165 * 0805 X.25 165 * 0805 X.25
166 * 0806 ARP 166 * 0806 ARP
167 * 8137 IPX 167 * 8137 IPX
168 * 0009 Localtalk 168 * 0009 Localtalk
169 * 86DD IPv6 169 * 86DD IPv6
170 */ 170 */
171 171
172 #define PTYPE_HASH_SIZE (16) 172 #define PTYPE_HASH_SIZE (16)
173 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1) 173 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
174 174
175 static DEFINE_SPINLOCK(ptype_lock); 175 static DEFINE_SPINLOCK(ptype_lock);
176 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 176 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
177 static struct list_head ptype_all __read_mostly; /* Taps */ 177 static struct list_head ptype_all __read_mostly; /* Taps */
178 178
179 /* 179 /*
180 * The @dev_base_head list is protected by @dev_base_lock and the rtnl 180 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
181 * semaphore. 181 * semaphore.
182 * 182 *
183 * Pure readers hold dev_base_lock for reading, or rcu_read_lock() 183 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
184 * 184 *
185 * Writers must hold the rtnl semaphore while they loop through the 185 * Writers must hold the rtnl semaphore while they loop through the
186 * dev_base_head list, and hold dev_base_lock for writing when they do the 186 * dev_base_head list, and hold dev_base_lock for writing when they do the
187 * actual updates. This allows pure readers to access the list even 187 * actual updates. This allows pure readers to access the list even
188 * while a writer is preparing to update it. 188 * while a writer is preparing to update it.
189 * 189 *
190 * To put it another way, dev_base_lock is held for writing only to 190 * To put it another way, dev_base_lock is held for writing only to
191 * protect against pure readers; the rtnl semaphore provides the 191 * protect against pure readers; the rtnl semaphore provides the
192 * protection against other writers. 192 * protection against other writers.
193 * 193 *
194 * See, for example usages, register_netdevice() and 194 * See, for example usages, register_netdevice() and
195 * unregister_netdevice(), which must be called with the rtnl 195 * unregister_netdevice(), which must be called with the rtnl
196 * semaphore held. 196 * semaphore held.
197 */ 197 */
198 DEFINE_RWLOCK(dev_base_lock); 198 DEFINE_RWLOCK(dev_base_lock);
199 EXPORT_SYMBOL(dev_base_lock); 199 EXPORT_SYMBOL(dev_base_lock);
200 200
201 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 201 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
202 { 202 {
203 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ)); 203 unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
204 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 204 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
205 } 205 }
206 206
207 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 207 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
208 { 208 {
209 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 209 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
210 } 210 }
211 211
212 static inline void rps_lock(struct softnet_data *sd) 212 static inline void rps_lock(struct softnet_data *sd)
213 { 213 {
214 #ifdef CONFIG_RPS 214 #ifdef CONFIG_RPS
215 spin_lock(&sd->input_pkt_queue.lock); 215 spin_lock(&sd->input_pkt_queue.lock);
216 #endif 216 #endif
217 } 217 }
218 218
219 static inline void rps_unlock(struct softnet_data *sd) 219 static inline void rps_unlock(struct softnet_data *sd)
220 { 220 {
221 #ifdef CONFIG_RPS 221 #ifdef CONFIG_RPS
222 spin_unlock(&sd->input_pkt_queue.lock); 222 spin_unlock(&sd->input_pkt_queue.lock);
223 #endif 223 #endif
224 } 224 }
225 225
226 /* Device list insertion */ 226 /* Device list insertion */
227 static int list_netdevice(struct net_device *dev) 227 static int list_netdevice(struct net_device *dev)
228 { 228 {
229 struct net *net = dev_net(dev); 229 struct net *net = dev_net(dev);
230 230
231 ASSERT_RTNL(); 231 ASSERT_RTNL();
232 232
233 write_lock_bh(&dev_base_lock); 233 write_lock_bh(&dev_base_lock);
234 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 234 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
235 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 235 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
236 hlist_add_head_rcu(&dev->index_hlist, 236 hlist_add_head_rcu(&dev->index_hlist,
237 dev_index_hash(net, dev->ifindex)); 237 dev_index_hash(net, dev->ifindex));
238 write_unlock_bh(&dev_base_lock); 238 write_unlock_bh(&dev_base_lock);
239 return 0; 239 return 0;
240 } 240 }
241 241
242 /* Device list removal 242 /* Device list removal
243 * caller must respect a RCU grace period before freeing/reusing dev 243 * caller must respect a RCU grace period before freeing/reusing dev
244 */ 244 */
245 static void unlist_netdevice(struct net_device *dev) 245 static void unlist_netdevice(struct net_device *dev)
246 { 246 {
247 ASSERT_RTNL(); 247 ASSERT_RTNL();
248 248
249 /* Unlink dev from the device chain */ 249 /* Unlink dev from the device chain */
250 write_lock_bh(&dev_base_lock); 250 write_lock_bh(&dev_base_lock);
251 list_del_rcu(&dev->dev_list); 251 list_del_rcu(&dev->dev_list);
252 hlist_del_rcu(&dev->name_hlist); 252 hlist_del_rcu(&dev->name_hlist);
253 hlist_del_rcu(&dev->index_hlist); 253 hlist_del_rcu(&dev->index_hlist);
254 write_unlock_bh(&dev_base_lock); 254 write_unlock_bh(&dev_base_lock);
255 } 255 }
256 256
257 /* 257 /*
258 * Our notifier list 258 * Our notifier list
259 */ 259 */
260 260
261 static RAW_NOTIFIER_HEAD(netdev_chain); 261 static RAW_NOTIFIER_HEAD(netdev_chain);
262 262
263 /* 263 /*
264 * Device drivers call our routines to queue packets here. We empty the 264 * Device drivers call our routines to queue packets here. We empty the
265 * queue in the local softnet handler. 265 * queue in the local softnet handler.
266 */ 266 */
267 267
268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); 268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
269 EXPORT_PER_CPU_SYMBOL(softnet_data); 269 EXPORT_PER_CPU_SYMBOL(softnet_data);
270 270
271 #ifdef CONFIG_LOCKDEP 271 #ifdef CONFIG_LOCKDEP
272 /* 272 /*
273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 273 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
274 * according to dev->type 274 * according to dev->type
275 */ 275 */
276 static const unsigned short netdev_lock_type[] = 276 static const unsigned short netdev_lock_type[] =
277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 277 {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 278 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 279 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 280 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 281 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 282 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 283 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 284 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 285 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 286 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 287 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 288 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211, 289 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
290 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, 290 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
291 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154, 291 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
292 ARPHRD_VOID, ARPHRD_NONE}; 292 ARPHRD_VOID, ARPHRD_NONE};
293 293
294 static const char *const netdev_lock_name[] = 294 static const char *const netdev_lock_name[] =
295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 295 {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 296 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 297 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 298 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 299 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 300 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 301 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 302 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 303 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 304 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 305 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 306 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
307 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211", 307 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
308 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", 308 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
309 "_xmit_PHONET_PIPE", "_xmit_IEEE802154", 309 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
310 "_xmit_VOID", "_xmit_NONE"}; 310 "_xmit_VOID", "_xmit_NONE"};
311 311
312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 312 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 313 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
314 314
315 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 315 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
316 { 316 {
317 int i; 317 int i;
318 318
319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 319 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
320 if (netdev_lock_type[i] == dev_type) 320 if (netdev_lock_type[i] == dev_type)
321 return i; 321 return i;
322 /* the last key is used by default */ 322 /* the last key is used by default */
323 return ARRAY_SIZE(netdev_lock_type) - 1; 323 return ARRAY_SIZE(netdev_lock_type) - 1;
324 } 324 }
325 325
326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 326 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
327 unsigned short dev_type) 327 unsigned short dev_type)
328 { 328 {
329 int i; 329 int i;
330 330
331 i = netdev_lock_pos(dev_type); 331 i = netdev_lock_pos(dev_type);
332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 332 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
333 netdev_lock_name[i]); 333 netdev_lock_name[i]);
334 } 334 }
335 335
336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 336 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
337 { 337 {
338 int i; 338 int i;
339 339
340 i = netdev_lock_pos(dev->type); 340 i = netdev_lock_pos(dev->type);
341 lockdep_set_class_and_name(&dev->addr_list_lock, 341 lockdep_set_class_and_name(&dev->addr_list_lock,
342 &netdev_addr_lock_key[i], 342 &netdev_addr_lock_key[i],
343 netdev_lock_name[i]); 343 netdev_lock_name[i]);
344 } 344 }
345 #else 345 #else
346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 346 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
347 unsigned short dev_type) 347 unsigned short dev_type)
348 { 348 {
349 } 349 }
350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 350 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
351 { 351 {
352 } 352 }
353 #endif 353 #endif
354 354
355 /******************************************************************************* 355 /*******************************************************************************
356 356
357 Protocol management and registration routines 357 Protocol management and registration routines
358 358
359 *******************************************************************************/ 359 *******************************************************************************/
360 360
361 /* 361 /*
362 * Add a protocol ID to the list. Now that the input handler is 362 * Add a protocol ID to the list. Now that the input handler is
363 * smarter we can dispense with all the messy stuff that used to be 363 * smarter we can dispense with all the messy stuff that used to be
364 * here. 364 * here.
365 * 365 *
366 * BEWARE!!! Protocol handlers, mangling input packets, 366 * BEWARE!!! Protocol handlers, mangling input packets,
367 * MUST BE last in hash buckets and checking protocol handlers 367 * MUST BE last in hash buckets and checking protocol handlers
368 * MUST start from promiscuous ptype_all chain in net_bh. 368 * MUST start from promiscuous ptype_all chain in net_bh.
369 * It is true now, do not change it. 369 * It is true now, do not change it.
370 * Explanation follows: if protocol handler, mangling packet, will 370 * Explanation follows: if protocol handler, mangling packet, will
371 * be the first on list, it is not able to sense, that packet 371 * be the first on list, it is not able to sense, that packet
372 * is cloned and should be copied-on-write, so that it will 372 * is cloned and should be copied-on-write, so that it will
373 * change it and subsequent readers will get broken packet. 373 * change it and subsequent readers will get broken packet.
374 * --ANK (980803) 374 * --ANK (980803)
375 */ 375 */
376 376
377 static inline struct list_head *ptype_head(const struct packet_type *pt) 377 static inline struct list_head *ptype_head(const struct packet_type *pt)
378 { 378 {
379 if (pt->type == htons(ETH_P_ALL)) 379 if (pt->type == htons(ETH_P_ALL))
380 return &ptype_all; 380 return &ptype_all;
381 else 381 else
382 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 382 return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
383 } 383 }
384 384
385 /** 385 /**
386 * dev_add_pack - add packet handler 386 * dev_add_pack - add packet handler
387 * @pt: packet type declaration 387 * @pt: packet type declaration
388 * 388 *
389 * Add a protocol handler to the networking stack. The passed &packet_type 389 * Add a protocol handler to the networking stack. The passed &packet_type
390 * is linked into kernel lists and may not be freed until it has been 390 * is linked into kernel lists and may not be freed until it has been
391 * removed from the kernel lists. 391 * removed from the kernel lists.
392 * 392 *
393 * This call does not sleep therefore it can not 393 * This call does not sleep therefore it can not
394 * guarantee all CPU's that are in middle of receiving packets 394 * guarantee all CPU's that are in middle of receiving packets
395 * will see the new packet type (until the next received packet). 395 * will see the new packet type (until the next received packet).
396 */ 396 */
397 397
398 void dev_add_pack(struct packet_type *pt) 398 void dev_add_pack(struct packet_type *pt)
399 { 399 {
400 struct list_head *head = ptype_head(pt); 400 struct list_head *head = ptype_head(pt);
401 401
402 spin_lock(&ptype_lock); 402 spin_lock(&ptype_lock);
403 list_add_rcu(&pt->list, head); 403 list_add_rcu(&pt->list, head);
404 spin_unlock(&ptype_lock); 404 spin_unlock(&ptype_lock);
405 } 405 }
406 EXPORT_SYMBOL(dev_add_pack); 406 EXPORT_SYMBOL(dev_add_pack);
407 407
408 /** 408 /**
409 * __dev_remove_pack - remove packet handler 409 * __dev_remove_pack - remove packet handler
410 * @pt: packet type declaration 410 * @pt: packet type declaration
411 * 411 *
412 * Remove a protocol handler that was previously added to the kernel 412 * Remove a protocol handler that was previously added to the kernel
413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 413 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
414 * from the kernel lists and can be freed or reused once this function 414 * from the kernel lists and can be freed or reused once this function
415 * returns. 415 * returns.
416 * 416 *
417 * The packet type might still be in use by receivers 417 * The packet type might still be in use by receivers
418 * and must not be freed until after all the CPU's have gone 418 * and must not be freed until after all the CPU's have gone
419 * through a quiescent state. 419 * through a quiescent state.
420 */ 420 */
421 void __dev_remove_pack(struct packet_type *pt) 421 void __dev_remove_pack(struct packet_type *pt)
422 { 422 {
423 struct list_head *head = ptype_head(pt); 423 struct list_head *head = ptype_head(pt);
424 struct packet_type *pt1; 424 struct packet_type *pt1;
425 425
426 spin_lock(&ptype_lock); 426 spin_lock(&ptype_lock);
427 427
428 list_for_each_entry(pt1, head, list) { 428 list_for_each_entry(pt1, head, list) {
429 if (pt == pt1) { 429 if (pt == pt1) {
430 list_del_rcu(&pt->list); 430 list_del_rcu(&pt->list);
431 goto out; 431 goto out;
432 } 432 }
433 } 433 }
434 434
435 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt); 435 printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
436 out: 436 out:
437 spin_unlock(&ptype_lock); 437 spin_unlock(&ptype_lock);
438 } 438 }
439 EXPORT_SYMBOL(__dev_remove_pack); 439 EXPORT_SYMBOL(__dev_remove_pack);
440 440
441 /** 441 /**
442 * dev_remove_pack - remove packet handler 442 * dev_remove_pack - remove packet handler
443 * @pt: packet type declaration 443 * @pt: packet type declaration
444 * 444 *
445 * Remove a protocol handler that was previously added to the kernel 445 * Remove a protocol handler that was previously added to the kernel
446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 446 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
447 * from the kernel lists and can be freed or reused once this function 447 * from the kernel lists and can be freed or reused once this function
448 * returns. 448 * returns.
449 * 449 *
450 * This call sleeps to guarantee that no CPU is looking at the packet 450 * This call sleeps to guarantee that no CPU is looking at the packet
451 * type after return. 451 * type after return.
452 */ 452 */
453 void dev_remove_pack(struct packet_type *pt) 453 void dev_remove_pack(struct packet_type *pt)
454 { 454 {
455 __dev_remove_pack(pt); 455 __dev_remove_pack(pt);
456 456
457 synchronize_net(); 457 synchronize_net();
458 } 458 }
459 EXPORT_SYMBOL(dev_remove_pack); 459 EXPORT_SYMBOL(dev_remove_pack);
460 460
461 /****************************************************************************** 461 /******************************************************************************
462 462
463 Device Boot-time Settings Routines 463 Device Boot-time Settings Routines
464 464
465 *******************************************************************************/ 465 *******************************************************************************/
466 466
467 /* Boot time configuration table */ 467 /* Boot time configuration table */
468 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; 468 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
469 469
470 /** 470 /**
471 * netdev_boot_setup_add - add new setup entry 471 * netdev_boot_setup_add - add new setup entry
472 * @name: name of the device 472 * @name: name of the device
473 * @map: configured settings for the device 473 * @map: configured settings for the device
474 * 474 *
475 * Adds new setup entry to the dev_boot_setup list. The function 475 * Adds new setup entry to the dev_boot_setup list. The function
476 * returns 0 on error and 1 on success. This is a generic routine to 476 * returns 0 on error and 1 on success. This is a generic routine to
477 * all netdevices. 477 * all netdevices.
478 */ 478 */
479 static int netdev_boot_setup_add(char *name, struct ifmap *map) 479 static int netdev_boot_setup_add(char *name, struct ifmap *map)
480 { 480 {
481 struct netdev_boot_setup *s; 481 struct netdev_boot_setup *s;
482 int i; 482 int i;
483 483
484 s = dev_boot_setup; 484 s = dev_boot_setup;
485 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 485 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
486 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { 486 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
487 memset(s[i].name, 0, sizeof(s[i].name)); 487 memset(s[i].name, 0, sizeof(s[i].name));
488 strlcpy(s[i].name, name, IFNAMSIZ); 488 strlcpy(s[i].name, name, IFNAMSIZ);
489 memcpy(&s[i].map, map, sizeof(s[i].map)); 489 memcpy(&s[i].map, map, sizeof(s[i].map));
490 break; 490 break;
491 } 491 }
492 } 492 }
493 493
494 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; 494 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
495 } 495 }
496 496
497 /** 497 /**
498 * netdev_boot_setup_check - check boot time settings 498 * netdev_boot_setup_check - check boot time settings
499 * @dev: the netdevice 499 * @dev: the netdevice
500 * 500 *
501 * Check boot time settings for the device. 501 * Check boot time settings for the device.
502 * The found settings are set for the device to be used 502 * The found settings are set for the device to be used
503 * later in the device probing. 503 * later in the device probing.
504 * Returns 0 if no settings found, 1 if they are. 504 * Returns 0 if no settings found, 1 if they are.
505 */ 505 */
506 int netdev_boot_setup_check(struct net_device *dev) 506 int netdev_boot_setup_check(struct net_device *dev)
507 { 507 {
508 struct netdev_boot_setup *s = dev_boot_setup; 508 struct netdev_boot_setup *s = dev_boot_setup;
509 int i; 509 int i;
510 510
511 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { 511 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
512 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && 512 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
513 !strcmp(dev->name, s[i].name)) { 513 !strcmp(dev->name, s[i].name)) {
514 dev->irq = s[i].map.irq; 514 dev->irq = s[i].map.irq;
515 dev->base_addr = s[i].map.base_addr; 515 dev->base_addr = s[i].map.base_addr;
516 dev->mem_start = s[i].map.mem_start; 516 dev->mem_start = s[i].map.mem_start;
517 dev->mem_end = s[i].map.mem_end; 517 dev->mem_end = s[i].map.mem_end;
518 return 1; 518 return 1;
519 } 519 }
520 } 520 }
521 return 0; 521 return 0;
522 } 522 }
523 EXPORT_SYMBOL(netdev_boot_setup_check); 523 EXPORT_SYMBOL(netdev_boot_setup_check);
524 524
525 525
526 /** 526 /**
527 * netdev_boot_base - get address from boot time settings 527 * netdev_boot_base - get address from boot time settings
528 * @prefix: prefix for network device 528 * @prefix: prefix for network device
529 * @unit: id for network device 529 * @unit: id for network device
530 * 530 *
531 * Check boot time settings for the base address of device. 531 * Check boot time settings for the base address of device.
532 * The found settings are set for the device to be used 532 * The found settings are set for the device to be used
533 * later in the device probing. 533 * later in the device probing.
534 * Returns 0 if no settings found. 534 * Returns 0 if no settings found.
535 */ 535 */
536 unsigned long netdev_boot_base(const char *prefix, int unit) 536 unsigned long netdev_boot_base(const char *prefix, int unit)
537 { 537 {
538 const struct netdev_boot_setup *s = dev_boot_setup; 538 const struct netdev_boot_setup *s = dev_boot_setup;
539 char name[IFNAMSIZ]; 539 char name[IFNAMSIZ];
540 int i; 540 int i;
541 541
542 sprintf(name, "%s%d", prefix, unit); 542 sprintf(name, "%s%d", prefix, unit);
543 543
544 /* 544 /*
545 * If device already registered then return base of 1 545 * If device already registered then return base of 1
546 * to indicate not to probe for this interface 546 * to indicate not to probe for this interface
547 */ 547 */
548 if (__dev_get_by_name(&init_net, name)) 548 if (__dev_get_by_name(&init_net, name))
549 return 1; 549 return 1;
550 550
551 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) 551 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
552 if (!strcmp(name, s[i].name)) 552 if (!strcmp(name, s[i].name))
553 return s[i].map.base_addr; 553 return s[i].map.base_addr;
554 return 0; 554 return 0;
555 } 555 }
556 556
557 /* 557 /*
558 * Saves at boot time configured settings for any netdevice. 558 * Saves at boot time configured settings for any netdevice.
559 */ 559 */
560 int __init netdev_boot_setup(char *str) 560 int __init netdev_boot_setup(char *str)
561 { 561 {
562 int ints[5]; 562 int ints[5];
563 struct ifmap map; 563 struct ifmap map;
564 564
565 str = get_options(str, ARRAY_SIZE(ints), ints); 565 str = get_options(str, ARRAY_SIZE(ints), ints);
566 if (!str || !*str) 566 if (!str || !*str)
567 return 0; 567 return 0;
568 568
569 /* Save settings */ 569 /* Save settings */
570 memset(&map, 0, sizeof(map)); 570 memset(&map, 0, sizeof(map));
571 if (ints[0] > 0) 571 if (ints[0] > 0)
572 map.irq = ints[1]; 572 map.irq = ints[1];
573 if (ints[0] > 1) 573 if (ints[0] > 1)
574 map.base_addr = ints[2]; 574 map.base_addr = ints[2];
575 if (ints[0] > 2) 575 if (ints[0] > 2)
576 map.mem_start = ints[3]; 576 map.mem_start = ints[3];
577 if (ints[0] > 3) 577 if (ints[0] > 3)
578 map.mem_end = ints[4]; 578 map.mem_end = ints[4];
579 579
580 /* Add new entry to the list */ 580 /* Add new entry to the list */
581 return netdev_boot_setup_add(str, &map); 581 return netdev_boot_setup_add(str, &map);
582 } 582 }
583 583
584 __setup("netdev=", netdev_boot_setup); 584 __setup("netdev=", netdev_boot_setup);
585 585
586 /******************************************************************************* 586 /*******************************************************************************
587 587
588 Device Interface Subroutines 588 Device Interface Subroutines
589 589
590 *******************************************************************************/ 590 *******************************************************************************/
591 591
592 /** 592 /**
593 * __dev_get_by_name - find a device by its name 593 * __dev_get_by_name - find a device by its name
594 * @net: the applicable net namespace 594 * @net: the applicable net namespace
595 * @name: name to find 595 * @name: name to find
596 * 596 *
597 * Find an interface by name. Must be called under RTNL semaphore 597 * Find an interface by name. Must be called under RTNL semaphore
598 * or @dev_base_lock. If the name is found a pointer to the device 598 * or @dev_base_lock. If the name is found a pointer to the device
599 * is returned. If the name is not found then %NULL is returned. The 599 * is returned. If the name is not found then %NULL is returned. The
600 * reference counters are not incremented so the caller must be 600 * reference counters are not incremented so the caller must be
601 * careful with locks. 601 * careful with locks.
602 */ 602 */
603 603
604 struct net_device *__dev_get_by_name(struct net *net, const char *name) 604 struct net_device *__dev_get_by_name(struct net *net, const char *name)
605 { 605 {
606 struct hlist_node *p; 606 struct hlist_node *p;
607 struct net_device *dev; 607 struct net_device *dev;
608 struct hlist_head *head = dev_name_hash(net, name); 608 struct hlist_head *head = dev_name_hash(net, name);
609 609
610 hlist_for_each_entry(dev, p, head, name_hlist) 610 hlist_for_each_entry(dev, p, head, name_hlist)
611 if (!strncmp(dev->name, name, IFNAMSIZ)) 611 if (!strncmp(dev->name, name, IFNAMSIZ))
612 return dev; 612 return dev;
613 613
614 return NULL; 614 return NULL;
615 } 615 }
616 EXPORT_SYMBOL(__dev_get_by_name); 616 EXPORT_SYMBOL(__dev_get_by_name);
617 617
618 /** 618 /**
619 * dev_get_by_name_rcu - find a device by its name 619 * dev_get_by_name_rcu - find a device by its name
620 * @net: the applicable net namespace 620 * @net: the applicable net namespace
621 * @name: name to find 621 * @name: name to find
622 * 622 *
623 * Find an interface by name. 623 * Find an interface by name.
624 * If the name is found a pointer to the device is returned. 624 * If the name is found a pointer to the device is returned.
625 * If the name is not found then %NULL is returned. 625 * If the name is not found then %NULL is returned.
626 * The reference counters are not incremented so the caller must be 626 * The reference counters are not incremented so the caller must be
627 * careful with locks. The caller must hold RCU lock. 627 * careful with locks. The caller must hold RCU lock.
628 */ 628 */
629 629
630 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 630 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
631 { 631 {
632 struct hlist_node *p; 632 struct hlist_node *p;
633 struct net_device *dev; 633 struct net_device *dev;
634 struct hlist_head *head = dev_name_hash(net, name); 634 struct hlist_head *head = dev_name_hash(net, name);
635 635
636 hlist_for_each_entry_rcu(dev, p, head, name_hlist) 636 hlist_for_each_entry_rcu(dev, p, head, name_hlist)
637 if (!strncmp(dev->name, name, IFNAMSIZ)) 637 if (!strncmp(dev->name, name, IFNAMSIZ))
638 return dev; 638 return dev;
639 639
640 return NULL; 640 return NULL;
641 } 641 }
642 EXPORT_SYMBOL(dev_get_by_name_rcu); 642 EXPORT_SYMBOL(dev_get_by_name_rcu);
643 643
644 /** 644 /**
645 * dev_get_by_name - find a device by its name 645 * dev_get_by_name - find a device by its name
646 * @net: the applicable net namespace 646 * @net: the applicable net namespace
647 * @name: name to find 647 * @name: name to find
648 * 648 *
649 * Find an interface by name. This can be called from any 649 * Find an interface by name. This can be called from any
650 * context and does its own locking. The returned handle has 650 * context and does its own locking. The returned handle has
651 * the usage count incremented and the caller must use dev_put() to 651 * the usage count incremented and the caller must use dev_put() to
652 * release it when it is no longer needed. %NULL is returned if no 652 * release it when it is no longer needed. %NULL is returned if no
653 * matching device is found. 653 * matching device is found.
654 */ 654 */
655 655
656 struct net_device *dev_get_by_name(struct net *net, const char *name) 656 struct net_device *dev_get_by_name(struct net *net, const char *name)
657 { 657 {
658 struct net_device *dev; 658 struct net_device *dev;
659 659
660 rcu_read_lock(); 660 rcu_read_lock();
661 dev = dev_get_by_name_rcu(net, name); 661 dev = dev_get_by_name_rcu(net, name);
662 if (dev) 662 if (dev)
663 dev_hold(dev); 663 dev_hold(dev);
664 rcu_read_unlock(); 664 rcu_read_unlock();
665 return dev; 665 return dev;
666 } 666 }
667 EXPORT_SYMBOL(dev_get_by_name); 667 EXPORT_SYMBOL(dev_get_by_name);
668 668
669 /** 669 /**
670 * __dev_get_by_index - find a device by its ifindex 670 * __dev_get_by_index - find a device by its ifindex
671 * @net: the applicable net namespace 671 * @net: the applicable net namespace
672 * @ifindex: index of device 672 * @ifindex: index of device
673 * 673 *
674 * Search for an interface by index. Returns %NULL if the device 674 * Search for an interface by index. Returns %NULL if the device
675 * is not found or a pointer to the device. The device has not 675 * is not found or a pointer to the device. The device has not
676 * had its reference counter increased so the caller must be careful 676 * had its reference counter increased so the caller must be careful
677 * about locking. The caller must hold either the RTNL semaphore 677 * about locking. The caller must hold either the RTNL semaphore
678 * or @dev_base_lock. 678 * or @dev_base_lock.
679 */ 679 */
680 680
681 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 681 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
682 { 682 {
683 struct hlist_node *p; 683 struct hlist_node *p;
684 struct net_device *dev; 684 struct net_device *dev;
685 struct hlist_head *head = dev_index_hash(net, ifindex); 685 struct hlist_head *head = dev_index_hash(net, ifindex);
686 686
687 hlist_for_each_entry(dev, p, head, index_hlist) 687 hlist_for_each_entry(dev, p, head, index_hlist)
688 if (dev->ifindex == ifindex) 688 if (dev->ifindex == ifindex)
689 return dev; 689 return dev;
690 690
691 return NULL; 691 return NULL;
692 } 692 }
693 EXPORT_SYMBOL(__dev_get_by_index); 693 EXPORT_SYMBOL(__dev_get_by_index);
694 694
695 /** 695 /**
696 * dev_get_by_index_rcu - find a device by its ifindex 696 * dev_get_by_index_rcu - find a device by its ifindex
697 * @net: the applicable net namespace 697 * @net: the applicable net namespace
698 * @ifindex: index of device 698 * @ifindex: index of device
699 * 699 *
700 * Search for an interface by index. Returns %NULL if the device 700 * Search for an interface by index. Returns %NULL if the device
701 * is not found or a pointer to the device. The device has not 701 * is not found or a pointer to the device. The device has not
702 * had its reference counter increased so the caller must be careful 702 * had its reference counter increased so the caller must be careful
703 * about locking. The caller must hold RCU lock. 703 * about locking. The caller must hold RCU lock.
704 */ 704 */
705 705
706 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 706 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
707 { 707 {
708 struct hlist_node *p; 708 struct hlist_node *p;
709 struct net_device *dev; 709 struct net_device *dev;
710 struct hlist_head *head = dev_index_hash(net, ifindex); 710 struct hlist_head *head = dev_index_hash(net, ifindex);
711 711
712 hlist_for_each_entry_rcu(dev, p, head, index_hlist) 712 hlist_for_each_entry_rcu(dev, p, head, index_hlist)
713 if (dev->ifindex == ifindex) 713 if (dev->ifindex == ifindex)
714 return dev; 714 return dev;
715 715
716 return NULL; 716 return NULL;
717 } 717 }
718 EXPORT_SYMBOL(dev_get_by_index_rcu); 718 EXPORT_SYMBOL(dev_get_by_index_rcu);
719 719
720 720
721 /** 721 /**
722 * dev_get_by_index - find a device by its ifindex 722 * dev_get_by_index - find a device by its ifindex
723 * @net: the applicable net namespace 723 * @net: the applicable net namespace
724 * @ifindex: index of device 724 * @ifindex: index of device
725 * 725 *
726 * Search for an interface by index. Returns NULL if the device 726 * Search for an interface by index. Returns NULL if the device
727 * is not found or a pointer to the device. The device returned has 727 * is not found or a pointer to the device. The device returned has
728 * had a reference added and the pointer is safe until the user calls 728 * had a reference added and the pointer is safe until the user calls
729 * dev_put to indicate they have finished with it. 729 * dev_put to indicate they have finished with it.
730 */ 730 */
731 731
732 struct net_device *dev_get_by_index(struct net *net, int ifindex) 732 struct net_device *dev_get_by_index(struct net *net, int ifindex)
733 { 733 {
734 struct net_device *dev; 734 struct net_device *dev;
735 735
736 rcu_read_lock(); 736 rcu_read_lock();
737 dev = dev_get_by_index_rcu(net, ifindex); 737 dev = dev_get_by_index_rcu(net, ifindex);
738 if (dev) 738 if (dev)
739 dev_hold(dev); 739 dev_hold(dev);
740 rcu_read_unlock(); 740 rcu_read_unlock();
741 return dev; 741 return dev;
742 } 742 }
743 EXPORT_SYMBOL(dev_get_by_index); 743 EXPORT_SYMBOL(dev_get_by_index);
744 744
745 /** 745 /**
746 * dev_getbyhwaddr_rcu - find a device by its hardware address 746 * dev_getbyhwaddr_rcu - find a device by its hardware address
747 * @net: the applicable net namespace 747 * @net: the applicable net namespace
748 * @type: media type of device 748 * @type: media type of device
749 * @ha: hardware address 749 * @ha: hardware address
750 * 750 *
751 * Search for an interface by MAC address. Returns NULL if the device 751 * Search for an interface by MAC address. Returns NULL if the device
752 * is not found or a pointer to the device. The caller must hold RCU 752 * is not found or a pointer to the device. The caller must hold RCU
753 * The returned device has not had its ref count increased 753 * The returned device has not had its ref count increased
754 * and the caller must therefore be careful about locking 754 * and the caller must therefore be careful about locking
755 * 755 *
756 */ 756 */
757 757
758 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 758 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
759 const char *ha) 759 const char *ha)
760 { 760 {
761 struct net_device *dev; 761 struct net_device *dev;
762 762
763 for_each_netdev_rcu(net, dev) 763 for_each_netdev_rcu(net, dev)
764 if (dev->type == type && 764 if (dev->type == type &&
765 !memcmp(dev->dev_addr, ha, dev->addr_len)) 765 !memcmp(dev->dev_addr, ha, dev->addr_len))
766 return dev; 766 return dev;
767 767
768 return NULL; 768 return NULL;
769 } 769 }
770 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 770 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
771 771
772 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) 772 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
773 { 773 {
774 struct net_device *dev; 774 struct net_device *dev;
775 775
776 ASSERT_RTNL(); 776 ASSERT_RTNL();
777 for_each_netdev(net, dev) 777 for_each_netdev(net, dev)
778 if (dev->type == type) 778 if (dev->type == type)
779 return dev; 779 return dev;
780 780
781 return NULL; 781 return NULL;
782 } 782 }
783 EXPORT_SYMBOL(__dev_getfirstbyhwtype); 783 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
784 784
785 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 785 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
786 { 786 {
787 struct net_device *dev, *ret = NULL; 787 struct net_device *dev, *ret = NULL;
788 788
789 rcu_read_lock(); 789 rcu_read_lock();
790 for_each_netdev_rcu(net, dev) 790 for_each_netdev_rcu(net, dev)
791 if (dev->type == type) { 791 if (dev->type == type) {
792 dev_hold(dev); 792 dev_hold(dev);
793 ret = dev; 793 ret = dev;
794 break; 794 break;
795 } 795 }
796 rcu_read_unlock(); 796 rcu_read_unlock();
797 return ret; 797 return ret;
798 } 798 }
799 EXPORT_SYMBOL(dev_getfirstbyhwtype); 799 EXPORT_SYMBOL(dev_getfirstbyhwtype);
800 800
801 /** 801 /**
802 * dev_get_by_flags_rcu - find any device with given flags 802 * dev_get_by_flags_rcu - find any device with given flags
803 * @net: the applicable net namespace 803 * @net: the applicable net namespace
804 * @if_flags: IFF_* values 804 * @if_flags: IFF_* values
805 * @mask: bitmask of bits in if_flags to check 805 * @mask: bitmask of bits in if_flags to check
806 * 806 *
807 * Search for any interface with the given flags. Returns NULL if a device 807 * Search for any interface with the given flags. Returns NULL if a device
808 * is not found or a pointer to the device. Must be called inside 808 * is not found or a pointer to the device. Must be called inside
809 * rcu_read_lock(), and result refcount is unchanged. 809 * rcu_read_lock(), and result refcount is unchanged.
810 */ 810 */
811 811
812 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags, 812 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
813 unsigned short mask) 813 unsigned short mask)
814 { 814 {
815 struct net_device *dev, *ret; 815 struct net_device *dev, *ret;
816 816
817 ret = NULL; 817 ret = NULL;
818 for_each_netdev_rcu(net, dev) { 818 for_each_netdev_rcu(net, dev) {
819 if (((dev->flags ^ if_flags) & mask) == 0) { 819 if (((dev->flags ^ if_flags) & mask) == 0) {
820 ret = dev; 820 ret = dev;
821 break; 821 break;
822 } 822 }
823 } 823 }
824 return ret; 824 return ret;
825 } 825 }
826 EXPORT_SYMBOL(dev_get_by_flags_rcu); 826 EXPORT_SYMBOL(dev_get_by_flags_rcu);
827 827
828 /** 828 /**
829 * dev_valid_name - check if name is okay for network device 829 * dev_valid_name - check if name is okay for network device
830 * @name: name string 830 * @name: name string
831 * 831 *
832 * Network device names need to be valid file names to 832 * Network device names need to be valid file names to
833 * to allow sysfs to work. We also disallow any kind of 833 * to allow sysfs to work. We also disallow any kind of
834 * whitespace. 834 * whitespace.
835 */ 835 */
836 int dev_valid_name(const char *name) 836 int dev_valid_name(const char *name)
837 { 837 {
838 if (*name == '\0') 838 if (*name == '\0')
839 return 0; 839 return 0;
840 if (strlen(name) >= IFNAMSIZ) 840 if (strlen(name) >= IFNAMSIZ)
841 return 0; 841 return 0;
842 if (!strcmp(name, ".") || !strcmp(name, "..")) 842 if (!strcmp(name, ".") || !strcmp(name, ".."))
843 return 0; 843 return 0;
844 844
845 while (*name) { 845 while (*name) {
846 if (*name == '/' || isspace(*name)) 846 if (*name == '/' || isspace(*name))
847 return 0; 847 return 0;
848 name++; 848 name++;
849 } 849 }
850 return 1; 850 return 1;
851 } 851 }
852 EXPORT_SYMBOL(dev_valid_name); 852 EXPORT_SYMBOL(dev_valid_name);
853 853
854 /** 854 /**
855 * __dev_alloc_name - allocate a name for a device 855 * __dev_alloc_name - allocate a name for a device
856 * @net: network namespace to allocate the device name in 856 * @net: network namespace to allocate the device name in
857 * @name: name format string 857 * @name: name format string
858 * @buf: scratch buffer and result name string 858 * @buf: scratch buffer and result name string
859 * 859 *
860 * Passed a format string - eg "lt%d" it will try and find a suitable 860 * Passed a format string - eg "lt%d" it will try and find a suitable
861 * id. It scans list of devices to build up a free map, then chooses 861 * id. It scans list of devices to build up a free map, then chooses
862 * the first empty slot. The caller must hold the dev_base or rtnl lock 862 * the first empty slot. The caller must hold the dev_base or rtnl lock
863 * while allocating the name and adding the device in order to avoid 863 * while allocating the name and adding the device in order to avoid
864 * duplicates. 864 * duplicates.
865 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 865 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
866 * Returns the number of the unit assigned or a negative errno code. 866 * Returns the number of the unit assigned or a negative errno code.
867 */ 867 */
868 868
869 static int __dev_alloc_name(struct net *net, const char *name, char *buf) 869 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
870 { 870 {
871 int i = 0; 871 int i = 0;
872 const char *p; 872 const char *p;
873 const int max_netdevices = 8*PAGE_SIZE; 873 const int max_netdevices = 8*PAGE_SIZE;
874 unsigned long *inuse; 874 unsigned long *inuse;
875 struct net_device *d; 875 struct net_device *d;
876 876
877 p = strnchr(name, IFNAMSIZ-1, '%'); 877 p = strnchr(name, IFNAMSIZ-1, '%');
878 if (p) { 878 if (p) {
879 /* 879 /*
880 * Verify the string as this thing may have come from 880 * Verify the string as this thing may have come from
881 * the user. There must be either one "%d" and no other "%" 881 * the user. There must be either one "%d" and no other "%"
882 * characters. 882 * characters.
883 */ 883 */
884 if (p[1] != 'd' || strchr(p + 2, '%')) 884 if (p[1] != 'd' || strchr(p + 2, '%'))
885 return -EINVAL; 885 return -EINVAL;
886 886
887 /* Use one page as a bit array of possible slots */ 887 /* Use one page as a bit array of possible slots */
888 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); 888 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
889 if (!inuse) 889 if (!inuse)
890 return -ENOMEM; 890 return -ENOMEM;
891 891
892 for_each_netdev(net, d) { 892 for_each_netdev(net, d) {
893 if (!sscanf(d->name, name, &i)) 893 if (!sscanf(d->name, name, &i))
894 continue; 894 continue;
895 if (i < 0 || i >= max_netdevices) 895 if (i < 0 || i >= max_netdevices)
896 continue; 896 continue;
897 897
898 /* avoid cases where sscanf is not exact inverse of printf */ 898 /* avoid cases where sscanf is not exact inverse of printf */
899 snprintf(buf, IFNAMSIZ, name, i); 899 snprintf(buf, IFNAMSIZ, name, i);
900 if (!strncmp(buf, d->name, IFNAMSIZ)) 900 if (!strncmp(buf, d->name, IFNAMSIZ))
901 set_bit(i, inuse); 901 set_bit(i, inuse);
902 } 902 }
903 903
904 i = find_first_zero_bit(inuse, max_netdevices); 904 i = find_first_zero_bit(inuse, max_netdevices);
905 free_page((unsigned long) inuse); 905 free_page((unsigned long) inuse);
906 } 906 }
907 907
908 if (buf != name) 908 if (buf != name)
909 snprintf(buf, IFNAMSIZ, name, i); 909 snprintf(buf, IFNAMSIZ, name, i);
910 if (!__dev_get_by_name(net, buf)) 910 if (!__dev_get_by_name(net, buf))
911 return i; 911 return i;
912 912
913 /* It is possible to run out of possible slots 913 /* It is possible to run out of possible slots
914 * when the name is long and there isn't enough space left 914 * when the name is long and there isn't enough space left
915 * for the digits, or if all bits are used. 915 * for the digits, or if all bits are used.
916 */ 916 */
917 return -ENFILE; 917 return -ENFILE;
918 } 918 }
919 919
920 /** 920 /**
921 * dev_alloc_name - allocate a name for a device 921 * dev_alloc_name - allocate a name for a device
922 * @dev: device 922 * @dev: device
923 * @name: name format string 923 * @name: name format string
924 * 924 *
925 * Passed a format string - eg "lt%d" it will try and find a suitable 925 * Passed a format string - eg "lt%d" it will try and find a suitable
926 * id. It scans list of devices to build up a free map, then chooses 926 * id. It scans list of devices to build up a free map, then chooses
927 * the first empty slot. The caller must hold the dev_base or rtnl lock 927 * the first empty slot. The caller must hold the dev_base or rtnl lock
928 * while allocating the name and adding the device in order to avoid 928 * while allocating the name and adding the device in order to avoid
929 * duplicates. 929 * duplicates.
930 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 930 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
931 * Returns the number of the unit assigned or a negative errno code. 931 * Returns the number of the unit assigned or a negative errno code.
932 */ 932 */
933 933
934 int dev_alloc_name(struct net_device *dev, const char *name) 934 int dev_alloc_name(struct net_device *dev, const char *name)
935 { 935 {
936 char buf[IFNAMSIZ]; 936 char buf[IFNAMSIZ];
937 struct net *net; 937 struct net *net;
938 int ret; 938 int ret;
939 939
940 BUG_ON(!dev_net(dev)); 940 BUG_ON(!dev_net(dev));
941 net = dev_net(dev); 941 net = dev_net(dev);
942 ret = __dev_alloc_name(net, name, buf); 942 ret = __dev_alloc_name(net, name, buf);
943 if (ret >= 0) 943 if (ret >= 0)
944 strlcpy(dev->name, buf, IFNAMSIZ); 944 strlcpy(dev->name, buf, IFNAMSIZ);
945 return ret; 945 return ret;
946 } 946 }
947 EXPORT_SYMBOL(dev_alloc_name); 947 EXPORT_SYMBOL(dev_alloc_name);
948 948
949 static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt) 949 static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
950 { 950 {
951 struct net *net; 951 struct net *net;
952 952
953 BUG_ON(!dev_net(dev)); 953 BUG_ON(!dev_net(dev));
954 net = dev_net(dev); 954 net = dev_net(dev);
955 955
956 if (!dev_valid_name(name)) 956 if (!dev_valid_name(name))
957 return -EINVAL; 957 return -EINVAL;
958 958
959 if (fmt && strchr(name, '%')) 959 if (fmt && strchr(name, '%'))
960 return dev_alloc_name(dev, name); 960 return dev_alloc_name(dev, name);
961 else if (__dev_get_by_name(net, name)) 961 else if (__dev_get_by_name(net, name))
962 return -EEXIST; 962 return -EEXIST;
963 else if (dev->name != name) 963 else if (dev->name != name)
964 strlcpy(dev->name, name, IFNAMSIZ); 964 strlcpy(dev->name, name, IFNAMSIZ);
965 965
966 return 0; 966 return 0;
967 } 967 }
968 968
969 /** 969 /**
970 * dev_change_name - change name of a device 970 * dev_change_name - change name of a device
971 * @dev: device 971 * @dev: device
972 * @newname: name (or format string) must be at least IFNAMSIZ 972 * @newname: name (or format string) must be at least IFNAMSIZ
973 * 973 *
974 * Change name of a device, can pass format strings "eth%d". 974 * Change name of a device, can pass format strings "eth%d".
975 * for wildcarding. 975 * for wildcarding.
976 */ 976 */
977 int dev_change_name(struct net_device *dev, const char *newname) 977 int dev_change_name(struct net_device *dev, const char *newname)
978 { 978 {
979 char oldname[IFNAMSIZ]; 979 char oldname[IFNAMSIZ];
980 int err = 0; 980 int err = 0;
981 int ret; 981 int ret;
982 struct net *net; 982 struct net *net;
983 983
984 ASSERT_RTNL(); 984 ASSERT_RTNL();
985 BUG_ON(!dev_net(dev)); 985 BUG_ON(!dev_net(dev));
986 986
987 net = dev_net(dev); 987 net = dev_net(dev);
988 if (dev->flags & IFF_UP) 988 if (dev->flags & IFF_UP)
989 return -EBUSY; 989 return -EBUSY;
990 990
991 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) 991 if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
992 return 0; 992 return 0;
993 993
994 memcpy(oldname, dev->name, IFNAMSIZ); 994 memcpy(oldname, dev->name, IFNAMSIZ);
995 995
996 err = dev_get_valid_name(dev, newname, 1); 996 err = dev_get_valid_name(dev, newname, 1);
997 if (err < 0) 997 if (err < 0)
998 return err; 998 return err;
999 999
1000 rollback: 1000 rollback:
1001 ret = device_rename(&dev->dev, dev->name); 1001 ret = device_rename(&dev->dev, dev->name);
1002 if (ret) { 1002 if (ret) {
1003 memcpy(dev->name, oldname, IFNAMSIZ); 1003 memcpy(dev->name, oldname, IFNAMSIZ);
1004 return ret; 1004 return ret;
1005 } 1005 }
1006 1006
1007 write_lock_bh(&dev_base_lock); 1007 write_lock_bh(&dev_base_lock);
1008 hlist_del(&dev->name_hlist); 1008 hlist_del(&dev->name_hlist);
1009 write_unlock_bh(&dev_base_lock); 1009 write_unlock_bh(&dev_base_lock);
1010 1010
1011 synchronize_rcu(); 1011 synchronize_rcu();
1012 1012
1013 write_lock_bh(&dev_base_lock); 1013 write_lock_bh(&dev_base_lock);
1014 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); 1014 hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1015 write_unlock_bh(&dev_base_lock); 1015 write_unlock_bh(&dev_base_lock);
1016 1016
1017 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1017 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1018 ret = notifier_to_errno(ret); 1018 ret = notifier_to_errno(ret);
1019 1019
1020 if (ret) { 1020 if (ret) {
1021 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1021 /* err >= 0 after dev_alloc_name() or stores the first errno */
1022 if (err >= 0) { 1022 if (err >= 0) {
1023 err = ret; 1023 err = ret;
1024 memcpy(dev->name, oldname, IFNAMSIZ); 1024 memcpy(dev->name, oldname, IFNAMSIZ);
1025 goto rollback; 1025 goto rollback;
1026 } else { 1026 } else {
1027 printk(KERN_ERR 1027 printk(KERN_ERR
1028 "%s: name change rollback failed: %d.\n", 1028 "%s: name change rollback failed: %d.\n",
1029 dev->name, ret); 1029 dev->name, ret);
1030 } 1030 }
1031 } 1031 }
1032 1032
1033 return err; 1033 return err;
1034 } 1034 }
1035 1035
1036 /** 1036 /**
1037 * dev_set_alias - change ifalias of a device 1037 * dev_set_alias - change ifalias of a device
1038 * @dev: device 1038 * @dev: device
1039 * @alias: name up to IFALIASZ 1039 * @alias: name up to IFALIASZ
1040 * @len: limit of bytes to copy from info 1040 * @len: limit of bytes to copy from info
1041 * 1041 *
1042 * Set ifalias for a device, 1042 * Set ifalias for a device,
1043 */ 1043 */
1044 int dev_set_alias(struct net_device *dev, const char *alias, size_t len) 1044 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1045 { 1045 {
1046 ASSERT_RTNL(); 1046 ASSERT_RTNL();
1047 1047
1048 if (len >= IFALIASZ) 1048 if (len >= IFALIASZ)
1049 return -EINVAL; 1049 return -EINVAL;
1050 1050
1051 if (!len) { 1051 if (!len) {
1052 if (dev->ifalias) { 1052 if (dev->ifalias) {
1053 kfree(dev->ifalias); 1053 kfree(dev->ifalias);
1054 dev->ifalias = NULL; 1054 dev->ifalias = NULL;
1055 } 1055 }
1056 return 0; 1056 return 0;
1057 } 1057 }
1058 1058
1059 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); 1059 dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1060 if (!dev->ifalias) 1060 if (!dev->ifalias)
1061 return -ENOMEM; 1061 return -ENOMEM;
1062 1062
1063 strlcpy(dev->ifalias, alias, len+1); 1063 strlcpy(dev->ifalias, alias, len+1);
1064 return len; 1064 return len;
1065 } 1065 }
1066 1066
1067 1067
1068 /** 1068 /**
1069 * netdev_features_change - device changes features 1069 * netdev_features_change - device changes features
1070 * @dev: device to cause notification 1070 * @dev: device to cause notification
1071 * 1071 *
1072 * Called to indicate a device has changed features. 1072 * Called to indicate a device has changed features.
1073 */ 1073 */
1074 void netdev_features_change(struct net_device *dev) 1074 void netdev_features_change(struct net_device *dev)
1075 { 1075 {
1076 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1076 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1077 } 1077 }
1078 EXPORT_SYMBOL(netdev_features_change); 1078 EXPORT_SYMBOL(netdev_features_change);
1079 1079
1080 /** 1080 /**
1081 * netdev_state_change - device changes state 1081 * netdev_state_change - device changes state
1082 * @dev: device to cause notification 1082 * @dev: device to cause notification
1083 * 1083 *
1084 * Called to indicate a device has changed state. This function calls 1084 * Called to indicate a device has changed state. This function calls
1085 * the notifier chains for netdev_chain and sends a NEWLINK message 1085 * the notifier chains for netdev_chain and sends a NEWLINK message
1086 * to the routing socket. 1086 * to the routing socket.
1087 */ 1087 */
1088 void netdev_state_change(struct net_device *dev) 1088 void netdev_state_change(struct net_device *dev)
1089 { 1089 {
1090 if (dev->flags & IFF_UP) { 1090 if (dev->flags & IFF_UP) {
1091 call_netdevice_notifiers(NETDEV_CHANGE, dev); 1091 call_netdevice_notifiers(NETDEV_CHANGE, dev);
1092 rtmsg_ifinfo(RTM_NEWLINK, dev, 0); 1092 rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
1093 } 1093 }
1094 } 1094 }
1095 EXPORT_SYMBOL(netdev_state_change); 1095 EXPORT_SYMBOL(netdev_state_change);
1096 1096
1097 int netdev_bonding_change(struct net_device *dev, unsigned long event) 1097 int netdev_bonding_change(struct net_device *dev, unsigned long event)
1098 { 1098 {
1099 return call_netdevice_notifiers(event, dev); 1099 return call_netdevice_notifiers(event, dev);
1100 } 1100 }
1101 EXPORT_SYMBOL(netdev_bonding_change); 1101 EXPORT_SYMBOL(netdev_bonding_change);
1102 1102
1103 /** 1103 /**
1104 * dev_load - load a network module 1104 * dev_load - load a network module
1105 * @net: the applicable net namespace 1105 * @net: the applicable net namespace
1106 * @name: name of interface 1106 * @name: name of interface
1107 * 1107 *
1108 * If a network interface is not present and the process has suitable 1108 * If a network interface is not present and the process has suitable
1109 * privileges this function loads the module. If module loading is not 1109 * privileges this function loads the module. If module loading is not
1110 * available in this kernel then it becomes a nop. 1110 * available in this kernel then it becomes a nop.
1111 */ 1111 */
1112 1112
1113 void dev_load(struct net *net, const char *name) 1113 void dev_load(struct net *net, const char *name)
1114 { 1114 {
1115 struct net_device *dev; 1115 struct net_device *dev;
1116 1116
1117 rcu_read_lock(); 1117 rcu_read_lock();
1118 dev = dev_get_by_name_rcu(net, name); 1118 dev = dev_get_by_name_rcu(net, name);
1119 rcu_read_unlock(); 1119 rcu_read_unlock();
1120 1120
1121 if (!dev && capable(CAP_NET_ADMIN)) 1121 if (!dev && capable(CAP_NET_ADMIN))
1122 request_module("%s", name); 1122 request_module("%s", name);
1123 } 1123 }
1124 EXPORT_SYMBOL(dev_load); 1124 EXPORT_SYMBOL(dev_load);
1125 1125
1126 static int __dev_open(struct net_device *dev) 1126 static int __dev_open(struct net_device *dev)
1127 { 1127 {
1128 const struct net_device_ops *ops = dev->netdev_ops; 1128 const struct net_device_ops *ops = dev->netdev_ops;
1129 int ret; 1129 int ret;
1130 1130
1131 ASSERT_RTNL(); 1131 ASSERT_RTNL();
1132 1132
1133 /* 1133 /*
1134 * Is it even present? 1134 * Is it even present?
1135 */ 1135 */
1136 if (!netif_device_present(dev)) 1136 if (!netif_device_present(dev))
1137 return -ENODEV; 1137 return -ENODEV;
1138 1138
1139 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); 1139 ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1140 ret = notifier_to_errno(ret); 1140 ret = notifier_to_errno(ret);
1141 if (ret) 1141 if (ret)
1142 return ret; 1142 return ret;
1143 1143
1144 /* 1144 /*
1145 * Call device private open method 1145 * Call device private open method
1146 */ 1146 */
1147 set_bit(__LINK_STATE_START, &dev->state); 1147 set_bit(__LINK_STATE_START, &dev->state);
1148 1148
1149 if (ops->ndo_validate_addr) 1149 if (ops->ndo_validate_addr)
1150 ret = ops->ndo_validate_addr(dev); 1150 ret = ops->ndo_validate_addr(dev);
1151 1151
1152 if (!ret && ops->ndo_open) 1152 if (!ret && ops->ndo_open)
1153 ret = ops->ndo_open(dev); 1153 ret = ops->ndo_open(dev);
1154 1154
1155 /* 1155 /*
1156 * If it went open OK then: 1156 * If it went open OK then:
1157 */ 1157 */
1158 1158
1159 if (ret) 1159 if (ret)
1160 clear_bit(__LINK_STATE_START, &dev->state); 1160 clear_bit(__LINK_STATE_START, &dev->state);
1161 else { 1161 else {
1162 /* 1162 /*
1163 * Set the flags. 1163 * Set the flags.
1164 */ 1164 */
1165 dev->flags |= IFF_UP; 1165 dev->flags |= IFF_UP;
1166 1166
1167 /* 1167 /*
1168 * Enable NET_DMA 1168 * Enable NET_DMA
1169 */ 1169 */
1170 net_dmaengine_get(); 1170 net_dmaengine_get();
1171 1171
1172 /* 1172 /*
1173 * Initialize multicasting status 1173 * Initialize multicasting status
1174 */ 1174 */
1175 dev_set_rx_mode(dev); 1175 dev_set_rx_mode(dev);
1176 1176
1177 /* 1177 /*
1178 * Wakeup transmit queue engine 1178 * Wakeup transmit queue engine
1179 */ 1179 */
1180 dev_activate(dev); 1180 dev_activate(dev);
1181 } 1181 }
1182 1182
1183 return ret; 1183 return ret;
1184 } 1184 }
1185 1185
1186 /** 1186 /**
1187 * dev_open - prepare an interface for use. 1187 * dev_open - prepare an interface for use.
1188 * @dev: device to open 1188 * @dev: device to open
1189 * 1189 *
1190 * Takes a device from down to up state. The device's private open 1190 * Takes a device from down to up state. The device's private open
1191 * function is invoked and then the multicast lists are loaded. Finally 1191 * function is invoked and then the multicast lists are loaded. Finally
1192 * the device is moved into the up state and a %NETDEV_UP message is 1192 * the device is moved into the up state and a %NETDEV_UP message is
1193 * sent to the netdev notifier chain. 1193 * sent to the netdev notifier chain.
1194 * 1194 *
1195 * Calling this function on an active interface is a nop. On a failure 1195 * Calling this function on an active interface is a nop. On a failure
1196 * a negative errno code is returned. 1196 * a negative errno code is returned.
1197 */ 1197 */
1198 int dev_open(struct net_device *dev) 1198 int dev_open(struct net_device *dev)
1199 { 1199 {
1200 int ret; 1200 int ret;
1201 1201
1202 /* 1202 /*
1203 * Is it already up? 1203 * Is it already up?
1204 */ 1204 */
1205 if (dev->flags & IFF_UP) 1205 if (dev->flags & IFF_UP)
1206 return 0; 1206 return 0;
1207 1207
1208 /* 1208 /*
1209 * Open device 1209 * Open device
1210 */ 1210 */
1211 ret = __dev_open(dev); 1211 ret = __dev_open(dev);
1212 if (ret < 0) 1212 if (ret < 0)
1213 return ret; 1213 return ret;
1214 1214
1215 /* 1215 /*
1216 * ... and announce new interface. 1216 * ... and announce new interface.
1217 */ 1217 */
1218 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING); 1218 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1219 call_netdevice_notifiers(NETDEV_UP, dev); 1219 call_netdevice_notifiers(NETDEV_UP, dev);
1220 1220
1221 return ret; 1221 return ret;
1222 } 1222 }
1223 EXPORT_SYMBOL(dev_open); 1223 EXPORT_SYMBOL(dev_open);
1224 1224
1225 static int __dev_close_many(struct list_head *head) 1225 static int __dev_close_many(struct list_head *head)
1226 { 1226 {
1227 struct net_device *dev; 1227 struct net_device *dev;
1228 1228
1229 ASSERT_RTNL(); 1229 ASSERT_RTNL();
1230 might_sleep(); 1230 might_sleep();
1231 1231
1232 list_for_each_entry(dev, head, unreg_list) { 1232 list_for_each_entry(dev, head, unreg_list) {
1233 /* 1233 /*
1234 * Tell people we are going down, so that they can 1234 * Tell people we are going down, so that they can
1235 * prepare to death, when device is still operating. 1235 * prepare to death, when device is still operating.
1236 */ 1236 */
1237 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1237 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1238 1238
1239 clear_bit(__LINK_STATE_START, &dev->state); 1239 clear_bit(__LINK_STATE_START, &dev->state);
1240 1240
1241 /* Synchronize to scheduled poll. We cannot touch poll list, it 1241 /* Synchronize to scheduled poll. We cannot touch poll list, it
1242 * can be even on different cpu. So just clear netif_running(). 1242 * can be even on different cpu. So just clear netif_running().
1243 * 1243 *
1244 * dev->stop() will invoke napi_disable() on all of it's 1244 * dev->stop() will invoke napi_disable() on all of it's
1245 * napi_struct instances on this device. 1245 * napi_struct instances on this device.
1246 */ 1246 */
1247 smp_mb__after_clear_bit(); /* Commit netif_running(). */ 1247 smp_mb__after_clear_bit(); /* Commit netif_running(). */
1248 } 1248 }
1249 1249
1250 dev_deactivate_many(head); 1250 dev_deactivate_many(head);
1251 1251
1252 list_for_each_entry(dev, head, unreg_list) { 1252 list_for_each_entry(dev, head, unreg_list) {
1253 const struct net_device_ops *ops = dev->netdev_ops; 1253 const struct net_device_ops *ops = dev->netdev_ops;
1254 1254
1255 /* 1255 /*
1256 * Call the device specific close. This cannot fail. 1256 * Call the device specific close. This cannot fail.
1257 * Only if device is UP 1257 * Only if device is UP
1258 * 1258 *
1259 * We allow it to be called even after a DETACH hot-plug 1259 * We allow it to be called even after a DETACH hot-plug
1260 * event. 1260 * event.
1261 */ 1261 */
1262 if (ops->ndo_stop) 1262 if (ops->ndo_stop)
1263 ops->ndo_stop(dev); 1263 ops->ndo_stop(dev);
1264 1264
1265 /* 1265 /*
1266 * Device is now down. 1266 * Device is now down.
1267 */ 1267 */
1268 1268
1269 dev->flags &= ~IFF_UP; 1269 dev->flags &= ~IFF_UP;
1270 1270
1271 /* 1271 /*
1272 * Shutdown NET_DMA 1272 * Shutdown NET_DMA
1273 */ 1273 */
1274 net_dmaengine_put(); 1274 net_dmaengine_put();
1275 } 1275 }
1276 1276
1277 return 0; 1277 return 0;
1278 } 1278 }
1279 1279
1280 static int __dev_close(struct net_device *dev) 1280 static int __dev_close(struct net_device *dev)
1281 { 1281 {
1282 LIST_HEAD(single); 1282 LIST_HEAD(single);
1283 1283
1284 list_add(&dev->unreg_list, &single); 1284 list_add(&dev->unreg_list, &single);
1285 return __dev_close_many(&single); 1285 return __dev_close_many(&single);
1286 } 1286 }
1287 1287
1288 static int dev_close_many(struct list_head *head) 1288 static int dev_close_many(struct list_head *head)
1289 { 1289 {
1290 struct net_device *dev, *tmp; 1290 struct net_device *dev, *tmp;
1291 LIST_HEAD(tmp_list); 1291 LIST_HEAD(tmp_list);
1292 1292
1293 list_for_each_entry_safe(dev, tmp, head, unreg_list) 1293 list_for_each_entry_safe(dev, tmp, head, unreg_list)
1294 if (!(dev->flags & IFF_UP)) 1294 if (!(dev->flags & IFF_UP))
1295 list_move(&dev->unreg_list, &tmp_list); 1295 list_move(&dev->unreg_list, &tmp_list);
1296 1296
1297 __dev_close_many(head); 1297 __dev_close_many(head);
1298 1298
1299 /* 1299 /*
1300 * Tell people we are down 1300 * Tell people we are down
1301 */ 1301 */
1302 list_for_each_entry(dev, head, unreg_list) { 1302 list_for_each_entry(dev, head, unreg_list) {
1303 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING); 1303 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
1304 call_netdevice_notifiers(NETDEV_DOWN, dev); 1304 call_netdevice_notifiers(NETDEV_DOWN, dev);
1305 } 1305 }
1306 1306
1307 /* rollback_registered_many needs the complete original list */ 1307 /* rollback_registered_many needs the complete original list */
1308 list_splice(&tmp_list, head); 1308 list_splice(&tmp_list, head);
1309 return 0; 1309 return 0;
1310 } 1310 }
1311 1311
1312 /** 1312 /**
1313 * dev_close - shutdown an interface. 1313 * dev_close - shutdown an interface.
1314 * @dev: device to shutdown 1314 * @dev: device to shutdown
1315 * 1315 *
1316 * This function moves an active device into down state. A 1316 * This function moves an active device into down state. A
1317 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device 1317 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1318 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier 1318 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1319 * chain. 1319 * chain.
1320 */ 1320 */
1321 int dev_close(struct net_device *dev) 1321 int dev_close(struct net_device *dev)
1322 { 1322 {
1323 LIST_HEAD(single); 1323 LIST_HEAD(single);
1324 1324
1325 list_add(&dev->unreg_list, &single); 1325 list_add(&dev->unreg_list, &single);
1326 dev_close_many(&single); 1326 dev_close_many(&single);
1327 1327
1328 return 0; 1328 return 0;
1329 } 1329 }
1330 EXPORT_SYMBOL(dev_close); 1330 EXPORT_SYMBOL(dev_close);
1331 1331
1332 1332
1333 /** 1333 /**
1334 * dev_disable_lro - disable Large Receive Offload on a device 1334 * dev_disable_lro - disable Large Receive Offload on a device
1335 * @dev: device 1335 * @dev: device
1336 * 1336 *
1337 * Disable Large Receive Offload (LRO) on a net device. Must be 1337 * Disable Large Receive Offload (LRO) on a net device. Must be
1338 * called under RTNL. This is needed if received packets may be 1338 * called under RTNL. This is needed if received packets may be
1339 * forwarded to another interface. 1339 * forwarded to another interface.
1340 */ 1340 */
1341 void dev_disable_lro(struct net_device *dev) 1341 void dev_disable_lro(struct net_device *dev)
1342 { 1342 {
1343 if (dev->ethtool_ops && dev->ethtool_ops->get_flags && 1343 if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
1344 dev->ethtool_ops->set_flags) { 1344 dev->ethtool_ops->set_flags) {
1345 u32 flags = dev->ethtool_ops->get_flags(dev); 1345 u32 flags = dev->ethtool_ops->get_flags(dev);
1346 if (flags & ETH_FLAG_LRO) { 1346 if (flags & ETH_FLAG_LRO) {
1347 flags &= ~ETH_FLAG_LRO; 1347 flags &= ~ETH_FLAG_LRO;
1348 dev->ethtool_ops->set_flags(dev, flags); 1348 dev->ethtool_ops->set_flags(dev, flags);
1349 } 1349 }
1350 } 1350 }
1351 WARN_ON(dev->features & NETIF_F_LRO); 1351 WARN_ON(dev->features & NETIF_F_LRO);
1352 } 1352 }
1353 EXPORT_SYMBOL(dev_disable_lro); 1353 EXPORT_SYMBOL(dev_disable_lro);
1354 1354
1355 1355
1356 static int dev_boot_phase = 1; 1356 static int dev_boot_phase = 1;
1357 1357
1358 /* 1358 /*
1359 * Device change register/unregister. These are not inline or static 1359 * Device change register/unregister. These are not inline or static
1360 * as we export them to the world. 1360 * as we export them to the world.
1361 */ 1361 */
1362 1362
1363 /** 1363 /**
1364 * register_netdevice_notifier - register a network notifier block 1364 * register_netdevice_notifier - register a network notifier block
1365 * @nb: notifier 1365 * @nb: notifier
1366 * 1366 *
1367 * Register a notifier to be called when network device events occur. 1367 * Register a notifier to be called when network device events occur.
1368 * The notifier passed is linked into the kernel structures and must 1368 * The notifier passed is linked into the kernel structures and must
1369 * not be reused until it has been unregistered. A negative errno code 1369 * not be reused until it has been unregistered. A negative errno code
1370 * is returned on a failure. 1370 * is returned on a failure.
1371 * 1371 *
1372 * When registered all registration and up events are replayed 1372 * When registered all registration and up events are replayed
1373 * to the new notifier to allow device to have a race free 1373 * to the new notifier to allow device to have a race free
1374 * view of the network device list. 1374 * view of the network device list.
1375 */ 1375 */
1376 1376
1377 int register_netdevice_notifier(struct notifier_block *nb) 1377 int register_netdevice_notifier(struct notifier_block *nb)
1378 { 1378 {
1379 struct net_device *dev; 1379 struct net_device *dev;
1380 struct net_device *last; 1380 struct net_device *last;
1381 struct net *net; 1381 struct net *net;
1382 int err; 1382 int err;
1383 1383
1384 rtnl_lock(); 1384 rtnl_lock();
1385 err = raw_notifier_chain_register(&netdev_chain, nb); 1385 err = raw_notifier_chain_register(&netdev_chain, nb);
1386 if (err) 1386 if (err)
1387 goto unlock; 1387 goto unlock;
1388 if (dev_boot_phase) 1388 if (dev_boot_phase)
1389 goto unlock; 1389 goto unlock;
1390 for_each_net(net) { 1390 for_each_net(net) {
1391 for_each_netdev(net, dev) { 1391 for_each_netdev(net, dev) {
1392 err = nb->notifier_call(nb, NETDEV_REGISTER, dev); 1392 err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
1393 err = notifier_to_errno(err); 1393 err = notifier_to_errno(err);
1394 if (err) 1394 if (err)
1395 goto rollback; 1395 goto rollback;
1396 1396
1397 if (!(dev->flags & IFF_UP)) 1397 if (!(dev->flags & IFF_UP))
1398 continue; 1398 continue;
1399 1399
1400 nb->notifier_call(nb, NETDEV_UP, dev); 1400 nb->notifier_call(nb, NETDEV_UP, dev);
1401 } 1401 }
1402 } 1402 }
1403 1403
1404 unlock: 1404 unlock:
1405 rtnl_unlock(); 1405 rtnl_unlock();
1406 return err; 1406 return err;
1407 1407
1408 rollback: 1408 rollback:
1409 last = dev; 1409 last = dev;
1410 for_each_net(net) { 1410 for_each_net(net) {
1411 for_each_netdev(net, dev) { 1411 for_each_netdev(net, dev) {
1412 if (dev == last) 1412 if (dev == last)
1413 break; 1413 break;
1414 1414
1415 if (dev->flags & IFF_UP) { 1415 if (dev->flags & IFF_UP) {
1416 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev); 1416 nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
1417 nb->notifier_call(nb, NETDEV_DOWN, dev); 1417 nb->notifier_call(nb, NETDEV_DOWN, dev);
1418 } 1418 }
1419 nb->notifier_call(nb, NETDEV_UNREGISTER, dev); 1419 nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
1420 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev); 1420 nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
1421 } 1421 }
1422 } 1422 }
1423 1423
1424 raw_notifier_chain_unregister(&netdev_chain, nb); 1424 raw_notifier_chain_unregister(&netdev_chain, nb);
1425 goto unlock; 1425 goto unlock;
1426 } 1426 }
1427 EXPORT_SYMBOL(register_netdevice_notifier); 1427 EXPORT_SYMBOL(register_netdevice_notifier);
1428 1428
1429 /** 1429 /**
1430 * unregister_netdevice_notifier - unregister a network notifier block 1430 * unregister_netdevice_notifier - unregister a network notifier block
1431 * @nb: notifier 1431 * @nb: notifier
1432 * 1432 *
1433 * Unregister a notifier previously registered by 1433 * Unregister a notifier previously registered by
1434 * register_netdevice_notifier(). The notifier is unlinked into the 1434 * register_netdevice_notifier(). The notifier is unlinked into the
1435 * kernel structures and may then be reused. A negative errno code 1435 * kernel structures and may then be reused. A negative errno code
1436 * is returned on a failure. 1436 * is returned on a failure.
1437 */ 1437 */
1438 1438
1439 int unregister_netdevice_notifier(struct notifier_block *nb) 1439 int unregister_netdevice_notifier(struct notifier_block *nb)
1440 { 1440 {
1441 int err; 1441 int err;
1442 1442
1443 rtnl_lock(); 1443 rtnl_lock();
1444 err = raw_notifier_chain_unregister(&netdev_chain, nb); 1444 err = raw_notifier_chain_unregister(&netdev_chain, nb);
1445 rtnl_unlock(); 1445 rtnl_unlock();
1446 return err; 1446 return err;
1447 } 1447 }
1448 EXPORT_SYMBOL(unregister_netdevice_notifier); 1448 EXPORT_SYMBOL(unregister_netdevice_notifier);
1449 1449
1450 /** 1450 /**
1451 * call_netdevice_notifiers - call all network notifier blocks 1451 * call_netdevice_notifiers - call all network notifier blocks
1452 * @val: value passed unmodified to notifier function 1452 * @val: value passed unmodified to notifier function
1453 * @dev: net_device pointer passed unmodified to notifier function 1453 * @dev: net_device pointer passed unmodified to notifier function
1454 * 1454 *
1455 * Call all network notifier blocks. Parameters and return value 1455 * Call all network notifier blocks. Parameters and return value
1456 * are as for raw_notifier_call_chain(). 1456 * are as for raw_notifier_call_chain().
1457 */ 1457 */
1458 1458
1459 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 1459 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1460 { 1460 {
1461 ASSERT_RTNL(); 1461 ASSERT_RTNL();
1462 return raw_notifier_call_chain(&netdev_chain, val, dev); 1462 return raw_notifier_call_chain(&netdev_chain, val, dev);
1463 } 1463 }
1464 1464
1465 /* When > 0 there are consumers of rx skb time stamps */ 1465 /* When > 0 there are consumers of rx skb time stamps */
1466 static atomic_t netstamp_needed = ATOMIC_INIT(0); 1466 static atomic_t netstamp_needed = ATOMIC_INIT(0);
1467 1467
1468 void net_enable_timestamp(void) 1468 void net_enable_timestamp(void)
1469 { 1469 {
1470 atomic_inc(&netstamp_needed); 1470 atomic_inc(&netstamp_needed);
1471 } 1471 }
1472 EXPORT_SYMBOL(net_enable_timestamp); 1472 EXPORT_SYMBOL(net_enable_timestamp);
1473 1473
1474 void net_disable_timestamp(void) 1474 void net_disable_timestamp(void)
1475 { 1475 {
1476 atomic_dec(&netstamp_needed); 1476 atomic_dec(&netstamp_needed);
1477 } 1477 }
1478 EXPORT_SYMBOL(net_disable_timestamp); 1478 EXPORT_SYMBOL(net_disable_timestamp);
1479 1479
1480 static inline void net_timestamp_set(struct sk_buff *skb) 1480 static inline void net_timestamp_set(struct sk_buff *skb)
1481 { 1481 {
1482 if (atomic_read(&netstamp_needed)) 1482 if (atomic_read(&netstamp_needed))
1483 __net_timestamp(skb); 1483 __net_timestamp(skb);
1484 else 1484 else
1485 skb->tstamp.tv64 = 0; 1485 skb->tstamp.tv64 = 0;
1486 } 1486 }
1487 1487
1488 static inline void net_timestamp_check(struct sk_buff *skb) 1488 static inline void net_timestamp_check(struct sk_buff *skb)
1489 { 1489 {
1490 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed)) 1490 if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
1491 __net_timestamp(skb); 1491 __net_timestamp(skb);
1492 } 1492 }
1493 1493
1494 /** 1494 /**
1495 * dev_forward_skb - loopback an skb to another netif 1495 * dev_forward_skb - loopback an skb to another netif
1496 * 1496 *
1497 * @dev: destination network device 1497 * @dev: destination network device
1498 * @skb: buffer to forward 1498 * @skb: buffer to forward
1499 * 1499 *
1500 * return values: 1500 * return values:
1501 * NET_RX_SUCCESS (no congestion) 1501 * NET_RX_SUCCESS (no congestion)
1502 * NET_RX_DROP (packet was dropped, but freed) 1502 * NET_RX_DROP (packet was dropped, but freed)
1503 * 1503 *
1504 * dev_forward_skb can be used for injecting an skb from the 1504 * dev_forward_skb can be used for injecting an skb from the
1505 * start_xmit function of one device into the receive queue 1505 * start_xmit function of one device into the receive queue
1506 * of another device. 1506 * of another device.
1507 * 1507 *
1508 * The receiving device may be in another namespace, so 1508 * The receiving device may be in another namespace, so
1509 * we have to clear all information in the skb that could 1509 * we have to clear all information in the skb that could
1510 * impact namespace isolation. 1510 * impact namespace isolation.
1511 */ 1511 */
1512 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 1512 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1513 { 1513 {
1514 skb_orphan(skb); 1514 skb_orphan(skb);
1515 nf_reset(skb); 1515 nf_reset(skb);
1516 1516
1517 if (unlikely(!(dev->flags & IFF_UP) || 1517 if (unlikely(!(dev->flags & IFF_UP) ||
1518 (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN)))) { 1518 (skb->len > (dev->mtu + dev->hard_header_len + VLAN_HLEN)))) {
1519 atomic_long_inc(&dev->rx_dropped); 1519 atomic_long_inc(&dev->rx_dropped);
1520 kfree_skb(skb); 1520 kfree_skb(skb);
1521 return NET_RX_DROP; 1521 return NET_RX_DROP;
1522 } 1522 }
1523 skb_set_dev(skb, dev); 1523 skb_set_dev(skb, dev);
1524 skb->tstamp.tv64 = 0; 1524 skb->tstamp.tv64 = 0;
1525 skb->pkt_type = PACKET_HOST; 1525 skb->pkt_type = PACKET_HOST;
1526 skb->protocol = eth_type_trans(skb, dev); 1526 skb->protocol = eth_type_trans(skb, dev);
1527 return netif_rx(skb); 1527 return netif_rx(skb);
1528 } 1528 }
1529 EXPORT_SYMBOL_GPL(dev_forward_skb); 1529 EXPORT_SYMBOL_GPL(dev_forward_skb);
1530 1530
1531 static inline int deliver_skb(struct sk_buff *skb, 1531 static inline int deliver_skb(struct sk_buff *skb,
1532 struct packet_type *pt_prev, 1532 struct packet_type *pt_prev,
1533 struct net_device *orig_dev) 1533 struct net_device *orig_dev)
1534 { 1534 {
1535 atomic_inc(&skb->users); 1535 atomic_inc(&skb->users);
1536 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 1536 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1537 } 1537 }
1538 1538
1539 /* 1539 /*
1540 * Support routine. Sends outgoing frames to any network 1540 * Support routine. Sends outgoing frames to any network
1541 * taps currently in use. 1541 * taps currently in use.
1542 */ 1542 */
1543 1543
1544 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 1544 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1545 { 1545 {
1546 struct packet_type *ptype; 1546 struct packet_type *ptype;
1547 struct sk_buff *skb2 = NULL; 1547 struct sk_buff *skb2 = NULL;
1548 struct packet_type *pt_prev = NULL; 1548 struct packet_type *pt_prev = NULL;
1549 1549
1550 rcu_read_lock(); 1550 rcu_read_lock();
1551 list_for_each_entry_rcu(ptype, &ptype_all, list) { 1551 list_for_each_entry_rcu(ptype, &ptype_all, list) {
1552 /* Never send packets back to the socket 1552 /* Never send packets back to the socket
1553 * they originated from - MvS (miquels@drinkel.ow.org) 1553 * they originated from - MvS (miquels@drinkel.ow.org)
1554 */ 1554 */
1555 if ((ptype->dev == dev || !ptype->dev) && 1555 if ((ptype->dev == dev || !ptype->dev) &&
1556 (ptype->af_packet_priv == NULL || 1556 (ptype->af_packet_priv == NULL ||
1557 (struct sock *)ptype->af_packet_priv != skb->sk)) { 1557 (struct sock *)ptype->af_packet_priv != skb->sk)) {
1558 if (pt_prev) { 1558 if (pt_prev) {
1559 deliver_skb(skb2, pt_prev, skb->dev); 1559 deliver_skb(skb2, pt_prev, skb->dev);
1560 pt_prev = ptype; 1560 pt_prev = ptype;
1561 continue; 1561 continue;
1562 } 1562 }
1563 1563
1564 skb2 = skb_clone(skb, GFP_ATOMIC); 1564 skb2 = skb_clone(skb, GFP_ATOMIC);
1565 if (!skb2) 1565 if (!skb2)
1566 break; 1566 break;
1567 1567
1568 net_timestamp_set(skb2); 1568 net_timestamp_set(skb2);
1569 1569
1570 /* skb->nh should be correctly 1570 /* skb->nh should be correctly
1571 set by sender, so that the second statement is 1571 set by sender, so that the second statement is
1572 just protection against buggy protocols. 1572 just protection against buggy protocols.
1573 */ 1573 */
1574 skb_reset_mac_header(skb2); 1574 skb_reset_mac_header(skb2);
1575 1575
1576 if (skb_network_header(skb2) < skb2->data || 1576 if (skb_network_header(skb2) < skb2->data ||
1577 skb2->network_header > skb2->tail) { 1577 skb2->network_header > skb2->tail) {
1578 if (net_ratelimit()) 1578 if (net_ratelimit())
1579 printk(KERN_CRIT "protocol %04x is " 1579 printk(KERN_CRIT "protocol %04x is "
1580 "buggy, dev %s\n", 1580 "buggy, dev %s\n",
1581 ntohs(skb2->protocol), 1581 ntohs(skb2->protocol),
1582 dev->name); 1582 dev->name);
1583 skb_reset_network_header(skb2); 1583 skb_reset_network_header(skb2);
1584 } 1584 }
1585 1585
1586 skb2->transport_header = skb2->network_header; 1586 skb2->transport_header = skb2->network_header;
1587 skb2->pkt_type = PACKET_OUTGOING; 1587 skb2->pkt_type = PACKET_OUTGOING;
1588 pt_prev = ptype; 1588 pt_prev = ptype;
1589 } 1589 }
1590 } 1590 }
1591 if (pt_prev) 1591 if (pt_prev)
1592 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 1592 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1593 rcu_read_unlock(); 1593 rcu_read_unlock();
1594 } 1594 }
1595 1595
1596 /* netif_setup_tc - Handle tc mappings on real_num_tx_queues change 1596 /* netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1597 * @dev: Network device 1597 * @dev: Network device
1598 * @txq: number of queues available 1598 * @txq: number of queues available
1599 * 1599 *
1600 * If real_num_tx_queues is changed the tc mappings may no longer be 1600 * If real_num_tx_queues is changed the tc mappings may no longer be
1601 * valid. To resolve this verify the tc mapping remains valid and if 1601 * valid. To resolve this verify the tc mapping remains valid and if
1602 * not NULL the mapping. With no priorities mapping to this 1602 * not NULL the mapping. With no priorities mapping to this
1603 * offset/count pair it will no longer be used. In the worst case TC0 1603 * offset/count pair it will no longer be used. In the worst case TC0
1604 * is invalid nothing can be done so disable priority mappings. If is 1604 * is invalid nothing can be done so disable priority mappings. If is
1605 * expected that drivers will fix this mapping if they can before 1605 * expected that drivers will fix this mapping if they can before
1606 * calling netif_set_real_num_tx_queues. 1606 * calling netif_set_real_num_tx_queues.
1607 */ 1607 */
1608 void netif_setup_tc(struct net_device *dev, unsigned int txq) 1608 void netif_setup_tc(struct net_device *dev, unsigned int txq)
1609 { 1609 {
1610 int i; 1610 int i;
1611 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 1611 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1612 1612
1613 /* If TC0 is invalidated disable TC mapping */ 1613 /* If TC0 is invalidated disable TC mapping */
1614 if (tc->offset + tc->count > txq) { 1614 if (tc->offset + tc->count > txq) {
1615 pr_warning("Number of in use tx queues changed " 1615 pr_warning("Number of in use tx queues changed "
1616 "invalidating tc mappings. Priority " 1616 "invalidating tc mappings. Priority "
1617 "traffic classification disabled!\n"); 1617 "traffic classification disabled!\n");
1618 dev->num_tc = 0; 1618 dev->num_tc = 0;
1619 return; 1619 return;
1620 } 1620 }
1621 1621
1622 /* Invalidated prio to tc mappings set to TC0 */ 1622 /* Invalidated prio to tc mappings set to TC0 */
1623 for (i = 1; i < TC_BITMASK + 1; i++) { 1623 for (i = 1; i < TC_BITMASK + 1; i++) {
1624 int q = netdev_get_prio_tc_map(dev, i); 1624 int q = netdev_get_prio_tc_map(dev, i);
1625 1625
1626 tc = &dev->tc_to_txq[q]; 1626 tc = &dev->tc_to_txq[q];
1627 if (tc->offset + tc->count > txq) { 1627 if (tc->offset + tc->count > txq) {
1628 pr_warning("Number of in use tx queues " 1628 pr_warning("Number of in use tx queues "
1629 "changed. Priority %i to tc " 1629 "changed. Priority %i to tc "
1630 "mapping %i is no longer valid " 1630 "mapping %i is no longer valid "
1631 "setting map to 0\n", 1631 "setting map to 0\n",
1632 i, q); 1632 i, q);
1633 netdev_set_prio_tc_map(dev, i, 0); 1633 netdev_set_prio_tc_map(dev, i, 0);
1634 } 1634 }
1635 } 1635 }
1636 } 1636 }
1637 1637
1638 /* 1638 /*
1639 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 1639 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
1640 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. 1640 * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
1641 */ 1641 */
1642 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 1642 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
1643 { 1643 {
1644 int rc; 1644 int rc;
1645 1645
1646 if (txq < 1 || txq > dev->num_tx_queues) 1646 if (txq < 1 || txq > dev->num_tx_queues)
1647 return -EINVAL; 1647 return -EINVAL;
1648 1648
1649 if (dev->reg_state == NETREG_REGISTERED) { 1649 if (dev->reg_state == NETREG_REGISTERED) {
1650 ASSERT_RTNL(); 1650 ASSERT_RTNL();
1651 1651
1652 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 1652 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
1653 txq); 1653 txq);
1654 if (rc) 1654 if (rc)
1655 return rc; 1655 return rc;
1656 1656
1657 if (dev->num_tc) 1657 if (dev->num_tc)
1658 netif_setup_tc(dev, txq); 1658 netif_setup_tc(dev, txq);
1659 1659
1660 if (txq < dev->real_num_tx_queues) 1660 if (txq < dev->real_num_tx_queues)
1661 qdisc_reset_all_tx_gt(dev, txq); 1661 qdisc_reset_all_tx_gt(dev, txq);
1662 } 1662 }
1663 1663
1664 dev->real_num_tx_queues = txq; 1664 dev->real_num_tx_queues = txq;
1665 return 0; 1665 return 0;
1666 } 1666 }
1667 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 1667 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
1668 1668
1669 #ifdef CONFIG_RPS 1669 #ifdef CONFIG_RPS
1670 /** 1670 /**
1671 * netif_set_real_num_rx_queues - set actual number of RX queues used 1671 * netif_set_real_num_rx_queues - set actual number of RX queues used
1672 * @dev: Network device 1672 * @dev: Network device
1673 * @rxq: Actual number of RX queues 1673 * @rxq: Actual number of RX queues
1674 * 1674 *
1675 * This must be called either with the rtnl_lock held or before 1675 * This must be called either with the rtnl_lock held or before
1676 * registration of the net device. Returns 0 on success, or a 1676 * registration of the net device. Returns 0 on success, or a
1677 * negative error code. If called before registration, it always 1677 * negative error code. If called before registration, it always
1678 * succeeds. 1678 * succeeds.
1679 */ 1679 */
1680 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 1680 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
1681 { 1681 {
1682 int rc; 1682 int rc;
1683 1683
1684 if (rxq < 1 || rxq > dev->num_rx_queues) 1684 if (rxq < 1 || rxq > dev->num_rx_queues)
1685 return -EINVAL; 1685 return -EINVAL;
1686 1686
1687 if (dev->reg_state == NETREG_REGISTERED) { 1687 if (dev->reg_state == NETREG_REGISTERED) {
1688 ASSERT_RTNL(); 1688 ASSERT_RTNL();
1689 1689
1690 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 1690 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
1691 rxq); 1691 rxq);
1692 if (rc) 1692 if (rc)
1693 return rc; 1693 return rc;
1694 } 1694 }
1695 1695
1696 dev->real_num_rx_queues = rxq; 1696 dev->real_num_rx_queues = rxq;
1697 return 0; 1697 return 0;
1698 } 1698 }
1699 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 1699 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
1700 #endif 1700 #endif
1701 1701
1702 static inline void __netif_reschedule(struct Qdisc *q) 1702 static inline void __netif_reschedule(struct Qdisc *q)
1703 { 1703 {
1704 struct softnet_data *sd; 1704 struct softnet_data *sd;
1705 unsigned long flags; 1705 unsigned long flags;
1706 1706
1707 local_irq_save(flags); 1707 local_irq_save(flags);
1708 sd = &__get_cpu_var(softnet_data); 1708 sd = &__get_cpu_var(softnet_data);
1709 q->next_sched = NULL; 1709 q->next_sched = NULL;
1710 *sd->output_queue_tailp = q; 1710 *sd->output_queue_tailp = q;
1711 sd->output_queue_tailp = &q->next_sched; 1711 sd->output_queue_tailp = &q->next_sched;
1712 raise_softirq_irqoff(NET_TX_SOFTIRQ); 1712 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1713 local_irq_restore(flags); 1713 local_irq_restore(flags);
1714 } 1714 }
1715 1715
1716 void __netif_schedule(struct Qdisc *q) 1716 void __netif_schedule(struct Qdisc *q)
1717 { 1717 {
1718 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 1718 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
1719 __netif_reschedule(q); 1719 __netif_reschedule(q);
1720 } 1720 }
1721 EXPORT_SYMBOL(__netif_schedule); 1721 EXPORT_SYMBOL(__netif_schedule);
1722 1722
1723 void dev_kfree_skb_irq(struct sk_buff *skb) 1723 void dev_kfree_skb_irq(struct sk_buff *skb)
1724 { 1724 {
1725 if (atomic_dec_and_test(&skb->users)) { 1725 if (atomic_dec_and_test(&skb->users)) {
1726 struct softnet_data *sd; 1726 struct softnet_data *sd;
1727 unsigned long flags; 1727 unsigned long flags;
1728 1728
1729 local_irq_save(flags); 1729 local_irq_save(flags);
1730 sd = &__get_cpu_var(softnet_data); 1730 sd = &__get_cpu_var(softnet_data);
1731 skb->next = sd->completion_queue; 1731 skb->next = sd->completion_queue;
1732 sd->completion_queue = skb; 1732 sd->completion_queue = skb;
1733 raise_softirq_irqoff(NET_TX_SOFTIRQ); 1733 raise_softirq_irqoff(NET_TX_SOFTIRQ);
1734 local_irq_restore(flags); 1734 local_irq_restore(flags);
1735 } 1735 }
1736 } 1736 }
1737 EXPORT_SYMBOL(dev_kfree_skb_irq); 1737 EXPORT_SYMBOL(dev_kfree_skb_irq);
1738 1738
1739 void dev_kfree_skb_any(struct sk_buff *skb) 1739 void dev_kfree_skb_any(struct sk_buff *skb)
1740 { 1740 {
1741 if (in_irq() || irqs_disabled()) 1741 if (in_irq() || irqs_disabled())
1742 dev_kfree_skb_irq(skb); 1742 dev_kfree_skb_irq(skb);
1743 else 1743 else
1744 dev_kfree_skb(skb); 1744 dev_kfree_skb(skb);
1745 } 1745 }
1746 EXPORT_SYMBOL(dev_kfree_skb_any); 1746 EXPORT_SYMBOL(dev_kfree_skb_any);
1747 1747
1748 1748
1749 /** 1749 /**
1750 * netif_device_detach - mark device as removed 1750 * netif_device_detach - mark device as removed
1751 * @dev: network device 1751 * @dev: network device
1752 * 1752 *
1753 * Mark device as removed from system and therefore no longer available. 1753 * Mark device as removed from system and therefore no longer available.
1754 */ 1754 */
1755 void netif_device_detach(struct net_device *dev) 1755 void netif_device_detach(struct net_device *dev)
1756 { 1756 {
1757 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 1757 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
1758 netif_running(dev)) { 1758 netif_running(dev)) {
1759 netif_tx_stop_all_queues(dev); 1759 netif_tx_stop_all_queues(dev);
1760 } 1760 }
1761 } 1761 }
1762 EXPORT_SYMBOL(netif_device_detach); 1762 EXPORT_SYMBOL(netif_device_detach);
1763 1763
1764 /** 1764 /**
1765 * netif_device_attach - mark device as attached 1765 * netif_device_attach - mark device as attached
1766 * @dev: network device 1766 * @dev: network device
1767 * 1767 *
1768 * Mark device as attached from system and restart if needed. 1768 * Mark device as attached from system and restart if needed.
1769 */ 1769 */
1770 void netif_device_attach(struct net_device *dev) 1770 void netif_device_attach(struct net_device *dev)
1771 { 1771 {
1772 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 1772 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
1773 netif_running(dev)) { 1773 netif_running(dev)) {
1774 netif_tx_wake_all_queues(dev); 1774 netif_tx_wake_all_queues(dev);
1775 __netdev_watchdog_up(dev); 1775 __netdev_watchdog_up(dev);
1776 } 1776 }
1777 } 1777 }
1778 EXPORT_SYMBOL(netif_device_attach); 1778 EXPORT_SYMBOL(netif_device_attach);
1779 1779
1780 /** 1780 /**
1781 * skb_dev_set -- assign a new device to a buffer 1781 * skb_dev_set -- assign a new device to a buffer
1782 * @skb: buffer for the new device 1782 * @skb: buffer for the new device
1783 * @dev: network device 1783 * @dev: network device
1784 * 1784 *
1785 * If an skb is owned by a device already, we have to reset 1785 * If an skb is owned by a device already, we have to reset
1786 * all data private to the namespace a device belongs to 1786 * all data private to the namespace a device belongs to
1787 * before assigning it a new device. 1787 * before assigning it a new device.
1788 */ 1788 */
1789 #ifdef CONFIG_NET_NS 1789 #ifdef CONFIG_NET_NS
1790 void skb_set_dev(struct sk_buff *skb, struct net_device *dev) 1790 void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
1791 { 1791 {
1792 skb_dst_drop(skb); 1792 skb_dst_drop(skb);
1793 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) { 1793 if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
1794 secpath_reset(skb); 1794 secpath_reset(skb);
1795 nf_reset(skb); 1795 nf_reset(skb);
1796 skb_init_secmark(skb); 1796 skb_init_secmark(skb);
1797 skb->mark = 0; 1797 skb->mark = 0;
1798 skb->priority = 0; 1798 skb->priority = 0;
1799 skb->nf_trace = 0; 1799 skb->nf_trace = 0;
1800 skb->ipvs_property = 0; 1800 skb->ipvs_property = 0;
1801 #ifdef CONFIG_NET_SCHED 1801 #ifdef CONFIG_NET_SCHED
1802 skb->tc_index = 0; 1802 skb->tc_index = 0;
1803 #endif 1803 #endif
1804 } 1804 }
1805 skb->dev = dev; 1805 skb->dev = dev;
1806 } 1806 }
1807 EXPORT_SYMBOL(skb_set_dev); 1807 EXPORT_SYMBOL(skb_set_dev);
1808 #endif /* CONFIG_NET_NS */ 1808 #endif /* CONFIG_NET_NS */
1809 1809
1810 /* 1810 /*
1811 * Invalidate hardware checksum when packet is to be mangled, and 1811 * Invalidate hardware checksum when packet is to be mangled, and
1812 * complete checksum manually on outgoing path. 1812 * complete checksum manually on outgoing path.
1813 */ 1813 */
1814 int skb_checksum_help(struct sk_buff *skb) 1814 int skb_checksum_help(struct sk_buff *skb)
1815 { 1815 {
1816 __wsum csum; 1816 __wsum csum;
1817 int ret = 0, offset; 1817 int ret = 0, offset;
1818 1818
1819 if (skb->ip_summed == CHECKSUM_COMPLETE) 1819 if (skb->ip_summed == CHECKSUM_COMPLETE)
1820 goto out_set_summed; 1820 goto out_set_summed;
1821 1821
1822 if (unlikely(skb_shinfo(skb)->gso_size)) { 1822 if (unlikely(skb_shinfo(skb)->gso_size)) {
1823 /* Let GSO fix up the checksum. */ 1823 /* Let GSO fix up the checksum. */
1824 goto out_set_summed; 1824 goto out_set_summed;
1825 } 1825 }
1826 1826
1827 offset = skb_checksum_start_offset(skb); 1827 offset = skb_checksum_start_offset(skb);
1828 BUG_ON(offset >= skb_headlen(skb)); 1828 BUG_ON(offset >= skb_headlen(skb));
1829 csum = skb_checksum(skb, offset, skb->len - offset, 0); 1829 csum = skb_checksum(skb, offset, skb->len - offset, 0);
1830 1830
1831 offset += skb->csum_offset; 1831 offset += skb->csum_offset;
1832 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); 1832 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
1833 1833
1834 if (skb_cloned(skb) && 1834 if (skb_cloned(skb) &&
1835 !skb_clone_writable(skb, offset + sizeof(__sum16))) { 1835 !skb_clone_writable(skb, offset + sizeof(__sum16))) {
1836 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 1836 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1837 if (ret) 1837 if (ret)
1838 goto out; 1838 goto out;
1839 } 1839 }
1840 1840
1841 *(__sum16 *)(skb->data + offset) = csum_fold(csum); 1841 *(__sum16 *)(skb->data + offset) = csum_fold(csum);
1842 out_set_summed: 1842 out_set_summed:
1843 skb->ip_summed = CHECKSUM_NONE; 1843 skb->ip_summed = CHECKSUM_NONE;
1844 out: 1844 out:
1845 return ret; 1845 return ret;
1846 } 1846 }
1847 EXPORT_SYMBOL(skb_checksum_help); 1847 EXPORT_SYMBOL(skb_checksum_help);
1848 1848
1849 /** 1849 /**
1850 * skb_gso_segment - Perform segmentation on skb. 1850 * skb_gso_segment - Perform segmentation on skb.
1851 * @skb: buffer to segment 1851 * @skb: buffer to segment
1852 * @features: features for the output path (see dev->features) 1852 * @features: features for the output path (see dev->features)
1853 * 1853 *
1854 * This function segments the given skb and returns a list of segments. 1854 * This function segments the given skb and returns a list of segments.
1855 * 1855 *
1856 * It may return NULL if the skb requires no segmentation. This is 1856 * It may return NULL if the skb requires no segmentation. This is
1857 * only possible when GSO is used for verifying header integrity. 1857 * only possible when GSO is used for verifying header integrity.
1858 */ 1858 */
1859 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features) 1859 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
1860 { 1860 {
1861 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); 1861 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
1862 struct packet_type *ptype; 1862 struct packet_type *ptype;
1863 __be16 type = skb->protocol; 1863 __be16 type = skb->protocol;
1864 int vlan_depth = ETH_HLEN; 1864 int vlan_depth = ETH_HLEN;
1865 int err; 1865 int err;
1866 1866
1867 while (type == htons(ETH_P_8021Q)) { 1867 while (type == htons(ETH_P_8021Q)) {
1868 struct vlan_hdr *vh; 1868 struct vlan_hdr *vh;
1869 1869
1870 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN))) 1870 if (unlikely(!pskb_may_pull(skb, vlan_depth + VLAN_HLEN)))
1871 return ERR_PTR(-EINVAL); 1871 return ERR_PTR(-EINVAL);
1872 1872
1873 vh = (struct vlan_hdr *)(skb->data + vlan_depth); 1873 vh = (struct vlan_hdr *)(skb->data + vlan_depth);
1874 type = vh->h_vlan_encapsulated_proto; 1874 type = vh->h_vlan_encapsulated_proto;
1875 vlan_depth += VLAN_HLEN; 1875 vlan_depth += VLAN_HLEN;
1876 } 1876 }
1877 1877
1878 skb_reset_mac_header(skb); 1878 skb_reset_mac_header(skb);
1879 skb->mac_len = skb->network_header - skb->mac_header; 1879 skb->mac_len = skb->network_header - skb->mac_header;
1880 __skb_pull(skb, skb->mac_len); 1880 __skb_pull(skb, skb->mac_len);
1881 1881
1882 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) { 1882 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1883 struct net_device *dev = skb->dev; 1883 struct net_device *dev = skb->dev;
1884 struct ethtool_drvinfo info = {}; 1884 struct ethtool_drvinfo info = {};
1885 1885
1886 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo) 1886 if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
1887 dev->ethtool_ops->get_drvinfo(dev, &info); 1887 dev->ethtool_ops->get_drvinfo(dev, &info);
1888 1888
1889 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n", 1889 WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d ip_summed=%d\n",
1890 info.driver, dev ? dev->features : 0L, 1890 info.driver, dev ? dev->features : 0L,
1891 skb->sk ? skb->sk->sk_route_caps : 0L, 1891 skb->sk ? skb->sk->sk_route_caps : 0L,
1892 skb->len, skb->data_len, skb->ip_summed); 1892 skb->len, skb->data_len, skb->ip_summed);
1893 1893
1894 if (skb_header_cloned(skb) && 1894 if (skb_header_cloned(skb) &&
1895 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))) 1895 (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
1896 return ERR_PTR(err); 1896 return ERR_PTR(err);
1897 } 1897 }
1898 1898
1899 rcu_read_lock(); 1899 rcu_read_lock();
1900 list_for_each_entry_rcu(ptype, 1900 list_for_each_entry_rcu(ptype,
1901 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) { 1901 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
1902 if (ptype->type == type && !ptype->dev && ptype->gso_segment) { 1902 if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
1903 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) { 1903 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
1904 err = ptype->gso_send_check(skb); 1904 err = ptype->gso_send_check(skb);
1905 segs = ERR_PTR(err); 1905 segs = ERR_PTR(err);
1906 if (err || skb_gso_ok(skb, features)) 1906 if (err || skb_gso_ok(skb, features))
1907 break; 1907 break;
1908 __skb_push(skb, (skb->data - 1908 __skb_push(skb, (skb->data -
1909 skb_network_header(skb))); 1909 skb_network_header(skb)));
1910 } 1910 }
1911 segs = ptype->gso_segment(skb, features); 1911 segs = ptype->gso_segment(skb, features);
1912 break; 1912 break;
1913 } 1913 }
1914 } 1914 }
1915 rcu_read_unlock(); 1915 rcu_read_unlock();
1916 1916
1917 __skb_push(skb, skb->data - skb_mac_header(skb)); 1917 __skb_push(skb, skb->data - skb_mac_header(skb));
1918 1918
1919 return segs; 1919 return segs;
1920 } 1920 }
1921 EXPORT_SYMBOL(skb_gso_segment); 1921 EXPORT_SYMBOL(skb_gso_segment);
1922 1922
1923 /* Take action when hardware reception checksum errors are detected. */ 1923 /* Take action when hardware reception checksum errors are detected. */
1924 #ifdef CONFIG_BUG 1924 #ifdef CONFIG_BUG
1925 void netdev_rx_csum_fault(struct net_device *dev) 1925 void netdev_rx_csum_fault(struct net_device *dev)
1926 { 1926 {
1927 if (net_ratelimit()) { 1927 if (net_ratelimit()) {
1928 printk(KERN_ERR "%s: hw csum failure.\n", 1928 printk(KERN_ERR "%s: hw csum failure.\n",
1929 dev ? dev->name : "<unknown>"); 1929 dev ? dev->name : "<unknown>");
1930 dump_stack(); 1930 dump_stack();
1931 } 1931 }
1932 } 1932 }
1933 EXPORT_SYMBOL(netdev_rx_csum_fault); 1933 EXPORT_SYMBOL(netdev_rx_csum_fault);
1934 #endif 1934 #endif
1935 1935
1936 /* Actually, we should eliminate this check as soon as we know, that: 1936 /* Actually, we should eliminate this check as soon as we know, that:
1937 * 1. IOMMU is present and allows to map all the memory. 1937 * 1. IOMMU is present and allows to map all the memory.
1938 * 2. No high memory really exists on this machine. 1938 * 2. No high memory really exists on this machine.
1939 */ 1939 */
1940 1940
1941 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 1941 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1942 { 1942 {
1943 #ifdef CONFIG_HIGHMEM 1943 #ifdef CONFIG_HIGHMEM
1944 int i; 1944 int i;
1945 if (!(dev->features & NETIF_F_HIGHDMA)) { 1945 if (!(dev->features & NETIF_F_HIGHDMA)) {
1946 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1946 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1947 if (PageHighMem(skb_shinfo(skb)->frags[i].page)) 1947 if (PageHighMem(skb_shinfo(skb)->frags[i].page))
1948 return 1; 1948 return 1;
1949 } 1949 }
1950 1950
1951 if (PCI_DMA_BUS_IS_PHYS) { 1951 if (PCI_DMA_BUS_IS_PHYS) {
1952 struct device *pdev = dev->dev.parent; 1952 struct device *pdev = dev->dev.parent;
1953 1953
1954 if (!pdev) 1954 if (!pdev)
1955 return 0; 1955 return 0;
1956 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1956 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1957 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page); 1957 dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
1958 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) 1958 if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
1959 return 1; 1959 return 1;
1960 } 1960 }
1961 } 1961 }
1962 #endif 1962 #endif
1963 return 0; 1963 return 0;
1964 } 1964 }
1965 1965
1966 struct dev_gso_cb { 1966 struct dev_gso_cb {
1967 void (*destructor)(struct sk_buff *skb); 1967 void (*destructor)(struct sk_buff *skb);
1968 }; 1968 };
1969 1969
1970 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb) 1970 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
1971 1971
1972 static void dev_gso_skb_destructor(struct sk_buff *skb) 1972 static void dev_gso_skb_destructor(struct sk_buff *skb)
1973 { 1973 {
1974 struct dev_gso_cb *cb; 1974 struct dev_gso_cb *cb;
1975 1975
1976 do { 1976 do {
1977 struct sk_buff *nskb = skb->next; 1977 struct sk_buff *nskb = skb->next;
1978 1978
1979 skb->next = nskb->next; 1979 skb->next = nskb->next;
1980 nskb->next = NULL; 1980 nskb->next = NULL;
1981 kfree_skb(nskb); 1981 kfree_skb(nskb);
1982 } while (skb->next); 1982 } while (skb->next);
1983 1983
1984 cb = DEV_GSO_CB(skb); 1984 cb = DEV_GSO_CB(skb);
1985 if (cb->destructor) 1985 if (cb->destructor)
1986 cb->destructor(skb); 1986 cb->destructor(skb);
1987 } 1987 }
1988 1988
1989 /** 1989 /**
1990 * dev_gso_segment - Perform emulated hardware segmentation on skb. 1990 * dev_gso_segment - Perform emulated hardware segmentation on skb.
1991 * @skb: buffer to segment 1991 * @skb: buffer to segment
1992 * @features: device features as applicable to this skb 1992 * @features: device features as applicable to this skb
1993 * 1993 *
1994 * This function segments the given skb and stores the list of segments 1994 * This function segments the given skb and stores the list of segments
1995 * in skb->next. 1995 * in skb->next.
1996 */ 1996 */
1997 static int dev_gso_segment(struct sk_buff *skb, int features) 1997 static int dev_gso_segment(struct sk_buff *skb, int features)
1998 { 1998 {
1999 struct sk_buff *segs; 1999 struct sk_buff *segs;
2000 2000
2001 segs = skb_gso_segment(skb, features); 2001 segs = skb_gso_segment(skb, features);
2002 2002
2003 /* Verifying header integrity only. */ 2003 /* Verifying header integrity only. */
2004 if (!segs) 2004 if (!segs)
2005 return 0; 2005 return 0;
2006 2006
2007 if (IS_ERR(segs)) 2007 if (IS_ERR(segs))
2008 return PTR_ERR(segs); 2008 return PTR_ERR(segs);
2009 2009
2010 skb->next = segs; 2010 skb->next = segs;
2011 DEV_GSO_CB(skb)->destructor = skb->destructor; 2011 DEV_GSO_CB(skb)->destructor = skb->destructor;
2012 skb->destructor = dev_gso_skb_destructor; 2012 skb->destructor = dev_gso_skb_destructor;
2013 2013
2014 return 0; 2014 return 0;
2015 } 2015 }
2016 2016
2017 /* 2017 /*
2018 * Try to orphan skb early, right before transmission by the device. 2018 * Try to orphan skb early, right before transmission by the device.
2019 * We cannot orphan skb if tx timestamp is requested or the sk-reference 2019 * We cannot orphan skb if tx timestamp is requested or the sk-reference
2020 * is needed on driver level for other reasons, e.g. see net/can/raw.c 2020 * is needed on driver level for other reasons, e.g. see net/can/raw.c
2021 */ 2021 */
2022 static inline void skb_orphan_try(struct sk_buff *skb) 2022 static inline void skb_orphan_try(struct sk_buff *skb)
2023 { 2023 {
2024 struct sock *sk = skb->sk; 2024 struct sock *sk = skb->sk;
2025 2025
2026 if (sk && !skb_shinfo(skb)->tx_flags) { 2026 if (sk && !skb_shinfo(skb)->tx_flags) {
2027 /* skb_tx_hash() wont be able to get sk. 2027 /* skb_tx_hash() wont be able to get sk.
2028 * We copy sk_hash into skb->rxhash 2028 * We copy sk_hash into skb->rxhash
2029 */ 2029 */
2030 if (!skb->rxhash) 2030 if (!skb->rxhash)
2031 skb->rxhash = sk->sk_hash; 2031 skb->rxhash = sk->sk_hash;
2032 skb_orphan(skb); 2032 skb_orphan(skb);
2033 } 2033 }
2034 } 2034 }
2035 2035
2036 static bool can_checksum_protocol(unsigned long features, __be16 protocol) 2036 static bool can_checksum_protocol(unsigned long features, __be16 protocol)
2037 { 2037 {
2038 return ((features & NETIF_F_GEN_CSUM) || 2038 return ((features & NETIF_F_GEN_CSUM) ||
2039 ((features & NETIF_F_V4_CSUM) && 2039 ((features & NETIF_F_V4_CSUM) &&
2040 protocol == htons(ETH_P_IP)) || 2040 protocol == htons(ETH_P_IP)) ||
2041 ((features & NETIF_F_V6_CSUM) && 2041 ((features & NETIF_F_V6_CSUM) &&
2042 protocol == htons(ETH_P_IPV6)) || 2042 protocol == htons(ETH_P_IPV6)) ||
2043 ((features & NETIF_F_FCOE_CRC) && 2043 ((features & NETIF_F_FCOE_CRC) &&
2044 protocol == htons(ETH_P_FCOE))); 2044 protocol == htons(ETH_P_FCOE)));
2045 } 2045 }
2046 2046
2047 static int harmonize_features(struct sk_buff *skb, __be16 protocol, int features) 2047 static int harmonize_features(struct sk_buff *skb, __be16 protocol, int features)
2048 { 2048 {
2049 if (!can_checksum_protocol(protocol, features)) { 2049 if (!can_checksum_protocol(protocol, features)) {
2050 features &= ~NETIF_F_ALL_CSUM; 2050 features &= ~NETIF_F_ALL_CSUM;
2051 features &= ~NETIF_F_SG; 2051 features &= ~NETIF_F_SG;
2052 } else if (illegal_highdma(skb->dev, skb)) { 2052 } else if (illegal_highdma(skb->dev, skb)) {
2053 features &= ~NETIF_F_SG; 2053 features &= ~NETIF_F_SG;
2054 } 2054 }
2055 2055
2056 return features; 2056 return features;
2057 } 2057 }
2058 2058
2059 int netif_skb_features(struct sk_buff *skb) 2059 int netif_skb_features(struct sk_buff *skb)
2060 { 2060 {
2061 __be16 protocol = skb->protocol; 2061 __be16 protocol = skb->protocol;
2062 int features = skb->dev->features; 2062 int features = skb->dev->features;
2063 2063
2064 if (protocol == htons(ETH_P_8021Q)) { 2064 if (protocol == htons(ETH_P_8021Q)) {
2065 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 2065 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
2066 protocol = veh->h_vlan_encapsulated_proto; 2066 protocol = veh->h_vlan_encapsulated_proto;
2067 } else if (!vlan_tx_tag_present(skb)) { 2067 } else if (!vlan_tx_tag_present(skb)) {
2068 return harmonize_features(skb, protocol, features); 2068 return harmonize_features(skb, protocol, features);
2069 } 2069 }
2070 2070
2071 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX); 2071 features &= (skb->dev->vlan_features | NETIF_F_HW_VLAN_TX);
2072 2072
2073 if (protocol != htons(ETH_P_8021Q)) { 2073 if (protocol != htons(ETH_P_8021Q)) {
2074 return harmonize_features(skb, protocol, features); 2074 return harmonize_features(skb, protocol, features);
2075 } else { 2075 } else {
2076 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | 2076 features &= NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST |
2077 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX; 2077 NETIF_F_GEN_CSUM | NETIF_F_HW_VLAN_TX;
2078 return harmonize_features(skb, protocol, features); 2078 return harmonize_features(skb, protocol, features);
2079 } 2079 }
2080 } 2080 }
2081 EXPORT_SYMBOL(netif_skb_features); 2081 EXPORT_SYMBOL(netif_skb_features);
2082 2082
2083 /* 2083 /*
2084 * Returns true if either: 2084 * Returns true if either:
2085 * 1. skb has frag_list and the device doesn't support FRAGLIST, or 2085 * 1. skb has frag_list and the device doesn't support FRAGLIST, or
2086 * 2. skb is fragmented and the device does not support SG, or if 2086 * 2. skb is fragmented and the device does not support SG, or if
2087 * at least one of fragments is in highmem and device does not 2087 * at least one of fragments is in highmem and device does not
2088 * support DMA from it. 2088 * support DMA from it.
2089 */ 2089 */
2090 static inline int skb_needs_linearize(struct sk_buff *skb, 2090 static inline int skb_needs_linearize(struct sk_buff *skb,
2091 int features) 2091 int features)
2092 { 2092 {
2093 return skb_is_nonlinear(skb) && 2093 return skb_is_nonlinear(skb) &&
2094 ((skb_has_frag_list(skb) && 2094 ((skb_has_frag_list(skb) &&
2095 !(features & NETIF_F_FRAGLIST)) || 2095 !(features & NETIF_F_FRAGLIST)) ||
2096 (skb_shinfo(skb)->nr_frags && 2096 (skb_shinfo(skb)->nr_frags &&
2097 !(features & NETIF_F_SG))); 2097 !(features & NETIF_F_SG)));
2098 } 2098 }
2099 2099
2100 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev, 2100 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
2101 struct netdev_queue *txq) 2101 struct netdev_queue *txq)
2102 { 2102 {
2103 const struct net_device_ops *ops = dev->netdev_ops; 2103 const struct net_device_ops *ops = dev->netdev_ops;
2104 int rc = NETDEV_TX_OK; 2104 int rc = NETDEV_TX_OK;
2105 2105
2106 if (likely(!skb->next)) { 2106 if (likely(!skb->next)) {
2107 int features; 2107 int features;
2108 2108
2109 /* 2109 /*
2110 * If device doesnt need skb->dst, release it right now while 2110 * If device doesnt need skb->dst, release it right now while
2111 * its hot in this cpu cache 2111 * its hot in this cpu cache
2112 */ 2112 */
2113 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 2113 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2114 skb_dst_drop(skb); 2114 skb_dst_drop(skb);
2115 2115
2116 if (!list_empty(&ptype_all)) 2116 if (!list_empty(&ptype_all))
2117 dev_queue_xmit_nit(skb, dev); 2117 dev_queue_xmit_nit(skb, dev);
2118 2118
2119 skb_orphan_try(skb); 2119 skb_orphan_try(skb);
2120 2120
2121 features = netif_skb_features(skb); 2121 features = netif_skb_features(skb);
2122 2122
2123 if (vlan_tx_tag_present(skb) && 2123 if (vlan_tx_tag_present(skb) &&
2124 !(features & NETIF_F_HW_VLAN_TX)) { 2124 !(features & NETIF_F_HW_VLAN_TX)) {
2125 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb)); 2125 skb = __vlan_put_tag(skb, vlan_tx_tag_get(skb));
2126 if (unlikely(!skb)) 2126 if (unlikely(!skb))
2127 goto out; 2127 goto out;
2128 2128
2129 skb->vlan_tci = 0; 2129 skb->vlan_tci = 0;
2130 } 2130 }
2131 2131
2132 if (netif_needs_gso(skb, features)) { 2132 if (netif_needs_gso(skb, features)) {
2133 if (unlikely(dev_gso_segment(skb, features))) 2133 if (unlikely(dev_gso_segment(skb, features)))
2134 goto out_kfree_skb; 2134 goto out_kfree_skb;
2135 if (skb->next) 2135 if (skb->next)
2136 goto gso; 2136 goto gso;
2137 } else { 2137 } else {
2138 if (skb_needs_linearize(skb, features) && 2138 if (skb_needs_linearize(skb, features) &&
2139 __skb_linearize(skb)) 2139 __skb_linearize(skb))
2140 goto out_kfree_skb; 2140 goto out_kfree_skb;
2141 2141
2142 /* If packet is not checksummed and device does not 2142 /* If packet is not checksummed and device does not
2143 * support checksumming for this protocol, complete 2143 * support checksumming for this protocol, complete
2144 * checksumming here. 2144 * checksumming here.
2145 */ 2145 */
2146 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2146 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2147 skb_set_transport_header(skb, 2147 skb_set_transport_header(skb,
2148 skb_checksum_start_offset(skb)); 2148 skb_checksum_start_offset(skb));
2149 if (!(features & NETIF_F_ALL_CSUM) && 2149 if (!(features & NETIF_F_ALL_CSUM) &&
2150 skb_checksum_help(skb)) 2150 skb_checksum_help(skb))
2151 goto out_kfree_skb; 2151 goto out_kfree_skb;
2152 } 2152 }
2153 } 2153 }
2154 2154
2155 rc = ops->ndo_start_xmit(skb, dev); 2155 rc = ops->ndo_start_xmit(skb, dev);
2156 trace_net_dev_xmit(skb, rc); 2156 trace_net_dev_xmit(skb, rc);
2157 if (rc == NETDEV_TX_OK) 2157 if (rc == NETDEV_TX_OK)
2158 txq_trans_update(txq); 2158 txq_trans_update(txq);
2159 return rc; 2159 return rc;
2160 } 2160 }
2161 2161
2162 gso: 2162 gso:
2163 do { 2163 do {
2164 struct sk_buff *nskb = skb->next; 2164 struct sk_buff *nskb = skb->next;
2165 2165
2166 skb->next = nskb->next; 2166 skb->next = nskb->next;
2167 nskb->next = NULL; 2167 nskb->next = NULL;
2168 2168
2169 /* 2169 /*
2170 * If device doesnt need nskb->dst, release it right now while 2170 * If device doesnt need nskb->dst, release it right now while
2171 * its hot in this cpu cache 2171 * its hot in this cpu cache
2172 */ 2172 */
2173 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 2173 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
2174 skb_dst_drop(nskb); 2174 skb_dst_drop(nskb);
2175 2175
2176 rc = ops->ndo_start_xmit(nskb, dev); 2176 rc = ops->ndo_start_xmit(nskb, dev);
2177 trace_net_dev_xmit(nskb, rc); 2177 trace_net_dev_xmit(nskb, rc);
2178 if (unlikely(rc != NETDEV_TX_OK)) { 2178 if (unlikely(rc != NETDEV_TX_OK)) {
2179 if (rc & ~NETDEV_TX_MASK) 2179 if (rc & ~NETDEV_TX_MASK)
2180 goto out_kfree_gso_skb; 2180 goto out_kfree_gso_skb;
2181 nskb->next = skb->next; 2181 nskb->next = skb->next;
2182 skb->next = nskb; 2182 skb->next = nskb;
2183 return rc; 2183 return rc;
2184 } 2184 }
2185 txq_trans_update(txq); 2185 txq_trans_update(txq);
2186 if (unlikely(netif_tx_queue_stopped(txq) && skb->next)) 2186 if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
2187 return NETDEV_TX_BUSY; 2187 return NETDEV_TX_BUSY;
2188 } while (skb->next); 2188 } while (skb->next);
2189 2189
2190 out_kfree_gso_skb: 2190 out_kfree_gso_skb:
2191 if (likely(skb->next == NULL)) 2191 if (likely(skb->next == NULL))
2192 skb->destructor = DEV_GSO_CB(skb)->destructor; 2192 skb->destructor = DEV_GSO_CB(skb)->destructor;
2193 out_kfree_skb: 2193 out_kfree_skb:
2194 kfree_skb(skb); 2194 kfree_skb(skb);
2195 out: 2195 out:
2196 return rc; 2196 return rc;
2197 } 2197 }
2198 2198
2199 static u32 hashrnd __read_mostly; 2199 static u32 hashrnd __read_mostly;
2200 2200
2201 /* 2201 /*
2202 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 2202 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2203 * to be used as a distribution range. 2203 * to be used as a distribution range.
2204 */ 2204 */
2205 u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb, 2205 u16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb,
2206 unsigned int num_tx_queues) 2206 unsigned int num_tx_queues)
2207 { 2207 {
2208 u32 hash; 2208 u32 hash;
2209 u16 qoffset = 0; 2209 u16 qoffset = 0;
2210 u16 qcount = num_tx_queues; 2210 u16 qcount = num_tx_queues;
2211 2211
2212 if (skb_rx_queue_recorded(skb)) { 2212 if (skb_rx_queue_recorded(skb)) {
2213 hash = skb_get_rx_queue(skb); 2213 hash = skb_get_rx_queue(skb);
2214 while (unlikely(hash >= num_tx_queues)) 2214 while (unlikely(hash >= num_tx_queues))
2215 hash -= num_tx_queues; 2215 hash -= num_tx_queues;
2216 return hash; 2216 return hash;
2217 } 2217 }
2218 2218
2219 if (dev->num_tc) { 2219 if (dev->num_tc) {
2220 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 2220 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2221 qoffset = dev->tc_to_txq[tc].offset; 2221 qoffset = dev->tc_to_txq[tc].offset;
2222 qcount = dev->tc_to_txq[tc].count; 2222 qcount = dev->tc_to_txq[tc].count;
2223 } 2223 }
2224 2224
2225 if (skb->sk && skb->sk->sk_hash) 2225 if (skb->sk && skb->sk->sk_hash)
2226 hash = skb->sk->sk_hash; 2226 hash = skb->sk->sk_hash;
2227 else 2227 else
2228 hash = (__force u16) skb->protocol ^ skb->rxhash; 2228 hash = (__force u16) skb->protocol ^ skb->rxhash;
2229 hash = jhash_1word(hash, hashrnd); 2229 hash = jhash_1word(hash, hashrnd);
2230 2230
2231 return (u16) (((u64) hash * qcount) >> 32) + qoffset; 2231 return (u16) (((u64) hash * qcount) >> 32) + qoffset;
2232 } 2232 }
2233 EXPORT_SYMBOL(__skb_tx_hash); 2233 EXPORT_SYMBOL(__skb_tx_hash);
2234 2234
2235 static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index) 2235 static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
2236 { 2236 {
2237 if (unlikely(queue_index >= dev->real_num_tx_queues)) { 2237 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
2238 if (net_ratelimit()) { 2238 if (net_ratelimit()) {
2239 pr_warning("%s selects TX queue %d, but " 2239 pr_warning("%s selects TX queue %d, but "
2240 "real number of TX queues is %d\n", 2240 "real number of TX queues is %d\n",
2241 dev->name, queue_index, dev->real_num_tx_queues); 2241 dev->name, queue_index, dev->real_num_tx_queues);
2242 } 2242 }
2243 return 0; 2243 return 0;
2244 } 2244 }
2245 return queue_index; 2245 return queue_index;
2246 } 2246 }
2247 2247
2248 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 2248 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
2249 { 2249 {
2250 #ifdef CONFIG_XPS 2250 #ifdef CONFIG_XPS
2251 struct xps_dev_maps *dev_maps; 2251 struct xps_dev_maps *dev_maps;
2252 struct xps_map *map; 2252 struct xps_map *map;
2253 int queue_index = -1; 2253 int queue_index = -1;
2254 2254
2255 rcu_read_lock(); 2255 rcu_read_lock();
2256 dev_maps = rcu_dereference(dev->xps_maps); 2256 dev_maps = rcu_dereference(dev->xps_maps);
2257 if (dev_maps) { 2257 if (dev_maps) {
2258 map = rcu_dereference( 2258 map = rcu_dereference(
2259 dev_maps->cpu_map[raw_smp_processor_id()]); 2259 dev_maps->cpu_map[raw_smp_processor_id()]);
2260 if (map) { 2260 if (map) {
2261 if (map->len == 1) 2261 if (map->len == 1)
2262 queue_index = map->queues[0]; 2262 queue_index = map->queues[0];
2263 else { 2263 else {
2264 u32 hash; 2264 u32 hash;
2265 if (skb->sk && skb->sk->sk_hash) 2265 if (skb->sk && skb->sk->sk_hash)
2266 hash = skb->sk->sk_hash; 2266 hash = skb->sk->sk_hash;
2267 else 2267 else
2268 hash = (__force u16) skb->protocol ^ 2268 hash = (__force u16) skb->protocol ^
2269 skb->rxhash; 2269 skb->rxhash;
2270 hash = jhash_1word(hash, hashrnd); 2270 hash = jhash_1word(hash, hashrnd);
2271 queue_index = map->queues[ 2271 queue_index = map->queues[
2272 ((u64)hash * map->len) >> 32]; 2272 ((u64)hash * map->len) >> 32];
2273 } 2273 }
2274 if (unlikely(queue_index >= dev->real_num_tx_queues)) 2274 if (unlikely(queue_index >= dev->real_num_tx_queues))
2275 queue_index = -1; 2275 queue_index = -1;
2276 } 2276 }
2277 } 2277 }
2278 rcu_read_unlock(); 2278 rcu_read_unlock();
2279 2279
2280 return queue_index; 2280 return queue_index;
2281 #else 2281 #else
2282 return -1; 2282 return -1;
2283 #endif 2283 #endif
2284 } 2284 }
2285 2285
2286 static struct netdev_queue *dev_pick_tx(struct net_device *dev, 2286 static struct netdev_queue *dev_pick_tx(struct net_device *dev,
2287 struct sk_buff *skb) 2287 struct sk_buff *skb)
2288 { 2288 {
2289 int queue_index; 2289 int queue_index;
2290 const struct net_device_ops *ops = dev->netdev_ops; 2290 const struct net_device_ops *ops = dev->netdev_ops;
2291 2291
2292 if (dev->real_num_tx_queues == 1) 2292 if (dev->real_num_tx_queues == 1)
2293 queue_index = 0; 2293 queue_index = 0;
2294 else if (ops->ndo_select_queue) { 2294 else if (ops->ndo_select_queue) {
2295 queue_index = ops->ndo_select_queue(dev, skb); 2295 queue_index = ops->ndo_select_queue(dev, skb);
2296 queue_index = dev_cap_txqueue(dev, queue_index); 2296 queue_index = dev_cap_txqueue(dev, queue_index);
2297 } else { 2297 } else {
2298 struct sock *sk = skb->sk; 2298 struct sock *sk = skb->sk;
2299 queue_index = sk_tx_queue_get(sk); 2299 queue_index = sk_tx_queue_get(sk);
2300 2300
2301 if (queue_index < 0 || skb->ooo_okay || 2301 if (queue_index < 0 || skb->ooo_okay ||
2302 queue_index >= dev->real_num_tx_queues) { 2302 queue_index >= dev->real_num_tx_queues) {
2303 int old_index = queue_index; 2303 int old_index = queue_index;
2304 2304
2305 queue_index = get_xps_queue(dev, skb); 2305 queue_index = get_xps_queue(dev, skb);
2306 if (queue_index < 0) 2306 if (queue_index < 0)
2307 queue_index = skb_tx_hash(dev, skb); 2307 queue_index = skb_tx_hash(dev, skb);
2308 2308
2309 if (queue_index != old_index && sk) { 2309 if (queue_index != old_index && sk) {
2310 struct dst_entry *dst = 2310 struct dst_entry *dst =
2311 rcu_dereference_check(sk->sk_dst_cache, 1); 2311 rcu_dereference_check(sk->sk_dst_cache, 1);
2312 2312
2313 if (dst && skb_dst(skb) == dst) 2313 if (dst && skb_dst(skb) == dst)
2314 sk_tx_queue_set(sk, queue_index); 2314 sk_tx_queue_set(sk, queue_index);
2315 } 2315 }
2316 } 2316 }
2317 } 2317 }
2318 2318
2319 skb_set_queue_mapping(skb, queue_index); 2319 skb_set_queue_mapping(skb, queue_index);
2320 return netdev_get_tx_queue(dev, queue_index); 2320 return netdev_get_tx_queue(dev, queue_index);
2321 } 2321 }
2322 2322
2323 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 2323 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
2324 struct net_device *dev, 2324 struct net_device *dev,
2325 struct netdev_queue *txq) 2325 struct netdev_queue *txq)
2326 { 2326 {
2327 spinlock_t *root_lock = qdisc_lock(q); 2327 spinlock_t *root_lock = qdisc_lock(q);
2328 bool contended = qdisc_is_running(q); 2328 bool contended;
2329 int rc; 2329 int rc;
2330 2330
2331 qdisc_skb_cb(skb)->pkt_len = skb->len;
2332 qdisc_calculate_pkt_len(skb, q);
2331 /* 2333 /*
2332 * Heuristic to force contended enqueues to serialize on a 2334 * Heuristic to force contended enqueues to serialize on a
2333 * separate lock before trying to get qdisc main lock. 2335 * separate lock before trying to get qdisc main lock.
2334 * This permits __QDISC_STATE_RUNNING owner to get the lock more often 2336 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
2335 * and dequeue packets faster. 2337 * and dequeue packets faster.
2336 */ 2338 */
2339 contended = qdisc_is_running(q);
2337 if (unlikely(contended)) 2340 if (unlikely(contended))
2338 spin_lock(&q->busylock); 2341 spin_lock(&q->busylock);
2339 2342
2340 spin_lock(root_lock); 2343 spin_lock(root_lock);
2341 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 2344 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
2342 kfree_skb(skb); 2345 kfree_skb(skb);
2343 rc = NET_XMIT_DROP; 2346 rc = NET_XMIT_DROP;
2344 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 2347 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
2345 qdisc_run_begin(q)) { 2348 qdisc_run_begin(q)) {
2346 /* 2349 /*
2347 * This is a work-conserving queue; there are no old skbs 2350 * This is a work-conserving queue; there are no old skbs
2348 * waiting to be sent out; and the qdisc is not running - 2351 * waiting to be sent out; and the qdisc is not running -
2349 * xmit the skb directly. 2352 * xmit the skb directly.
2350 */ 2353 */
2351 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE)) 2354 if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
2352 skb_dst_force(skb); 2355 skb_dst_force(skb);
2353 2356
2354 qdisc_skb_cb(skb)->pkt_len = skb->len;
2355 qdisc_bstats_update(q, skb); 2357 qdisc_bstats_update(q, skb);
2356 2358
2357 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) { 2359 if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
2358 if (unlikely(contended)) { 2360 if (unlikely(contended)) {
2359 spin_unlock(&q->busylock); 2361 spin_unlock(&q->busylock);
2360 contended = false; 2362 contended = false;
2361 } 2363 }
2362 __qdisc_run(q); 2364 __qdisc_run(q);
2363 } else 2365 } else
2364 qdisc_run_end(q); 2366 qdisc_run_end(q);
2365 2367
2366 rc = NET_XMIT_SUCCESS; 2368 rc = NET_XMIT_SUCCESS;
2367 } else { 2369 } else {
2368 skb_dst_force(skb); 2370 skb_dst_force(skb);
2369 rc = qdisc_enqueue_root(skb, q); 2371 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
2370 if (qdisc_run_begin(q)) { 2372 if (qdisc_run_begin(q)) {
2371 if (unlikely(contended)) { 2373 if (unlikely(contended)) {
2372 spin_unlock(&q->busylock); 2374 spin_unlock(&q->busylock);
2373 contended = false; 2375 contended = false;
2374 } 2376 }
2375 __qdisc_run(q); 2377 __qdisc_run(q);
2376 } 2378 }
2377 } 2379 }
2378 spin_unlock(root_lock); 2380 spin_unlock(root_lock);
2379 if (unlikely(contended)) 2381 if (unlikely(contended))
2380 spin_unlock(&q->busylock); 2382 spin_unlock(&q->busylock);
2381 return rc; 2383 return rc;
2382 } 2384 }
2383 2385
2384 static DEFINE_PER_CPU(int, xmit_recursion); 2386 static DEFINE_PER_CPU(int, xmit_recursion);
2385 #define RECURSION_LIMIT 10 2387 #define RECURSION_LIMIT 10
2386 2388
2387 /** 2389 /**
2388 * dev_queue_xmit - transmit a buffer 2390 * dev_queue_xmit - transmit a buffer
2389 * @skb: buffer to transmit 2391 * @skb: buffer to transmit
2390 * 2392 *
2391 * Queue a buffer for transmission to a network device. The caller must 2393 * Queue a buffer for transmission to a network device. The caller must
2392 * have set the device and priority and built the buffer before calling 2394 * have set the device and priority and built the buffer before calling
2393 * this function. The function can be called from an interrupt. 2395 * this function. The function can be called from an interrupt.
2394 * 2396 *
2395 * A negative errno code is returned on a failure. A success does not 2397 * A negative errno code is returned on a failure. A success does not
2396 * guarantee the frame will be transmitted as it may be dropped due 2398 * guarantee the frame will be transmitted as it may be dropped due
2397 * to congestion or traffic shaping. 2399 * to congestion or traffic shaping.
2398 * 2400 *
2399 * ----------------------------------------------------------------------------------- 2401 * -----------------------------------------------------------------------------------
2400 * I notice this method can also return errors from the queue disciplines, 2402 * I notice this method can also return errors from the queue disciplines,
2401 * including NET_XMIT_DROP, which is a positive value. So, errors can also 2403 * including NET_XMIT_DROP, which is a positive value. So, errors can also
2402 * be positive. 2404 * be positive.
2403 * 2405 *
2404 * Regardless of the return value, the skb is consumed, so it is currently 2406 * Regardless of the return value, the skb is consumed, so it is currently
2405 * difficult to retry a send to this method. (You can bump the ref count 2407 * difficult to retry a send to this method. (You can bump the ref count
2406 * before sending to hold a reference for retry if you are careful.) 2408 * before sending to hold a reference for retry if you are careful.)
2407 * 2409 *
2408 * When calling this method, interrupts MUST be enabled. This is because 2410 * When calling this method, interrupts MUST be enabled. This is because
2409 * the BH enable code must have IRQs enabled so that it will not deadlock. 2411 * the BH enable code must have IRQs enabled so that it will not deadlock.
2410 * --BLG 2412 * --BLG
2411 */ 2413 */
2412 int dev_queue_xmit(struct sk_buff *skb) 2414 int dev_queue_xmit(struct sk_buff *skb)
2413 { 2415 {
2414 struct net_device *dev = skb->dev; 2416 struct net_device *dev = skb->dev;
2415 struct netdev_queue *txq; 2417 struct netdev_queue *txq;
2416 struct Qdisc *q; 2418 struct Qdisc *q;
2417 int rc = -ENOMEM; 2419 int rc = -ENOMEM;
2418 2420
2419 /* Disable soft irqs for various locks below. Also 2421 /* Disable soft irqs for various locks below. Also
2420 * stops preemption for RCU. 2422 * stops preemption for RCU.
2421 */ 2423 */
2422 rcu_read_lock_bh(); 2424 rcu_read_lock_bh();
2423 2425
2424 txq = dev_pick_tx(dev, skb); 2426 txq = dev_pick_tx(dev, skb);
2425 q = rcu_dereference_bh(txq->qdisc); 2427 q = rcu_dereference_bh(txq->qdisc);
2426 2428
2427 #ifdef CONFIG_NET_CLS_ACT 2429 #ifdef CONFIG_NET_CLS_ACT
2428 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); 2430 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
2429 #endif 2431 #endif
2430 trace_net_dev_queue(skb); 2432 trace_net_dev_queue(skb);
2431 if (q->enqueue) { 2433 if (q->enqueue) {
2432 rc = __dev_xmit_skb(skb, q, dev, txq); 2434 rc = __dev_xmit_skb(skb, q, dev, txq);
2433 goto out; 2435 goto out;
2434 } 2436 }
2435 2437
2436 /* The device has no queue. Common case for software devices: 2438 /* The device has no queue. Common case for software devices:
2437 loopback, all the sorts of tunnels... 2439 loopback, all the sorts of tunnels...
2438 2440
2439 Really, it is unlikely that netif_tx_lock protection is necessary 2441 Really, it is unlikely that netif_tx_lock protection is necessary
2440 here. (f.e. loopback and IP tunnels are clean ignoring statistics 2442 here. (f.e. loopback and IP tunnels are clean ignoring statistics
2441 counters.) 2443 counters.)
2442 However, it is possible, that they rely on protection 2444 However, it is possible, that they rely on protection
2443 made by us here. 2445 made by us here.
2444 2446
2445 Check this and shot the lock. It is not prone from deadlocks. 2447 Check this and shot the lock. It is not prone from deadlocks.
2446 Either shot noqueue qdisc, it is even simpler 8) 2448 Either shot noqueue qdisc, it is even simpler 8)
2447 */ 2449 */
2448 if (dev->flags & IFF_UP) { 2450 if (dev->flags & IFF_UP) {
2449 int cpu = smp_processor_id(); /* ok because BHs are off */ 2451 int cpu = smp_processor_id(); /* ok because BHs are off */
2450 2452
2451 if (txq->xmit_lock_owner != cpu) { 2453 if (txq->xmit_lock_owner != cpu) {
2452 2454
2453 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT) 2455 if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
2454 goto recursion_alert; 2456 goto recursion_alert;
2455 2457
2456 HARD_TX_LOCK(dev, txq, cpu); 2458 HARD_TX_LOCK(dev, txq, cpu);
2457 2459
2458 if (!netif_tx_queue_stopped(txq)) { 2460 if (!netif_tx_queue_stopped(txq)) {
2459 __this_cpu_inc(xmit_recursion); 2461 __this_cpu_inc(xmit_recursion);
2460 rc = dev_hard_start_xmit(skb, dev, txq); 2462 rc = dev_hard_start_xmit(skb, dev, txq);
2461 __this_cpu_dec(xmit_recursion); 2463 __this_cpu_dec(xmit_recursion);
2462 if (dev_xmit_complete(rc)) { 2464 if (dev_xmit_complete(rc)) {
2463 HARD_TX_UNLOCK(dev, txq); 2465 HARD_TX_UNLOCK(dev, txq);
2464 goto out; 2466 goto out;
2465 } 2467 }
2466 } 2468 }
2467 HARD_TX_UNLOCK(dev, txq); 2469 HARD_TX_UNLOCK(dev, txq);
2468 if (net_ratelimit()) 2470 if (net_ratelimit())
2469 printk(KERN_CRIT "Virtual device %s asks to " 2471 printk(KERN_CRIT "Virtual device %s asks to "
2470 "queue packet!\n", dev->name); 2472 "queue packet!\n", dev->name);
2471 } else { 2473 } else {
2472 /* Recursion is detected! It is possible, 2474 /* Recursion is detected! It is possible,
2473 * unfortunately 2475 * unfortunately
2474 */ 2476 */
2475 recursion_alert: 2477 recursion_alert:
2476 if (net_ratelimit()) 2478 if (net_ratelimit())
2477 printk(KERN_CRIT "Dead loop on virtual device " 2479 printk(KERN_CRIT "Dead loop on virtual device "
2478 "%s, fix it urgently!\n", dev->name); 2480 "%s, fix it urgently!\n", dev->name);
2479 } 2481 }
2480 } 2482 }
2481 2483
2482 rc = -ENETDOWN; 2484 rc = -ENETDOWN;
2483 rcu_read_unlock_bh(); 2485 rcu_read_unlock_bh();
2484 2486
2485 kfree_skb(skb); 2487 kfree_skb(skb);
2486 return rc; 2488 return rc;
2487 out: 2489 out:
2488 rcu_read_unlock_bh(); 2490 rcu_read_unlock_bh();
2489 return rc; 2491 return rc;
2490 } 2492 }
2491 EXPORT_SYMBOL(dev_queue_xmit); 2493 EXPORT_SYMBOL(dev_queue_xmit);
2492 2494
2493 2495
2494 /*======================================================================= 2496 /*=======================================================================
2495 Receiver routines 2497 Receiver routines
2496 =======================================================================*/ 2498 =======================================================================*/
2497 2499
2498 int netdev_max_backlog __read_mostly = 1000; 2500 int netdev_max_backlog __read_mostly = 1000;
2499 int netdev_tstamp_prequeue __read_mostly = 1; 2501 int netdev_tstamp_prequeue __read_mostly = 1;
2500 int netdev_budget __read_mostly = 300; 2502 int netdev_budget __read_mostly = 300;
2501 int weight_p __read_mostly = 64; /* old backlog weight */ 2503 int weight_p __read_mostly = 64; /* old backlog weight */
2502 2504
2503 /* Called with irq disabled */ 2505 /* Called with irq disabled */
2504 static inline void ____napi_schedule(struct softnet_data *sd, 2506 static inline void ____napi_schedule(struct softnet_data *sd,
2505 struct napi_struct *napi) 2507 struct napi_struct *napi)
2506 { 2508 {
2507 list_add_tail(&napi->poll_list, &sd->poll_list); 2509 list_add_tail(&napi->poll_list, &sd->poll_list);
2508 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 2510 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2509 } 2511 }
2510 2512
2511 /* 2513 /*
2512 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses 2514 * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
2513 * and src/dst port numbers. Returns a non-zero hash number on success 2515 * and src/dst port numbers. Returns a non-zero hash number on success
2514 * and 0 on failure. 2516 * and 0 on failure.
2515 */ 2517 */
2516 __u32 __skb_get_rxhash(struct sk_buff *skb) 2518 __u32 __skb_get_rxhash(struct sk_buff *skb)
2517 { 2519 {
2518 int nhoff, hash = 0, poff; 2520 int nhoff, hash = 0, poff;
2519 struct ipv6hdr *ip6; 2521 struct ipv6hdr *ip6;
2520 struct iphdr *ip; 2522 struct iphdr *ip;
2521 u8 ip_proto; 2523 u8 ip_proto;
2522 u32 addr1, addr2, ihl; 2524 u32 addr1, addr2, ihl;
2523 union { 2525 union {
2524 u32 v32; 2526 u32 v32;
2525 u16 v16[2]; 2527 u16 v16[2];
2526 } ports; 2528 } ports;
2527 2529
2528 nhoff = skb_network_offset(skb); 2530 nhoff = skb_network_offset(skb);
2529 2531
2530 switch (skb->protocol) { 2532 switch (skb->protocol) {
2531 case __constant_htons(ETH_P_IP): 2533 case __constant_htons(ETH_P_IP):
2532 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff)) 2534 if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
2533 goto done; 2535 goto done;
2534 2536
2535 ip = (struct iphdr *) (skb->data + nhoff); 2537 ip = (struct iphdr *) (skb->data + nhoff);
2536 if (ip->frag_off & htons(IP_MF | IP_OFFSET)) 2538 if (ip->frag_off & htons(IP_MF | IP_OFFSET))
2537 ip_proto = 0; 2539 ip_proto = 0;
2538 else 2540 else
2539 ip_proto = ip->protocol; 2541 ip_proto = ip->protocol;
2540 addr1 = (__force u32) ip->saddr; 2542 addr1 = (__force u32) ip->saddr;
2541 addr2 = (__force u32) ip->daddr; 2543 addr2 = (__force u32) ip->daddr;
2542 ihl = ip->ihl; 2544 ihl = ip->ihl;
2543 break; 2545 break;
2544 case __constant_htons(ETH_P_IPV6): 2546 case __constant_htons(ETH_P_IPV6):
2545 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff)) 2547 if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
2546 goto done; 2548 goto done;
2547 2549
2548 ip6 = (struct ipv6hdr *) (skb->data + nhoff); 2550 ip6 = (struct ipv6hdr *) (skb->data + nhoff);
2549 ip_proto = ip6->nexthdr; 2551 ip_proto = ip6->nexthdr;
2550 addr1 = (__force u32) ip6->saddr.s6_addr32[3]; 2552 addr1 = (__force u32) ip6->saddr.s6_addr32[3];
2551 addr2 = (__force u32) ip6->daddr.s6_addr32[3]; 2553 addr2 = (__force u32) ip6->daddr.s6_addr32[3];
2552 ihl = (40 >> 2); 2554 ihl = (40 >> 2);
2553 break; 2555 break;
2554 default: 2556 default:
2555 goto done; 2557 goto done;
2556 } 2558 }
2557 2559
2558 ports.v32 = 0; 2560 ports.v32 = 0;
2559 poff = proto_ports_offset(ip_proto); 2561 poff = proto_ports_offset(ip_proto);
2560 if (poff >= 0) { 2562 if (poff >= 0) {
2561 nhoff += ihl * 4 + poff; 2563 nhoff += ihl * 4 + poff;
2562 if (pskb_may_pull(skb, nhoff + 4)) { 2564 if (pskb_may_pull(skb, nhoff + 4)) {
2563 ports.v32 = * (__force u32 *) (skb->data + nhoff); 2565 ports.v32 = * (__force u32 *) (skb->data + nhoff);
2564 if (ports.v16[1] < ports.v16[0]) 2566 if (ports.v16[1] < ports.v16[0])
2565 swap(ports.v16[0], ports.v16[1]); 2567 swap(ports.v16[0], ports.v16[1]);
2566 } 2568 }
2567 } 2569 }
2568 2570
2569 /* get a consistent hash (same value on both flow directions) */ 2571 /* get a consistent hash (same value on both flow directions) */
2570 if (addr2 < addr1) 2572 if (addr2 < addr1)
2571 swap(addr1, addr2); 2573 swap(addr1, addr2);
2572 2574
2573 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd); 2575 hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
2574 if (!hash) 2576 if (!hash)
2575 hash = 1; 2577 hash = 1;
2576 2578
2577 done: 2579 done:
2578 return hash; 2580 return hash;
2579 } 2581 }
2580 EXPORT_SYMBOL(__skb_get_rxhash); 2582 EXPORT_SYMBOL(__skb_get_rxhash);
2581 2583
2582 #ifdef CONFIG_RPS 2584 #ifdef CONFIG_RPS
2583 2585
2584 /* One global table that all flow-based protocols share. */ 2586 /* One global table that all flow-based protocols share. */
2585 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; 2587 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
2586 EXPORT_SYMBOL(rps_sock_flow_table); 2588 EXPORT_SYMBOL(rps_sock_flow_table);
2587 2589
2588 /* 2590 /*
2589 * get_rps_cpu is called from netif_receive_skb and returns the target 2591 * get_rps_cpu is called from netif_receive_skb and returns the target
2590 * CPU from the RPS map of the receiving queue for a given skb. 2592 * CPU from the RPS map of the receiving queue for a given skb.
2591 * rcu_read_lock must be held on entry. 2593 * rcu_read_lock must be held on entry.
2592 */ 2594 */
2593 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 2595 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
2594 struct rps_dev_flow **rflowp) 2596 struct rps_dev_flow **rflowp)
2595 { 2597 {
2596 struct netdev_rx_queue *rxqueue; 2598 struct netdev_rx_queue *rxqueue;
2597 struct rps_map *map; 2599 struct rps_map *map;
2598 struct rps_dev_flow_table *flow_table; 2600 struct rps_dev_flow_table *flow_table;
2599 struct rps_sock_flow_table *sock_flow_table; 2601 struct rps_sock_flow_table *sock_flow_table;
2600 int cpu = -1; 2602 int cpu = -1;
2601 u16 tcpu; 2603 u16 tcpu;
2602 2604
2603 if (skb_rx_queue_recorded(skb)) { 2605 if (skb_rx_queue_recorded(skb)) {
2604 u16 index = skb_get_rx_queue(skb); 2606 u16 index = skb_get_rx_queue(skb);
2605 if (unlikely(index >= dev->real_num_rx_queues)) { 2607 if (unlikely(index >= dev->real_num_rx_queues)) {
2606 WARN_ONCE(dev->real_num_rx_queues > 1, 2608 WARN_ONCE(dev->real_num_rx_queues > 1,
2607 "%s received packet on queue %u, but number " 2609 "%s received packet on queue %u, but number "
2608 "of RX queues is %u\n", 2610 "of RX queues is %u\n",
2609 dev->name, index, dev->real_num_rx_queues); 2611 dev->name, index, dev->real_num_rx_queues);
2610 goto done; 2612 goto done;
2611 } 2613 }
2612 rxqueue = dev->_rx + index; 2614 rxqueue = dev->_rx + index;
2613 } else 2615 } else
2614 rxqueue = dev->_rx; 2616 rxqueue = dev->_rx;
2615 2617
2616 map = rcu_dereference(rxqueue->rps_map); 2618 map = rcu_dereference(rxqueue->rps_map);
2617 if (map) { 2619 if (map) {
2618 if (map->len == 1) { 2620 if (map->len == 1) {
2619 tcpu = map->cpus[0]; 2621 tcpu = map->cpus[0];
2620 if (cpu_online(tcpu)) 2622 if (cpu_online(tcpu))
2621 cpu = tcpu; 2623 cpu = tcpu;
2622 goto done; 2624 goto done;
2623 } 2625 }
2624 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) { 2626 } else if (!rcu_dereference_raw(rxqueue->rps_flow_table)) {
2625 goto done; 2627 goto done;
2626 } 2628 }
2627 2629
2628 skb_reset_network_header(skb); 2630 skb_reset_network_header(skb);
2629 if (!skb_get_rxhash(skb)) 2631 if (!skb_get_rxhash(skb))
2630 goto done; 2632 goto done;
2631 2633
2632 flow_table = rcu_dereference(rxqueue->rps_flow_table); 2634 flow_table = rcu_dereference(rxqueue->rps_flow_table);
2633 sock_flow_table = rcu_dereference(rps_sock_flow_table); 2635 sock_flow_table = rcu_dereference(rps_sock_flow_table);
2634 if (flow_table && sock_flow_table) { 2636 if (flow_table && sock_flow_table) {
2635 u16 next_cpu; 2637 u16 next_cpu;
2636 struct rps_dev_flow *rflow; 2638 struct rps_dev_flow *rflow;
2637 2639
2638 rflow = &flow_table->flows[skb->rxhash & flow_table->mask]; 2640 rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
2639 tcpu = rflow->cpu; 2641 tcpu = rflow->cpu;
2640 2642
2641 next_cpu = sock_flow_table->ents[skb->rxhash & 2643 next_cpu = sock_flow_table->ents[skb->rxhash &
2642 sock_flow_table->mask]; 2644 sock_flow_table->mask];
2643 2645
2644 /* 2646 /*
2645 * If the desired CPU (where last recvmsg was done) is 2647 * If the desired CPU (where last recvmsg was done) is
2646 * different from current CPU (one in the rx-queue flow 2648 * different from current CPU (one in the rx-queue flow
2647 * table entry), switch if one of the following holds: 2649 * table entry), switch if one of the following holds:
2648 * - Current CPU is unset (equal to RPS_NO_CPU). 2650 * - Current CPU is unset (equal to RPS_NO_CPU).
2649 * - Current CPU is offline. 2651 * - Current CPU is offline.
2650 * - The current CPU's queue tail has advanced beyond the 2652 * - The current CPU's queue tail has advanced beyond the
2651 * last packet that was enqueued using this table entry. 2653 * last packet that was enqueued using this table entry.
2652 * This guarantees that all previous packets for the flow 2654 * This guarantees that all previous packets for the flow
2653 * have been dequeued, thus preserving in order delivery. 2655 * have been dequeued, thus preserving in order delivery.
2654 */ 2656 */
2655 if (unlikely(tcpu != next_cpu) && 2657 if (unlikely(tcpu != next_cpu) &&
2656 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) || 2658 (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
2657 ((int)(per_cpu(softnet_data, tcpu).input_queue_head - 2659 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
2658 rflow->last_qtail)) >= 0)) { 2660 rflow->last_qtail)) >= 0)) {
2659 tcpu = rflow->cpu = next_cpu; 2661 tcpu = rflow->cpu = next_cpu;
2660 if (tcpu != RPS_NO_CPU) 2662 if (tcpu != RPS_NO_CPU)
2661 rflow->last_qtail = per_cpu(softnet_data, 2663 rflow->last_qtail = per_cpu(softnet_data,
2662 tcpu).input_queue_head; 2664 tcpu).input_queue_head;
2663 } 2665 }
2664 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) { 2666 if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
2665 *rflowp = rflow; 2667 *rflowp = rflow;
2666 cpu = tcpu; 2668 cpu = tcpu;
2667 goto done; 2669 goto done;
2668 } 2670 }
2669 } 2671 }
2670 2672
2671 if (map) { 2673 if (map) {
2672 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32]; 2674 tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
2673 2675
2674 if (cpu_online(tcpu)) { 2676 if (cpu_online(tcpu)) {
2675 cpu = tcpu; 2677 cpu = tcpu;
2676 goto done; 2678 goto done;
2677 } 2679 }
2678 } 2680 }
2679 2681
2680 done: 2682 done:
2681 return cpu; 2683 return cpu;
2682 } 2684 }
2683 2685
2684 /* Called from hardirq (IPI) context */ 2686 /* Called from hardirq (IPI) context */
2685 static void rps_trigger_softirq(void *data) 2687 static void rps_trigger_softirq(void *data)
2686 { 2688 {
2687 struct softnet_data *sd = data; 2689 struct softnet_data *sd = data;
2688 2690
2689 ____napi_schedule(sd, &sd->backlog); 2691 ____napi_schedule(sd, &sd->backlog);
2690 sd->received_rps++; 2692 sd->received_rps++;
2691 } 2693 }
2692 2694
2693 #endif /* CONFIG_RPS */ 2695 #endif /* CONFIG_RPS */
2694 2696
2695 /* 2697 /*
2696 * Check if this softnet_data structure is another cpu one 2698 * Check if this softnet_data structure is another cpu one
2697 * If yes, queue it to our IPI list and return 1 2699 * If yes, queue it to our IPI list and return 1
2698 * If no, return 0 2700 * If no, return 0
2699 */ 2701 */
2700 static int rps_ipi_queued(struct softnet_data *sd) 2702 static int rps_ipi_queued(struct softnet_data *sd)
2701 { 2703 {
2702 #ifdef CONFIG_RPS 2704 #ifdef CONFIG_RPS
2703 struct softnet_data *mysd = &__get_cpu_var(softnet_data); 2705 struct softnet_data *mysd = &__get_cpu_var(softnet_data);
2704 2706
2705 if (sd != mysd) { 2707 if (sd != mysd) {
2706 sd->rps_ipi_next = mysd->rps_ipi_list; 2708 sd->rps_ipi_next = mysd->rps_ipi_list;
2707 mysd->rps_ipi_list = sd; 2709 mysd->rps_ipi_list = sd;
2708 2710
2709 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 2711 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
2710 return 1; 2712 return 1;
2711 } 2713 }
2712 #endif /* CONFIG_RPS */ 2714 #endif /* CONFIG_RPS */
2713 return 0; 2715 return 0;
2714 } 2716 }
2715 2717
2716 /* 2718 /*
2717 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 2719 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
2718 * queue (may be a remote CPU queue). 2720 * queue (may be a remote CPU queue).
2719 */ 2721 */
2720 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 2722 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
2721 unsigned int *qtail) 2723 unsigned int *qtail)
2722 { 2724 {
2723 struct softnet_data *sd; 2725 struct softnet_data *sd;
2724 unsigned long flags; 2726 unsigned long flags;
2725 2727
2726 sd = &per_cpu(softnet_data, cpu); 2728 sd = &per_cpu(softnet_data, cpu);
2727 2729
2728 local_irq_save(flags); 2730 local_irq_save(flags);
2729 2731
2730 rps_lock(sd); 2732 rps_lock(sd);
2731 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) { 2733 if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
2732 if (skb_queue_len(&sd->input_pkt_queue)) { 2734 if (skb_queue_len(&sd->input_pkt_queue)) {
2733 enqueue: 2735 enqueue:
2734 __skb_queue_tail(&sd->input_pkt_queue, skb); 2736 __skb_queue_tail(&sd->input_pkt_queue, skb);
2735 input_queue_tail_incr_save(sd, qtail); 2737 input_queue_tail_incr_save(sd, qtail);
2736 rps_unlock(sd); 2738 rps_unlock(sd);
2737 local_irq_restore(flags); 2739 local_irq_restore(flags);
2738 return NET_RX_SUCCESS; 2740 return NET_RX_SUCCESS;
2739 } 2741 }
2740 2742
2741 /* Schedule NAPI for backlog device 2743 /* Schedule NAPI for backlog device
2742 * We can use non atomic operation since we own the queue lock 2744 * We can use non atomic operation since we own the queue lock
2743 */ 2745 */
2744 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { 2746 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
2745 if (!rps_ipi_queued(sd)) 2747 if (!rps_ipi_queued(sd))
2746 ____napi_schedule(sd, &sd->backlog); 2748 ____napi_schedule(sd, &sd->backlog);
2747 } 2749 }
2748 goto enqueue; 2750 goto enqueue;
2749 } 2751 }
2750 2752
2751 sd->dropped++; 2753 sd->dropped++;
2752 rps_unlock(sd); 2754 rps_unlock(sd);
2753 2755
2754 local_irq_restore(flags); 2756 local_irq_restore(flags);
2755 2757
2756 atomic_long_inc(&skb->dev->rx_dropped); 2758 atomic_long_inc(&skb->dev->rx_dropped);
2757 kfree_skb(skb); 2759 kfree_skb(skb);
2758 return NET_RX_DROP; 2760 return NET_RX_DROP;
2759 } 2761 }
2760 2762
2761 /** 2763 /**
2762 * netif_rx - post buffer to the network code 2764 * netif_rx - post buffer to the network code
2763 * @skb: buffer to post 2765 * @skb: buffer to post
2764 * 2766 *
2765 * This function receives a packet from a device driver and queues it for 2767 * This function receives a packet from a device driver and queues it for
2766 * the upper (protocol) levels to process. It always succeeds. The buffer 2768 * the upper (protocol) levels to process. It always succeeds. The buffer
2767 * may be dropped during processing for congestion control or by the 2769 * may be dropped during processing for congestion control or by the
2768 * protocol layers. 2770 * protocol layers.
2769 * 2771 *
2770 * return values: 2772 * return values:
2771 * NET_RX_SUCCESS (no congestion) 2773 * NET_RX_SUCCESS (no congestion)
2772 * NET_RX_DROP (packet was dropped) 2774 * NET_RX_DROP (packet was dropped)
2773 * 2775 *
2774 */ 2776 */
2775 2777
2776 int netif_rx(struct sk_buff *skb) 2778 int netif_rx(struct sk_buff *skb)
2777 { 2779 {
2778 int ret; 2780 int ret;
2779 2781
2780 /* if netpoll wants it, pretend we never saw it */ 2782 /* if netpoll wants it, pretend we never saw it */
2781 if (netpoll_rx(skb)) 2783 if (netpoll_rx(skb))
2782 return NET_RX_DROP; 2784 return NET_RX_DROP;
2783 2785
2784 if (netdev_tstamp_prequeue) 2786 if (netdev_tstamp_prequeue)
2785 net_timestamp_check(skb); 2787 net_timestamp_check(skb);
2786 2788
2787 trace_netif_rx(skb); 2789 trace_netif_rx(skb);
2788 #ifdef CONFIG_RPS 2790 #ifdef CONFIG_RPS
2789 { 2791 {
2790 struct rps_dev_flow voidflow, *rflow = &voidflow; 2792 struct rps_dev_flow voidflow, *rflow = &voidflow;
2791 int cpu; 2793 int cpu;
2792 2794
2793 preempt_disable(); 2795 preempt_disable();
2794 rcu_read_lock(); 2796 rcu_read_lock();
2795 2797
2796 cpu = get_rps_cpu(skb->dev, skb, &rflow); 2798 cpu = get_rps_cpu(skb->dev, skb, &rflow);
2797 if (cpu < 0) 2799 if (cpu < 0)
2798 cpu = smp_processor_id(); 2800 cpu = smp_processor_id();
2799 2801
2800 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 2802 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
2801 2803
2802 rcu_read_unlock(); 2804 rcu_read_unlock();
2803 preempt_enable(); 2805 preempt_enable();
2804 } 2806 }
2805 #else 2807 #else
2806 { 2808 {
2807 unsigned int qtail; 2809 unsigned int qtail;
2808 ret = enqueue_to_backlog(skb, get_cpu(), &qtail); 2810 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
2809 put_cpu(); 2811 put_cpu();
2810 } 2812 }
2811 #endif 2813 #endif
2812 return ret; 2814 return ret;
2813 } 2815 }
2814 EXPORT_SYMBOL(netif_rx); 2816 EXPORT_SYMBOL(netif_rx);
2815 2817
2816 int netif_rx_ni(struct sk_buff *skb) 2818 int netif_rx_ni(struct sk_buff *skb)
2817 { 2819 {
2818 int err; 2820 int err;
2819 2821
2820 preempt_disable(); 2822 preempt_disable();
2821 err = netif_rx(skb); 2823 err = netif_rx(skb);
2822 if (local_softirq_pending()) 2824 if (local_softirq_pending())
2823 do_softirq(); 2825 do_softirq();
2824 preempt_enable(); 2826 preempt_enable();
2825 2827
2826 return err; 2828 return err;
2827 } 2829 }
2828 EXPORT_SYMBOL(netif_rx_ni); 2830 EXPORT_SYMBOL(netif_rx_ni);
2829 2831
2830 static void net_tx_action(struct softirq_action *h) 2832 static void net_tx_action(struct softirq_action *h)
2831 { 2833 {
2832 struct softnet_data *sd = &__get_cpu_var(softnet_data); 2834 struct softnet_data *sd = &__get_cpu_var(softnet_data);
2833 2835
2834 if (sd->completion_queue) { 2836 if (sd->completion_queue) {
2835 struct sk_buff *clist; 2837 struct sk_buff *clist;
2836 2838
2837 local_irq_disable(); 2839 local_irq_disable();
2838 clist = sd->completion_queue; 2840 clist = sd->completion_queue;
2839 sd->completion_queue = NULL; 2841 sd->completion_queue = NULL;
2840 local_irq_enable(); 2842 local_irq_enable();
2841 2843
2842 while (clist) { 2844 while (clist) {
2843 struct sk_buff *skb = clist; 2845 struct sk_buff *skb = clist;
2844 clist = clist->next; 2846 clist = clist->next;
2845 2847
2846 WARN_ON(atomic_read(&skb->users)); 2848 WARN_ON(atomic_read(&skb->users));
2847 trace_kfree_skb(skb, net_tx_action); 2849 trace_kfree_skb(skb, net_tx_action);
2848 __kfree_skb(skb); 2850 __kfree_skb(skb);
2849 } 2851 }
2850 } 2852 }
2851 2853
2852 if (sd->output_queue) { 2854 if (sd->output_queue) {
2853 struct Qdisc *head; 2855 struct Qdisc *head;
2854 2856
2855 local_irq_disable(); 2857 local_irq_disable();
2856 head = sd->output_queue; 2858 head = sd->output_queue;
2857 sd->output_queue = NULL; 2859 sd->output_queue = NULL;
2858 sd->output_queue_tailp = &sd->output_queue; 2860 sd->output_queue_tailp = &sd->output_queue;
2859 local_irq_enable(); 2861 local_irq_enable();
2860 2862
2861 while (head) { 2863 while (head) {
2862 struct Qdisc *q = head; 2864 struct Qdisc *q = head;
2863 spinlock_t *root_lock; 2865 spinlock_t *root_lock;
2864 2866
2865 head = head->next_sched; 2867 head = head->next_sched;
2866 2868
2867 root_lock = qdisc_lock(q); 2869 root_lock = qdisc_lock(q);
2868 if (spin_trylock(root_lock)) { 2870 if (spin_trylock(root_lock)) {
2869 smp_mb__before_clear_bit(); 2871 smp_mb__before_clear_bit();
2870 clear_bit(__QDISC_STATE_SCHED, 2872 clear_bit(__QDISC_STATE_SCHED,
2871 &q->state); 2873 &q->state);
2872 qdisc_run(q); 2874 qdisc_run(q);
2873 spin_unlock(root_lock); 2875 spin_unlock(root_lock);
2874 } else { 2876 } else {
2875 if (!test_bit(__QDISC_STATE_DEACTIVATED, 2877 if (!test_bit(__QDISC_STATE_DEACTIVATED,
2876 &q->state)) { 2878 &q->state)) {
2877 __netif_reschedule(q); 2879 __netif_reschedule(q);
2878 } else { 2880 } else {
2879 smp_mb__before_clear_bit(); 2881 smp_mb__before_clear_bit();
2880 clear_bit(__QDISC_STATE_SCHED, 2882 clear_bit(__QDISC_STATE_SCHED,
2881 &q->state); 2883 &q->state);
2882 } 2884 }
2883 } 2885 }
2884 } 2886 }
2885 } 2887 }
2886 } 2888 }
2887 2889
2888 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ 2890 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
2889 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) 2891 (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
2890 /* This hook is defined here for ATM LANE */ 2892 /* This hook is defined here for ATM LANE */
2891 int (*br_fdb_test_addr_hook)(struct net_device *dev, 2893 int (*br_fdb_test_addr_hook)(struct net_device *dev,
2892 unsigned char *addr) __read_mostly; 2894 unsigned char *addr) __read_mostly;
2893 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); 2895 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
2894 #endif 2896 #endif
2895 2897
2896 #ifdef CONFIG_NET_CLS_ACT 2898 #ifdef CONFIG_NET_CLS_ACT
2897 /* TODO: Maybe we should just force sch_ingress to be compiled in 2899 /* TODO: Maybe we should just force sch_ingress to be compiled in
2898 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions 2900 * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
2899 * a compare and 2 stores extra right now if we dont have it on 2901 * a compare and 2 stores extra right now if we dont have it on
2900 * but have CONFIG_NET_CLS_ACT 2902 * but have CONFIG_NET_CLS_ACT
2901 * NOTE: This doesnt stop any functionality; if you dont have 2903 * NOTE: This doesnt stop any functionality; if you dont have
2902 * the ingress scheduler, you just cant add policies on ingress. 2904 * the ingress scheduler, you just cant add policies on ingress.
2903 * 2905 *
2904 */ 2906 */
2905 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq) 2907 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
2906 { 2908 {
2907 struct net_device *dev = skb->dev; 2909 struct net_device *dev = skb->dev;
2908 u32 ttl = G_TC_RTTL(skb->tc_verd); 2910 u32 ttl = G_TC_RTTL(skb->tc_verd);
2909 int result = TC_ACT_OK; 2911 int result = TC_ACT_OK;
2910 struct Qdisc *q; 2912 struct Qdisc *q;
2911 2913
2912 if (unlikely(MAX_RED_LOOP < ttl++)) { 2914 if (unlikely(MAX_RED_LOOP < ttl++)) {
2913 if (net_ratelimit()) 2915 if (net_ratelimit())
2914 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n", 2916 pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
2915 skb->skb_iif, dev->ifindex); 2917 skb->skb_iif, dev->ifindex);
2916 return TC_ACT_SHOT; 2918 return TC_ACT_SHOT;
2917 } 2919 }
2918 2920
2919 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl); 2921 skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
2920 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); 2922 skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
2921 2923
2922 q = rxq->qdisc; 2924 q = rxq->qdisc;
2923 if (q != &noop_qdisc) { 2925 if (q != &noop_qdisc) {
2924 spin_lock(qdisc_lock(q)); 2926 spin_lock(qdisc_lock(q));
2925 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) 2927 if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
2926 result = qdisc_enqueue_root(skb, q); 2928 result = qdisc_enqueue_root(skb, q);
2927 spin_unlock(qdisc_lock(q)); 2929 spin_unlock(qdisc_lock(q));
2928 } 2930 }
2929 2931
2930 return result; 2932 return result;
2931 } 2933 }
2932 2934
2933 static inline struct sk_buff *handle_ing(struct sk_buff *skb, 2935 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
2934 struct packet_type **pt_prev, 2936 struct packet_type **pt_prev,
2935 int *ret, struct net_device *orig_dev) 2937 int *ret, struct net_device *orig_dev)
2936 { 2938 {
2937 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue); 2939 struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
2938 2940
2939 if (!rxq || rxq->qdisc == &noop_qdisc) 2941 if (!rxq || rxq->qdisc == &noop_qdisc)
2940 goto out; 2942 goto out;
2941 2943
2942 if (*pt_prev) { 2944 if (*pt_prev) {
2943 *ret = deliver_skb(skb, *pt_prev, orig_dev); 2945 *ret = deliver_skb(skb, *pt_prev, orig_dev);
2944 *pt_prev = NULL; 2946 *pt_prev = NULL;
2945 } 2947 }
2946 2948
2947 switch (ing_filter(skb, rxq)) { 2949 switch (ing_filter(skb, rxq)) {
2948 case TC_ACT_SHOT: 2950 case TC_ACT_SHOT:
2949 case TC_ACT_STOLEN: 2951 case TC_ACT_STOLEN:
2950 kfree_skb(skb); 2952 kfree_skb(skb);
2951 return NULL; 2953 return NULL;
2952 } 2954 }
2953 2955
2954 out: 2956 out:
2955 skb->tc_verd = 0; 2957 skb->tc_verd = 0;
2956 return skb; 2958 return skb;
2957 } 2959 }
2958 #endif 2960 #endif
2959 2961
2960 /** 2962 /**
2961 * netdev_rx_handler_register - register receive handler 2963 * netdev_rx_handler_register - register receive handler
2962 * @dev: device to register a handler for 2964 * @dev: device to register a handler for
2963 * @rx_handler: receive handler to register 2965 * @rx_handler: receive handler to register
2964 * @rx_handler_data: data pointer that is used by rx handler 2966 * @rx_handler_data: data pointer that is used by rx handler
2965 * 2967 *
2966 * Register a receive hander for a device. This handler will then be 2968 * Register a receive hander for a device. This handler will then be
2967 * called from __netif_receive_skb. A negative errno code is returned 2969 * called from __netif_receive_skb. A negative errno code is returned
2968 * on a failure. 2970 * on a failure.
2969 * 2971 *
2970 * The caller must hold the rtnl_mutex. 2972 * The caller must hold the rtnl_mutex.
2971 */ 2973 */
2972 int netdev_rx_handler_register(struct net_device *dev, 2974 int netdev_rx_handler_register(struct net_device *dev,
2973 rx_handler_func_t *rx_handler, 2975 rx_handler_func_t *rx_handler,
2974 void *rx_handler_data) 2976 void *rx_handler_data)
2975 { 2977 {
2976 ASSERT_RTNL(); 2978 ASSERT_RTNL();
2977 2979
2978 if (dev->rx_handler) 2980 if (dev->rx_handler)
2979 return -EBUSY; 2981 return -EBUSY;
2980 2982
2981 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 2983 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
2982 rcu_assign_pointer(dev->rx_handler, rx_handler); 2984 rcu_assign_pointer(dev->rx_handler, rx_handler);
2983 2985
2984 return 0; 2986 return 0;
2985 } 2987 }
2986 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 2988 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
2987 2989
2988 /** 2990 /**
2989 * netdev_rx_handler_unregister - unregister receive handler 2991 * netdev_rx_handler_unregister - unregister receive handler
2990 * @dev: device to unregister a handler from 2992 * @dev: device to unregister a handler from
2991 * 2993 *
2992 * Unregister a receive hander from a device. 2994 * Unregister a receive hander from a device.
2993 * 2995 *
2994 * The caller must hold the rtnl_mutex. 2996 * The caller must hold the rtnl_mutex.
2995 */ 2997 */
2996 void netdev_rx_handler_unregister(struct net_device *dev) 2998 void netdev_rx_handler_unregister(struct net_device *dev)
2997 { 2999 {
2998 3000
2999 ASSERT_RTNL(); 3001 ASSERT_RTNL();
3000 rcu_assign_pointer(dev->rx_handler, NULL); 3002 rcu_assign_pointer(dev->rx_handler, NULL);
3001 rcu_assign_pointer(dev->rx_handler_data, NULL); 3003 rcu_assign_pointer(dev->rx_handler_data, NULL);
3002 } 3004 }
3003 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 3005 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3004 3006
3005 static inline void skb_bond_set_mac_by_master(struct sk_buff *skb, 3007 static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,
3006 struct net_device *master) 3008 struct net_device *master)
3007 { 3009 {
3008 if (skb->pkt_type == PACKET_HOST) { 3010 if (skb->pkt_type == PACKET_HOST) {
3009 u16 *dest = (u16 *) eth_hdr(skb)->h_dest; 3011 u16 *dest = (u16 *) eth_hdr(skb)->h_dest;
3010 3012
3011 memcpy(dest, master->dev_addr, ETH_ALEN); 3013 memcpy(dest, master->dev_addr, ETH_ALEN);
3012 } 3014 }
3013 } 3015 }
3014 3016
3015 /* On bonding slaves other than the currently active slave, suppress 3017 /* On bonding slaves other than the currently active slave, suppress
3016 * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and 3018 * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
3017 * ARP on active-backup slaves with arp_validate enabled. 3019 * ARP on active-backup slaves with arp_validate enabled.
3018 */ 3020 */
3019 int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master) 3021 int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)
3020 { 3022 {
3021 struct net_device *dev = skb->dev; 3023 struct net_device *dev = skb->dev;
3022 3024
3023 if (master->priv_flags & IFF_MASTER_ARPMON) 3025 if (master->priv_flags & IFF_MASTER_ARPMON)
3024 dev->last_rx = jiffies; 3026 dev->last_rx = jiffies;
3025 3027
3026 if ((master->priv_flags & IFF_MASTER_ALB) && 3028 if ((master->priv_flags & IFF_MASTER_ALB) &&
3027 (master->priv_flags & IFF_BRIDGE_PORT)) { 3029 (master->priv_flags & IFF_BRIDGE_PORT)) {
3028 /* Do address unmangle. The local destination address 3030 /* Do address unmangle. The local destination address
3029 * will be always the one master has. Provides the right 3031 * will be always the one master has. Provides the right
3030 * functionality in a bridge. 3032 * functionality in a bridge.
3031 */ 3033 */
3032 skb_bond_set_mac_by_master(skb, master); 3034 skb_bond_set_mac_by_master(skb, master);
3033 } 3035 }
3034 3036
3035 if (dev->priv_flags & IFF_SLAVE_INACTIVE) { 3037 if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
3036 if ((dev->priv_flags & IFF_SLAVE_NEEDARP) && 3038 if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&
3037 skb->protocol == __cpu_to_be16(ETH_P_ARP)) 3039 skb->protocol == __cpu_to_be16(ETH_P_ARP))
3038 return 0; 3040 return 0;
3039 3041
3040 if (master->priv_flags & IFF_MASTER_ALB) { 3042 if (master->priv_flags & IFF_MASTER_ALB) {
3041 if (skb->pkt_type != PACKET_BROADCAST && 3043 if (skb->pkt_type != PACKET_BROADCAST &&
3042 skb->pkt_type != PACKET_MULTICAST) 3044 skb->pkt_type != PACKET_MULTICAST)
3043 return 0; 3045 return 0;
3044 } 3046 }
3045 if (master->priv_flags & IFF_MASTER_8023AD && 3047 if (master->priv_flags & IFF_MASTER_8023AD &&
3046 skb->protocol == __cpu_to_be16(ETH_P_SLOW)) 3048 skb->protocol == __cpu_to_be16(ETH_P_SLOW))
3047 return 0; 3049 return 0;
3048 3050
3049 return 1; 3051 return 1;
3050 } 3052 }
3051 return 0; 3053 return 0;
3052 } 3054 }
3053 EXPORT_SYMBOL(__skb_bond_should_drop); 3055 EXPORT_SYMBOL(__skb_bond_should_drop);
3054 3056
3055 static int __netif_receive_skb(struct sk_buff *skb) 3057 static int __netif_receive_skb(struct sk_buff *skb)
3056 { 3058 {
3057 struct packet_type *ptype, *pt_prev; 3059 struct packet_type *ptype, *pt_prev;
3058 rx_handler_func_t *rx_handler; 3060 rx_handler_func_t *rx_handler;
3059 struct net_device *orig_dev; 3061 struct net_device *orig_dev;
3060 struct net_device *master; 3062 struct net_device *master;
3061 struct net_device *null_or_orig; 3063 struct net_device *null_or_orig;
3062 struct net_device *orig_or_bond; 3064 struct net_device *orig_or_bond;
3063 int ret = NET_RX_DROP; 3065 int ret = NET_RX_DROP;
3064 __be16 type; 3066 __be16 type;
3065 3067
3066 if (!netdev_tstamp_prequeue) 3068 if (!netdev_tstamp_prequeue)
3067 net_timestamp_check(skb); 3069 net_timestamp_check(skb);
3068 3070
3069 trace_netif_receive_skb(skb); 3071 trace_netif_receive_skb(skb);
3070 3072
3071 /* if we've gotten here through NAPI, check netpoll */ 3073 /* if we've gotten here through NAPI, check netpoll */
3072 if (netpoll_receive_skb(skb)) 3074 if (netpoll_receive_skb(skb))
3073 return NET_RX_DROP; 3075 return NET_RX_DROP;
3074 3076
3075 if (!skb->skb_iif) 3077 if (!skb->skb_iif)
3076 skb->skb_iif = skb->dev->ifindex; 3078 skb->skb_iif = skb->dev->ifindex;
3077 3079
3078 /* 3080 /*
3079 * bonding note: skbs received on inactive slaves should only 3081 * bonding note: skbs received on inactive slaves should only
3080 * be delivered to pkt handlers that are exact matches. Also 3082 * be delivered to pkt handlers that are exact matches. Also
3081 * the deliver_no_wcard flag will be set. If packet handlers 3083 * the deliver_no_wcard flag will be set. If packet handlers
3082 * are sensitive to duplicate packets these skbs will need to 3084 * are sensitive to duplicate packets these skbs will need to
3083 * be dropped at the handler. 3085 * be dropped at the handler.
3084 */ 3086 */
3085 null_or_orig = NULL; 3087 null_or_orig = NULL;
3086 orig_dev = skb->dev; 3088 orig_dev = skb->dev;
3087 master = ACCESS_ONCE(orig_dev->master); 3089 master = ACCESS_ONCE(orig_dev->master);
3088 if (skb->deliver_no_wcard) 3090 if (skb->deliver_no_wcard)
3089 null_or_orig = orig_dev; 3091 null_or_orig = orig_dev;
3090 else if (master) { 3092 else if (master) {
3091 if (skb_bond_should_drop(skb, master)) { 3093 if (skb_bond_should_drop(skb, master)) {
3092 skb->deliver_no_wcard = 1; 3094 skb->deliver_no_wcard = 1;
3093 null_or_orig = orig_dev; /* deliver only exact match */ 3095 null_or_orig = orig_dev; /* deliver only exact match */
3094 } else 3096 } else
3095 skb->dev = master; 3097 skb->dev = master;
3096 } 3098 }
3097 3099
3098 __this_cpu_inc(softnet_data.processed); 3100 __this_cpu_inc(softnet_data.processed);
3099 skb_reset_network_header(skb); 3101 skb_reset_network_header(skb);
3100 skb_reset_transport_header(skb); 3102 skb_reset_transport_header(skb);
3101 skb->mac_len = skb->network_header - skb->mac_header; 3103 skb->mac_len = skb->network_header - skb->mac_header;
3102 3104
3103 pt_prev = NULL; 3105 pt_prev = NULL;
3104 3106
3105 rcu_read_lock(); 3107 rcu_read_lock();
3106 3108
3107 #ifdef CONFIG_NET_CLS_ACT 3109 #ifdef CONFIG_NET_CLS_ACT
3108 if (skb->tc_verd & TC_NCLS) { 3110 if (skb->tc_verd & TC_NCLS) {
3109 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); 3111 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
3110 goto ncls; 3112 goto ncls;
3111 } 3113 }
3112 #endif 3114 #endif
3113 3115
3114 list_for_each_entry_rcu(ptype, &ptype_all, list) { 3116 list_for_each_entry_rcu(ptype, &ptype_all, list) {
3115 if (ptype->dev == null_or_orig || ptype->dev == skb->dev || 3117 if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
3116 ptype->dev == orig_dev) { 3118 ptype->dev == orig_dev) {
3117 if (pt_prev) 3119 if (pt_prev)
3118 ret = deliver_skb(skb, pt_prev, orig_dev); 3120 ret = deliver_skb(skb, pt_prev, orig_dev);
3119 pt_prev = ptype; 3121 pt_prev = ptype;
3120 } 3122 }
3121 } 3123 }
3122 3124
3123 #ifdef CONFIG_NET_CLS_ACT 3125 #ifdef CONFIG_NET_CLS_ACT
3124 skb = handle_ing(skb, &pt_prev, &ret, orig_dev); 3126 skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
3125 if (!skb) 3127 if (!skb)
3126 goto out; 3128 goto out;
3127 ncls: 3129 ncls:
3128 #endif 3130 #endif
3129 3131
3130 /* Handle special case of bridge or macvlan */ 3132 /* Handle special case of bridge or macvlan */
3131 rx_handler = rcu_dereference(skb->dev->rx_handler); 3133 rx_handler = rcu_dereference(skb->dev->rx_handler);
3132 if (rx_handler) { 3134 if (rx_handler) {
3133 if (pt_prev) { 3135 if (pt_prev) {
3134 ret = deliver_skb(skb, pt_prev, orig_dev); 3136 ret = deliver_skb(skb, pt_prev, orig_dev);
3135 pt_prev = NULL; 3137 pt_prev = NULL;
3136 } 3138 }
3137 skb = rx_handler(skb); 3139 skb = rx_handler(skb);
3138 if (!skb) 3140 if (!skb)
3139 goto out; 3141 goto out;
3140 } 3142 }
3141 3143
3142 if (vlan_tx_tag_present(skb)) { 3144 if (vlan_tx_tag_present(skb)) {
3143 if (pt_prev) { 3145 if (pt_prev) {
3144 ret = deliver_skb(skb, pt_prev, orig_dev); 3146 ret = deliver_skb(skb, pt_prev, orig_dev);
3145 pt_prev = NULL; 3147 pt_prev = NULL;
3146 } 3148 }
3147 if (vlan_hwaccel_do_receive(&skb)) { 3149 if (vlan_hwaccel_do_receive(&skb)) {
3148 ret = __netif_receive_skb(skb); 3150 ret = __netif_receive_skb(skb);
3149 goto out; 3151 goto out;
3150 } else if (unlikely(!skb)) 3152 } else if (unlikely(!skb))
3151 goto out; 3153 goto out;
3152 } 3154 }
3153 3155
3154 /* 3156 /*
3155 * Make sure frames received on VLAN interfaces stacked on 3157 * Make sure frames received on VLAN interfaces stacked on
3156 * bonding interfaces still make their way to any base bonding 3158 * bonding interfaces still make their way to any base bonding
3157 * device that may have registered for a specific ptype. The 3159 * device that may have registered for a specific ptype. The
3158 * handler may have to adjust skb->dev and orig_dev. 3160 * handler may have to adjust skb->dev and orig_dev.
3159 */ 3161 */
3160 orig_or_bond = orig_dev; 3162 orig_or_bond = orig_dev;
3161 if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) && 3163 if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
3162 (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) { 3164 (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
3163 orig_or_bond = vlan_dev_real_dev(skb->dev); 3165 orig_or_bond = vlan_dev_real_dev(skb->dev);
3164 } 3166 }
3165 3167
3166 type = skb->protocol; 3168 type = skb->protocol;
3167 list_for_each_entry_rcu(ptype, 3169 list_for_each_entry_rcu(ptype,
3168 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) { 3170 &ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
3169 if (ptype->type == type && (ptype->dev == null_or_orig || 3171 if (ptype->type == type && (ptype->dev == null_or_orig ||
3170 ptype->dev == skb->dev || ptype->dev == orig_dev || 3172 ptype->dev == skb->dev || ptype->dev == orig_dev ||
3171 ptype->dev == orig_or_bond)) { 3173 ptype->dev == orig_or_bond)) {
3172 if (pt_prev) 3174 if (pt_prev)
3173 ret = deliver_skb(skb, pt_prev, orig_dev); 3175 ret = deliver_skb(skb, pt_prev, orig_dev);
3174 pt_prev = ptype; 3176 pt_prev = ptype;
3175 } 3177 }
3176 } 3178 }
3177 3179
3178 if (pt_prev) { 3180 if (pt_prev) {
3179 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 3181 ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
3180 } else { 3182 } else {
3181 atomic_long_inc(&skb->dev->rx_dropped); 3183 atomic_long_inc(&skb->dev->rx_dropped);
3182 kfree_skb(skb); 3184 kfree_skb(skb);
3183 /* Jamal, now you will not able to escape explaining 3185 /* Jamal, now you will not able to escape explaining
3184 * me how you were going to use this. :-) 3186 * me how you were going to use this. :-)
3185 */ 3187 */
3186 ret = NET_RX_DROP; 3188 ret = NET_RX_DROP;
3187 } 3189 }
3188 3190
3189 out: 3191 out:
3190 rcu_read_unlock(); 3192 rcu_read_unlock();
3191 return ret; 3193 return ret;
3192 } 3194 }
3193 3195
3194 /** 3196 /**
3195 * netif_receive_skb - process receive buffer from network 3197 * netif_receive_skb - process receive buffer from network
3196 * @skb: buffer to process 3198 * @skb: buffer to process
3197 * 3199 *
3198 * netif_receive_skb() is the main receive data processing function. 3200 * netif_receive_skb() is the main receive data processing function.
3199 * It always succeeds. The buffer may be dropped during processing 3201 * It always succeeds. The buffer may be dropped during processing
3200 * for congestion control or by the protocol layers. 3202 * for congestion control or by the protocol layers.
3201 * 3203 *
3202 * This function may only be called from softirq context and interrupts 3204 * This function may only be called from softirq context and interrupts
3203 * should be enabled. 3205 * should be enabled.
3204 * 3206 *
3205 * Return values (usually ignored): 3207 * Return values (usually ignored):
3206 * NET_RX_SUCCESS: no congestion 3208 * NET_RX_SUCCESS: no congestion
3207 * NET_RX_DROP: packet was dropped 3209 * NET_RX_DROP: packet was dropped
3208 */ 3210 */
3209 int netif_receive_skb(struct sk_buff *skb) 3211 int netif_receive_skb(struct sk_buff *skb)
3210 { 3212 {
3211 if (netdev_tstamp_prequeue) 3213 if (netdev_tstamp_prequeue)
3212 net_timestamp_check(skb); 3214 net_timestamp_check(skb);
3213 3215
3214 if (skb_defer_rx_timestamp(skb)) 3216 if (skb_defer_rx_timestamp(skb))
3215 return NET_RX_SUCCESS; 3217 return NET_RX_SUCCESS;
3216 3218
3217 #ifdef CONFIG_RPS 3219 #ifdef CONFIG_RPS
3218 { 3220 {
3219 struct rps_dev_flow voidflow, *rflow = &voidflow; 3221 struct rps_dev_flow voidflow, *rflow = &voidflow;
3220 int cpu, ret; 3222 int cpu, ret;
3221 3223
3222 rcu_read_lock(); 3224 rcu_read_lock();
3223 3225
3224 cpu = get_rps_cpu(skb->dev, skb, &rflow); 3226 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3225 3227
3226 if (cpu >= 0) { 3228 if (cpu >= 0) {
3227 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 3229 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3228 rcu_read_unlock(); 3230 rcu_read_unlock();
3229 } else { 3231 } else {
3230 rcu_read_unlock(); 3232 rcu_read_unlock();
3231 ret = __netif_receive_skb(skb); 3233 ret = __netif_receive_skb(skb);
3232 } 3234 }
3233 3235
3234 return ret; 3236 return ret;
3235 } 3237 }
3236 #else 3238 #else
3237 return __netif_receive_skb(skb); 3239 return __netif_receive_skb(skb);
3238 #endif 3240 #endif
3239 } 3241 }
3240 EXPORT_SYMBOL(netif_receive_skb); 3242 EXPORT_SYMBOL(netif_receive_skb);
3241 3243
3242 /* Network device is going away, flush any packets still pending 3244 /* Network device is going away, flush any packets still pending
3243 * Called with irqs disabled. 3245 * Called with irqs disabled.
3244 */ 3246 */
3245 static void flush_backlog(void *arg) 3247 static void flush_backlog(void *arg)
3246 { 3248 {
3247 struct net_device *dev = arg; 3249 struct net_device *dev = arg;
3248 struct softnet_data *sd = &__get_cpu_var(softnet_data); 3250 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3249 struct sk_buff *skb, *tmp; 3251 struct sk_buff *skb, *tmp;
3250 3252
3251 rps_lock(sd); 3253 rps_lock(sd);
3252 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 3254 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
3253 if (skb->dev == dev) { 3255 if (skb->dev == dev) {
3254 __skb_unlink(skb, &sd->input_pkt_queue); 3256 __skb_unlink(skb, &sd->input_pkt_queue);
3255 kfree_skb(skb); 3257 kfree_skb(skb);
3256 input_queue_head_incr(sd); 3258 input_queue_head_incr(sd);
3257 } 3259 }
3258 } 3260 }
3259 rps_unlock(sd); 3261 rps_unlock(sd);
3260 3262
3261 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 3263 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
3262 if (skb->dev == dev) { 3264 if (skb->dev == dev) {
3263 __skb_unlink(skb, &sd->process_queue); 3265 __skb_unlink(skb, &sd->process_queue);
3264 kfree_skb(skb); 3266 kfree_skb(skb);
3265 input_queue_head_incr(sd); 3267 input_queue_head_incr(sd);
3266 } 3268 }
3267 } 3269 }
3268 } 3270 }
3269 3271
3270 static int napi_gro_complete(struct sk_buff *skb) 3272 static int napi_gro_complete(struct sk_buff *skb)
3271 { 3273 {
3272 struct packet_type *ptype; 3274 struct packet_type *ptype;
3273 __be16 type = skb->protocol; 3275 __be16 type = skb->protocol;
3274 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK]; 3276 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3275 int err = -ENOENT; 3277 int err = -ENOENT;
3276 3278
3277 if (NAPI_GRO_CB(skb)->count == 1) { 3279 if (NAPI_GRO_CB(skb)->count == 1) {
3278 skb_shinfo(skb)->gso_size = 0; 3280 skb_shinfo(skb)->gso_size = 0;
3279 goto out; 3281 goto out;
3280 } 3282 }
3281 3283
3282 rcu_read_lock(); 3284 rcu_read_lock();
3283 list_for_each_entry_rcu(ptype, head, list) { 3285 list_for_each_entry_rcu(ptype, head, list) {
3284 if (ptype->type != type || ptype->dev || !ptype->gro_complete) 3286 if (ptype->type != type || ptype->dev || !ptype->gro_complete)
3285 continue; 3287 continue;
3286 3288
3287 err = ptype->gro_complete(skb); 3289 err = ptype->gro_complete(skb);
3288 break; 3290 break;
3289 } 3291 }
3290 rcu_read_unlock(); 3292 rcu_read_unlock();
3291 3293
3292 if (err) { 3294 if (err) {
3293 WARN_ON(&ptype->list == head); 3295 WARN_ON(&ptype->list == head);
3294 kfree_skb(skb); 3296 kfree_skb(skb);
3295 return NET_RX_SUCCESS; 3297 return NET_RX_SUCCESS;
3296 } 3298 }
3297 3299
3298 out: 3300 out:
3299 return netif_receive_skb(skb); 3301 return netif_receive_skb(skb);
3300 } 3302 }
3301 3303
3302 inline void napi_gro_flush(struct napi_struct *napi) 3304 inline void napi_gro_flush(struct napi_struct *napi)
3303 { 3305 {
3304 struct sk_buff *skb, *next; 3306 struct sk_buff *skb, *next;
3305 3307
3306 for (skb = napi->gro_list; skb; skb = next) { 3308 for (skb = napi->gro_list; skb; skb = next) {
3307 next = skb->next; 3309 next = skb->next;
3308 skb->next = NULL; 3310 skb->next = NULL;
3309 napi_gro_complete(skb); 3311 napi_gro_complete(skb);
3310 } 3312 }
3311 3313
3312 napi->gro_count = 0; 3314 napi->gro_count = 0;
3313 napi->gro_list = NULL; 3315 napi->gro_list = NULL;
3314 } 3316 }
3315 EXPORT_SYMBOL(napi_gro_flush); 3317 EXPORT_SYMBOL(napi_gro_flush);
3316 3318
3317 enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 3319 enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3318 { 3320 {
3319 struct sk_buff **pp = NULL; 3321 struct sk_buff **pp = NULL;
3320 struct packet_type *ptype; 3322 struct packet_type *ptype;
3321 __be16 type = skb->protocol; 3323 __be16 type = skb->protocol;
3322 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK]; 3324 struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
3323 int same_flow; 3325 int same_flow;
3324 int mac_len; 3326 int mac_len;
3325 enum gro_result ret; 3327 enum gro_result ret;
3326 3328
3327 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb)) 3329 if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
3328 goto normal; 3330 goto normal;
3329 3331
3330 if (skb_is_gso(skb) || skb_has_frag_list(skb)) 3332 if (skb_is_gso(skb) || skb_has_frag_list(skb))
3331 goto normal; 3333 goto normal;
3332 3334
3333 rcu_read_lock(); 3335 rcu_read_lock();
3334 list_for_each_entry_rcu(ptype, head, list) { 3336 list_for_each_entry_rcu(ptype, head, list) {
3335 if (ptype->type != type || ptype->dev || !ptype->gro_receive) 3337 if (ptype->type != type || ptype->dev || !ptype->gro_receive)
3336 continue; 3338 continue;
3337 3339
3338 skb_set_network_header(skb, skb_gro_offset(skb)); 3340 skb_set_network_header(skb, skb_gro_offset(skb));
3339 mac_len = skb->network_header - skb->mac_header; 3341 mac_len = skb->network_header - skb->mac_header;
3340 skb->mac_len = mac_len; 3342 skb->mac_len = mac_len;
3341 NAPI_GRO_CB(skb)->same_flow = 0; 3343 NAPI_GRO_CB(skb)->same_flow = 0;
3342 NAPI_GRO_CB(skb)->flush = 0; 3344 NAPI_GRO_CB(skb)->flush = 0;
3343 NAPI_GRO_CB(skb)->free = 0; 3345 NAPI_GRO_CB(skb)->free = 0;
3344 3346
3345 pp = ptype->gro_receive(&napi->gro_list, skb); 3347 pp = ptype->gro_receive(&napi->gro_list, skb);
3346 break; 3348 break;
3347 } 3349 }
3348 rcu_read_unlock(); 3350 rcu_read_unlock();
3349 3351
3350 if (&ptype->list == head) 3352 if (&ptype->list == head)
3351 goto normal; 3353 goto normal;
3352 3354
3353 same_flow = NAPI_GRO_CB(skb)->same_flow; 3355 same_flow = NAPI_GRO_CB(skb)->same_flow;
3354 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; 3356 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
3355 3357
3356 if (pp) { 3358 if (pp) {
3357 struct sk_buff *nskb = *pp; 3359 struct sk_buff *nskb = *pp;
3358 3360
3359 *pp = nskb->next; 3361 *pp = nskb->next;
3360 nskb->next = NULL; 3362 nskb->next = NULL;
3361 napi_gro_complete(nskb); 3363 napi_gro_complete(nskb);
3362 napi->gro_count--; 3364 napi->gro_count--;
3363 } 3365 }
3364 3366
3365 if (same_flow) 3367 if (same_flow)
3366 goto ok; 3368 goto ok;
3367 3369
3368 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS) 3370 if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
3369 goto normal; 3371 goto normal;
3370 3372
3371 napi->gro_count++; 3373 napi->gro_count++;
3372 NAPI_GRO_CB(skb)->count = 1; 3374 NAPI_GRO_CB(skb)->count = 1;
3373 skb_shinfo(skb)->gso_size = skb_gro_len(skb); 3375 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
3374 skb->next = napi->gro_list; 3376 skb->next = napi->gro_list;
3375 napi->gro_list = skb; 3377 napi->gro_list = skb;
3376 ret = GRO_HELD; 3378 ret = GRO_HELD;
3377 3379
3378 pull: 3380 pull:
3379 if (skb_headlen(skb) < skb_gro_offset(skb)) { 3381 if (skb_headlen(skb) < skb_gro_offset(skb)) {
3380 int grow = skb_gro_offset(skb) - skb_headlen(skb); 3382 int grow = skb_gro_offset(skb) - skb_headlen(skb);
3381 3383
3382 BUG_ON(skb->end - skb->tail < grow); 3384 BUG_ON(skb->end - skb->tail < grow);
3383 3385
3384 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); 3386 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
3385 3387
3386 skb->tail += grow; 3388 skb->tail += grow;
3387 skb->data_len -= grow; 3389 skb->data_len -= grow;
3388 3390
3389 skb_shinfo(skb)->frags[0].page_offset += grow; 3391 skb_shinfo(skb)->frags[0].page_offset += grow;
3390 skb_shinfo(skb)->frags[0].size -= grow; 3392 skb_shinfo(skb)->frags[0].size -= grow;
3391 3393
3392 if (unlikely(!skb_shinfo(skb)->frags[0].size)) { 3394 if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
3393 put_page(skb_shinfo(skb)->frags[0].page); 3395 put_page(skb_shinfo(skb)->frags[0].page);
3394 memmove(skb_shinfo(skb)->frags, 3396 memmove(skb_shinfo(skb)->frags,
3395 skb_shinfo(skb)->frags + 1, 3397 skb_shinfo(skb)->frags + 1,
3396 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t)); 3398 --skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
3397 } 3399 }
3398 } 3400 }
3399 3401
3400 ok: 3402 ok:
3401 return ret; 3403 return ret;
3402 3404
3403 normal: 3405 normal:
3404 ret = GRO_NORMAL; 3406 ret = GRO_NORMAL;
3405 goto pull; 3407 goto pull;
3406 } 3408 }
3407 EXPORT_SYMBOL(dev_gro_receive); 3409 EXPORT_SYMBOL(dev_gro_receive);
3408 3410
3409 static inline gro_result_t 3411 static inline gro_result_t
3410 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 3412 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3411 { 3413 {
3412 struct sk_buff *p; 3414 struct sk_buff *p;
3413 3415
3414 for (p = napi->gro_list; p; p = p->next) { 3416 for (p = napi->gro_list; p; p = p->next) {
3415 unsigned long diffs; 3417 unsigned long diffs;
3416 3418
3417 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; 3419 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
3418 diffs |= p->vlan_tci ^ skb->vlan_tci; 3420 diffs |= p->vlan_tci ^ skb->vlan_tci;
3419 diffs |= compare_ether_header(skb_mac_header(p), 3421 diffs |= compare_ether_header(skb_mac_header(p),
3420 skb_gro_mac_header(skb)); 3422 skb_gro_mac_header(skb));
3421 NAPI_GRO_CB(p)->same_flow = !diffs; 3423 NAPI_GRO_CB(p)->same_flow = !diffs;
3422 NAPI_GRO_CB(p)->flush = 0; 3424 NAPI_GRO_CB(p)->flush = 0;
3423 } 3425 }
3424 3426
3425 return dev_gro_receive(napi, skb); 3427 return dev_gro_receive(napi, skb);
3426 } 3428 }
3427 3429
3428 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) 3430 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
3429 { 3431 {
3430 switch (ret) { 3432 switch (ret) {
3431 case GRO_NORMAL: 3433 case GRO_NORMAL:
3432 if (netif_receive_skb(skb)) 3434 if (netif_receive_skb(skb))
3433 ret = GRO_DROP; 3435 ret = GRO_DROP;
3434 break; 3436 break;
3435 3437
3436 case GRO_DROP: 3438 case GRO_DROP:
3437 case GRO_MERGED_FREE: 3439 case GRO_MERGED_FREE:
3438 kfree_skb(skb); 3440 kfree_skb(skb);
3439 break; 3441 break;
3440 3442
3441 case GRO_HELD: 3443 case GRO_HELD:
3442 case GRO_MERGED: 3444 case GRO_MERGED:
3443 break; 3445 break;
3444 } 3446 }
3445 3447
3446 return ret; 3448 return ret;
3447 } 3449 }
3448 EXPORT_SYMBOL(napi_skb_finish); 3450 EXPORT_SYMBOL(napi_skb_finish);
3449 3451
3450 void skb_gro_reset_offset(struct sk_buff *skb) 3452 void skb_gro_reset_offset(struct sk_buff *skb)
3451 { 3453 {
3452 NAPI_GRO_CB(skb)->data_offset = 0; 3454 NAPI_GRO_CB(skb)->data_offset = 0;
3453 NAPI_GRO_CB(skb)->frag0 = NULL; 3455 NAPI_GRO_CB(skb)->frag0 = NULL;
3454 NAPI_GRO_CB(skb)->frag0_len = 0; 3456 NAPI_GRO_CB(skb)->frag0_len = 0;
3455 3457
3456 if (skb->mac_header == skb->tail && 3458 if (skb->mac_header == skb->tail &&
3457 !PageHighMem(skb_shinfo(skb)->frags[0].page)) { 3459 !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
3458 NAPI_GRO_CB(skb)->frag0 = 3460 NAPI_GRO_CB(skb)->frag0 =
3459 page_address(skb_shinfo(skb)->frags[0].page) + 3461 page_address(skb_shinfo(skb)->frags[0].page) +
3460 skb_shinfo(skb)->frags[0].page_offset; 3462 skb_shinfo(skb)->frags[0].page_offset;
3461 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size; 3463 NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
3462 } 3464 }
3463 } 3465 }
3464 EXPORT_SYMBOL(skb_gro_reset_offset); 3466 EXPORT_SYMBOL(skb_gro_reset_offset);
3465 3467
3466 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) 3468 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
3467 { 3469 {
3468 skb_gro_reset_offset(skb); 3470 skb_gro_reset_offset(skb);
3469 3471
3470 return napi_skb_finish(__napi_gro_receive(napi, skb), skb); 3472 return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
3471 } 3473 }
3472 EXPORT_SYMBOL(napi_gro_receive); 3474 EXPORT_SYMBOL(napi_gro_receive);
3473 3475
3474 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) 3476 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
3475 { 3477 {
3476 __skb_pull(skb, skb_headlen(skb)); 3478 __skb_pull(skb, skb_headlen(skb));
3477 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb)); 3479 skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
3478 skb->vlan_tci = 0; 3480 skb->vlan_tci = 0;
3479 3481
3480 napi->skb = skb; 3482 napi->skb = skb;
3481 } 3483 }
3482 3484
3483 struct sk_buff *napi_get_frags(struct napi_struct *napi) 3485 struct sk_buff *napi_get_frags(struct napi_struct *napi)
3484 { 3486 {
3485 struct sk_buff *skb = napi->skb; 3487 struct sk_buff *skb = napi->skb;
3486 3488
3487 if (!skb) { 3489 if (!skb) {
3488 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD); 3490 skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
3489 if (skb) 3491 if (skb)
3490 napi->skb = skb; 3492 napi->skb = skb;
3491 } 3493 }
3492 return skb; 3494 return skb;
3493 } 3495 }
3494 EXPORT_SYMBOL(napi_get_frags); 3496 EXPORT_SYMBOL(napi_get_frags);
3495 3497
3496 gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, 3498 gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
3497 gro_result_t ret) 3499 gro_result_t ret)
3498 { 3500 {
3499 switch (ret) { 3501 switch (ret) {
3500 case GRO_NORMAL: 3502 case GRO_NORMAL:
3501 case GRO_HELD: 3503 case GRO_HELD:
3502 skb->protocol = eth_type_trans(skb, skb->dev); 3504 skb->protocol = eth_type_trans(skb, skb->dev);
3503 3505
3504 if (ret == GRO_HELD) 3506 if (ret == GRO_HELD)
3505 skb_gro_pull(skb, -ETH_HLEN); 3507 skb_gro_pull(skb, -ETH_HLEN);
3506 else if (netif_receive_skb(skb)) 3508 else if (netif_receive_skb(skb))
3507 ret = GRO_DROP; 3509 ret = GRO_DROP;
3508 break; 3510 break;
3509 3511
3510 case GRO_DROP: 3512 case GRO_DROP:
3511 case GRO_MERGED_FREE: 3513 case GRO_MERGED_FREE:
3512 napi_reuse_skb(napi, skb); 3514 napi_reuse_skb(napi, skb);
3513 break; 3515 break;
3514 3516
3515 case GRO_MERGED: 3517 case GRO_MERGED:
3516 break; 3518 break;
3517 } 3519 }
3518 3520
3519 return ret; 3521 return ret;
3520 } 3522 }
3521 EXPORT_SYMBOL(napi_frags_finish); 3523 EXPORT_SYMBOL(napi_frags_finish);
3522 3524
3523 struct sk_buff *napi_frags_skb(struct napi_struct *napi) 3525 struct sk_buff *napi_frags_skb(struct napi_struct *napi)
3524 { 3526 {
3525 struct sk_buff *skb = napi->skb; 3527 struct sk_buff *skb = napi->skb;
3526 struct ethhdr *eth; 3528 struct ethhdr *eth;
3527 unsigned int hlen; 3529 unsigned int hlen;
3528 unsigned int off; 3530 unsigned int off;
3529 3531
3530 napi->skb = NULL; 3532 napi->skb = NULL;
3531 3533
3532 skb_reset_mac_header(skb); 3534 skb_reset_mac_header(skb);
3533 skb_gro_reset_offset(skb); 3535 skb_gro_reset_offset(skb);
3534 3536
3535 off = skb_gro_offset(skb); 3537 off = skb_gro_offset(skb);
3536 hlen = off + sizeof(*eth); 3538 hlen = off + sizeof(*eth);
3537 eth = skb_gro_header_fast(skb, off); 3539 eth = skb_gro_header_fast(skb, off);
3538 if (skb_gro_header_hard(skb, hlen)) { 3540 if (skb_gro_header_hard(skb, hlen)) {
3539 eth = skb_gro_header_slow(skb, hlen, off); 3541 eth = skb_gro_header_slow(skb, hlen, off);
3540 if (unlikely(!eth)) { 3542 if (unlikely(!eth)) {
3541 napi_reuse_skb(napi, skb); 3543 napi_reuse_skb(napi, skb);
3542 skb = NULL; 3544 skb = NULL;
3543 goto out; 3545 goto out;
3544 } 3546 }
3545 } 3547 }
3546 3548
3547 skb_gro_pull(skb, sizeof(*eth)); 3549 skb_gro_pull(skb, sizeof(*eth));
3548 3550
3549 /* 3551 /*
3550 * This works because the only protocols we care about don't require 3552 * This works because the only protocols we care about don't require
3551 * special handling. We'll fix it up properly at the end. 3553 * special handling. We'll fix it up properly at the end.
3552 */ 3554 */
3553 skb->protocol = eth->h_proto; 3555 skb->protocol = eth->h_proto;
3554 3556
3555 out: 3557 out:
3556 return skb; 3558 return skb;
3557 } 3559 }
3558 EXPORT_SYMBOL(napi_frags_skb); 3560 EXPORT_SYMBOL(napi_frags_skb);
3559 3561
3560 gro_result_t napi_gro_frags(struct napi_struct *napi) 3562 gro_result_t napi_gro_frags(struct napi_struct *napi)
3561 { 3563 {
3562 struct sk_buff *skb = napi_frags_skb(napi); 3564 struct sk_buff *skb = napi_frags_skb(napi);
3563 3565
3564 if (!skb) 3566 if (!skb)
3565 return GRO_DROP; 3567 return GRO_DROP;
3566 3568
3567 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb)); 3569 return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
3568 } 3570 }
3569 EXPORT_SYMBOL(napi_gro_frags); 3571 EXPORT_SYMBOL(napi_gro_frags);
3570 3572
3571 /* 3573 /*
3572 * net_rps_action sends any pending IPI's for rps. 3574 * net_rps_action sends any pending IPI's for rps.
3573 * Note: called with local irq disabled, but exits with local irq enabled. 3575 * Note: called with local irq disabled, but exits with local irq enabled.
3574 */ 3576 */
3575 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 3577 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
3576 { 3578 {
3577 #ifdef CONFIG_RPS 3579 #ifdef CONFIG_RPS
3578 struct softnet_data *remsd = sd->rps_ipi_list; 3580 struct softnet_data *remsd = sd->rps_ipi_list;
3579 3581
3580 if (remsd) { 3582 if (remsd) {
3581 sd->rps_ipi_list = NULL; 3583 sd->rps_ipi_list = NULL;
3582 3584
3583 local_irq_enable(); 3585 local_irq_enable();
3584 3586
3585 /* Send pending IPI's to kick RPS processing on remote cpus. */ 3587 /* Send pending IPI's to kick RPS processing on remote cpus. */
3586 while (remsd) { 3588 while (remsd) {
3587 struct softnet_data *next = remsd->rps_ipi_next; 3589 struct softnet_data *next = remsd->rps_ipi_next;
3588 3590
3589 if (cpu_online(remsd->cpu)) 3591 if (cpu_online(remsd->cpu))
3590 __smp_call_function_single(remsd->cpu, 3592 __smp_call_function_single(remsd->cpu,
3591 &remsd->csd, 0); 3593 &remsd->csd, 0);
3592 remsd = next; 3594 remsd = next;
3593 } 3595 }
3594 } else 3596 } else
3595 #endif 3597 #endif
3596 local_irq_enable(); 3598 local_irq_enable();
3597 } 3599 }
3598 3600
3599 static int process_backlog(struct napi_struct *napi, int quota) 3601 static int process_backlog(struct napi_struct *napi, int quota)
3600 { 3602 {
3601 int work = 0; 3603 int work = 0;
3602 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 3604 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
3603 3605
3604 #ifdef CONFIG_RPS 3606 #ifdef CONFIG_RPS
3605 /* Check if we have pending ipi, its better to send them now, 3607 /* Check if we have pending ipi, its better to send them now,
3606 * not waiting net_rx_action() end. 3608 * not waiting net_rx_action() end.
3607 */ 3609 */
3608 if (sd->rps_ipi_list) { 3610 if (sd->rps_ipi_list) {
3609 local_irq_disable(); 3611 local_irq_disable();
3610 net_rps_action_and_irq_enable(sd); 3612 net_rps_action_and_irq_enable(sd);
3611 } 3613 }
3612 #endif 3614 #endif
3613 napi->weight = weight_p; 3615 napi->weight = weight_p;
3614 local_irq_disable(); 3616 local_irq_disable();
3615 while (work < quota) { 3617 while (work < quota) {
3616 struct sk_buff *skb; 3618 struct sk_buff *skb;
3617 unsigned int qlen; 3619 unsigned int qlen;
3618 3620
3619 while ((skb = __skb_dequeue(&sd->process_queue))) { 3621 while ((skb = __skb_dequeue(&sd->process_queue))) {
3620 local_irq_enable(); 3622 local_irq_enable();
3621 __netif_receive_skb(skb); 3623 __netif_receive_skb(skb);
3622 local_irq_disable(); 3624 local_irq_disable();
3623 input_queue_head_incr(sd); 3625 input_queue_head_incr(sd);
3624 if (++work >= quota) { 3626 if (++work >= quota) {
3625 local_irq_enable(); 3627 local_irq_enable();
3626 return work; 3628 return work;
3627 } 3629 }
3628 } 3630 }
3629 3631
3630 rps_lock(sd); 3632 rps_lock(sd);
3631 qlen = skb_queue_len(&sd->input_pkt_queue); 3633 qlen = skb_queue_len(&sd->input_pkt_queue);
3632 if (qlen) 3634 if (qlen)
3633 skb_queue_splice_tail_init(&sd->input_pkt_queue, 3635 skb_queue_splice_tail_init(&sd->input_pkt_queue,
3634 &sd->process_queue); 3636 &sd->process_queue);
3635 3637
3636 if (qlen < quota - work) { 3638 if (qlen < quota - work) {
3637 /* 3639 /*
3638 * Inline a custom version of __napi_complete(). 3640 * Inline a custom version of __napi_complete().
3639 * only current cpu owns and manipulates this napi, 3641 * only current cpu owns and manipulates this napi,
3640 * and NAPI_STATE_SCHED is the only possible flag set on backlog. 3642 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
3641 * we can use a plain write instead of clear_bit(), 3643 * we can use a plain write instead of clear_bit(),
3642 * and we dont need an smp_mb() memory barrier. 3644 * and we dont need an smp_mb() memory barrier.
3643 */ 3645 */
3644 list_del(&napi->poll_list); 3646 list_del(&napi->poll_list);
3645 napi->state = 0; 3647 napi->state = 0;
3646 3648
3647 quota = work + qlen; 3649 quota = work + qlen;
3648 } 3650 }
3649 rps_unlock(sd); 3651 rps_unlock(sd);
3650 } 3652 }
3651 local_irq_enable(); 3653 local_irq_enable();
3652 3654
3653 return work; 3655 return work;
3654 } 3656 }
3655 3657
3656 /** 3658 /**
3657 * __napi_schedule - schedule for receive 3659 * __napi_schedule - schedule for receive
3658 * @n: entry to schedule 3660 * @n: entry to schedule
3659 * 3661 *
3660 * The entry's receive function will be scheduled to run 3662 * The entry's receive function will be scheduled to run
3661 */ 3663 */
3662 void __napi_schedule(struct napi_struct *n) 3664 void __napi_schedule(struct napi_struct *n)
3663 { 3665 {
3664 unsigned long flags; 3666 unsigned long flags;
3665 3667
3666 local_irq_save(flags); 3668 local_irq_save(flags);
3667 ____napi_schedule(&__get_cpu_var(softnet_data), n); 3669 ____napi_schedule(&__get_cpu_var(softnet_data), n);
3668 local_irq_restore(flags); 3670 local_irq_restore(flags);
3669 } 3671 }
3670 EXPORT_SYMBOL(__napi_schedule); 3672 EXPORT_SYMBOL(__napi_schedule);
3671 3673
3672 void __napi_complete(struct napi_struct *n) 3674 void __napi_complete(struct napi_struct *n)
3673 { 3675 {
3674 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); 3676 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
3675 BUG_ON(n->gro_list); 3677 BUG_ON(n->gro_list);
3676 3678
3677 list_del(&n->poll_list); 3679 list_del(&n->poll_list);
3678 smp_mb__before_clear_bit(); 3680 smp_mb__before_clear_bit();
3679 clear_bit(NAPI_STATE_SCHED, &n->state); 3681 clear_bit(NAPI_STATE_SCHED, &n->state);
3680 } 3682 }
3681 EXPORT_SYMBOL(__napi_complete); 3683 EXPORT_SYMBOL(__napi_complete);
3682 3684
3683 void napi_complete(struct napi_struct *n) 3685 void napi_complete(struct napi_struct *n)
3684 { 3686 {
3685 unsigned long flags; 3687 unsigned long flags;
3686 3688
3687 /* 3689 /*
3688 * don't let napi dequeue from the cpu poll list 3690 * don't let napi dequeue from the cpu poll list
3689 * just in case its running on a different cpu 3691 * just in case its running on a different cpu
3690 */ 3692 */
3691 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) 3693 if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
3692 return; 3694 return;
3693 3695
3694 napi_gro_flush(n); 3696 napi_gro_flush(n);
3695 local_irq_save(flags); 3697 local_irq_save(flags);
3696 __napi_complete(n); 3698 __napi_complete(n);
3697 local_irq_restore(flags); 3699 local_irq_restore(flags);
3698 } 3700 }
3699 EXPORT_SYMBOL(napi_complete); 3701 EXPORT_SYMBOL(napi_complete);
3700 3702
3701 void netif_napi_add(struct net_device *dev, struct napi_struct *napi, 3703 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
3702 int (*poll)(struct napi_struct *, int), int weight) 3704 int (*poll)(struct napi_struct *, int), int weight)
3703 { 3705 {
3704 INIT_LIST_HEAD(&napi->poll_list); 3706 INIT_LIST_HEAD(&napi->poll_list);
3705 napi->gro_count = 0; 3707 napi->gro_count = 0;
3706 napi->gro_list = NULL; 3708 napi->gro_list = NULL;
3707 napi->skb = NULL; 3709 napi->skb = NULL;
3708 napi->poll = poll; 3710 napi->poll = poll;
3709 napi->weight = weight; 3711 napi->weight = weight;
3710 list_add(&napi->dev_list, &dev->napi_list); 3712 list_add(&napi->dev_list, &dev->napi_list);
3711 napi->dev = dev; 3713 napi->dev = dev;
3712 #ifdef CONFIG_NETPOLL 3714 #ifdef CONFIG_NETPOLL
3713 spin_lock_init(&napi->poll_lock); 3715 spin_lock_init(&napi->poll_lock);
3714 napi->poll_owner = -1; 3716 napi->poll_owner = -1;
3715 #endif 3717 #endif
3716 set_bit(NAPI_STATE_SCHED, &napi->state); 3718 set_bit(NAPI_STATE_SCHED, &napi->state);
3717 } 3719 }
3718 EXPORT_SYMBOL(netif_napi_add); 3720 EXPORT_SYMBOL(netif_napi_add);
3719 3721
3720 void netif_napi_del(struct napi_struct *napi) 3722 void netif_napi_del(struct napi_struct *napi)
3721 { 3723 {
3722 struct sk_buff *skb, *next; 3724 struct sk_buff *skb, *next;
3723 3725
3724 list_del_init(&napi->dev_list); 3726 list_del_init(&napi->dev_list);
3725 napi_free_frags(napi); 3727 napi_free_frags(napi);
3726 3728
3727 for (skb = napi->gro_list; skb; skb = next) { 3729 for (skb = napi->gro_list; skb; skb = next) {
3728 next = skb->next; 3730 next = skb->next;
3729 skb->next = NULL; 3731 skb->next = NULL;
3730 kfree_skb(skb); 3732 kfree_skb(skb);
3731 } 3733 }
3732 3734
3733 napi->gro_list = NULL; 3735 napi->gro_list = NULL;
3734 napi->gro_count = 0; 3736 napi->gro_count = 0;
3735 } 3737 }
3736 EXPORT_SYMBOL(netif_napi_del); 3738 EXPORT_SYMBOL(netif_napi_del);
3737 3739
3738 static void net_rx_action(struct softirq_action *h) 3740 static void net_rx_action(struct softirq_action *h)
3739 { 3741 {
3740 struct softnet_data *sd = &__get_cpu_var(softnet_data); 3742 struct softnet_data *sd = &__get_cpu_var(softnet_data);
3741 unsigned long time_limit = jiffies + 2; 3743 unsigned long time_limit = jiffies + 2;
3742 int budget = netdev_budget; 3744 int budget = netdev_budget;
3743 void *have; 3745 void *have;
3744 3746
3745 local_irq_disable(); 3747 local_irq_disable();
3746 3748
3747 while (!list_empty(&sd->poll_list)) { 3749 while (!list_empty(&sd->poll_list)) {
3748 struct napi_struct *n; 3750 struct napi_struct *n;
3749 int work, weight; 3751 int work, weight;
3750 3752
3751 /* If softirq window is exhuasted then punt. 3753 /* If softirq window is exhuasted then punt.
3752 * Allow this to run for 2 jiffies since which will allow 3754 * Allow this to run for 2 jiffies since which will allow
3753 * an average latency of 1.5/HZ. 3755 * an average latency of 1.5/HZ.
3754 */ 3756 */
3755 if (unlikely(budget <= 0 || time_after(jiffies, time_limit))) 3757 if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
3756 goto softnet_break; 3758 goto softnet_break;
3757 3759
3758 local_irq_enable(); 3760 local_irq_enable();
3759 3761
3760 /* Even though interrupts have been re-enabled, this 3762 /* Even though interrupts have been re-enabled, this
3761 * access is safe because interrupts can only add new 3763 * access is safe because interrupts can only add new
3762 * entries to the tail of this list, and only ->poll() 3764 * entries to the tail of this list, and only ->poll()
3763 * calls can remove this head entry from the list. 3765 * calls can remove this head entry from the list.
3764 */ 3766 */
3765 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list); 3767 n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
3766 3768
3767 have = netpoll_poll_lock(n); 3769 have = netpoll_poll_lock(n);
3768 3770
3769 weight = n->weight; 3771 weight = n->weight;
3770 3772
3771 /* This NAPI_STATE_SCHED test is for avoiding a race 3773 /* This NAPI_STATE_SCHED test is for avoiding a race
3772 * with netpoll's poll_napi(). Only the entity which 3774 * with netpoll's poll_napi(). Only the entity which
3773 * obtains the lock and sees NAPI_STATE_SCHED set will 3775 * obtains the lock and sees NAPI_STATE_SCHED set will
3774 * actually make the ->poll() call. Therefore we avoid 3776 * actually make the ->poll() call. Therefore we avoid
3775 * accidently calling ->poll() when NAPI is not scheduled. 3777 * accidently calling ->poll() when NAPI is not scheduled.
3776 */ 3778 */
3777 work = 0; 3779 work = 0;
3778 if (test_bit(NAPI_STATE_SCHED, &n->state)) { 3780 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
3779 work = n->poll(n, weight); 3781 work = n->poll(n, weight);
3780 trace_napi_poll(n); 3782 trace_napi_poll(n);
3781 } 3783 }
3782 3784
3783 WARN_ON_ONCE(work > weight); 3785 WARN_ON_ONCE(work > weight);
3784 3786
3785 budget -= work; 3787 budget -= work;
3786 3788
3787 local_irq_disable(); 3789 local_irq_disable();
3788 3790
3789 /* Drivers must not modify the NAPI state if they 3791 /* Drivers must not modify the NAPI state if they
3790 * consume the entire weight. In such cases this code 3792 * consume the entire weight. In such cases this code
3791 * still "owns" the NAPI instance and therefore can 3793 * still "owns" the NAPI instance and therefore can
3792 * move the instance around on the list at-will. 3794 * move the instance around on the list at-will.
3793 */ 3795 */
3794 if (unlikely(work == weight)) { 3796 if (unlikely(work == weight)) {
3795 if (unlikely(napi_disable_pending(n))) { 3797 if (unlikely(napi_disable_pending(n))) {
3796 local_irq_enable(); 3798 local_irq_enable();
3797 napi_complete(n); 3799 napi_complete(n);
3798 local_irq_disable(); 3800 local_irq_disable();
3799 } else 3801 } else
3800 list_move_tail(&n->poll_list, &sd->poll_list); 3802 list_move_tail(&n->poll_list, &sd->poll_list);
3801 } 3803 }
3802 3804
3803 netpoll_poll_unlock(have); 3805 netpoll_poll_unlock(have);
3804 } 3806 }
3805 out: 3807 out:
3806 net_rps_action_and_irq_enable(sd); 3808 net_rps_action_and_irq_enable(sd);
3807 3809
3808 #ifdef CONFIG_NET_DMA 3810 #ifdef CONFIG_NET_DMA
3809 /* 3811 /*
3810 * There may not be any more sk_buffs coming right now, so push 3812 * There may not be any more sk_buffs coming right now, so push
3811 * any pending DMA copies to hardware 3813 * any pending DMA copies to hardware
3812 */ 3814 */
3813 dma_issue_pending_all(); 3815 dma_issue_pending_all();
3814 #endif 3816 #endif
3815 3817
3816 return; 3818 return;
3817 3819
3818 softnet_break: 3820 softnet_break:
3819 sd->time_squeeze++; 3821 sd->time_squeeze++;
3820 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 3822 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3821 goto out; 3823 goto out;
3822 } 3824 }
3823 3825
3824 static gifconf_func_t *gifconf_list[NPROTO]; 3826 static gifconf_func_t *gifconf_list[NPROTO];
3825 3827
3826 /** 3828 /**
3827 * register_gifconf - register a SIOCGIF handler 3829 * register_gifconf - register a SIOCGIF handler
3828 * @family: Address family 3830 * @family: Address family
3829 * @gifconf: Function handler 3831 * @gifconf: Function handler
3830 * 3832 *
3831 * Register protocol dependent address dumping routines. The handler 3833 * Register protocol dependent address dumping routines. The handler
3832 * that is passed must not be freed or reused until it has been replaced 3834 * that is passed must not be freed or reused until it has been replaced
3833 * by another handler. 3835 * by another handler.
3834 */ 3836 */
3835 int register_gifconf(unsigned int family, gifconf_func_t *gifconf) 3837 int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
3836 { 3838 {
3837 if (family >= NPROTO) 3839 if (family >= NPROTO)
3838 return -EINVAL; 3840 return -EINVAL;
3839 gifconf_list[family] = gifconf; 3841 gifconf_list[family] = gifconf;
3840 return 0; 3842 return 0;
3841 } 3843 }
3842 EXPORT_SYMBOL(register_gifconf); 3844 EXPORT_SYMBOL(register_gifconf);
3843 3845
3844 3846
3845 /* 3847 /*
3846 * Map an interface index to its name (SIOCGIFNAME) 3848 * Map an interface index to its name (SIOCGIFNAME)
3847 */ 3849 */
3848 3850
3849 /* 3851 /*
3850 * We need this ioctl for efficient implementation of the 3852 * We need this ioctl for efficient implementation of the
3851 * if_indextoname() function required by the IPv6 API. Without 3853 * if_indextoname() function required by the IPv6 API. Without
3852 * it, we would have to search all the interfaces to find a 3854 * it, we would have to search all the interfaces to find a
3853 * match. --pb 3855 * match. --pb
3854 */ 3856 */
3855 3857
3856 static int dev_ifname(struct net *net, struct ifreq __user *arg) 3858 static int dev_ifname(struct net *net, struct ifreq __user *arg)
3857 { 3859 {
3858 struct net_device *dev; 3860 struct net_device *dev;
3859 struct ifreq ifr; 3861 struct ifreq ifr;
3860 3862
3861 /* 3863 /*
3862 * Fetch the caller's info block. 3864 * Fetch the caller's info block.
3863 */ 3865 */
3864 3866
3865 if (copy_from_user(&ifr, arg, sizeof(struct ifreq))) 3867 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
3866 return -EFAULT; 3868 return -EFAULT;
3867 3869
3868 rcu_read_lock(); 3870 rcu_read_lock();
3869 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex); 3871 dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
3870 if (!dev) { 3872 if (!dev) {
3871 rcu_read_unlock(); 3873 rcu_read_unlock();
3872 return -ENODEV; 3874 return -ENODEV;
3873 } 3875 }
3874 3876
3875 strcpy(ifr.ifr_name, dev->name); 3877 strcpy(ifr.ifr_name, dev->name);
3876 rcu_read_unlock(); 3878 rcu_read_unlock();
3877 3879
3878 if (copy_to_user(arg, &ifr, sizeof(struct ifreq))) 3880 if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
3879 return -EFAULT; 3881 return -EFAULT;
3880 return 0; 3882 return 0;
3881 } 3883 }
3882 3884
3883 /* 3885 /*
3884 * Perform a SIOCGIFCONF call. This structure will change 3886 * Perform a SIOCGIFCONF call. This structure will change
3885 * size eventually, and there is nothing I can do about it. 3887 * size eventually, and there is nothing I can do about it.
3886 * Thus we will need a 'compatibility mode'. 3888 * Thus we will need a 'compatibility mode'.
3887 */ 3889 */
3888 3890
3889 static int dev_ifconf(struct net *net, char __user *arg) 3891 static int dev_ifconf(struct net *net, char __user *arg)
3890 { 3892 {
3891 struct ifconf ifc; 3893 struct ifconf ifc;
3892 struct net_device *dev; 3894 struct net_device *dev;
3893 char __user *pos; 3895 char __user *pos;
3894 int len; 3896 int len;
3895 int total; 3897 int total;
3896 int i; 3898 int i;
3897 3899
3898 /* 3900 /*
3899 * Fetch the caller's info block. 3901 * Fetch the caller's info block.
3900 */ 3902 */
3901 3903
3902 if (copy_from_user(&ifc, arg, sizeof(struct ifconf))) 3904 if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
3903 return -EFAULT; 3905 return -EFAULT;
3904 3906
3905 pos = ifc.ifc_buf; 3907 pos = ifc.ifc_buf;
3906 len = ifc.ifc_len; 3908 len = ifc.ifc_len;
3907 3909
3908 /* 3910 /*
3909 * Loop over the interfaces, and write an info block for each. 3911 * Loop over the interfaces, and write an info block for each.
3910 */ 3912 */
3911 3913
3912 total = 0; 3914 total = 0;
3913 for_each_netdev(net, dev) { 3915 for_each_netdev(net, dev) {
3914 for (i = 0; i < NPROTO; i++) { 3916 for (i = 0; i < NPROTO; i++) {
3915 if (gifconf_list[i]) { 3917 if (gifconf_list[i]) {
3916 int done; 3918 int done;
3917 if (!pos) 3919 if (!pos)
3918 done = gifconf_list[i](dev, NULL, 0); 3920 done = gifconf_list[i](dev, NULL, 0);
3919 else 3921 else
3920 done = gifconf_list[i](dev, pos + total, 3922 done = gifconf_list[i](dev, pos + total,
3921 len - total); 3923 len - total);
3922 if (done < 0) 3924 if (done < 0)
3923 return -EFAULT; 3925 return -EFAULT;
3924 total += done; 3926 total += done;
3925 } 3927 }
3926 } 3928 }
3927 } 3929 }
3928 3930
3929 /* 3931 /*
3930 * All done. Write the updated control block back to the caller. 3932 * All done. Write the updated control block back to the caller.
3931 */ 3933 */
3932 ifc.ifc_len = total; 3934 ifc.ifc_len = total;
3933 3935
3934 /* 3936 /*
3935 * Both BSD and Solaris return 0 here, so we do too. 3937 * Both BSD and Solaris return 0 here, so we do too.
3936 */ 3938 */
3937 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0; 3939 return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
3938 } 3940 }
3939 3941
3940 #ifdef CONFIG_PROC_FS 3942 #ifdef CONFIG_PROC_FS
3941 /* 3943 /*
3942 * This is invoked by the /proc filesystem handler to display a device 3944 * This is invoked by the /proc filesystem handler to display a device
3943 * in detail. 3945 * in detail.
3944 */ 3946 */
3945 void *dev_seq_start(struct seq_file *seq, loff_t *pos) 3947 void *dev_seq_start(struct seq_file *seq, loff_t *pos)
3946 __acquires(RCU) 3948 __acquires(RCU)
3947 { 3949 {
3948 struct net *net = seq_file_net(seq); 3950 struct net *net = seq_file_net(seq);
3949 loff_t off; 3951 loff_t off;
3950 struct net_device *dev; 3952 struct net_device *dev;
3951 3953
3952 rcu_read_lock(); 3954 rcu_read_lock();
3953 if (!*pos) 3955 if (!*pos)
3954 return SEQ_START_TOKEN; 3956 return SEQ_START_TOKEN;
3955 3957
3956 off = 1; 3958 off = 1;
3957 for_each_netdev_rcu(net, dev) 3959 for_each_netdev_rcu(net, dev)
3958 if (off++ == *pos) 3960 if (off++ == *pos)
3959 return dev; 3961 return dev;
3960 3962
3961 return NULL; 3963 return NULL;
3962 } 3964 }
3963 3965
3964 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos) 3966 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3965 { 3967 {
3966 struct net_device *dev = (v == SEQ_START_TOKEN) ? 3968 struct net_device *dev = (v == SEQ_START_TOKEN) ?
3967 first_net_device(seq_file_net(seq)) : 3969 first_net_device(seq_file_net(seq)) :
3968 next_net_device((struct net_device *)v); 3970 next_net_device((struct net_device *)v);
3969 3971
3970 ++*pos; 3972 ++*pos;
3971 return rcu_dereference(dev); 3973 return rcu_dereference(dev);
3972 } 3974 }
3973 3975
3974 void dev_seq_stop(struct seq_file *seq, void *v) 3976 void dev_seq_stop(struct seq_file *seq, void *v)
3975 __releases(RCU) 3977 __releases(RCU)
3976 { 3978 {
3977 rcu_read_unlock(); 3979 rcu_read_unlock();
3978 } 3980 }
3979 3981
3980 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev) 3982 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
3981 { 3983 {
3982 struct rtnl_link_stats64 temp; 3984 struct rtnl_link_stats64 temp;
3983 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp); 3985 const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
3984 3986
3985 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu " 3987 seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
3986 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n", 3988 "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
3987 dev->name, stats->rx_bytes, stats->rx_packets, 3989 dev->name, stats->rx_bytes, stats->rx_packets,
3988 stats->rx_errors, 3990 stats->rx_errors,
3989 stats->rx_dropped + stats->rx_missed_errors, 3991 stats->rx_dropped + stats->rx_missed_errors,
3990 stats->rx_fifo_errors, 3992 stats->rx_fifo_errors,
3991 stats->rx_length_errors + stats->rx_over_errors + 3993 stats->rx_length_errors + stats->rx_over_errors +
3992 stats->rx_crc_errors + stats->rx_frame_errors, 3994 stats->rx_crc_errors + stats->rx_frame_errors,
3993 stats->rx_compressed, stats->multicast, 3995 stats->rx_compressed, stats->multicast,
3994 stats->tx_bytes, stats->tx_packets, 3996 stats->tx_bytes, stats->tx_packets,
3995 stats->tx_errors, stats->tx_dropped, 3997 stats->tx_errors, stats->tx_dropped,
3996 stats->tx_fifo_errors, stats->collisions, 3998 stats->tx_fifo_errors, stats->collisions,
3997 stats->tx_carrier_errors + 3999 stats->tx_carrier_errors +
3998 stats->tx_aborted_errors + 4000 stats->tx_aborted_errors +
3999 stats->tx_window_errors + 4001 stats->tx_window_errors +
4000 stats->tx_heartbeat_errors, 4002 stats->tx_heartbeat_errors,
4001 stats->tx_compressed); 4003 stats->tx_compressed);
4002 } 4004 }
4003 4005
4004 /* 4006 /*
4005 * Called from the PROCfs module. This now uses the new arbitrary sized 4007 * Called from the PROCfs module. This now uses the new arbitrary sized
4006 * /proc/net interface to create /proc/net/dev 4008 * /proc/net interface to create /proc/net/dev
4007 */ 4009 */
4008 static int dev_seq_show(struct seq_file *seq, void *v) 4010 static int dev_seq_show(struct seq_file *seq, void *v)
4009 { 4011 {
4010 if (v == SEQ_START_TOKEN) 4012 if (v == SEQ_START_TOKEN)
4011 seq_puts(seq, "Inter-| Receive " 4013 seq_puts(seq, "Inter-| Receive "
4012 " | Transmit\n" 4014 " | Transmit\n"
4013 " face |bytes packets errs drop fifo frame " 4015 " face |bytes packets errs drop fifo frame "
4014 "compressed multicast|bytes packets errs " 4016 "compressed multicast|bytes packets errs "
4015 "drop fifo colls carrier compressed\n"); 4017 "drop fifo colls carrier compressed\n");
4016 else 4018 else
4017 dev_seq_printf_stats(seq, v); 4019 dev_seq_printf_stats(seq, v);
4018 return 0; 4020 return 0;
4019 } 4021 }
4020 4022
4021 static struct softnet_data *softnet_get_online(loff_t *pos) 4023 static struct softnet_data *softnet_get_online(loff_t *pos)
4022 { 4024 {
4023 struct softnet_data *sd = NULL; 4025 struct softnet_data *sd = NULL;
4024 4026
4025 while (*pos < nr_cpu_ids) 4027 while (*pos < nr_cpu_ids)
4026 if (cpu_online(*pos)) { 4028 if (cpu_online(*pos)) {
4027 sd = &per_cpu(softnet_data, *pos); 4029 sd = &per_cpu(softnet_data, *pos);
4028 break; 4030 break;
4029 } else 4031 } else
4030 ++*pos; 4032 ++*pos;
4031 return sd; 4033 return sd;
4032 } 4034 }
4033 4035
4034 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos) 4036 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
4035 { 4037 {
4036 return softnet_get_online(pos); 4038 return softnet_get_online(pos);
4037 } 4039 }
4038 4040
4039 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4041 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4040 { 4042 {
4041 ++*pos; 4043 ++*pos;
4042 return softnet_get_online(pos); 4044 return softnet_get_online(pos);
4043 } 4045 }
4044 4046
4045 static void softnet_seq_stop(struct seq_file *seq, void *v) 4047 static void softnet_seq_stop(struct seq_file *seq, void *v)
4046 { 4048 {
4047 } 4049 }
4048 4050
4049 static int softnet_seq_show(struct seq_file *seq, void *v) 4051 static int softnet_seq_show(struct seq_file *seq, void *v)
4050 { 4052 {
4051 struct softnet_data *sd = v; 4053 struct softnet_data *sd = v;
4052 4054
4053 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n", 4055 seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
4054 sd->processed, sd->dropped, sd->time_squeeze, 0, 4056 sd->processed, sd->dropped, sd->time_squeeze, 0,
4055 0, 0, 0, 0, /* was fastroute */ 4057 0, 0, 0, 0, /* was fastroute */
4056 sd->cpu_collision, sd->received_rps); 4058 sd->cpu_collision, sd->received_rps);
4057 return 0; 4059 return 0;
4058 } 4060 }
4059 4061
4060 static const struct seq_operations dev_seq_ops = { 4062 static const struct seq_operations dev_seq_ops = {
4061 .start = dev_seq_start, 4063 .start = dev_seq_start,
4062 .next = dev_seq_next, 4064 .next = dev_seq_next,
4063 .stop = dev_seq_stop, 4065 .stop = dev_seq_stop,
4064 .show = dev_seq_show, 4066 .show = dev_seq_show,
4065 }; 4067 };
4066 4068
4067 static int dev_seq_open(struct inode *inode, struct file *file) 4069 static int dev_seq_open(struct inode *inode, struct file *file)
4068 { 4070 {
4069 return seq_open_net(inode, file, &dev_seq_ops, 4071 return seq_open_net(inode, file, &dev_seq_ops,
4070 sizeof(struct seq_net_private)); 4072 sizeof(struct seq_net_private));
4071 } 4073 }
4072 4074
4073 static const struct file_operations dev_seq_fops = { 4075 static const struct file_operations dev_seq_fops = {
4074 .owner = THIS_MODULE, 4076 .owner = THIS_MODULE,
4075 .open = dev_seq_open, 4077 .open = dev_seq_open,
4076 .read = seq_read, 4078 .read = seq_read,
4077 .llseek = seq_lseek, 4079 .llseek = seq_lseek,
4078 .release = seq_release_net, 4080 .release = seq_release_net,
4079 }; 4081 };
4080 4082
4081 static const struct seq_operations softnet_seq_ops = { 4083 static const struct seq_operations softnet_seq_ops = {
4082 .start = softnet_seq_start, 4084 .start = softnet_seq_start,
4083 .next = softnet_seq_next, 4085 .next = softnet_seq_next,
4084 .stop = softnet_seq_stop, 4086 .stop = softnet_seq_stop,
4085 .show = softnet_seq_show, 4087 .show = softnet_seq_show,
4086 }; 4088 };
4087 4089
4088 static int softnet_seq_open(struct inode *inode, struct file *file) 4090 static int softnet_seq_open(struct inode *inode, struct file *file)
4089 { 4091 {
4090 return seq_open(file, &softnet_seq_ops); 4092 return seq_open(file, &softnet_seq_ops);
4091 } 4093 }
4092 4094
4093 static const struct file_operations softnet_seq_fops = { 4095 static const struct file_operations softnet_seq_fops = {
4094 .owner = THIS_MODULE, 4096 .owner = THIS_MODULE,
4095 .open = softnet_seq_open, 4097 .open = softnet_seq_open,
4096 .read = seq_read, 4098 .read = seq_read,
4097 .llseek = seq_lseek, 4099 .llseek = seq_lseek,
4098 .release = seq_release, 4100 .release = seq_release,
4099 }; 4101 };
4100 4102
4101 static void *ptype_get_idx(loff_t pos) 4103 static void *ptype_get_idx(loff_t pos)
4102 { 4104 {
4103 struct packet_type *pt = NULL; 4105 struct packet_type *pt = NULL;
4104 loff_t i = 0; 4106 loff_t i = 0;
4105 int t; 4107 int t;
4106 4108
4107 list_for_each_entry_rcu(pt, &ptype_all, list) { 4109 list_for_each_entry_rcu(pt, &ptype_all, list) {
4108 if (i == pos) 4110 if (i == pos)
4109 return pt; 4111 return pt;
4110 ++i; 4112 ++i;
4111 } 4113 }
4112 4114
4113 for (t = 0; t < PTYPE_HASH_SIZE; t++) { 4115 for (t = 0; t < PTYPE_HASH_SIZE; t++) {
4114 list_for_each_entry_rcu(pt, &ptype_base[t], list) { 4116 list_for_each_entry_rcu(pt, &ptype_base[t], list) {
4115 if (i == pos) 4117 if (i == pos)
4116 return pt; 4118 return pt;
4117 ++i; 4119 ++i;
4118 } 4120 }
4119 } 4121 }
4120 return NULL; 4122 return NULL;
4121 } 4123 }
4122 4124
4123 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos) 4125 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
4124 __acquires(RCU) 4126 __acquires(RCU)
4125 { 4127 {
4126 rcu_read_lock(); 4128 rcu_read_lock();
4127 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN; 4129 return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
4128 } 4130 }
4129 4131
4130 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos) 4132 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
4131 { 4133 {
4132 struct packet_type *pt; 4134 struct packet_type *pt;
4133 struct list_head *nxt; 4135 struct list_head *nxt;
4134 int hash; 4136 int hash;
4135 4137
4136 ++*pos; 4138 ++*pos;
4137 if (v == SEQ_START_TOKEN) 4139 if (v == SEQ_START_TOKEN)
4138 return ptype_get_idx(0); 4140 return ptype_get_idx(0);
4139 4141
4140 pt = v; 4142 pt = v;
4141 nxt = pt->list.next; 4143 nxt = pt->list.next;
4142 if (pt->type == htons(ETH_P_ALL)) { 4144 if (pt->type == htons(ETH_P_ALL)) {
4143 if (nxt != &ptype_all) 4145 if (nxt != &ptype_all)
4144 goto found; 4146 goto found;
4145 hash = 0; 4147 hash = 0;
4146 nxt = ptype_base[0].next; 4148 nxt = ptype_base[0].next;
4147 } else 4149 } else
4148 hash = ntohs(pt->type) & PTYPE_HASH_MASK; 4150 hash = ntohs(pt->type) & PTYPE_HASH_MASK;
4149 4151
4150 while (nxt == &ptype_base[hash]) { 4152 while (nxt == &ptype_base[hash]) {
4151 if (++hash >= PTYPE_HASH_SIZE) 4153 if (++hash >= PTYPE_HASH_SIZE)
4152 return NULL; 4154 return NULL;
4153 nxt = ptype_base[hash].next; 4155 nxt = ptype_base[hash].next;
4154 } 4156 }
4155 found: 4157 found:
4156 return list_entry(nxt, struct packet_type, list); 4158 return list_entry(nxt, struct packet_type, list);
4157 } 4159 }
4158 4160
4159 static void ptype_seq_stop(struct seq_file *seq, void *v) 4161 static void ptype_seq_stop(struct seq_file *seq, void *v)
4160 __releases(RCU) 4162 __releases(RCU)
4161 { 4163 {
4162 rcu_read_unlock(); 4164 rcu_read_unlock();
4163 } 4165 }
4164 4166
4165 static int ptype_seq_show(struct seq_file *seq, void *v) 4167 static int ptype_seq_show(struct seq_file *seq, void *v)
4166 { 4168 {
4167 struct packet_type *pt = v; 4169 struct packet_type *pt = v;
4168 4170
4169 if (v == SEQ_START_TOKEN) 4171 if (v == SEQ_START_TOKEN)
4170 seq_puts(seq, "Type Device Function\n"); 4172 seq_puts(seq, "Type Device Function\n");
4171 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) { 4173 else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
4172 if (pt->type == htons(ETH_P_ALL)) 4174 if (pt->type == htons(ETH_P_ALL))
4173 seq_puts(seq, "ALL "); 4175 seq_puts(seq, "ALL ");
4174 else 4176 else
4175 seq_printf(seq, "%04x", ntohs(pt->type)); 4177 seq_printf(seq, "%04x", ntohs(pt->type));
4176 4178
4177 seq_printf(seq, " %-8s %pF\n", 4179 seq_printf(seq, " %-8s %pF\n",
4178 pt->dev ? pt->dev->name : "", pt->func); 4180 pt->dev ? pt->dev->name : "", pt->func);
4179 } 4181 }
4180 4182
4181 return 0; 4183 return 0;
4182 } 4184 }
4183 4185
4184 static const struct seq_operations ptype_seq_ops = { 4186 static const struct seq_operations ptype_seq_ops = {
4185 .start = ptype_seq_start, 4187 .start = ptype_seq_start,
4186 .next = ptype_seq_next, 4188 .next = ptype_seq_next,
4187 .stop = ptype_seq_stop, 4189 .stop = ptype_seq_stop,
4188 .show = ptype_seq_show, 4190 .show = ptype_seq_show,
4189 }; 4191 };
4190 4192
4191 static int ptype_seq_open(struct inode *inode, struct file *file) 4193 static int ptype_seq_open(struct inode *inode, struct file *file)
4192 { 4194 {
4193 return seq_open_net(inode, file, &ptype_seq_ops, 4195 return seq_open_net(inode, file, &ptype_seq_ops,
4194 sizeof(struct seq_net_private)); 4196 sizeof(struct seq_net_private));
4195 } 4197 }
4196 4198
4197 static const struct file_operations ptype_seq_fops = { 4199 static const struct file_operations ptype_seq_fops = {
4198 .owner = THIS_MODULE, 4200 .owner = THIS_MODULE,
4199 .open = ptype_seq_open, 4201 .open = ptype_seq_open,
4200 .read = seq_read, 4202 .read = seq_read,
4201 .llseek = seq_lseek, 4203 .llseek = seq_lseek,
4202 .release = seq_release_net, 4204 .release = seq_release_net,
4203 }; 4205 };
4204 4206
4205 4207
4206 static int __net_init dev_proc_net_init(struct net *net) 4208 static int __net_init dev_proc_net_init(struct net *net)
4207 { 4209 {
4208 int rc = -ENOMEM; 4210 int rc = -ENOMEM;
4209 4211
4210 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops)) 4212 if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
4211 goto out; 4213 goto out;
4212 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops)) 4214 if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
4213 goto out_dev; 4215 goto out_dev;
4214 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops)) 4216 if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
4215 goto out_softnet; 4217 goto out_softnet;
4216 4218
4217 if (wext_proc_init(net)) 4219 if (wext_proc_init(net))
4218 goto out_ptype; 4220 goto out_ptype;
4219 rc = 0; 4221 rc = 0;
4220 out: 4222 out:
4221 return rc; 4223 return rc;
4222 out_ptype: 4224 out_ptype:
4223 proc_net_remove(net, "ptype"); 4225 proc_net_remove(net, "ptype");
4224 out_softnet: 4226 out_softnet:
4225 proc_net_remove(net, "softnet_stat"); 4227 proc_net_remove(net, "softnet_stat");
4226 out_dev: 4228 out_dev:
4227 proc_net_remove(net, "dev"); 4229 proc_net_remove(net, "dev");
4228 goto out; 4230 goto out;
4229 } 4231 }
4230 4232
4231 static void __net_exit dev_proc_net_exit(struct net *net) 4233 static void __net_exit dev_proc_net_exit(struct net *net)
4232 { 4234 {
4233 wext_proc_exit(net); 4235 wext_proc_exit(net);
4234 4236
4235 proc_net_remove(net, "ptype"); 4237 proc_net_remove(net, "ptype");
4236 proc_net_remove(net, "softnet_stat"); 4238 proc_net_remove(net, "softnet_stat");
4237 proc_net_remove(net, "dev"); 4239 proc_net_remove(net, "dev");
4238 } 4240 }
4239 4241
4240 static struct pernet_operations __net_initdata dev_proc_ops = { 4242 static struct pernet_operations __net_initdata dev_proc_ops = {
4241 .init = dev_proc_net_init, 4243 .init = dev_proc_net_init,
4242 .exit = dev_proc_net_exit, 4244 .exit = dev_proc_net_exit,
4243 }; 4245 };
4244 4246
4245 static int __init dev_proc_init(void) 4247 static int __init dev_proc_init(void)
4246 { 4248 {
4247 return register_pernet_subsys(&dev_proc_ops); 4249 return register_pernet_subsys(&dev_proc_ops);
4248 } 4250 }
4249 #else 4251 #else
4250 #define dev_proc_init() 0 4252 #define dev_proc_init() 0
4251 #endif /* CONFIG_PROC_FS */ 4253 #endif /* CONFIG_PROC_FS */
4252 4254
4253 4255
4254 /** 4256 /**
4255 * netdev_set_master - set up master/slave pair 4257 * netdev_set_master - set up master/slave pair
4256 * @slave: slave device 4258 * @slave: slave device
4257 * @master: new master device 4259 * @master: new master device
4258 * 4260 *
4259 * Changes the master device of the slave. Pass %NULL to break the 4261 * Changes the master device of the slave. Pass %NULL to break the
4260 * bonding. The caller must hold the RTNL semaphore. On a failure 4262 * bonding. The caller must hold the RTNL semaphore. On a failure
4261 * a negative errno code is returned. On success the reference counts 4263 * a negative errno code is returned. On success the reference counts
4262 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the 4264 * are adjusted, %RTM_NEWLINK is sent to the routing socket and the
4263 * function returns zero. 4265 * function returns zero.
4264 */ 4266 */
4265 int netdev_set_master(struct net_device *slave, struct net_device *master) 4267 int netdev_set_master(struct net_device *slave, struct net_device *master)
4266 { 4268 {
4267 struct net_device *old = slave->master; 4269 struct net_device *old = slave->master;
4268 4270
4269 ASSERT_RTNL(); 4271 ASSERT_RTNL();
4270 4272
4271 if (master) { 4273 if (master) {
4272 if (old) 4274 if (old)
4273 return -EBUSY; 4275 return -EBUSY;
4274 dev_hold(master); 4276 dev_hold(master);
4275 } 4277 }
4276 4278
4277 slave->master = master; 4279 slave->master = master;
4278 4280
4279 if (old) { 4281 if (old) {
4280 synchronize_net(); 4282 synchronize_net();
4281 dev_put(old); 4283 dev_put(old);
4282 } 4284 }
4283 if (master) 4285 if (master)
4284 slave->flags |= IFF_SLAVE; 4286 slave->flags |= IFF_SLAVE;
4285 else 4287 else
4286 slave->flags &= ~IFF_SLAVE; 4288 slave->flags &= ~IFF_SLAVE;
4287 4289
4288 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE); 4290 rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
4289 return 0; 4291 return 0;
4290 } 4292 }
4291 EXPORT_SYMBOL(netdev_set_master); 4293 EXPORT_SYMBOL(netdev_set_master);
4292 4294
4293 static void dev_change_rx_flags(struct net_device *dev, int flags) 4295 static void dev_change_rx_flags(struct net_device *dev, int flags)
4294 { 4296 {
4295 const struct net_device_ops *ops = dev->netdev_ops; 4297 const struct net_device_ops *ops = dev->netdev_ops;
4296 4298
4297 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags) 4299 if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
4298 ops->ndo_change_rx_flags(dev, flags); 4300 ops->ndo_change_rx_flags(dev, flags);
4299 } 4301 }
4300 4302
4301 static int __dev_set_promiscuity(struct net_device *dev, int inc) 4303 static int __dev_set_promiscuity(struct net_device *dev, int inc)
4302 { 4304 {
4303 unsigned short old_flags = dev->flags; 4305 unsigned short old_flags = dev->flags;
4304 uid_t uid; 4306 uid_t uid;
4305 gid_t gid; 4307 gid_t gid;
4306 4308
4307 ASSERT_RTNL(); 4309 ASSERT_RTNL();
4308 4310
4309 dev->flags |= IFF_PROMISC; 4311 dev->flags |= IFF_PROMISC;
4310 dev->promiscuity += inc; 4312 dev->promiscuity += inc;
4311 if (dev->promiscuity == 0) { 4313 if (dev->promiscuity == 0) {
4312 /* 4314 /*
4313 * Avoid overflow. 4315 * Avoid overflow.
4314 * If inc causes overflow, untouch promisc and return error. 4316 * If inc causes overflow, untouch promisc and return error.
4315 */ 4317 */
4316 if (inc < 0) 4318 if (inc < 0)
4317 dev->flags &= ~IFF_PROMISC; 4319 dev->flags &= ~IFF_PROMISC;
4318 else { 4320 else {
4319 dev->promiscuity -= inc; 4321 dev->promiscuity -= inc;
4320 printk(KERN_WARNING "%s: promiscuity touches roof, " 4322 printk(KERN_WARNING "%s: promiscuity touches roof, "
4321 "set promiscuity failed, promiscuity feature " 4323 "set promiscuity failed, promiscuity feature "
4322 "of device might be broken.\n", dev->name); 4324 "of device might be broken.\n", dev->name);
4323 return -EOVERFLOW; 4325 return -EOVERFLOW;
4324 } 4326 }
4325 } 4327 }
4326 if (dev->flags != old_flags) { 4328 if (dev->flags != old_flags) {
4327 printk(KERN_INFO "device %s %s promiscuous mode\n", 4329 printk(KERN_INFO "device %s %s promiscuous mode\n",
4328 dev->name, (dev->flags & IFF_PROMISC) ? "entered" : 4330 dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
4329 "left"); 4331 "left");
4330 if (audit_enabled) { 4332 if (audit_enabled) {
4331 current_uid_gid(&uid, &gid); 4333 current_uid_gid(&uid, &gid);
4332 audit_log(current->audit_context, GFP_ATOMIC, 4334 audit_log(current->audit_context, GFP_ATOMIC,
4333 AUDIT_ANOM_PROMISCUOUS, 4335 AUDIT_ANOM_PROMISCUOUS,
4334 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 4336 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
4335 dev->name, (dev->flags & IFF_PROMISC), 4337 dev->name, (dev->flags & IFF_PROMISC),
4336 (old_flags & IFF_PROMISC), 4338 (old_flags & IFF_PROMISC),
4337 audit_get_loginuid(current), 4339 audit_get_loginuid(current),
4338 uid, gid, 4340 uid, gid,
4339 audit_get_sessionid(current)); 4341 audit_get_sessionid(current));
4340 } 4342 }
4341 4343
4342 dev_change_rx_flags(dev, IFF_PROMISC); 4344 dev_change_rx_flags(dev, IFF_PROMISC);
4343 } 4345 }
4344 return 0; 4346 return 0;
4345 } 4347 }
4346 4348
4347 /** 4349 /**
4348 * dev_set_promiscuity - update promiscuity count on a device 4350 * dev_set_promiscuity - update promiscuity count on a device
4349 * @dev: device 4351 * @dev: device
4350 * @inc: modifier 4352 * @inc: modifier
4351 * 4353 *
4352 * Add or remove promiscuity from a device. While the count in the device 4354 * Add or remove promiscuity from a device. While the count in the device
4353 * remains above zero the interface remains promiscuous. Once it hits zero 4355 * remains above zero the interface remains promiscuous. Once it hits zero
4354 * the device reverts back to normal filtering operation. A negative inc 4356 * the device reverts back to normal filtering operation. A negative inc
4355 * value is used to drop promiscuity on the device. 4357 * value is used to drop promiscuity on the device.
4356 * Return 0 if successful or a negative errno code on error. 4358 * Return 0 if successful or a negative errno code on error.
4357 */ 4359 */
4358 int dev_set_promiscuity(struct net_device *dev, int inc) 4360 int dev_set_promiscuity(struct net_device *dev, int inc)
4359 { 4361 {
4360 unsigned short old_flags = dev->flags; 4362 unsigned short old_flags = dev->flags;
4361 int err; 4363 int err;
4362 4364
4363 err = __dev_set_promiscuity(dev, inc); 4365 err = __dev_set_promiscuity(dev, inc);
4364 if (err < 0) 4366 if (err < 0)
4365 return err; 4367 return err;
4366 if (dev->flags != old_flags) 4368 if (dev->flags != old_flags)
4367 dev_set_rx_mode(dev); 4369 dev_set_rx_mode(dev);
4368 return err; 4370 return err;
4369 } 4371 }
4370 EXPORT_SYMBOL(dev_set_promiscuity); 4372 EXPORT_SYMBOL(dev_set_promiscuity);
4371 4373
4372 /** 4374 /**
4373 * dev_set_allmulti - update allmulti count on a device 4375 * dev_set_allmulti - update allmulti count on a device
4374 * @dev: device 4376 * @dev: device
4375 * @inc: modifier 4377 * @inc: modifier
4376 * 4378 *
4377 * Add or remove reception of all multicast frames to a device. While the 4379 * Add or remove reception of all multicast frames to a device. While the
4378 * count in the device remains above zero the interface remains listening 4380 * count in the device remains above zero the interface remains listening
4379 * to all interfaces. Once it hits zero the device reverts back to normal 4381 * to all interfaces. Once it hits zero the device reverts back to normal
4380 * filtering operation. A negative @inc value is used to drop the counter 4382 * filtering operation. A negative @inc value is used to drop the counter
4381 * when releasing a resource needing all multicasts. 4383 * when releasing a resource needing all multicasts.
4382 * Return 0 if successful or a negative errno code on error. 4384 * Return 0 if successful or a negative errno code on error.
4383 */ 4385 */
4384 4386
4385 int dev_set_allmulti(struct net_device *dev, int inc) 4387 int dev_set_allmulti(struct net_device *dev, int inc)
4386 { 4388 {
4387 unsigned short old_flags = dev->flags; 4389 unsigned short old_flags = dev->flags;
4388 4390
4389 ASSERT_RTNL(); 4391 ASSERT_RTNL();
4390 4392
4391 dev->flags |= IFF_ALLMULTI; 4393 dev->flags |= IFF_ALLMULTI;
4392 dev->allmulti += inc; 4394 dev->allmulti += inc;
4393 if (dev->allmulti == 0) { 4395 if (dev->allmulti == 0) {
4394 /* 4396 /*
4395 * Avoid overflow. 4397 * Avoid overflow.
4396 * If inc causes overflow, untouch allmulti and return error. 4398 * If inc causes overflow, untouch allmulti and return error.
4397 */ 4399 */
4398 if (inc < 0) 4400 if (inc < 0)
4399 dev->flags &= ~IFF_ALLMULTI; 4401 dev->flags &= ~IFF_ALLMULTI;
4400 else { 4402 else {
4401 dev->allmulti -= inc; 4403 dev->allmulti -= inc;
4402 printk(KERN_WARNING "%s: allmulti touches roof, " 4404 printk(KERN_WARNING "%s: allmulti touches roof, "
4403 "set allmulti failed, allmulti feature of " 4405 "set allmulti failed, allmulti feature of "
4404 "device might be broken.\n", dev->name); 4406 "device might be broken.\n", dev->name);
4405 return -EOVERFLOW; 4407 return -EOVERFLOW;
4406 } 4408 }
4407 } 4409 }
4408 if (dev->flags ^ old_flags) { 4410 if (dev->flags ^ old_flags) {
4409 dev_change_rx_flags(dev, IFF_ALLMULTI); 4411 dev_change_rx_flags(dev, IFF_ALLMULTI);
4410 dev_set_rx_mode(dev); 4412 dev_set_rx_mode(dev);
4411 } 4413 }
4412 return 0; 4414 return 0;
4413 } 4415 }
4414 EXPORT_SYMBOL(dev_set_allmulti); 4416 EXPORT_SYMBOL(dev_set_allmulti);
4415 4417
4416 /* 4418 /*
4417 * Upload unicast and multicast address lists to device and 4419 * Upload unicast and multicast address lists to device and
4418 * configure RX filtering. When the device doesn't support unicast 4420 * configure RX filtering. When the device doesn't support unicast
4419 * filtering it is put in promiscuous mode while unicast addresses 4421 * filtering it is put in promiscuous mode while unicast addresses
4420 * are present. 4422 * are present.
4421 */ 4423 */
4422 void __dev_set_rx_mode(struct net_device *dev) 4424 void __dev_set_rx_mode(struct net_device *dev)
4423 { 4425 {
4424 const struct net_device_ops *ops = dev->netdev_ops; 4426 const struct net_device_ops *ops = dev->netdev_ops;
4425 4427
4426 /* dev_open will call this function so the list will stay sane. */ 4428 /* dev_open will call this function so the list will stay sane. */
4427 if (!(dev->flags&IFF_UP)) 4429 if (!(dev->flags&IFF_UP))
4428 return; 4430 return;
4429 4431
4430 if (!netif_device_present(dev)) 4432 if (!netif_device_present(dev))
4431 return; 4433 return;
4432 4434
4433 if (ops->ndo_set_rx_mode) 4435 if (ops->ndo_set_rx_mode)
4434 ops->ndo_set_rx_mode(dev); 4436 ops->ndo_set_rx_mode(dev);
4435 else { 4437 else {
4436 /* Unicast addresses changes may only happen under the rtnl, 4438 /* Unicast addresses changes may only happen under the rtnl,
4437 * therefore calling __dev_set_promiscuity here is safe. 4439 * therefore calling __dev_set_promiscuity here is safe.
4438 */ 4440 */
4439 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { 4441 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
4440 __dev_set_promiscuity(dev, 1); 4442 __dev_set_promiscuity(dev, 1);
4441 dev->uc_promisc = 1; 4443 dev->uc_promisc = 1;
4442 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { 4444 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
4443 __dev_set_promiscuity(dev, -1); 4445 __dev_set_promiscuity(dev, -1);
4444 dev->uc_promisc = 0; 4446 dev->uc_promisc = 0;
4445 } 4447 }
4446 4448
4447 if (ops->ndo_set_multicast_list) 4449 if (ops->ndo_set_multicast_list)
4448 ops->ndo_set_multicast_list(dev); 4450 ops->ndo_set_multicast_list(dev);
4449 } 4451 }
4450 } 4452 }
4451 4453
4452 void dev_set_rx_mode(struct net_device *dev) 4454 void dev_set_rx_mode(struct net_device *dev)
4453 { 4455 {
4454 netif_addr_lock_bh(dev); 4456 netif_addr_lock_bh(dev);
4455 __dev_set_rx_mode(dev); 4457 __dev_set_rx_mode(dev);
4456 netif_addr_unlock_bh(dev); 4458 netif_addr_unlock_bh(dev);
4457 } 4459 }
4458 4460
4459 /** 4461 /**
4460 * dev_get_flags - get flags reported to userspace 4462 * dev_get_flags - get flags reported to userspace
4461 * @dev: device 4463 * @dev: device
4462 * 4464 *
4463 * Get the combination of flag bits exported through APIs to userspace. 4465 * Get the combination of flag bits exported through APIs to userspace.
4464 */ 4466 */
4465 unsigned dev_get_flags(const struct net_device *dev) 4467 unsigned dev_get_flags(const struct net_device *dev)
4466 { 4468 {
4467 unsigned flags; 4469 unsigned flags;
4468 4470
4469 flags = (dev->flags & ~(IFF_PROMISC | 4471 flags = (dev->flags & ~(IFF_PROMISC |
4470 IFF_ALLMULTI | 4472 IFF_ALLMULTI |
4471 IFF_RUNNING | 4473 IFF_RUNNING |
4472 IFF_LOWER_UP | 4474 IFF_LOWER_UP |
4473 IFF_DORMANT)) | 4475 IFF_DORMANT)) |
4474 (dev->gflags & (IFF_PROMISC | 4476 (dev->gflags & (IFF_PROMISC |
4475 IFF_ALLMULTI)); 4477 IFF_ALLMULTI));
4476 4478
4477 if (netif_running(dev)) { 4479 if (netif_running(dev)) {
4478 if (netif_oper_up(dev)) 4480 if (netif_oper_up(dev))
4479 flags |= IFF_RUNNING; 4481 flags |= IFF_RUNNING;
4480 if (netif_carrier_ok(dev)) 4482 if (netif_carrier_ok(dev))
4481 flags |= IFF_LOWER_UP; 4483 flags |= IFF_LOWER_UP;
4482 if (netif_dormant(dev)) 4484 if (netif_dormant(dev))
4483 flags |= IFF_DORMANT; 4485 flags |= IFF_DORMANT;
4484 } 4486 }
4485 4487
4486 return flags; 4488 return flags;
4487 } 4489 }
4488 EXPORT_SYMBOL(dev_get_flags); 4490 EXPORT_SYMBOL(dev_get_flags);
4489 4491
4490 int __dev_change_flags(struct net_device *dev, unsigned int flags) 4492 int __dev_change_flags(struct net_device *dev, unsigned int flags)
4491 { 4493 {
4492 int old_flags = dev->flags; 4494 int old_flags = dev->flags;
4493 int ret; 4495 int ret;
4494 4496
4495 ASSERT_RTNL(); 4497 ASSERT_RTNL();
4496 4498
4497 /* 4499 /*
4498 * Set the flags on our device. 4500 * Set the flags on our device.
4499 */ 4501 */
4500 4502
4501 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 4503 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
4502 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 4504 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
4503 IFF_AUTOMEDIA)) | 4505 IFF_AUTOMEDIA)) |
4504 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 4506 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
4505 IFF_ALLMULTI)); 4507 IFF_ALLMULTI));
4506 4508
4507 /* 4509 /*
4508 * Load in the correct multicast list now the flags have changed. 4510 * Load in the correct multicast list now the flags have changed.
4509 */ 4511 */
4510 4512
4511 if ((old_flags ^ flags) & IFF_MULTICAST) 4513 if ((old_flags ^ flags) & IFF_MULTICAST)
4512 dev_change_rx_flags(dev, IFF_MULTICAST); 4514 dev_change_rx_flags(dev, IFF_MULTICAST);
4513 4515
4514 dev_set_rx_mode(dev); 4516 dev_set_rx_mode(dev);
4515 4517
4516 /* 4518 /*
4517 * Have we downed the interface. We handle IFF_UP ourselves 4519 * Have we downed the interface. We handle IFF_UP ourselves
4518 * according to user attempts to set it, rather than blindly 4520 * according to user attempts to set it, rather than blindly
4519 * setting it. 4521 * setting it.
4520 */ 4522 */
4521 4523
4522 ret = 0; 4524 ret = 0;
4523 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */ 4525 if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */
4524 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); 4526 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
4525 4527
4526 if (!ret) 4528 if (!ret)
4527 dev_set_rx_mode(dev); 4529 dev_set_rx_mode(dev);
4528 } 4530 }
4529 4531
4530 if ((flags ^ dev->gflags) & IFF_PROMISC) { 4532 if ((flags ^ dev->gflags) & IFF_PROMISC) {
4531 int inc = (flags & IFF_PROMISC) ? 1 : -1; 4533 int inc = (flags & IFF_PROMISC) ? 1 : -1;
4532 4534
4533 dev->gflags ^= IFF_PROMISC; 4535 dev->gflags ^= IFF_PROMISC;
4534 dev_set_promiscuity(dev, inc); 4536 dev_set_promiscuity(dev, inc);
4535 } 4537 }
4536 4538
4537 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 4539 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
4538 is important. Some (broken) drivers set IFF_PROMISC, when 4540 is important. Some (broken) drivers set IFF_PROMISC, when
4539 IFF_ALLMULTI is requested not asking us and not reporting. 4541 IFF_ALLMULTI is requested not asking us and not reporting.
4540 */ 4542 */
4541 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 4543 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
4542 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 4544 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
4543 4545
4544 dev->gflags ^= IFF_ALLMULTI; 4546 dev->gflags ^= IFF_ALLMULTI;
4545 dev_set_allmulti(dev, inc); 4547 dev_set_allmulti(dev, inc);
4546 } 4548 }
4547 4549
4548 return ret; 4550 return ret;
4549 } 4551 }
4550 4552
4551 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags) 4553 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
4552 { 4554 {
4553 unsigned int changes = dev->flags ^ old_flags; 4555 unsigned int changes = dev->flags ^ old_flags;
4554 4556
4555 if (changes & IFF_UP) { 4557 if (changes & IFF_UP) {
4556 if (dev->flags & IFF_UP) 4558 if (dev->flags & IFF_UP)
4557 call_netdevice_notifiers(NETDEV_UP, dev); 4559 call_netdevice_notifiers(NETDEV_UP, dev);
4558 else 4560 else
4559 call_netdevice_notifiers(NETDEV_DOWN, dev); 4561 call_netdevice_notifiers(NETDEV_DOWN, dev);
4560 } 4562 }
4561 4563
4562 if (dev->flags & IFF_UP && 4564 if (dev->flags & IFF_UP &&
4563 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) 4565 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
4564 call_netdevice_notifiers(NETDEV_CHANGE, dev); 4566 call_netdevice_notifiers(NETDEV_CHANGE, dev);
4565 } 4567 }
4566 4568
4567 /** 4569 /**
4568 * dev_change_flags - change device settings 4570 * dev_change_flags - change device settings
4569 * @dev: device 4571 * @dev: device
4570 * @flags: device state flags 4572 * @flags: device state flags
4571 * 4573 *
4572 * Change settings on device based state flags. The flags are 4574 * Change settings on device based state flags. The flags are
4573 * in the userspace exported format. 4575 * in the userspace exported format.
4574 */ 4576 */
4575 int dev_change_flags(struct net_device *dev, unsigned flags) 4577 int dev_change_flags(struct net_device *dev, unsigned flags)
4576 { 4578 {
4577 int ret, changes; 4579 int ret, changes;
4578 int old_flags = dev->flags; 4580 int old_flags = dev->flags;
4579 4581
4580 ret = __dev_change_flags(dev, flags); 4582 ret = __dev_change_flags(dev, flags);
4581 if (ret < 0) 4583 if (ret < 0)
4582 return ret; 4584 return ret;
4583 4585
4584 changes = old_flags ^ dev->flags; 4586 changes = old_flags ^ dev->flags;
4585 if (changes) 4587 if (changes)
4586 rtmsg_ifinfo(RTM_NEWLINK, dev, changes); 4588 rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
4587 4589
4588 __dev_notify_flags(dev, old_flags); 4590 __dev_notify_flags(dev, old_flags);
4589 return ret; 4591 return ret;
4590 } 4592 }
4591 EXPORT_SYMBOL(dev_change_flags); 4593 EXPORT_SYMBOL(dev_change_flags);
4592 4594
4593 /** 4595 /**
4594 * dev_set_mtu - Change maximum transfer unit 4596 * dev_set_mtu - Change maximum transfer unit
4595 * @dev: device 4597 * @dev: device
4596 * @new_mtu: new transfer unit 4598 * @new_mtu: new transfer unit
4597 * 4599 *
4598 * Change the maximum transfer size of the network device. 4600 * Change the maximum transfer size of the network device.
4599 */ 4601 */
4600 int dev_set_mtu(struct net_device *dev, int new_mtu) 4602 int dev_set_mtu(struct net_device *dev, int new_mtu)
4601 { 4603 {
4602 const struct net_device_ops *ops = dev->netdev_ops; 4604 const struct net_device_ops *ops = dev->netdev_ops;
4603 int err; 4605 int err;
4604 4606
4605 if (new_mtu == dev->mtu) 4607 if (new_mtu == dev->mtu)
4606 return 0; 4608 return 0;
4607 4609
4608 /* MTU must be positive. */ 4610 /* MTU must be positive. */
4609 if (new_mtu < 0) 4611 if (new_mtu < 0)
4610 return -EINVAL; 4612 return -EINVAL;
4611 4613
4612 if (!netif_device_present(dev)) 4614 if (!netif_device_present(dev))
4613 return -ENODEV; 4615 return -ENODEV;
4614 4616
4615 err = 0; 4617 err = 0;
4616 if (ops->ndo_change_mtu) 4618 if (ops->ndo_change_mtu)
4617 err = ops->ndo_change_mtu(dev, new_mtu); 4619 err = ops->ndo_change_mtu(dev, new_mtu);
4618 else 4620 else
4619 dev->mtu = new_mtu; 4621 dev->mtu = new_mtu;
4620 4622
4621 if (!err && dev->flags & IFF_UP) 4623 if (!err && dev->flags & IFF_UP)
4622 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); 4624 call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
4623 return err; 4625 return err;
4624 } 4626 }
4625 EXPORT_SYMBOL(dev_set_mtu); 4627 EXPORT_SYMBOL(dev_set_mtu);
4626 4628
4627 /** 4629 /**
4628 * dev_set_group - Change group this device belongs to 4630 * dev_set_group - Change group this device belongs to
4629 * @dev: device 4631 * @dev: device
4630 * @new_group: group this device should belong to 4632 * @new_group: group this device should belong to
4631 */ 4633 */
4632 void dev_set_group(struct net_device *dev, int new_group) 4634 void dev_set_group(struct net_device *dev, int new_group)
4633 { 4635 {
4634 dev->group = new_group; 4636 dev->group = new_group;
4635 } 4637 }
4636 EXPORT_SYMBOL(dev_set_group); 4638 EXPORT_SYMBOL(dev_set_group);
4637 4639
4638 /** 4640 /**
4639 * dev_set_mac_address - Change Media Access Control Address 4641 * dev_set_mac_address - Change Media Access Control Address
4640 * @dev: device 4642 * @dev: device
4641 * @sa: new address 4643 * @sa: new address
4642 * 4644 *
4643 * Change the hardware (MAC) address of the device 4645 * Change the hardware (MAC) address of the device
4644 */ 4646 */
4645 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) 4647 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
4646 { 4648 {
4647 const struct net_device_ops *ops = dev->netdev_ops; 4649 const struct net_device_ops *ops = dev->netdev_ops;
4648 int err; 4650 int err;
4649 4651
4650 if (!ops->ndo_set_mac_address) 4652 if (!ops->ndo_set_mac_address)
4651 return -EOPNOTSUPP; 4653 return -EOPNOTSUPP;
4652 if (sa->sa_family != dev->type) 4654 if (sa->sa_family != dev->type)
4653 return -EINVAL; 4655 return -EINVAL;
4654 if (!netif_device_present(dev)) 4656 if (!netif_device_present(dev))
4655 return -ENODEV; 4657 return -ENODEV;
4656 err = ops->ndo_set_mac_address(dev, sa); 4658 err = ops->ndo_set_mac_address(dev, sa);
4657 if (!err) 4659 if (!err)
4658 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 4660 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4659 return err; 4661 return err;
4660 } 4662 }
4661 EXPORT_SYMBOL(dev_set_mac_address); 4663 EXPORT_SYMBOL(dev_set_mac_address);
4662 4664
4663 /* 4665 /*
4664 * Perform the SIOCxIFxxx calls, inside rcu_read_lock() 4666 * Perform the SIOCxIFxxx calls, inside rcu_read_lock()
4665 */ 4667 */
4666 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd) 4668 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
4667 { 4669 {
4668 int err; 4670 int err;
4669 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name); 4671 struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
4670 4672
4671 if (!dev) 4673 if (!dev)
4672 return -ENODEV; 4674 return -ENODEV;
4673 4675
4674 switch (cmd) { 4676 switch (cmd) {
4675 case SIOCGIFFLAGS: /* Get interface flags */ 4677 case SIOCGIFFLAGS: /* Get interface flags */
4676 ifr->ifr_flags = (short) dev_get_flags(dev); 4678 ifr->ifr_flags = (short) dev_get_flags(dev);
4677 return 0; 4679 return 0;
4678 4680
4679 case SIOCGIFMETRIC: /* Get the metric on the interface 4681 case SIOCGIFMETRIC: /* Get the metric on the interface
4680 (currently unused) */ 4682 (currently unused) */
4681 ifr->ifr_metric = 0; 4683 ifr->ifr_metric = 0;
4682 return 0; 4684 return 0;
4683 4685
4684 case SIOCGIFMTU: /* Get the MTU of a device */ 4686 case SIOCGIFMTU: /* Get the MTU of a device */
4685 ifr->ifr_mtu = dev->mtu; 4687 ifr->ifr_mtu = dev->mtu;
4686 return 0; 4688 return 0;
4687 4689
4688 case SIOCGIFHWADDR: 4690 case SIOCGIFHWADDR:
4689 if (!dev->addr_len) 4691 if (!dev->addr_len)
4690 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data); 4692 memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
4691 else 4693 else
4692 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr, 4694 memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
4693 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len)); 4695 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4694 ifr->ifr_hwaddr.sa_family = dev->type; 4696 ifr->ifr_hwaddr.sa_family = dev->type;
4695 return 0; 4697 return 0;
4696 4698
4697 case SIOCGIFSLAVE: 4699 case SIOCGIFSLAVE:
4698 err = -EINVAL; 4700 err = -EINVAL;
4699 break; 4701 break;
4700 4702
4701 case SIOCGIFMAP: 4703 case SIOCGIFMAP:
4702 ifr->ifr_map.mem_start = dev->mem_start; 4704 ifr->ifr_map.mem_start = dev->mem_start;
4703 ifr->ifr_map.mem_end = dev->mem_end; 4705 ifr->ifr_map.mem_end = dev->mem_end;
4704 ifr->ifr_map.base_addr = dev->base_addr; 4706 ifr->ifr_map.base_addr = dev->base_addr;
4705 ifr->ifr_map.irq = dev->irq; 4707 ifr->ifr_map.irq = dev->irq;
4706 ifr->ifr_map.dma = dev->dma; 4708 ifr->ifr_map.dma = dev->dma;
4707 ifr->ifr_map.port = dev->if_port; 4709 ifr->ifr_map.port = dev->if_port;
4708 return 0; 4710 return 0;
4709 4711
4710 case SIOCGIFINDEX: 4712 case SIOCGIFINDEX:
4711 ifr->ifr_ifindex = dev->ifindex; 4713 ifr->ifr_ifindex = dev->ifindex;
4712 return 0; 4714 return 0;
4713 4715
4714 case SIOCGIFTXQLEN: 4716 case SIOCGIFTXQLEN:
4715 ifr->ifr_qlen = dev->tx_queue_len; 4717 ifr->ifr_qlen = dev->tx_queue_len;
4716 return 0; 4718 return 0;
4717 4719
4718 default: 4720 default:
4719 /* dev_ioctl() should ensure this case 4721 /* dev_ioctl() should ensure this case
4720 * is never reached 4722 * is never reached
4721 */ 4723 */
4722 WARN_ON(1); 4724 WARN_ON(1);
4723 err = -EINVAL; 4725 err = -EINVAL;
4724 break; 4726 break;
4725 4727
4726 } 4728 }
4727 return err; 4729 return err;
4728 } 4730 }
4729 4731
4730 /* 4732 /*
4731 * Perform the SIOCxIFxxx calls, inside rtnl_lock() 4733 * Perform the SIOCxIFxxx calls, inside rtnl_lock()
4732 */ 4734 */
4733 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd) 4735 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
4734 { 4736 {
4735 int err; 4737 int err;
4736 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name); 4738 struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
4737 const struct net_device_ops *ops; 4739 const struct net_device_ops *ops;
4738 4740
4739 if (!dev) 4741 if (!dev)
4740 return -ENODEV; 4742 return -ENODEV;
4741 4743
4742 ops = dev->netdev_ops; 4744 ops = dev->netdev_ops;
4743 4745
4744 switch (cmd) { 4746 switch (cmd) {
4745 case SIOCSIFFLAGS: /* Set interface flags */ 4747 case SIOCSIFFLAGS: /* Set interface flags */
4746 return dev_change_flags(dev, ifr->ifr_flags); 4748 return dev_change_flags(dev, ifr->ifr_flags);
4747 4749
4748 case SIOCSIFMETRIC: /* Set the metric on the interface 4750 case SIOCSIFMETRIC: /* Set the metric on the interface
4749 (currently unused) */ 4751 (currently unused) */
4750 return -EOPNOTSUPP; 4752 return -EOPNOTSUPP;
4751 4753
4752 case SIOCSIFMTU: /* Set the MTU of a device */ 4754 case SIOCSIFMTU: /* Set the MTU of a device */
4753 return dev_set_mtu(dev, ifr->ifr_mtu); 4755 return dev_set_mtu(dev, ifr->ifr_mtu);
4754 4756
4755 case SIOCSIFHWADDR: 4757 case SIOCSIFHWADDR:
4756 return dev_set_mac_address(dev, &ifr->ifr_hwaddr); 4758 return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
4757 4759
4758 case SIOCSIFHWBROADCAST: 4760 case SIOCSIFHWBROADCAST:
4759 if (ifr->ifr_hwaddr.sa_family != dev->type) 4761 if (ifr->ifr_hwaddr.sa_family != dev->type)
4760 return -EINVAL; 4762 return -EINVAL;
4761 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data, 4763 memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
4762 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len)); 4764 min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
4763 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 4765 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
4764 return 0; 4766 return 0;
4765 4767
4766 case SIOCSIFMAP: 4768 case SIOCSIFMAP:
4767 if (ops->ndo_set_config) { 4769 if (ops->ndo_set_config) {
4768 if (!netif_device_present(dev)) 4770 if (!netif_device_present(dev))
4769 return -ENODEV; 4771 return -ENODEV;
4770 return ops->ndo_set_config(dev, &ifr->ifr_map); 4772 return ops->ndo_set_config(dev, &ifr->ifr_map);
4771 } 4773 }
4772 return -EOPNOTSUPP; 4774 return -EOPNOTSUPP;
4773 4775
4774 case SIOCADDMULTI: 4776 case SIOCADDMULTI:
4775 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) || 4777 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4776 ifr->ifr_hwaddr.sa_family != AF_UNSPEC) 4778 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4777 return -EINVAL; 4779 return -EINVAL;
4778 if (!netif_device_present(dev)) 4780 if (!netif_device_present(dev))
4779 return -ENODEV; 4781 return -ENODEV;
4780 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data); 4782 return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
4781 4783
4782 case SIOCDELMULTI: 4784 case SIOCDELMULTI:
4783 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) || 4785 if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
4784 ifr->ifr_hwaddr.sa_family != AF_UNSPEC) 4786 ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
4785 return -EINVAL; 4787 return -EINVAL;
4786 if (!netif_device_present(dev)) 4788 if (!netif_device_present(dev))
4787 return -ENODEV; 4789 return -ENODEV;
4788 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data); 4790 return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
4789 4791
4790 case SIOCSIFTXQLEN: 4792 case SIOCSIFTXQLEN:
4791 if (ifr->ifr_qlen < 0) 4793 if (ifr->ifr_qlen < 0)
4792 return -EINVAL; 4794 return -EINVAL;
4793 dev->tx_queue_len = ifr->ifr_qlen; 4795 dev->tx_queue_len = ifr->ifr_qlen;
4794 return 0; 4796 return 0;
4795 4797
4796 case SIOCSIFNAME: 4798 case SIOCSIFNAME:
4797 ifr->ifr_newname[IFNAMSIZ-1] = '\0'; 4799 ifr->ifr_newname[IFNAMSIZ-1] = '\0';
4798 return dev_change_name(dev, ifr->ifr_newname); 4800 return dev_change_name(dev, ifr->ifr_newname);
4799 4801
4800 /* 4802 /*
4801 * Unknown or private ioctl 4803 * Unknown or private ioctl
4802 */ 4804 */
4803 default: 4805 default:
4804 if ((cmd >= SIOCDEVPRIVATE && 4806 if ((cmd >= SIOCDEVPRIVATE &&
4805 cmd <= SIOCDEVPRIVATE + 15) || 4807 cmd <= SIOCDEVPRIVATE + 15) ||
4806 cmd == SIOCBONDENSLAVE || 4808 cmd == SIOCBONDENSLAVE ||
4807 cmd == SIOCBONDRELEASE || 4809 cmd == SIOCBONDRELEASE ||
4808 cmd == SIOCBONDSETHWADDR || 4810 cmd == SIOCBONDSETHWADDR ||
4809 cmd == SIOCBONDSLAVEINFOQUERY || 4811 cmd == SIOCBONDSLAVEINFOQUERY ||
4810 cmd == SIOCBONDINFOQUERY || 4812 cmd == SIOCBONDINFOQUERY ||
4811 cmd == SIOCBONDCHANGEACTIVE || 4813 cmd == SIOCBONDCHANGEACTIVE ||
4812 cmd == SIOCGMIIPHY || 4814 cmd == SIOCGMIIPHY ||
4813 cmd == SIOCGMIIREG || 4815 cmd == SIOCGMIIREG ||
4814 cmd == SIOCSMIIREG || 4816 cmd == SIOCSMIIREG ||
4815 cmd == SIOCBRADDIF || 4817 cmd == SIOCBRADDIF ||
4816 cmd == SIOCBRDELIF || 4818 cmd == SIOCBRDELIF ||
4817 cmd == SIOCSHWTSTAMP || 4819 cmd == SIOCSHWTSTAMP ||
4818 cmd == SIOCWANDEV) { 4820 cmd == SIOCWANDEV) {
4819 err = -EOPNOTSUPP; 4821 err = -EOPNOTSUPP;
4820 if (ops->ndo_do_ioctl) { 4822 if (ops->ndo_do_ioctl) {
4821 if (netif_device_present(dev)) 4823 if (netif_device_present(dev))
4822 err = ops->ndo_do_ioctl(dev, ifr, cmd); 4824 err = ops->ndo_do_ioctl(dev, ifr, cmd);
4823 else 4825 else
4824 err = -ENODEV; 4826 err = -ENODEV;
4825 } 4827 }
4826 } else 4828 } else
4827 err = -EINVAL; 4829 err = -EINVAL;
4828 4830
4829 } 4831 }
4830 return err; 4832 return err;
4831 } 4833 }
4832 4834
4833 /* 4835 /*
4834 * This function handles all "interface"-type I/O control requests. The actual 4836 * This function handles all "interface"-type I/O control requests. The actual
4835 * 'doing' part of this is dev_ifsioc above. 4837 * 'doing' part of this is dev_ifsioc above.
4836 */ 4838 */
4837 4839
4838 /** 4840 /**
4839 * dev_ioctl - network device ioctl 4841 * dev_ioctl - network device ioctl
4840 * @net: the applicable net namespace 4842 * @net: the applicable net namespace
4841 * @cmd: command to issue 4843 * @cmd: command to issue
4842 * @arg: pointer to a struct ifreq in user space 4844 * @arg: pointer to a struct ifreq in user space
4843 * 4845 *
4844 * Issue ioctl functions to devices. This is normally called by the 4846 * Issue ioctl functions to devices. This is normally called by the
4845 * user space syscall interfaces but can sometimes be useful for 4847 * user space syscall interfaces but can sometimes be useful for
4846 * other purposes. The return value is the return from the syscall if 4848 * other purposes. The return value is the return from the syscall if
4847 * positive or a negative errno code on error. 4849 * positive or a negative errno code on error.
4848 */ 4850 */
4849 4851
4850 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg) 4852 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
4851 { 4853 {
4852 struct ifreq ifr; 4854 struct ifreq ifr;
4853 int ret; 4855 int ret;
4854 char *colon; 4856 char *colon;
4855 4857
4856 /* One special case: SIOCGIFCONF takes ifconf argument 4858 /* One special case: SIOCGIFCONF takes ifconf argument
4857 and requires shared lock, because it sleeps writing 4859 and requires shared lock, because it sleeps writing
4858 to user space. 4860 to user space.
4859 */ 4861 */
4860 4862
4861 if (cmd == SIOCGIFCONF) { 4863 if (cmd == SIOCGIFCONF) {
4862 rtnl_lock(); 4864 rtnl_lock();
4863 ret = dev_ifconf(net, (char __user *) arg); 4865 ret = dev_ifconf(net, (char __user *) arg);
4864 rtnl_unlock(); 4866 rtnl_unlock();
4865 return ret; 4867 return ret;
4866 } 4868 }
4867 if (cmd == SIOCGIFNAME) 4869 if (cmd == SIOCGIFNAME)
4868 return dev_ifname(net, (struct ifreq __user *)arg); 4870 return dev_ifname(net, (struct ifreq __user *)arg);
4869 4871
4870 if (copy_from_user(&ifr, arg, sizeof(struct ifreq))) 4872 if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
4871 return -EFAULT; 4873 return -EFAULT;
4872 4874
4873 ifr.ifr_name[IFNAMSIZ-1] = 0; 4875 ifr.ifr_name[IFNAMSIZ-1] = 0;
4874 4876
4875 colon = strchr(ifr.ifr_name, ':'); 4877 colon = strchr(ifr.ifr_name, ':');
4876 if (colon) 4878 if (colon)
4877 *colon = 0; 4879 *colon = 0;
4878 4880
4879 /* 4881 /*
4880 * See which interface the caller is talking about. 4882 * See which interface the caller is talking about.
4881 */ 4883 */
4882 4884
4883 switch (cmd) { 4885 switch (cmd) {
4884 /* 4886 /*
4885 * These ioctl calls: 4887 * These ioctl calls:
4886 * - can be done by all. 4888 * - can be done by all.
4887 * - atomic and do not require locking. 4889 * - atomic and do not require locking.
4888 * - return a value 4890 * - return a value
4889 */ 4891 */
4890 case SIOCGIFFLAGS: 4892 case SIOCGIFFLAGS:
4891 case SIOCGIFMETRIC: 4893 case SIOCGIFMETRIC:
4892 case SIOCGIFMTU: 4894 case SIOCGIFMTU:
4893 case SIOCGIFHWADDR: 4895 case SIOCGIFHWADDR:
4894 case SIOCGIFSLAVE: 4896 case SIOCGIFSLAVE:
4895 case SIOCGIFMAP: 4897 case SIOCGIFMAP:
4896 case SIOCGIFINDEX: 4898 case SIOCGIFINDEX:
4897 case SIOCGIFTXQLEN: 4899 case SIOCGIFTXQLEN:
4898 dev_load(net, ifr.ifr_name); 4900 dev_load(net, ifr.ifr_name);
4899 rcu_read_lock(); 4901 rcu_read_lock();
4900 ret = dev_ifsioc_locked(net, &ifr, cmd); 4902 ret = dev_ifsioc_locked(net, &ifr, cmd);
4901 rcu_read_unlock(); 4903 rcu_read_unlock();
4902 if (!ret) { 4904 if (!ret) {
4903 if (colon) 4905 if (colon)
4904 *colon = ':'; 4906 *colon = ':';
4905 if (copy_to_user(arg, &ifr, 4907 if (copy_to_user(arg, &ifr,
4906 sizeof(struct ifreq))) 4908 sizeof(struct ifreq)))
4907 ret = -EFAULT; 4909 ret = -EFAULT;
4908 } 4910 }
4909 return ret; 4911 return ret;
4910 4912
4911 case SIOCETHTOOL: 4913 case SIOCETHTOOL:
4912 dev_load(net, ifr.ifr_name); 4914 dev_load(net, ifr.ifr_name);
4913 rtnl_lock(); 4915 rtnl_lock();
4914 ret = dev_ethtool(net, &ifr); 4916 ret = dev_ethtool(net, &ifr);
4915 rtnl_unlock(); 4917 rtnl_unlock();
4916 if (!ret) { 4918 if (!ret) {
4917 if (colon) 4919 if (colon)
4918 *colon = ':'; 4920 *colon = ':';
4919 if (copy_to_user(arg, &ifr, 4921 if (copy_to_user(arg, &ifr,
4920 sizeof(struct ifreq))) 4922 sizeof(struct ifreq)))
4921 ret = -EFAULT; 4923 ret = -EFAULT;
4922 } 4924 }
4923 return ret; 4925 return ret;
4924 4926
4925 /* 4927 /*
4926 * These ioctl calls: 4928 * These ioctl calls:
4927 * - require superuser power. 4929 * - require superuser power.
4928 * - require strict serialization. 4930 * - require strict serialization.
4929 * - return a value 4931 * - return a value
4930 */ 4932 */
4931 case SIOCGMIIPHY: 4933 case SIOCGMIIPHY:
4932 case SIOCGMIIREG: 4934 case SIOCGMIIREG:
4933 case SIOCSIFNAME: 4935 case SIOCSIFNAME:
4934 if (!capable(CAP_NET_ADMIN)) 4936 if (!capable(CAP_NET_ADMIN))
4935 return -EPERM; 4937 return -EPERM;
4936 dev_load(net, ifr.ifr_name); 4938 dev_load(net, ifr.ifr_name);
4937 rtnl_lock(); 4939 rtnl_lock();
4938 ret = dev_ifsioc(net, &ifr, cmd); 4940 ret = dev_ifsioc(net, &ifr, cmd);
4939 rtnl_unlock(); 4941 rtnl_unlock();
4940 if (!ret) { 4942 if (!ret) {
4941 if (colon) 4943 if (colon)
4942 *colon = ':'; 4944 *colon = ':';
4943 if (copy_to_user(arg, &ifr, 4945 if (copy_to_user(arg, &ifr,
4944 sizeof(struct ifreq))) 4946 sizeof(struct ifreq)))
4945 ret = -EFAULT; 4947 ret = -EFAULT;
4946 } 4948 }
4947 return ret; 4949 return ret;
4948 4950
4949 /* 4951 /*
4950 * These ioctl calls: 4952 * These ioctl calls:
4951 * - require superuser power. 4953 * - require superuser power.
4952 * - require strict serialization. 4954 * - require strict serialization.
4953 * - do not return a value 4955 * - do not return a value
4954 */ 4956 */
4955 case SIOCSIFFLAGS: 4957 case SIOCSIFFLAGS:
4956 case SIOCSIFMETRIC: 4958 case SIOCSIFMETRIC:
4957 case SIOCSIFMTU: 4959 case SIOCSIFMTU:
4958 case SIOCSIFMAP: 4960 case SIOCSIFMAP:
4959 case SIOCSIFHWADDR: 4961 case SIOCSIFHWADDR:
4960 case SIOCSIFSLAVE: 4962 case SIOCSIFSLAVE:
4961 case SIOCADDMULTI: 4963 case SIOCADDMULTI:
4962 case SIOCDELMULTI: 4964 case SIOCDELMULTI:
4963 case SIOCSIFHWBROADCAST: 4965 case SIOCSIFHWBROADCAST:
4964 case SIOCSIFTXQLEN: 4966 case SIOCSIFTXQLEN:
4965 case SIOCSMIIREG: 4967 case SIOCSMIIREG:
4966 case SIOCBONDENSLAVE: 4968 case SIOCBONDENSLAVE:
4967 case SIOCBONDRELEASE: 4969 case SIOCBONDRELEASE:
4968 case SIOCBONDSETHWADDR: 4970 case SIOCBONDSETHWADDR:
4969 case SIOCBONDCHANGEACTIVE: 4971 case SIOCBONDCHANGEACTIVE:
4970 case SIOCBRADDIF: 4972 case SIOCBRADDIF:
4971 case SIOCBRDELIF: 4973 case SIOCBRDELIF:
4972 case SIOCSHWTSTAMP: 4974 case SIOCSHWTSTAMP:
4973 if (!capable(CAP_NET_ADMIN)) 4975 if (!capable(CAP_NET_ADMIN))
4974 return -EPERM; 4976 return -EPERM;
4975 /* fall through */ 4977 /* fall through */
4976 case SIOCBONDSLAVEINFOQUERY: 4978 case SIOCBONDSLAVEINFOQUERY:
4977 case SIOCBONDINFOQUERY: 4979 case SIOCBONDINFOQUERY:
4978 dev_load(net, ifr.ifr_name); 4980 dev_load(net, ifr.ifr_name);
4979 rtnl_lock(); 4981 rtnl_lock();
4980 ret = dev_ifsioc(net, &ifr, cmd); 4982 ret = dev_ifsioc(net, &ifr, cmd);
4981 rtnl_unlock(); 4983 rtnl_unlock();
4982 return ret; 4984 return ret;
4983 4985
4984 case SIOCGIFMEM: 4986 case SIOCGIFMEM:
4985 /* Get the per device memory space. We can add this but 4987 /* Get the per device memory space. We can add this but
4986 * currently do not support it */ 4988 * currently do not support it */
4987 case SIOCSIFMEM: 4989 case SIOCSIFMEM:
4988 /* Set the per device memory buffer space. 4990 /* Set the per device memory buffer space.
4989 * Not applicable in our case */ 4991 * Not applicable in our case */
4990 case SIOCSIFLINK: 4992 case SIOCSIFLINK:
4991 return -EINVAL; 4993 return -EINVAL;
4992 4994
4993 /* 4995 /*
4994 * Unknown or private ioctl. 4996 * Unknown or private ioctl.
4995 */ 4997 */
4996 default: 4998 default:
4997 if (cmd == SIOCWANDEV || 4999 if (cmd == SIOCWANDEV ||
4998 (cmd >= SIOCDEVPRIVATE && 5000 (cmd >= SIOCDEVPRIVATE &&
4999 cmd <= SIOCDEVPRIVATE + 15)) { 5001 cmd <= SIOCDEVPRIVATE + 15)) {
5000 dev_load(net, ifr.ifr_name); 5002 dev_load(net, ifr.ifr_name);
5001 rtnl_lock(); 5003 rtnl_lock();
5002 ret = dev_ifsioc(net, &ifr, cmd); 5004 ret = dev_ifsioc(net, &ifr, cmd);
5003 rtnl_unlock(); 5005 rtnl_unlock();
5004 if (!ret && copy_to_user(arg, &ifr, 5006 if (!ret && copy_to_user(arg, &ifr,
5005 sizeof(struct ifreq))) 5007 sizeof(struct ifreq)))
5006 ret = -EFAULT; 5008 ret = -EFAULT;
5007 return ret; 5009 return ret;
5008 } 5010 }
5009 /* Take care of Wireless Extensions */ 5011 /* Take care of Wireless Extensions */
5010 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST) 5012 if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
5011 return wext_handle_ioctl(net, &ifr, cmd, arg); 5013 return wext_handle_ioctl(net, &ifr, cmd, arg);
5012 return -EINVAL; 5014 return -EINVAL;
5013 } 5015 }
5014 } 5016 }
5015 5017
5016 5018
5017 /** 5019 /**
5018 * dev_new_index - allocate an ifindex 5020 * dev_new_index - allocate an ifindex
5019 * @net: the applicable net namespace 5021 * @net: the applicable net namespace
5020 * 5022 *
5021 * Returns a suitable unique value for a new device interface 5023 * Returns a suitable unique value for a new device interface
5022 * number. The caller must hold the rtnl semaphore or the 5024 * number. The caller must hold the rtnl semaphore or the
5023 * dev_base_lock to be sure it remains unique. 5025 * dev_base_lock to be sure it remains unique.
5024 */ 5026 */
5025 static int dev_new_index(struct net *net) 5027 static int dev_new_index(struct net *net)
5026 { 5028 {
5027 static int ifindex; 5029 static int ifindex;
5028 for (;;) { 5030 for (;;) {
5029 if (++ifindex <= 0) 5031 if (++ifindex <= 0)
5030 ifindex = 1; 5032 ifindex = 1;
5031 if (!__dev_get_by_index(net, ifindex)) 5033 if (!__dev_get_by_index(net, ifindex))
5032 return ifindex; 5034 return ifindex;
5033 } 5035 }
5034 } 5036 }
5035 5037
5036 /* Delayed registration/unregisteration */ 5038 /* Delayed registration/unregisteration */
5037 static LIST_HEAD(net_todo_list); 5039 static LIST_HEAD(net_todo_list);
5038 5040
5039 static void net_set_todo(struct net_device *dev) 5041 static void net_set_todo(struct net_device *dev)
5040 { 5042 {
5041 list_add_tail(&dev->todo_list, &net_todo_list); 5043 list_add_tail(&dev->todo_list, &net_todo_list);
5042 } 5044 }
5043 5045
5044 static void rollback_registered_many(struct list_head *head) 5046 static void rollback_registered_many(struct list_head *head)
5045 { 5047 {
5046 struct net_device *dev, *tmp; 5048 struct net_device *dev, *tmp;
5047 5049
5048 BUG_ON(dev_boot_phase); 5050 BUG_ON(dev_boot_phase);
5049 ASSERT_RTNL(); 5051 ASSERT_RTNL();
5050 5052
5051 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 5053 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
5052 /* Some devices call without registering 5054 /* Some devices call without registering
5053 * for initialization unwind. Remove those 5055 * for initialization unwind. Remove those
5054 * devices and proceed with the remaining. 5056 * devices and proceed with the remaining.
5055 */ 5057 */
5056 if (dev->reg_state == NETREG_UNINITIALIZED) { 5058 if (dev->reg_state == NETREG_UNINITIALIZED) {
5057 pr_debug("unregister_netdevice: device %s/%p never " 5059 pr_debug("unregister_netdevice: device %s/%p never "
5058 "was registered\n", dev->name, dev); 5060 "was registered\n", dev->name, dev);
5059 5061
5060 WARN_ON(1); 5062 WARN_ON(1);
5061 list_del(&dev->unreg_list); 5063 list_del(&dev->unreg_list);
5062 continue; 5064 continue;
5063 } 5065 }
5064 5066
5065 BUG_ON(dev->reg_state != NETREG_REGISTERED); 5067 BUG_ON(dev->reg_state != NETREG_REGISTERED);
5066 } 5068 }
5067 5069
5068 /* If device is running, close it first. */ 5070 /* If device is running, close it first. */
5069 dev_close_many(head); 5071 dev_close_many(head);
5070 5072
5071 list_for_each_entry(dev, head, unreg_list) { 5073 list_for_each_entry(dev, head, unreg_list) {
5072 /* And unlink it from device chain. */ 5074 /* And unlink it from device chain. */
5073 unlist_netdevice(dev); 5075 unlist_netdevice(dev);
5074 5076
5075 dev->reg_state = NETREG_UNREGISTERING; 5077 dev->reg_state = NETREG_UNREGISTERING;
5076 } 5078 }
5077 5079
5078 synchronize_net(); 5080 synchronize_net();
5079 5081
5080 list_for_each_entry(dev, head, unreg_list) { 5082 list_for_each_entry(dev, head, unreg_list) {
5081 /* Shutdown queueing discipline. */ 5083 /* Shutdown queueing discipline. */
5082 dev_shutdown(dev); 5084 dev_shutdown(dev);
5083 5085
5084 5086
5085 /* Notify protocols, that we are about to destroy 5087 /* Notify protocols, that we are about to destroy
5086 this device. They should clean all the things. 5088 this device. They should clean all the things.
5087 */ 5089 */
5088 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 5090 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5089 5091
5090 if (!dev->rtnl_link_ops || 5092 if (!dev->rtnl_link_ops ||
5091 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 5093 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5092 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U); 5094 rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
5093 5095
5094 /* 5096 /*
5095 * Flush the unicast and multicast chains 5097 * Flush the unicast and multicast chains
5096 */ 5098 */
5097 dev_uc_flush(dev); 5099 dev_uc_flush(dev);
5098 dev_mc_flush(dev); 5100 dev_mc_flush(dev);
5099 5101
5100 if (dev->netdev_ops->ndo_uninit) 5102 if (dev->netdev_ops->ndo_uninit)
5101 dev->netdev_ops->ndo_uninit(dev); 5103 dev->netdev_ops->ndo_uninit(dev);
5102 5104
5103 /* Notifier chain MUST detach us from master device. */ 5105 /* Notifier chain MUST detach us from master device. */
5104 WARN_ON(dev->master); 5106 WARN_ON(dev->master);
5105 5107
5106 /* Remove entries from kobject tree */ 5108 /* Remove entries from kobject tree */
5107 netdev_unregister_kobject(dev); 5109 netdev_unregister_kobject(dev);
5108 } 5110 }
5109 5111
5110 /* Process any work delayed until the end of the batch */ 5112 /* Process any work delayed until the end of the batch */
5111 dev = list_first_entry(head, struct net_device, unreg_list); 5113 dev = list_first_entry(head, struct net_device, unreg_list);
5112 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev); 5114 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5113 5115
5114 rcu_barrier(); 5116 rcu_barrier();
5115 5117
5116 list_for_each_entry(dev, head, unreg_list) 5118 list_for_each_entry(dev, head, unreg_list)
5117 dev_put(dev); 5119 dev_put(dev);
5118 } 5120 }
5119 5121
5120 static void rollback_registered(struct net_device *dev) 5122 static void rollback_registered(struct net_device *dev)
5121 { 5123 {
5122 LIST_HEAD(single); 5124 LIST_HEAD(single);
5123 5125
5124 list_add(&dev->unreg_list, &single); 5126 list_add(&dev->unreg_list, &single);
5125 rollback_registered_many(&single); 5127 rollback_registered_many(&single);
5126 } 5128 }
5127 5129
5128 unsigned long netdev_fix_features(unsigned long features, const char *name) 5130 unsigned long netdev_fix_features(unsigned long features, const char *name)
5129 { 5131 {
5130 /* Fix illegal SG+CSUM combinations. */ 5132 /* Fix illegal SG+CSUM combinations. */
5131 if ((features & NETIF_F_SG) && 5133 if ((features & NETIF_F_SG) &&
5132 !(features & NETIF_F_ALL_CSUM)) { 5134 !(features & NETIF_F_ALL_CSUM)) {
5133 if (name) 5135 if (name)
5134 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no " 5136 printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
5135 "checksum feature.\n", name); 5137 "checksum feature.\n", name);
5136 features &= ~NETIF_F_SG; 5138 features &= ~NETIF_F_SG;
5137 } 5139 }
5138 5140
5139 /* TSO requires that SG is present as well. */ 5141 /* TSO requires that SG is present as well. */
5140 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) { 5142 if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
5141 if (name) 5143 if (name)
5142 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no " 5144 printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
5143 "SG feature.\n", name); 5145 "SG feature.\n", name);
5144 features &= ~NETIF_F_TSO; 5146 features &= ~NETIF_F_TSO;
5145 } 5147 }
5146 5148
5147 if (features & NETIF_F_UFO) { 5149 if (features & NETIF_F_UFO) {
5148 /* maybe split UFO into V4 and V6? */ 5150 /* maybe split UFO into V4 and V6? */
5149 if (!((features & NETIF_F_GEN_CSUM) || 5151 if (!((features & NETIF_F_GEN_CSUM) ||
5150 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM)) 5152 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))
5151 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 5153 == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5152 if (name) 5154 if (name)
5153 printk(KERN_ERR "%s: Dropping NETIF_F_UFO " 5155 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
5154 "since no checksum offload features.\n", 5156 "since no checksum offload features.\n",
5155 name); 5157 name);
5156 features &= ~NETIF_F_UFO; 5158 features &= ~NETIF_F_UFO;
5157 } 5159 }
5158 5160
5159 if (!(features & NETIF_F_SG)) { 5161 if (!(features & NETIF_F_SG)) {
5160 if (name) 5162 if (name)
5161 printk(KERN_ERR "%s: Dropping NETIF_F_UFO " 5163 printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
5162 "since no NETIF_F_SG feature.\n", name); 5164 "since no NETIF_F_SG feature.\n", name);
5163 features &= ~NETIF_F_UFO; 5165 features &= ~NETIF_F_UFO;
5164 } 5166 }
5165 } 5167 }
5166 5168
5167 return features; 5169 return features;
5168 } 5170 }
5169 EXPORT_SYMBOL(netdev_fix_features); 5171 EXPORT_SYMBOL(netdev_fix_features);
5170 5172
5171 /** 5173 /**
5172 * netif_stacked_transfer_operstate - transfer operstate 5174 * netif_stacked_transfer_operstate - transfer operstate
5173 * @rootdev: the root or lower level device to transfer state from 5175 * @rootdev: the root or lower level device to transfer state from
5174 * @dev: the device to transfer operstate to 5176 * @dev: the device to transfer operstate to
5175 * 5177 *
5176 * Transfer operational state from root to device. This is normally 5178 * Transfer operational state from root to device. This is normally
5177 * called when a stacking relationship exists between the root 5179 * called when a stacking relationship exists between the root
5178 * device and the device(a leaf device). 5180 * device and the device(a leaf device).
5179 */ 5181 */
5180 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 5182 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5181 struct net_device *dev) 5183 struct net_device *dev)
5182 { 5184 {
5183 if (rootdev->operstate == IF_OPER_DORMANT) 5185 if (rootdev->operstate == IF_OPER_DORMANT)
5184 netif_dormant_on(dev); 5186 netif_dormant_on(dev);
5185 else 5187 else
5186 netif_dormant_off(dev); 5188 netif_dormant_off(dev);
5187 5189
5188 if (netif_carrier_ok(rootdev)) { 5190 if (netif_carrier_ok(rootdev)) {
5189 if (!netif_carrier_ok(dev)) 5191 if (!netif_carrier_ok(dev))
5190 netif_carrier_on(dev); 5192 netif_carrier_on(dev);
5191 } else { 5193 } else {
5192 if (netif_carrier_ok(dev)) 5194 if (netif_carrier_ok(dev))
5193 netif_carrier_off(dev); 5195 netif_carrier_off(dev);
5194 } 5196 }
5195 } 5197 }
5196 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 5198 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
5197 5199
5198 #ifdef CONFIG_RPS 5200 #ifdef CONFIG_RPS
5199 static int netif_alloc_rx_queues(struct net_device *dev) 5201 static int netif_alloc_rx_queues(struct net_device *dev)
5200 { 5202 {
5201 unsigned int i, count = dev->num_rx_queues; 5203 unsigned int i, count = dev->num_rx_queues;
5202 struct netdev_rx_queue *rx; 5204 struct netdev_rx_queue *rx;
5203 5205
5204 BUG_ON(count < 1); 5206 BUG_ON(count < 1);
5205 5207
5206 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL); 5208 rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
5207 if (!rx) { 5209 if (!rx) {
5208 pr_err("netdev: Unable to allocate %u rx queues.\n", count); 5210 pr_err("netdev: Unable to allocate %u rx queues.\n", count);
5209 return -ENOMEM; 5211 return -ENOMEM;
5210 } 5212 }
5211 dev->_rx = rx; 5213 dev->_rx = rx;
5212 5214
5213 for (i = 0; i < count; i++) 5215 for (i = 0; i < count; i++)
5214 rx[i].dev = dev; 5216 rx[i].dev = dev;
5215 return 0; 5217 return 0;
5216 } 5218 }
5217 #endif 5219 #endif
5218 5220
5219 static void netdev_init_one_queue(struct net_device *dev, 5221 static void netdev_init_one_queue(struct net_device *dev,
5220 struct netdev_queue *queue, void *_unused) 5222 struct netdev_queue *queue, void *_unused)
5221 { 5223 {
5222 /* Initialize queue lock */ 5224 /* Initialize queue lock */
5223 spin_lock_init(&queue->_xmit_lock); 5225 spin_lock_init(&queue->_xmit_lock);
5224 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 5226 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
5225 queue->xmit_lock_owner = -1; 5227 queue->xmit_lock_owner = -1;
5226 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 5228 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
5227 queue->dev = dev; 5229 queue->dev = dev;
5228 } 5230 }
5229 5231
5230 static int netif_alloc_netdev_queues(struct net_device *dev) 5232 static int netif_alloc_netdev_queues(struct net_device *dev)
5231 { 5233 {
5232 unsigned int count = dev->num_tx_queues; 5234 unsigned int count = dev->num_tx_queues;
5233 struct netdev_queue *tx; 5235 struct netdev_queue *tx;
5234 5236
5235 BUG_ON(count < 1); 5237 BUG_ON(count < 1);
5236 5238
5237 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL); 5239 tx = kcalloc(count, sizeof(struct netdev_queue), GFP_KERNEL);
5238 if (!tx) { 5240 if (!tx) {
5239 pr_err("netdev: Unable to allocate %u tx queues.\n", 5241 pr_err("netdev: Unable to allocate %u tx queues.\n",
5240 count); 5242 count);
5241 return -ENOMEM; 5243 return -ENOMEM;
5242 } 5244 }
5243 dev->_tx = tx; 5245 dev->_tx = tx;
5244 5246
5245 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 5247 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
5246 spin_lock_init(&dev->tx_global_lock); 5248 spin_lock_init(&dev->tx_global_lock);
5247 5249
5248 return 0; 5250 return 0;
5249 } 5251 }
5250 5252
5251 /** 5253 /**
5252 * register_netdevice - register a network device 5254 * register_netdevice - register a network device
5253 * @dev: device to register 5255 * @dev: device to register
5254 * 5256 *
5255 * Take a completed network device structure and add it to the kernel 5257 * Take a completed network device structure and add it to the kernel
5256 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 5258 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5257 * chain. 0 is returned on success. A negative errno code is returned 5259 * chain. 0 is returned on success. A negative errno code is returned
5258 * on a failure to set up the device, or if the name is a duplicate. 5260 * on a failure to set up the device, or if the name is a duplicate.
5259 * 5261 *
5260 * Callers must hold the rtnl semaphore. You may want 5262 * Callers must hold the rtnl semaphore. You may want
5261 * register_netdev() instead of this. 5263 * register_netdev() instead of this.
5262 * 5264 *
5263 * BUGS: 5265 * BUGS:
5264 * The locking appears insufficient to guarantee two parallel registers 5266 * The locking appears insufficient to guarantee two parallel registers
5265 * will not get the same name. 5267 * will not get the same name.
5266 */ 5268 */
5267 5269
5268 int register_netdevice(struct net_device *dev) 5270 int register_netdevice(struct net_device *dev)
5269 { 5271 {
5270 int ret; 5272 int ret;
5271 struct net *net = dev_net(dev); 5273 struct net *net = dev_net(dev);
5272 5274
5273 BUG_ON(dev_boot_phase); 5275 BUG_ON(dev_boot_phase);
5274 ASSERT_RTNL(); 5276 ASSERT_RTNL();
5275 5277
5276 might_sleep(); 5278 might_sleep();
5277 5279
5278 /* When net_device's are persistent, this will be fatal. */ 5280 /* When net_device's are persistent, this will be fatal. */
5279 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 5281 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
5280 BUG_ON(!net); 5282 BUG_ON(!net);
5281 5283
5282 spin_lock_init(&dev->addr_list_lock); 5284 spin_lock_init(&dev->addr_list_lock);
5283 netdev_set_addr_lockdep_class(dev); 5285 netdev_set_addr_lockdep_class(dev);
5284 5286
5285 dev->iflink = -1; 5287 dev->iflink = -1;
5286 5288
5287 /* Init, if this function is available */ 5289 /* Init, if this function is available */
5288 if (dev->netdev_ops->ndo_init) { 5290 if (dev->netdev_ops->ndo_init) {
5289 ret = dev->netdev_ops->ndo_init(dev); 5291 ret = dev->netdev_ops->ndo_init(dev);
5290 if (ret) { 5292 if (ret) {
5291 if (ret > 0) 5293 if (ret > 0)
5292 ret = -EIO; 5294 ret = -EIO;
5293 goto out; 5295 goto out;
5294 } 5296 }
5295 } 5297 }
5296 5298
5297 ret = dev_get_valid_name(dev, dev->name, 0); 5299 ret = dev_get_valid_name(dev, dev->name, 0);
5298 if (ret) 5300 if (ret)
5299 goto err_uninit; 5301 goto err_uninit;
5300 5302
5301 dev->ifindex = dev_new_index(net); 5303 dev->ifindex = dev_new_index(net);
5302 if (dev->iflink == -1) 5304 if (dev->iflink == -1)
5303 dev->iflink = dev->ifindex; 5305 dev->iflink = dev->ifindex;
5304 5306
5305 /* Fix illegal checksum combinations */ 5307 /* Fix illegal checksum combinations */
5306 if ((dev->features & NETIF_F_HW_CSUM) && 5308 if ((dev->features & NETIF_F_HW_CSUM) &&
5307 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 5309 (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5308 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n", 5310 printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
5309 dev->name); 5311 dev->name);
5310 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 5312 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5311 } 5313 }
5312 5314
5313 if ((dev->features & NETIF_F_NO_CSUM) && 5315 if ((dev->features & NETIF_F_NO_CSUM) &&
5314 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 5316 (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
5315 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n", 5317 printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
5316 dev->name); 5318 dev->name);
5317 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM); 5319 dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
5318 } 5320 }
5319 5321
5320 dev->features = netdev_fix_features(dev->features, dev->name); 5322 dev->features = netdev_fix_features(dev->features, dev->name);
5321 5323
5322 /* Enable software GSO if SG is supported. */ 5324 /* Enable software GSO if SG is supported. */
5323 if (dev->features & NETIF_F_SG) 5325 if (dev->features & NETIF_F_SG)
5324 dev->features |= NETIF_F_GSO; 5326 dev->features |= NETIF_F_GSO;
5325 5327
5326 /* Enable GRO and NETIF_F_HIGHDMA for vlans by default, 5328 /* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
5327 * vlan_dev_init() will do the dev->features check, so these features 5329 * vlan_dev_init() will do the dev->features check, so these features
5328 * are enabled only if supported by underlying device. 5330 * are enabled only if supported by underlying device.
5329 */ 5331 */
5330 dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA); 5332 dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);
5331 5333
5332 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 5334 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
5333 ret = notifier_to_errno(ret); 5335 ret = notifier_to_errno(ret);
5334 if (ret) 5336 if (ret)
5335 goto err_uninit; 5337 goto err_uninit;
5336 5338
5337 ret = netdev_register_kobject(dev); 5339 ret = netdev_register_kobject(dev);
5338 if (ret) 5340 if (ret)
5339 goto err_uninit; 5341 goto err_uninit;
5340 dev->reg_state = NETREG_REGISTERED; 5342 dev->reg_state = NETREG_REGISTERED;
5341 5343
5342 /* 5344 /*
5343 * Default initial state at registry is that the 5345 * Default initial state at registry is that the
5344 * device is present. 5346 * device is present.
5345 */ 5347 */
5346 5348
5347 set_bit(__LINK_STATE_PRESENT, &dev->state); 5349 set_bit(__LINK_STATE_PRESENT, &dev->state);
5348 5350
5349 dev_init_scheduler(dev); 5351 dev_init_scheduler(dev);
5350 dev_hold(dev); 5352 dev_hold(dev);
5351 list_netdevice(dev); 5353 list_netdevice(dev);
5352 5354
5353 /* Notify protocols, that a new device appeared. */ 5355 /* Notify protocols, that a new device appeared. */
5354 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 5356 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
5355 ret = notifier_to_errno(ret); 5357 ret = notifier_to_errno(ret);
5356 if (ret) { 5358 if (ret) {
5357 rollback_registered(dev); 5359 rollback_registered(dev);
5358 dev->reg_state = NETREG_UNREGISTERED; 5360 dev->reg_state = NETREG_UNREGISTERED;
5359 } 5361 }
5360 /* 5362 /*
5361 * Prevent userspace races by waiting until the network 5363 * Prevent userspace races by waiting until the network
5362 * device is fully setup before sending notifications. 5364 * device is fully setup before sending notifications.
5363 */ 5365 */
5364 if (!dev->rtnl_link_ops || 5366 if (!dev->rtnl_link_ops ||
5365 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) 5367 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
5366 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U); 5368 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5367 5369
5368 out: 5370 out:
5369 return ret; 5371 return ret;
5370 5372
5371 err_uninit: 5373 err_uninit:
5372 if (dev->netdev_ops->ndo_uninit) 5374 if (dev->netdev_ops->ndo_uninit)
5373 dev->netdev_ops->ndo_uninit(dev); 5375 dev->netdev_ops->ndo_uninit(dev);
5374 goto out; 5376 goto out;
5375 } 5377 }
5376 EXPORT_SYMBOL(register_netdevice); 5378 EXPORT_SYMBOL(register_netdevice);
5377 5379
5378 /** 5380 /**
5379 * init_dummy_netdev - init a dummy network device for NAPI 5381 * init_dummy_netdev - init a dummy network device for NAPI
5380 * @dev: device to init 5382 * @dev: device to init
5381 * 5383 *
5382 * This takes a network device structure and initialize the minimum 5384 * This takes a network device structure and initialize the minimum
5383 * amount of fields so it can be used to schedule NAPI polls without 5385 * amount of fields so it can be used to schedule NAPI polls without
5384 * registering a full blown interface. This is to be used by drivers 5386 * registering a full blown interface. This is to be used by drivers
5385 * that need to tie several hardware interfaces to a single NAPI 5387 * that need to tie several hardware interfaces to a single NAPI
5386 * poll scheduler due to HW limitations. 5388 * poll scheduler due to HW limitations.
5387 */ 5389 */
5388 int init_dummy_netdev(struct net_device *dev) 5390 int init_dummy_netdev(struct net_device *dev)
5389 { 5391 {
5390 /* Clear everything. Note we don't initialize spinlocks 5392 /* Clear everything. Note we don't initialize spinlocks
5391 * are they aren't supposed to be taken by any of the 5393 * are they aren't supposed to be taken by any of the
5392 * NAPI code and this dummy netdev is supposed to be 5394 * NAPI code and this dummy netdev is supposed to be
5393 * only ever used for NAPI polls 5395 * only ever used for NAPI polls
5394 */ 5396 */
5395 memset(dev, 0, sizeof(struct net_device)); 5397 memset(dev, 0, sizeof(struct net_device));
5396 5398
5397 /* make sure we BUG if trying to hit standard 5399 /* make sure we BUG if trying to hit standard
5398 * register/unregister code path 5400 * register/unregister code path
5399 */ 5401 */
5400 dev->reg_state = NETREG_DUMMY; 5402 dev->reg_state = NETREG_DUMMY;
5401 5403
5402 /* NAPI wants this */ 5404 /* NAPI wants this */
5403 INIT_LIST_HEAD(&dev->napi_list); 5405 INIT_LIST_HEAD(&dev->napi_list);
5404 5406
5405 /* a dummy interface is started by default */ 5407 /* a dummy interface is started by default */
5406 set_bit(__LINK_STATE_PRESENT, &dev->state); 5408 set_bit(__LINK_STATE_PRESENT, &dev->state);
5407 set_bit(__LINK_STATE_START, &dev->state); 5409 set_bit(__LINK_STATE_START, &dev->state);
5408 5410
5409 /* Note : We dont allocate pcpu_refcnt for dummy devices, 5411 /* Note : We dont allocate pcpu_refcnt for dummy devices,
5410 * because users of this 'device' dont need to change 5412 * because users of this 'device' dont need to change
5411 * its refcount. 5413 * its refcount.
5412 */ 5414 */
5413 5415
5414 return 0; 5416 return 0;
5415 } 5417 }
5416 EXPORT_SYMBOL_GPL(init_dummy_netdev); 5418 EXPORT_SYMBOL_GPL(init_dummy_netdev);
5417 5419
5418 5420
5419 /** 5421 /**
5420 * register_netdev - register a network device 5422 * register_netdev - register a network device
5421 * @dev: device to register 5423 * @dev: device to register
5422 * 5424 *
5423 * Take a completed network device structure and add it to the kernel 5425 * Take a completed network device structure and add it to the kernel
5424 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 5426 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
5425 * chain. 0 is returned on success. A negative errno code is returned 5427 * chain. 0 is returned on success. A negative errno code is returned
5426 * on a failure to set up the device, or if the name is a duplicate. 5428 * on a failure to set up the device, or if the name is a duplicate.
5427 * 5429 *
5428 * This is a wrapper around register_netdevice that takes the rtnl semaphore 5430 * This is a wrapper around register_netdevice that takes the rtnl semaphore
5429 * and expands the device name if you passed a format string to 5431 * and expands the device name if you passed a format string to
5430 * alloc_netdev. 5432 * alloc_netdev.
5431 */ 5433 */
5432 int register_netdev(struct net_device *dev) 5434 int register_netdev(struct net_device *dev)
5433 { 5435 {
5434 int err; 5436 int err;
5435 5437
5436 rtnl_lock(); 5438 rtnl_lock();
5437 5439
5438 /* 5440 /*
5439 * If the name is a format string the caller wants us to do a 5441 * If the name is a format string the caller wants us to do a
5440 * name allocation. 5442 * name allocation.
5441 */ 5443 */
5442 if (strchr(dev->name, '%')) { 5444 if (strchr(dev->name, '%')) {
5443 err = dev_alloc_name(dev, dev->name); 5445 err = dev_alloc_name(dev, dev->name);
5444 if (err < 0) 5446 if (err < 0)
5445 goto out; 5447 goto out;
5446 } 5448 }
5447 5449
5448 err = register_netdevice(dev); 5450 err = register_netdevice(dev);
5449 out: 5451 out:
5450 rtnl_unlock(); 5452 rtnl_unlock();
5451 return err; 5453 return err;
5452 } 5454 }
5453 EXPORT_SYMBOL(register_netdev); 5455 EXPORT_SYMBOL(register_netdev);
5454 5456
5455 int netdev_refcnt_read(const struct net_device *dev) 5457 int netdev_refcnt_read(const struct net_device *dev)
5456 { 5458 {
5457 int i, refcnt = 0; 5459 int i, refcnt = 0;
5458 5460
5459 for_each_possible_cpu(i) 5461 for_each_possible_cpu(i)
5460 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 5462 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
5461 return refcnt; 5463 return refcnt;
5462 } 5464 }
5463 EXPORT_SYMBOL(netdev_refcnt_read); 5465 EXPORT_SYMBOL(netdev_refcnt_read);
5464 5466
5465 /* 5467 /*
5466 * netdev_wait_allrefs - wait until all references are gone. 5468 * netdev_wait_allrefs - wait until all references are gone.
5467 * 5469 *
5468 * This is called when unregistering network devices. 5470 * This is called when unregistering network devices.
5469 * 5471 *
5470 * Any protocol or device that holds a reference should register 5472 * Any protocol or device that holds a reference should register
5471 * for netdevice notification, and cleanup and put back the 5473 * for netdevice notification, and cleanup and put back the
5472 * reference if they receive an UNREGISTER event. 5474 * reference if they receive an UNREGISTER event.
5473 * We can get stuck here if buggy protocols don't correctly 5475 * We can get stuck here if buggy protocols don't correctly
5474 * call dev_put. 5476 * call dev_put.
5475 */ 5477 */
5476 static void netdev_wait_allrefs(struct net_device *dev) 5478 static void netdev_wait_allrefs(struct net_device *dev)
5477 { 5479 {
5478 unsigned long rebroadcast_time, warning_time; 5480 unsigned long rebroadcast_time, warning_time;
5479 int refcnt; 5481 int refcnt;
5480 5482
5481 linkwatch_forget_dev(dev); 5483 linkwatch_forget_dev(dev);
5482 5484
5483 rebroadcast_time = warning_time = jiffies; 5485 rebroadcast_time = warning_time = jiffies;
5484 refcnt = netdev_refcnt_read(dev); 5486 refcnt = netdev_refcnt_read(dev);
5485 5487
5486 while (refcnt != 0) { 5488 while (refcnt != 0) {
5487 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 5489 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
5488 rtnl_lock(); 5490 rtnl_lock();
5489 5491
5490 /* Rebroadcast unregister notification */ 5492 /* Rebroadcast unregister notification */
5491 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 5493 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5492 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users 5494 /* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
5493 * should have already handle it the first time */ 5495 * should have already handle it the first time */
5494 5496
5495 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 5497 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
5496 &dev->state)) { 5498 &dev->state)) {
5497 /* We must not have linkwatch events 5499 /* We must not have linkwatch events
5498 * pending on unregister. If this 5500 * pending on unregister. If this
5499 * happens, we simply run the queue 5501 * happens, we simply run the queue
5500 * unscheduled, resulting in a noop 5502 * unscheduled, resulting in a noop
5501 * for this device. 5503 * for this device.
5502 */ 5504 */
5503 linkwatch_run_queue(); 5505 linkwatch_run_queue();
5504 } 5506 }
5505 5507
5506 __rtnl_unlock(); 5508 __rtnl_unlock();
5507 5509
5508 rebroadcast_time = jiffies; 5510 rebroadcast_time = jiffies;
5509 } 5511 }
5510 5512
5511 msleep(250); 5513 msleep(250);
5512 5514
5513 refcnt = netdev_refcnt_read(dev); 5515 refcnt = netdev_refcnt_read(dev);
5514 5516
5515 if (time_after(jiffies, warning_time + 10 * HZ)) { 5517 if (time_after(jiffies, warning_time + 10 * HZ)) {
5516 printk(KERN_EMERG "unregister_netdevice: " 5518 printk(KERN_EMERG "unregister_netdevice: "
5517 "waiting for %s to become free. Usage " 5519 "waiting for %s to become free. Usage "
5518 "count = %d\n", 5520 "count = %d\n",
5519 dev->name, refcnt); 5521 dev->name, refcnt);
5520 warning_time = jiffies; 5522 warning_time = jiffies;
5521 } 5523 }
5522 } 5524 }
5523 } 5525 }
5524 5526
5525 /* The sequence is: 5527 /* The sequence is:
5526 * 5528 *
5527 * rtnl_lock(); 5529 * rtnl_lock();
5528 * ... 5530 * ...
5529 * register_netdevice(x1); 5531 * register_netdevice(x1);
5530 * register_netdevice(x2); 5532 * register_netdevice(x2);
5531 * ... 5533 * ...
5532 * unregister_netdevice(y1); 5534 * unregister_netdevice(y1);
5533 * unregister_netdevice(y2); 5535 * unregister_netdevice(y2);
5534 * ... 5536 * ...
5535 * rtnl_unlock(); 5537 * rtnl_unlock();
5536 * free_netdev(y1); 5538 * free_netdev(y1);
5537 * free_netdev(y2); 5539 * free_netdev(y2);
5538 * 5540 *
5539 * We are invoked by rtnl_unlock(). 5541 * We are invoked by rtnl_unlock().
5540 * This allows us to deal with problems: 5542 * This allows us to deal with problems:
5541 * 1) We can delete sysfs objects which invoke hotplug 5543 * 1) We can delete sysfs objects which invoke hotplug
5542 * without deadlocking with linkwatch via keventd. 5544 * without deadlocking with linkwatch via keventd.
5543 * 2) Since we run with the RTNL semaphore not held, we can sleep 5545 * 2) Since we run with the RTNL semaphore not held, we can sleep
5544 * safely in order to wait for the netdev refcnt to drop to zero. 5546 * safely in order to wait for the netdev refcnt to drop to zero.
5545 * 5547 *
5546 * We must not return until all unregister events added during 5548 * We must not return until all unregister events added during
5547 * the interval the lock was held have been completed. 5549 * the interval the lock was held have been completed.
5548 */ 5550 */
5549 void netdev_run_todo(void) 5551 void netdev_run_todo(void)
5550 { 5552 {
5551 struct list_head list; 5553 struct list_head list;
5552 5554
5553 /* Snapshot list, allow later requests */ 5555 /* Snapshot list, allow later requests */
5554 list_replace_init(&net_todo_list, &list); 5556 list_replace_init(&net_todo_list, &list);
5555 5557
5556 __rtnl_unlock(); 5558 __rtnl_unlock();
5557 5559
5558 while (!list_empty(&list)) { 5560 while (!list_empty(&list)) {
5559 struct net_device *dev 5561 struct net_device *dev
5560 = list_first_entry(&list, struct net_device, todo_list); 5562 = list_first_entry(&list, struct net_device, todo_list);
5561 list_del(&dev->todo_list); 5563 list_del(&dev->todo_list);
5562 5564
5563 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 5565 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
5564 printk(KERN_ERR "network todo '%s' but state %d\n", 5566 printk(KERN_ERR "network todo '%s' but state %d\n",
5565 dev->name, dev->reg_state); 5567 dev->name, dev->reg_state);
5566 dump_stack(); 5568 dump_stack();
5567 continue; 5569 continue;
5568 } 5570 }
5569 5571
5570 dev->reg_state = NETREG_UNREGISTERED; 5572 dev->reg_state = NETREG_UNREGISTERED;
5571 5573
5572 on_each_cpu(flush_backlog, dev, 1); 5574 on_each_cpu(flush_backlog, dev, 1);
5573 5575
5574 netdev_wait_allrefs(dev); 5576 netdev_wait_allrefs(dev);
5575 5577
5576 /* paranoia */ 5578 /* paranoia */
5577 BUG_ON(netdev_refcnt_read(dev)); 5579 BUG_ON(netdev_refcnt_read(dev));
5578 WARN_ON(rcu_dereference_raw(dev->ip_ptr)); 5580 WARN_ON(rcu_dereference_raw(dev->ip_ptr));
5579 WARN_ON(rcu_dereference_raw(dev->ip6_ptr)); 5581 WARN_ON(rcu_dereference_raw(dev->ip6_ptr));
5580 WARN_ON(dev->dn_ptr); 5582 WARN_ON(dev->dn_ptr);
5581 5583
5582 if (dev->destructor) 5584 if (dev->destructor)
5583 dev->destructor(dev); 5585 dev->destructor(dev);
5584 5586
5585 /* Free network device */ 5587 /* Free network device */
5586 kobject_put(&dev->dev.kobj); 5588 kobject_put(&dev->dev.kobj);
5587 } 5589 }
5588 } 5590 }
5589 5591
5590 /* Convert net_device_stats to rtnl_link_stats64. They have the same 5592 /* Convert net_device_stats to rtnl_link_stats64. They have the same
5591 * fields in the same order, with only the type differing. 5593 * fields in the same order, with only the type differing.
5592 */ 5594 */
5593 static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 5595 static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5594 const struct net_device_stats *netdev_stats) 5596 const struct net_device_stats *netdev_stats)
5595 { 5597 {
5596 #if BITS_PER_LONG == 64 5598 #if BITS_PER_LONG == 64
5597 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats)); 5599 BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
5598 memcpy(stats64, netdev_stats, sizeof(*stats64)); 5600 memcpy(stats64, netdev_stats, sizeof(*stats64));
5599 #else 5601 #else
5600 size_t i, n = sizeof(*stats64) / sizeof(u64); 5602 size_t i, n = sizeof(*stats64) / sizeof(u64);
5601 const unsigned long *src = (const unsigned long *)netdev_stats; 5603 const unsigned long *src = (const unsigned long *)netdev_stats;
5602 u64 *dst = (u64 *)stats64; 5604 u64 *dst = (u64 *)stats64;
5603 5605
5604 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) != 5606 BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
5605 sizeof(*stats64) / sizeof(u64)); 5607 sizeof(*stats64) / sizeof(u64));
5606 for (i = 0; i < n; i++) 5608 for (i = 0; i < n; i++)
5607 dst[i] = src[i]; 5609 dst[i] = src[i];
5608 #endif 5610 #endif
5609 } 5611 }
5610 5612
5611 /** 5613 /**
5612 * dev_get_stats - get network device statistics 5614 * dev_get_stats - get network device statistics
5613 * @dev: device to get statistics from 5615 * @dev: device to get statistics from
5614 * @storage: place to store stats 5616 * @storage: place to store stats
5615 * 5617 *
5616 * Get network statistics from device. Return @storage. 5618 * Get network statistics from device. Return @storage.
5617 * The device driver may provide its own method by setting 5619 * The device driver may provide its own method by setting
5618 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 5620 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
5619 * otherwise the internal statistics structure is used. 5621 * otherwise the internal statistics structure is used.
5620 */ 5622 */
5621 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 5623 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5622 struct rtnl_link_stats64 *storage) 5624 struct rtnl_link_stats64 *storage)
5623 { 5625 {
5624 const struct net_device_ops *ops = dev->netdev_ops; 5626 const struct net_device_ops *ops = dev->netdev_ops;
5625 5627
5626 if (ops->ndo_get_stats64) { 5628 if (ops->ndo_get_stats64) {
5627 memset(storage, 0, sizeof(*storage)); 5629 memset(storage, 0, sizeof(*storage));
5628 ops->ndo_get_stats64(dev, storage); 5630 ops->ndo_get_stats64(dev, storage);
5629 } else if (ops->ndo_get_stats) { 5631 } else if (ops->ndo_get_stats) {
5630 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 5632 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
5631 } else { 5633 } else {
5632 netdev_stats_to_stats64(storage, &dev->stats); 5634 netdev_stats_to_stats64(storage, &dev->stats);
5633 } 5635 }
5634 storage->rx_dropped += atomic_long_read(&dev->rx_dropped); 5636 storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
5635 return storage; 5637 return storage;
5636 } 5638 }
5637 EXPORT_SYMBOL(dev_get_stats); 5639 EXPORT_SYMBOL(dev_get_stats);
5638 5640
5639 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 5641 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
5640 { 5642 {
5641 struct netdev_queue *queue = dev_ingress_queue(dev); 5643 struct netdev_queue *queue = dev_ingress_queue(dev);
5642 5644
5643 #ifdef CONFIG_NET_CLS_ACT 5645 #ifdef CONFIG_NET_CLS_ACT
5644 if (queue) 5646 if (queue)
5645 return queue; 5647 return queue;
5646 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 5648 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
5647 if (!queue) 5649 if (!queue)
5648 return NULL; 5650 return NULL;
5649 netdev_init_one_queue(dev, queue, NULL); 5651 netdev_init_one_queue(dev, queue, NULL);
5650 queue->qdisc = &noop_qdisc; 5652 queue->qdisc = &noop_qdisc;
5651 queue->qdisc_sleeping = &noop_qdisc; 5653 queue->qdisc_sleeping = &noop_qdisc;
5652 rcu_assign_pointer(dev->ingress_queue, queue); 5654 rcu_assign_pointer(dev->ingress_queue, queue);
5653 #endif 5655 #endif
5654 return queue; 5656 return queue;
5655 } 5657 }
5656 5658
5657 /** 5659 /**
5658 * alloc_netdev_mqs - allocate network device 5660 * alloc_netdev_mqs - allocate network device
5659 * @sizeof_priv: size of private data to allocate space for 5661 * @sizeof_priv: size of private data to allocate space for
5660 * @name: device name format string 5662 * @name: device name format string
5661 * @setup: callback to initialize device 5663 * @setup: callback to initialize device
5662 * @txqs: the number of TX subqueues to allocate 5664 * @txqs: the number of TX subqueues to allocate
5663 * @rxqs: the number of RX subqueues to allocate 5665 * @rxqs: the number of RX subqueues to allocate
5664 * 5666 *
5665 * Allocates a struct net_device with private data area for driver use 5667 * Allocates a struct net_device with private data area for driver use
5666 * and performs basic initialization. Also allocates subquue structs 5668 * and performs basic initialization. Also allocates subquue structs
5667 * for each queue on the device. 5669 * for each queue on the device.
5668 */ 5670 */
5669 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 5671 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
5670 void (*setup)(struct net_device *), 5672 void (*setup)(struct net_device *),
5671 unsigned int txqs, unsigned int rxqs) 5673 unsigned int txqs, unsigned int rxqs)
5672 { 5674 {
5673 struct net_device *dev; 5675 struct net_device *dev;
5674 size_t alloc_size; 5676 size_t alloc_size;
5675 struct net_device *p; 5677 struct net_device *p;
5676 5678
5677 BUG_ON(strlen(name) >= sizeof(dev->name)); 5679 BUG_ON(strlen(name) >= sizeof(dev->name));
5678 5680
5679 if (txqs < 1) { 5681 if (txqs < 1) {
5680 pr_err("alloc_netdev: Unable to allocate device " 5682 pr_err("alloc_netdev: Unable to allocate device "
5681 "with zero queues.\n"); 5683 "with zero queues.\n");
5682 return NULL; 5684 return NULL;
5683 } 5685 }
5684 5686
5685 #ifdef CONFIG_RPS 5687 #ifdef CONFIG_RPS
5686 if (rxqs < 1) { 5688 if (rxqs < 1) {
5687 pr_err("alloc_netdev: Unable to allocate device " 5689 pr_err("alloc_netdev: Unable to allocate device "
5688 "with zero RX queues.\n"); 5690 "with zero RX queues.\n");
5689 return NULL; 5691 return NULL;
5690 } 5692 }
5691 #endif 5693 #endif
5692 5694
5693 alloc_size = sizeof(struct net_device); 5695 alloc_size = sizeof(struct net_device);
5694 if (sizeof_priv) { 5696 if (sizeof_priv) {
5695 /* ensure 32-byte alignment of private area */ 5697 /* ensure 32-byte alignment of private area */
5696 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); 5698 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
5697 alloc_size += sizeof_priv; 5699 alloc_size += sizeof_priv;
5698 } 5700 }
5699 /* ensure 32-byte alignment of whole construct */ 5701 /* ensure 32-byte alignment of whole construct */
5700 alloc_size += NETDEV_ALIGN - 1; 5702 alloc_size += NETDEV_ALIGN - 1;
5701 5703
5702 p = kzalloc(alloc_size, GFP_KERNEL); 5704 p = kzalloc(alloc_size, GFP_KERNEL);
5703 if (!p) { 5705 if (!p) {
5704 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n"); 5706 printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
5705 return NULL; 5707 return NULL;
5706 } 5708 }
5707 5709
5708 dev = PTR_ALIGN(p, NETDEV_ALIGN); 5710 dev = PTR_ALIGN(p, NETDEV_ALIGN);
5709 dev->padded = (char *)dev - (char *)p; 5711 dev->padded = (char *)dev - (char *)p;
5710 5712
5711 dev->pcpu_refcnt = alloc_percpu(int); 5713 dev->pcpu_refcnt = alloc_percpu(int);
5712 if (!dev->pcpu_refcnt) 5714 if (!dev->pcpu_refcnt)
5713 goto free_p; 5715 goto free_p;
5714 5716
5715 if (dev_addr_init(dev)) 5717 if (dev_addr_init(dev))
5716 goto free_pcpu; 5718 goto free_pcpu;
5717 5719
5718 dev_mc_init(dev); 5720 dev_mc_init(dev);
5719 dev_uc_init(dev); 5721 dev_uc_init(dev);
5720 5722
5721 dev_net_set(dev, &init_net); 5723 dev_net_set(dev, &init_net);
5722 5724
5723 dev->num_tx_queues = txqs; 5725 dev->num_tx_queues = txqs;
5724 dev->real_num_tx_queues = txqs; 5726 dev->real_num_tx_queues = txqs;
5725 if (netif_alloc_netdev_queues(dev)) 5727 if (netif_alloc_netdev_queues(dev))
5726 goto free_pcpu; 5728 goto free_pcpu;
5727 5729
5728 #ifdef CONFIG_RPS 5730 #ifdef CONFIG_RPS
5729 dev->num_rx_queues = rxqs; 5731 dev->num_rx_queues = rxqs;
5730 dev->real_num_rx_queues = rxqs; 5732 dev->real_num_rx_queues = rxqs;
5731 if (netif_alloc_rx_queues(dev)) 5733 if (netif_alloc_rx_queues(dev))
5732 goto free_pcpu; 5734 goto free_pcpu;
5733 #endif 5735 #endif
5734 5736
5735 dev->gso_max_size = GSO_MAX_SIZE; 5737 dev->gso_max_size = GSO_MAX_SIZE;
5736 5738
5737 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list); 5739 INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
5738 dev->ethtool_ntuple_list.count = 0; 5740 dev->ethtool_ntuple_list.count = 0;
5739 INIT_LIST_HEAD(&dev->napi_list); 5741 INIT_LIST_HEAD(&dev->napi_list);
5740 INIT_LIST_HEAD(&dev->unreg_list); 5742 INIT_LIST_HEAD(&dev->unreg_list);
5741 INIT_LIST_HEAD(&dev->link_watch_list); 5743 INIT_LIST_HEAD(&dev->link_watch_list);
5742 dev->priv_flags = IFF_XMIT_DST_RELEASE; 5744 dev->priv_flags = IFF_XMIT_DST_RELEASE;
5743 setup(dev); 5745 setup(dev);
5744 strcpy(dev->name, name); 5746 strcpy(dev->name, name);
5745 dev->group = INIT_NETDEV_GROUP; 5747 dev->group = INIT_NETDEV_GROUP;
5746 return dev; 5748 return dev;
5747 5749
5748 free_pcpu: 5750 free_pcpu:
5749 free_percpu(dev->pcpu_refcnt); 5751 free_percpu(dev->pcpu_refcnt);
5750 kfree(dev->_tx); 5752 kfree(dev->_tx);
5751 #ifdef CONFIG_RPS 5753 #ifdef CONFIG_RPS
5752 kfree(dev->_rx); 5754 kfree(dev->_rx);
5753 #endif 5755 #endif
5754 5756
5755 free_p: 5757 free_p:
5756 kfree(p); 5758 kfree(p);
5757 return NULL; 5759 return NULL;
5758 } 5760 }
5759 EXPORT_SYMBOL(alloc_netdev_mqs); 5761 EXPORT_SYMBOL(alloc_netdev_mqs);
5760 5762
5761 /** 5763 /**
5762 * free_netdev - free network device 5764 * free_netdev - free network device
5763 * @dev: device 5765 * @dev: device
5764 * 5766 *
5765 * This function does the last stage of destroying an allocated device 5767 * This function does the last stage of destroying an allocated device
5766 * interface. The reference to the device object is released. 5768 * interface. The reference to the device object is released.
5767 * If this is the last reference then it will be freed. 5769 * If this is the last reference then it will be freed.
5768 */ 5770 */
5769 void free_netdev(struct net_device *dev) 5771 void free_netdev(struct net_device *dev)
5770 { 5772 {
5771 struct napi_struct *p, *n; 5773 struct napi_struct *p, *n;
5772 5774
5773 release_net(dev_net(dev)); 5775 release_net(dev_net(dev));
5774 5776
5775 kfree(dev->_tx); 5777 kfree(dev->_tx);
5776 #ifdef CONFIG_RPS 5778 #ifdef CONFIG_RPS
5777 kfree(dev->_rx); 5779 kfree(dev->_rx);
5778 #endif 5780 #endif
5779 5781
5780 kfree(rcu_dereference_raw(dev->ingress_queue)); 5782 kfree(rcu_dereference_raw(dev->ingress_queue));
5781 5783
5782 /* Flush device addresses */ 5784 /* Flush device addresses */
5783 dev_addr_flush(dev); 5785 dev_addr_flush(dev);
5784 5786
5785 /* Clear ethtool n-tuple list */ 5787 /* Clear ethtool n-tuple list */
5786 ethtool_ntuple_flush(dev); 5788 ethtool_ntuple_flush(dev);
5787 5789
5788 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 5790 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
5789 netif_napi_del(p); 5791 netif_napi_del(p);
5790 5792
5791 free_percpu(dev->pcpu_refcnt); 5793 free_percpu(dev->pcpu_refcnt);
5792 dev->pcpu_refcnt = NULL; 5794 dev->pcpu_refcnt = NULL;
5793 5795
5794 /* Compatibility with error handling in drivers */ 5796 /* Compatibility with error handling in drivers */
5795 if (dev->reg_state == NETREG_UNINITIALIZED) { 5797 if (dev->reg_state == NETREG_UNINITIALIZED) {
5796 kfree((char *)dev - dev->padded); 5798 kfree((char *)dev - dev->padded);
5797 return; 5799 return;
5798 } 5800 }
5799 5801
5800 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 5802 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
5801 dev->reg_state = NETREG_RELEASED; 5803 dev->reg_state = NETREG_RELEASED;
5802 5804
5803 /* will free via device release */ 5805 /* will free via device release */
5804 put_device(&dev->dev); 5806 put_device(&dev->dev);
5805 } 5807 }
5806 EXPORT_SYMBOL(free_netdev); 5808 EXPORT_SYMBOL(free_netdev);
5807 5809
5808 /** 5810 /**
5809 * synchronize_net - Synchronize with packet receive processing 5811 * synchronize_net - Synchronize with packet receive processing
5810 * 5812 *
5811 * Wait for packets currently being received to be done. 5813 * Wait for packets currently being received to be done.
5812 * Does not block later packets from starting. 5814 * Does not block later packets from starting.
5813 */ 5815 */
5814 void synchronize_net(void) 5816 void synchronize_net(void)
5815 { 5817 {
5816 might_sleep(); 5818 might_sleep();
5817 synchronize_rcu(); 5819 synchronize_rcu();
5818 } 5820 }
5819 EXPORT_SYMBOL(synchronize_net); 5821 EXPORT_SYMBOL(synchronize_net);
5820 5822
5821 /** 5823 /**
5822 * unregister_netdevice_queue - remove device from the kernel 5824 * unregister_netdevice_queue - remove device from the kernel
5823 * @dev: device 5825 * @dev: device
5824 * @head: list 5826 * @head: list
5825 * 5827 *
5826 * This function shuts down a device interface and removes it 5828 * This function shuts down a device interface and removes it
5827 * from the kernel tables. 5829 * from the kernel tables.
5828 * If head not NULL, device is queued to be unregistered later. 5830 * If head not NULL, device is queued to be unregistered later.
5829 * 5831 *
5830 * Callers must hold the rtnl semaphore. You may want 5832 * Callers must hold the rtnl semaphore. You may want
5831 * unregister_netdev() instead of this. 5833 * unregister_netdev() instead of this.
5832 */ 5834 */
5833 5835
5834 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 5836 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
5835 { 5837 {
5836 ASSERT_RTNL(); 5838 ASSERT_RTNL();
5837 5839
5838 if (head) { 5840 if (head) {
5839 list_move_tail(&dev->unreg_list, head); 5841 list_move_tail(&dev->unreg_list, head);
5840 } else { 5842 } else {
5841 rollback_registered(dev); 5843 rollback_registered(dev);
5842 /* Finish processing unregister after unlock */ 5844 /* Finish processing unregister after unlock */
5843 net_set_todo(dev); 5845 net_set_todo(dev);
5844 } 5846 }
5845 } 5847 }
5846 EXPORT_SYMBOL(unregister_netdevice_queue); 5848 EXPORT_SYMBOL(unregister_netdevice_queue);
5847 5849
5848 /** 5850 /**
5849 * unregister_netdevice_many - unregister many devices 5851 * unregister_netdevice_many - unregister many devices
5850 * @head: list of devices 5852 * @head: list of devices
5851 */ 5853 */
5852 void unregister_netdevice_many(struct list_head *head) 5854 void unregister_netdevice_many(struct list_head *head)
5853 { 5855 {
5854 struct net_device *dev; 5856 struct net_device *dev;
5855 5857
5856 if (!list_empty(head)) { 5858 if (!list_empty(head)) {
5857 rollback_registered_many(head); 5859 rollback_registered_many(head);
5858 list_for_each_entry(dev, head, unreg_list) 5860 list_for_each_entry(dev, head, unreg_list)
5859 net_set_todo(dev); 5861 net_set_todo(dev);
5860 } 5862 }
5861 } 5863 }
5862 EXPORT_SYMBOL(unregister_netdevice_many); 5864 EXPORT_SYMBOL(unregister_netdevice_many);
5863 5865
5864 /** 5866 /**
5865 * unregister_netdev - remove device from the kernel 5867 * unregister_netdev - remove device from the kernel
5866 * @dev: device 5868 * @dev: device
5867 * 5869 *
5868 * This function shuts down a device interface and removes it 5870 * This function shuts down a device interface and removes it
5869 * from the kernel tables. 5871 * from the kernel tables.
5870 * 5872 *
5871 * This is just a wrapper for unregister_netdevice that takes 5873 * This is just a wrapper for unregister_netdevice that takes
5872 * the rtnl semaphore. In general you want to use this and not 5874 * the rtnl semaphore. In general you want to use this and not
5873 * unregister_netdevice. 5875 * unregister_netdevice.
5874 */ 5876 */
5875 void unregister_netdev(struct net_device *dev) 5877 void unregister_netdev(struct net_device *dev)
5876 { 5878 {
5877 rtnl_lock(); 5879 rtnl_lock();
5878 unregister_netdevice(dev); 5880 unregister_netdevice(dev);
5879 rtnl_unlock(); 5881 rtnl_unlock();
5880 } 5882 }
5881 EXPORT_SYMBOL(unregister_netdev); 5883 EXPORT_SYMBOL(unregister_netdev);
5882 5884
5883 /** 5885 /**
5884 * dev_change_net_namespace - move device to different nethost namespace 5886 * dev_change_net_namespace - move device to different nethost namespace
5885 * @dev: device 5887 * @dev: device
5886 * @net: network namespace 5888 * @net: network namespace
5887 * @pat: If not NULL name pattern to try if the current device name 5889 * @pat: If not NULL name pattern to try if the current device name
5888 * is already taken in the destination network namespace. 5890 * is already taken in the destination network namespace.
5889 * 5891 *
5890 * This function shuts down a device interface and moves it 5892 * This function shuts down a device interface and moves it
5891 * to a new network namespace. On success 0 is returned, on 5893 * to a new network namespace. On success 0 is returned, on
5892 * a failure a netagive errno code is returned. 5894 * a failure a netagive errno code is returned.
5893 * 5895 *
5894 * Callers must hold the rtnl semaphore. 5896 * Callers must hold the rtnl semaphore.
5895 */ 5897 */
5896 5898
5897 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) 5899 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
5898 { 5900 {
5899 int err; 5901 int err;
5900 5902
5901 ASSERT_RTNL(); 5903 ASSERT_RTNL();
5902 5904
5903 /* Don't allow namespace local devices to be moved. */ 5905 /* Don't allow namespace local devices to be moved. */
5904 err = -EINVAL; 5906 err = -EINVAL;
5905 if (dev->features & NETIF_F_NETNS_LOCAL) 5907 if (dev->features & NETIF_F_NETNS_LOCAL)
5906 goto out; 5908 goto out;
5907 5909
5908 /* Ensure the device has been registrered */ 5910 /* Ensure the device has been registrered */
5909 err = -EINVAL; 5911 err = -EINVAL;
5910 if (dev->reg_state != NETREG_REGISTERED) 5912 if (dev->reg_state != NETREG_REGISTERED)
5911 goto out; 5913 goto out;
5912 5914
5913 /* Get out if there is nothing todo */ 5915 /* Get out if there is nothing todo */
5914 err = 0; 5916 err = 0;
5915 if (net_eq(dev_net(dev), net)) 5917 if (net_eq(dev_net(dev), net))
5916 goto out; 5918 goto out;
5917 5919
5918 /* Pick the destination device name, and ensure 5920 /* Pick the destination device name, and ensure
5919 * we can use it in the destination network namespace. 5921 * we can use it in the destination network namespace.
5920 */ 5922 */
5921 err = -EEXIST; 5923 err = -EEXIST;
5922 if (__dev_get_by_name(net, dev->name)) { 5924 if (__dev_get_by_name(net, dev->name)) {
5923 /* We get here if we can't use the current device name */ 5925 /* We get here if we can't use the current device name */
5924 if (!pat) 5926 if (!pat)
5925 goto out; 5927 goto out;
5926 if (dev_get_valid_name(dev, pat, 1)) 5928 if (dev_get_valid_name(dev, pat, 1))
5927 goto out; 5929 goto out;
5928 } 5930 }
5929 5931
5930 /* 5932 /*
5931 * And now a mini version of register_netdevice unregister_netdevice. 5933 * And now a mini version of register_netdevice unregister_netdevice.
5932 */ 5934 */
5933 5935
5934 /* If device is running close it first. */ 5936 /* If device is running close it first. */
5935 dev_close(dev); 5937 dev_close(dev);
5936 5938
5937 /* And unlink it from device chain */ 5939 /* And unlink it from device chain */
5938 err = -ENODEV; 5940 err = -ENODEV;
5939 unlist_netdevice(dev); 5941 unlist_netdevice(dev);
5940 5942
5941 synchronize_net(); 5943 synchronize_net();
5942 5944
5943 /* Shutdown queueing discipline. */ 5945 /* Shutdown queueing discipline. */
5944 dev_shutdown(dev); 5946 dev_shutdown(dev);
5945 5947
5946 /* Notify protocols, that we are about to destroy 5948 /* Notify protocols, that we are about to destroy
5947 this device. They should clean all the things. 5949 this device. They should clean all the things.
5948 5950
5949 Note that dev->reg_state stays at NETREG_REGISTERED. 5951 Note that dev->reg_state stays at NETREG_REGISTERED.
5950 This is wanted because this way 8021q and macvlan know 5952 This is wanted because this way 8021q and macvlan know
5951 the device is just moving and can keep their slaves up. 5953 the device is just moving and can keep their slaves up.
5952 */ 5954 */
5953 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 5955 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
5954 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev); 5956 call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
5955 5957
5956 /* 5958 /*
5957 * Flush the unicast and multicast chains 5959 * Flush the unicast and multicast chains
5958 */ 5960 */
5959 dev_uc_flush(dev); 5961 dev_uc_flush(dev);
5960 dev_mc_flush(dev); 5962 dev_mc_flush(dev);
5961 5963
5962 /* Actually switch the network namespace */ 5964 /* Actually switch the network namespace */
5963 dev_net_set(dev, net); 5965 dev_net_set(dev, net);
5964 5966
5965 /* If there is an ifindex conflict assign a new one */ 5967 /* If there is an ifindex conflict assign a new one */
5966 if (__dev_get_by_index(net, dev->ifindex)) { 5968 if (__dev_get_by_index(net, dev->ifindex)) {
5967 int iflink = (dev->iflink == dev->ifindex); 5969 int iflink = (dev->iflink == dev->ifindex);
5968 dev->ifindex = dev_new_index(net); 5970 dev->ifindex = dev_new_index(net);
5969 if (iflink) 5971 if (iflink)
5970 dev->iflink = dev->ifindex; 5972 dev->iflink = dev->ifindex;
5971 } 5973 }
5972 5974
5973 /* Fixup kobjects */ 5975 /* Fixup kobjects */
5974 err = device_rename(&dev->dev, dev->name); 5976 err = device_rename(&dev->dev, dev->name);
5975 WARN_ON(err); 5977 WARN_ON(err);
5976 5978
5977 /* Add the device back in the hashes */ 5979 /* Add the device back in the hashes */
5978 list_netdevice(dev); 5980 list_netdevice(dev);
5979 5981
5980 /* Notify protocols, that a new device appeared. */ 5982 /* Notify protocols, that a new device appeared. */
5981 call_netdevice_notifiers(NETDEV_REGISTER, dev); 5983 call_netdevice_notifiers(NETDEV_REGISTER, dev);
5982 5984
5983 /* 5985 /*
5984 * Prevent userspace races by waiting until the network 5986 * Prevent userspace races by waiting until the network
5985 * device is fully setup before sending notifications. 5987 * device is fully setup before sending notifications.
5986 */ 5988 */
5987 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U); 5989 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
5988 5990
5989 synchronize_net(); 5991 synchronize_net();
5990 err = 0; 5992 err = 0;
5991 out: 5993 out:
5992 return err; 5994 return err;
5993 } 5995 }
5994 EXPORT_SYMBOL_GPL(dev_change_net_namespace); 5996 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
5995 5997
5996 static int dev_cpu_callback(struct notifier_block *nfb, 5998 static int dev_cpu_callback(struct notifier_block *nfb,
5997 unsigned long action, 5999 unsigned long action,
5998 void *ocpu) 6000 void *ocpu)
5999 { 6001 {
6000 struct sk_buff **list_skb; 6002 struct sk_buff **list_skb;
6001 struct sk_buff *skb; 6003 struct sk_buff *skb;
6002 unsigned int cpu, oldcpu = (unsigned long)ocpu; 6004 unsigned int cpu, oldcpu = (unsigned long)ocpu;
6003 struct softnet_data *sd, *oldsd; 6005 struct softnet_data *sd, *oldsd;
6004 6006
6005 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) 6007 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
6006 return NOTIFY_OK; 6008 return NOTIFY_OK;
6007 6009
6008 local_irq_disable(); 6010 local_irq_disable();
6009 cpu = smp_processor_id(); 6011 cpu = smp_processor_id();
6010 sd = &per_cpu(softnet_data, cpu); 6012 sd = &per_cpu(softnet_data, cpu);
6011 oldsd = &per_cpu(softnet_data, oldcpu); 6013 oldsd = &per_cpu(softnet_data, oldcpu);
6012 6014
6013 /* Find end of our completion_queue. */ 6015 /* Find end of our completion_queue. */
6014 list_skb = &sd->completion_queue; 6016 list_skb = &sd->completion_queue;
6015 while (*list_skb) 6017 while (*list_skb)
6016 list_skb = &(*list_skb)->next; 6018 list_skb = &(*list_skb)->next;
6017 /* Append completion queue from offline CPU. */ 6019 /* Append completion queue from offline CPU. */
6018 *list_skb = oldsd->completion_queue; 6020 *list_skb = oldsd->completion_queue;
6019 oldsd->completion_queue = NULL; 6021 oldsd->completion_queue = NULL;
6020 6022
6021 /* Append output queue from offline CPU. */ 6023 /* Append output queue from offline CPU. */
6022 if (oldsd->output_queue) { 6024 if (oldsd->output_queue) {
6023 *sd->output_queue_tailp = oldsd->output_queue; 6025 *sd->output_queue_tailp = oldsd->output_queue;
6024 sd->output_queue_tailp = oldsd->output_queue_tailp; 6026 sd->output_queue_tailp = oldsd->output_queue_tailp;
6025 oldsd->output_queue = NULL; 6027 oldsd->output_queue = NULL;
6026 oldsd->output_queue_tailp = &oldsd->output_queue; 6028 oldsd->output_queue_tailp = &oldsd->output_queue;
6027 } 6029 }
6028 6030
6029 raise_softirq_irqoff(NET_TX_SOFTIRQ); 6031 raise_softirq_irqoff(NET_TX_SOFTIRQ);
6030 local_irq_enable(); 6032 local_irq_enable();
6031 6033
6032 /* Process offline CPU's input_pkt_queue */ 6034 /* Process offline CPU's input_pkt_queue */
6033 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 6035 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
6034 netif_rx(skb); 6036 netif_rx(skb);
6035 input_queue_head_incr(oldsd); 6037 input_queue_head_incr(oldsd);
6036 } 6038 }
6037 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) { 6039 while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
6038 netif_rx(skb); 6040 netif_rx(skb);
6039 input_queue_head_incr(oldsd); 6041 input_queue_head_incr(oldsd);
6040 } 6042 }
6041 6043
6042 return NOTIFY_OK; 6044 return NOTIFY_OK;
6043 } 6045 }
6044 6046
6045 6047
6046 /** 6048 /**
6047 * netdev_increment_features - increment feature set by one 6049 * netdev_increment_features - increment feature set by one
6048 * @all: current feature set 6050 * @all: current feature set
6049 * @one: new feature set 6051 * @one: new feature set
6050 * @mask: mask feature set 6052 * @mask: mask feature set
6051 * 6053 *
6052 * Computes a new feature set after adding a device with feature set 6054 * Computes a new feature set after adding a device with feature set
6053 * @one to the master device with current feature set @all. Will not 6055 * @one to the master device with current feature set @all. Will not
6054 * enable anything that is off in @mask. Returns the new feature set. 6056 * enable anything that is off in @mask. Returns the new feature set.
6055 */ 6057 */
6056 unsigned long netdev_increment_features(unsigned long all, unsigned long one, 6058 unsigned long netdev_increment_features(unsigned long all, unsigned long one,
6057 unsigned long mask) 6059 unsigned long mask)
6058 { 6060 {
6059 /* If device needs checksumming, downgrade to it. */ 6061 /* If device needs checksumming, downgrade to it. */
6060 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM)) 6062 if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
6061 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM); 6063 all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
6062 else if (mask & NETIF_F_ALL_CSUM) { 6064 else if (mask & NETIF_F_ALL_CSUM) {
6063 /* If one device supports v4/v6 checksumming, set for all. */ 6065 /* If one device supports v4/v6 checksumming, set for all. */
6064 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) && 6066 if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
6065 !(all & NETIF_F_GEN_CSUM)) { 6067 !(all & NETIF_F_GEN_CSUM)) {
6066 all &= ~NETIF_F_ALL_CSUM; 6068 all &= ~NETIF_F_ALL_CSUM;
6067 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 6069 all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
6068 } 6070 }
6069 6071
6070 /* If one device supports hw checksumming, set for all. */ 6072 /* If one device supports hw checksumming, set for all. */
6071 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) { 6073 if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
6072 all &= ~NETIF_F_ALL_CSUM; 6074 all &= ~NETIF_F_ALL_CSUM;
6073 all |= NETIF_F_HW_CSUM; 6075 all |= NETIF_F_HW_CSUM;
6074 } 6076 }
6075 } 6077 }
6076 6078
6077 one |= NETIF_F_ALL_CSUM; 6079 one |= NETIF_F_ALL_CSUM;
6078 6080
6079 one |= all & NETIF_F_ONE_FOR_ALL; 6081 one |= all & NETIF_F_ONE_FOR_ALL;
6080 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO; 6082 all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
6081 all |= one & mask & NETIF_F_ONE_FOR_ALL; 6083 all |= one & mask & NETIF_F_ONE_FOR_ALL;
6082 6084
6083 return all; 6085 return all;
6084 } 6086 }
6085 EXPORT_SYMBOL(netdev_increment_features); 6087 EXPORT_SYMBOL(netdev_increment_features);
6086 6088
6087 static struct hlist_head *netdev_create_hash(void) 6089 static struct hlist_head *netdev_create_hash(void)
6088 { 6090 {
6089 int i; 6091 int i;
6090 struct hlist_head *hash; 6092 struct hlist_head *hash;
6091 6093
6092 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); 6094 hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
6093 if (hash != NULL) 6095 if (hash != NULL)
6094 for (i = 0; i < NETDEV_HASHENTRIES; i++) 6096 for (i = 0; i < NETDEV_HASHENTRIES; i++)
6095 INIT_HLIST_HEAD(&hash[i]); 6097 INIT_HLIST_HEAD(&hash[i]);
6096 6098
6097 return hash; 6099 return hash;
6098 } 6100 }
6099 6101
6100 /* Initialize per network namespace state */ 6102 /* Initialize per network namespace state */
6101 static int __net_init netdev_init(struct net *net) 6103 static int __net_init netdev_init(struct net *net)
6102 { 6104 {
6103 INIT_LIST_HEAD(&net->dev_base_head); 6105 INIT_LIST_HEAD(&net->dev_base_head);
6104 6106
6105 net->dev_name_head = netdev_create_hash(); 6107 net->dev_name_head = netdev_create_hash();
6106 if (net->dev_name_head == NULL) 6108 if (net->dev_name_head == NULL)
6107 goto err_name; 6109 goto err_name;
6108 6110
6109 net->dev_index_head = netdev_create_hash(); 6111 net->dev_index_head = netdev_create_hash();
6110 if (net->dev_index_head == NULL) 6112 if (net->dev_index_head == NULL)
6111 goto err_idx; 6113 goto err_idx;
6112 6114
6113 return 0; 6115 return 0;
6114 6116
6115 err_idx: 6117 err_idx:
6116 kfree(net->dev_name_head); 6118 kfree(net->dev_name_head);
6117 err_name: 6119 err_name:
6118 return -ENOMEM; 6120 return -ENOMEM;
6119 } 6121 }
6120 6122
6121 /** 6123 /**
6122 * netdev_drivername - network driver for the device 6124 * netdev_drivername - network driver for the device
6123 * @dev: network device 6125 * @dev: network device
6124 * @buffer: buffer for resulting name 6126 * @buffer: buffer for resulting name
6125 * @len: size of buffer 6127 * @len: size of buffer
6126 * 6128 *
6127 * Determine network driver for device. 6129 * Determine network driver for device.
6128 */ 6130 */
6129 char *netdev_drivername(const struct net_device *dev, char *buffer, int len) 6131 char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
6130 { 6132 {
6131 const struct device_driver *driver; 6133 const struct device_driver *driver;
6132 const struct device *parent; 6134 const struct device *parent;
6133 6135
6134 if (len <= 0 || !buffer) 6136 if (len <= 0 || !buffer)
6135 return buffer; 6137 return buffer;
6136 buffer[0] = 0; 6138 buffer[0] = 0;
6137 6139
6138 parent = dev->dev.parent; 6140 parent = dev->dev.parent;
6139 6141
6140 if (!parent) 6142 if (!parent)
6141 return buffer; 6143 return buffer;
6142 6144
6143 driver = parent->driver; 6145 driver = parent->driver;
6144 if (driver && driver->name) 6146 if (driver && driver->name)
6145 strlcpy(buffer, driver->name, len); 6147 strlcpy(buffer, driver->name, len);
6146 return buffer; 6148 return buffer;
6147 } 6149 }
6148 6150
6149 static int __netdev_printk(const char *level, const struct net_device *dev, 6151 static int __netdev_printk(const char *level, const struct net_device *dev,
6150 struct va_format *vaf) 6152 struct va_format *vaf)
6151 { 6153 {
6152 int r; 6154 int r;
6153 6155
6154 if (dev && dev->dev.parent) 6156 if (dev && dev->dev.parent)
6155 r = dev_printk(level, dev->dev.parent, "%s: %pV", 6157 r = dev_printk(level, dev->dev.parent, "%s: %pV",
6156 netdev_name(dev), vaf); 6158 netdev_name(dev), vaf);
6157 else if (dev) 6159 else if (dev)
6158 r = printk("%s%s: %pV", level, netdev_name(dev), vaf); 6160 r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
6159 else 6161 else
6160 r = printk("%s(NULL net_device): %pV", level, vaf); 6162 r = printk("%s(NULL net_device): %pV", level, vaf);
6161 6163
6162 return r; 6164 return r;
6163 } 6165 }
6164 6166
6165 int netdev_printk(const char *level, const struct net_device *dev, 6167 int netdev_printk(const char *level, const struct net_device *dev,
6166 const char *format, ...) 6168 const char *format, ...)
6167 { 6169 {
6168 struct va_format vaf; 6170 struct va_format vaf;
6169 va_list args; 6171 va_list args;
6170 int r; 6172 int r;
6171 6173
6172 va_start(args, format); 6174 va_start(args, format);
6173 6175
6174 vaf.fmt = format; 6176 vaf.fmt = format;
6175 vaf.va = &args; 6177 vaf.va = &args;
6176 6178
6177 r = __netdev_printk(level, dev, &vaf); 6179 r = __netdev_printk(level, dev, &vaf);
6178 va_end(args); 6180 va_end(args);
6179 6181
6180 return r; 6182 return r;
6181 } 6183 }
6182 EXPORT_SYMBOL(netdev_printk); 6184 EXPORT_SYMBOL(netdev_printk);
6183 6185
6184 #define define_netdev_printk_level(func, level) \ 6186 #define define_netdev_printk_level(func, level) \
6185 int func(const struct net_device *dev, const char *fmt, ...) \ 6187 int func(const struct net_device *dev, const char *fmt, ...) \
6186 { \ 6188 { \
6187 int r; \ 6189 int r; \
6188 struct va_format vaf; \ 6190 struct va_format vaf; \
6189 va_list args; \ 6191 va_list args; \
6190 \ 6192 \
6191 va_start(args, fmt); \ 6193 va_start(args, fmt); \
6192 \ 6194 \
6193 vaf.fmt = fmt; \ 6195 vaf.fmt = fmt; \
6194 vaf.va = &args; \ 6196 vaf.va = &args; \
6195 \ 6197 \
6196 r = __netdev_printk(level, dev, &vaf); \ 6198 r = __netdev_printk(level, dev, &vaf); \
6197 va_end(args); \ 6199 va_end(args); \
6198 \ 6200 \
6199 return r; \ 6201 return r; \
6200 } \ 6202 } \
6201 EXPORT_SYMBOL(func); 6203 EXPORT_SYMBOL(func);
6202 6204
6203 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 6205 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
6204 define_netdev_printk_level(netdev_alert, KERN_ALERT); 6206 define_netdev_printk_level(netdev_alert, KERN_ALERT);
6205 define_netdev_printk_level(netdev_crit, KERN_CRIT); 6207 define_netdev_printk_level(netdev_crit, KERN_CRIT);
6206 define_netdev_printk_level(netdev_err, KERN_ERR); 6208 define_netdev_printk_level(netdev_err, KERN_ERR);
6207 define_netdev_printk_level(netdev_warn, KERN_WARNING); 6209 define_netdev_printk_level(netdev_warn, KERN_WARNING);
6208 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 6210 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
6209 define_netdev_printk_level(netdev_info, KERN_INFO); 6211 define_netdev_printk_level(netdev_info, KERN_INFO);
6210 6212
6211 static void __net_exit netdev_exit(struct net *net) 6213 static void __net_exit netdev_exit(struct net *net)
6212 { 6214 {
6213 kfree(net->dev_name_head); 6215 kfree(net->dev_name_head);
6214 kfree(net->dev_index_head); 6216 kfree(net->dev_index_head);
6215 } 6217 }
6216 6218
6217 static struct pernet_operations __net_initdata netdev_net_ops = { 6219 static struct pernet_operations __net_initdata netdev_net_ops = {
6218 .init = netdev_init, 6220 .init = netdev_init,
6219 .exit = netdev_exit, 6221 .exit = netdev_exit,
6220 }; 6222 };
6221 6223
6222 static void __net_exit default_device_exit(struct net *net) 6224 static void __net_exit default_device_exit(struct net *net)
6223 { 6225 {
6224 struct net_device *dev, *aux; 6226 struct net_device *dev, *aux;
6225 /* 6227 /*
6226 * Push all migratable network devices back to the 6228 * Push all migratable network devices back to the
6227 * initial network namespace 6229 * initial network namespace
6228 */ 6230 */
6229 rtnl_lock(); 6231 rtnl_lock();
6230 for_each_netdev_safe(net, dev, aux) { 6232 for_each_netdev_safe(net, dev, aux) {
6231 int err; 6233 int err;
6232 char fb_name[IFNAMSIZ]; 6234 char fb_name[IFNAMSIZ];
6233 6235
6234 /* Ignore unmoveable devices (i.e. loopback) */ 6236 /* Ignore unmoveable devices (i.e. loopback) */
6235 if (dev->features & NETIF_F_NETNS_LOCAL) 6237 if (dev->features & NETIF_F_NETNS_LOCAL)
6236 continue; 6238 continue;
6237 6239
6238 /* Leave virtual devices for the generic cleanup */ 6240 /* Leave virtual devices for the generic cleanup */
6239 if (dev->rtnl_link_ops) 6241 if (dev->rtnl_link_ops)
6240 continue; 6242 continue;
6241 6243
6242 /* Push remaing network devices to init_net */ 6244 /* Push remaing network devices to init_net */
6243 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 6245 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
6244 err = dev_change_net_namespace(dev, &init_net, fb_name); 6246 err = dev_change_net_namespace(dev, &init_net, fb_name);
6245 if (err) { 6247 if (err) {
6246 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n", 6248 printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
6247 __func__, dev->name, err); 6249 __func__, dev->name, err);
6248 BUG(); 6250 BUG();
6249 } 6251 }
6250 } 6252 }
6251 rtnl_unlock(); 6253 rtnl_unlock();
6252 } 6254 }
6253 6255
6254 static void __net_exit default_device_exit_batch(struct list_head *net_list) 6256 static void __net_exit default_device_exit_batch(struct list_head *net_list)
6255 { 6257 {
6256 /* At exit all network devices most be removed from a network 6258 /* At exit all network devices most be removed from a network
6257 * namespace. Do this in the reverse order of registration. 6259 * namespace. Do this in the reverse order of registration.
6258 * Do this across as many network namespaces as possible to 6260 * Do this across as many network namespaces as possible to
6259 * improve batching efficiency. 6261 * improve batching efficiency.
6260 */ 6262 */
6261 struct net_device *dev; 6263 struct net_device *dev;
6262 struct net *net; 6264 struct net *net;
6263 LIST_HEAD(dev_kill_list); 6265 LIST_HEAD(dev_kill_list);
6264 6266
6265 rtnl_lock(); 6267 rtnl_lock();
6266 list_for_each_entry(net, net_list, exit_list) { 6268 list_for_each_entry(net, net_list, exit_list) {
6267 for_each_netdev_reverse(net, dev) { 6269 for_each_netdev_reverse(net, dev) {
6268 if (dev->rtnl_link_ops) 6270 if (dev->rtnl_link_ops)
6269 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 6271 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
6270 else 6272 else
6271 unregister_netdevice_queue(dev, &dev_kill_list); 6273 unregister_netdevice_queue(dev, &dev_kill_list);
6272 } 6274 }
6273 } 6275 }
6274 unregister_netdevice_many(&dev_kill_list); 6276 unregister_netdevice_many(&dev_kill_list);
6275 rtnl_unlock(); 6277 rtnl_unlock();
6276 } 6278 }
6277 6279
6278 static struct pernet_operations __net_initdata default_device_ops = { 6280 static struct pernet_operations __net_initdata default_device_ops = {
6279 .exit = default_device_exit, 6281 .exit = default_device_exit,
6280 .exit_batch = default_device_exit_batch, 6282 .exit_batch = default_device_exit_batch,
6281 }; 6283 };
6282 6284
6283 /* 6285 /*
6284 * Initialize the DEV module. At boot time this walks the device list and 6286 * Initialize the DEV module. At boot time this walks the device list and
6285 * unhooks any devices that fail to initialise (normally hardware not 6287 * unhooks any devices that fail to initialise (normally hardware not
6286 * present) and leaves us with a valid list of present and active devices. 6288 * present) and leaves us with a valid list of present and active devices.
6287 * 6289 *
6288 */ 6290 */
6289 6291
6290 /* 6292 /*
6291 * This is called single threaded during boot, so no need 6293 * This is called single threaded during boot, so no need
6292 * to take the rtnl semaphore. 6294 * to take the rtnl semaphore.
6293 */ 6295 */
6294 static int __init net_dev_init(void) 6296 static int __init net_dev_init(void)
6295 { 6297 {
6296 int i, rc = -ENOMEM; 6298 int i, rc = -ENOMEM;
6297 6299
6298 BUG_ON(!dev_boot_phase); 6300 BUG_ON(!dev_boot_phase);
6299 6301
6300 if (dev_proc_init()) 6302 if (dev_proc_init())
6301 goto out; 6303 goto out;
6302 6304
6303 if (netdev_kobject_init()) 6305 if (netdev_kobject_init())
6304 goto out; 6306 goto out;
6305 6307
6306 INIT_LIST_HEAD(&ptype_all); 6308 INIT_LIST_HEAD(&ptype_all);
6307 for (i = 0; i < PTYPE_HASH_SIZE; i++) 6309 for (i = 0; i < PTYPE_HASH_SIZE; i++)
6308 INIT_LIST_HEAD(&ptype_base[i]); 6310 INIT_LIST_HEAD(&ptype_base[i]);
6309 6311
6310 if (register_pernet_subsys(&netdev_net_ops)) 6312 if (register_pernet_subsys(&netdev_net_ops))
6311 goto out; 6313 goto out;
6312 6314
6313 /* 6315 /*
6314 * Initialise the packet receive queues. 6316 * Initialise the packet receive queues.
6315 */ 6317 */
6316 6318
6317 for_each_possible_cpu(i) { 6319 for_each_possible_cpu(i) {
6318 struct softnet_data *sd = &per_cpu(softnet_data, i); 6320 struct softnet_data *sd = &per_cpu(softnet_data, i);
6319 6321
6320 memset(sd, 0, sizeof(*sd)); 6322 memset(sd, 0, sizeof(*sd));
6321 skb_queue_head_init(&sd->input_pkt_queue); 6323 skb_queue_head_init(&sd->input_pkt_queue);
6322 skb_queue_head_init(&sd->process_queue); 6324 skb_queue_head_init(&sd->process_queue);
6323 sd->completion_queue = NULL; 6325 sd->completion_queue = NULL;
6324 INIT_LIST_HEAD(&sd->poll_list); 6326 INIT_LIST_HEAD(&sd->poll_list);
6325 sd->output_queue = NULL; 6327 sd->output_queue = NULL;
6326 sd->output_queue_tailp = &sd->output_queue; 6328 sd->output_queue_tailp = &sd->output_queue;
6327 #ifdef CONFIG_RPS 6329 #ifdef CONFIG_RPS
6328 sd->csd.func = rps_trigger_softirq; 6330 sd->csd.func = rps_trigger_softirq;
6329 sd->csd.info = sd; 6331 sd->csd.info = sd;
6330 sd->csd.flags = 0; 6332 sd->csd.flags = 0;
6331 sd->cpu = i; 6333 sd->cpu = i;
6332 #endif 6334 #endif
6333 6335
6334 sd->backlog.poll = process_backlog; 6336 sd->backlog.poll = process_backlog;
6335 sd->backlog.weight = weight_p; 6337 sd->backlog.weight = weight_p;
6336 sd->backlog.gro_list = NULL; 6338 sd->backlog.gro_list = NULL;
6337 sd->backlog.gro_count = 0; 6339 sd->backlog.gro_count = 0;
6338 } 6340 }
6339 6341
6340 dev_boot_phase = 0; 6342 dev_boot_phase = 0;
6341 6343
6342 /* The loopback device is special if any other network devices 6344 /* The loopback device is special if any other network devices
6343 * is present in a network namespace the loopback device must 6345 * is present in a network namespace the loopback device must
6344 * be present. Since we now dynamically allocate and free the 6346 * be present. Since we now dynamically allocate and free the
6345 * loopback device ensure this invariant is maintained by 6347 * loopback device ensure this invariant is maintained by
6346 * keeping the loopback device as the first device on the 6348 * keeping the loopback device as the first device on the
6347 * list of network devices. Ensuring the loopback devices 6349 * list of network devices. Ensuring the loopback devices
6348 * is the first device that appears and the last network device 6350 * is the first device that appears and the last network device
6349 * that disappears. 6351 * that disappears.
6350 */ 6352 */
6351 if (register_pernet_device(&loopback_net_ops)) 6353 if (register_pernet_device(&loopback_net_ops))
6352 goto out; 6354 goto out;
6353 6355
6354 if (register_pernet_device(&default_device_ops)) 6356 if (register_pernet_device(&default_device_ops))
6355 goto out; 6357 goto out;
6356 6358
6357 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 6359 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
6358 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 6360 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
6359 6361
6360 hotcpu_notifier(dev_cpu_callback, 0); 6362 hotcpu_notifier(dev_cpu_callback, 0);
6361 dst_init(); 6363 dst_init();
6362 dev_mcast_init(); 6364 dev_mcast_init();
6363 rc = 0; 6365 rc = 0;
6364 out: 6366 out:
6365 return rc; 6367 return rc;
6366 } 6368 }
6367 6369
6368 subsys_initcall(net_dev_init); 6370 subsys_initcall(net_dev_init);
6369 6371
6370 static int __init initialize_hashrnd(void) 6372 static int __init initialize_hashrnd(void)
6371 { 6373 {
6372 get_random_bytes(&hashrnd, sizeof(hashrnd)); 6374 get_random_bytes(&hashrnd, sizeof(hashrnd));
6373 return 0; 6375 return 0;
6374 } 6376 }
6375 6377
6376 late_initcall_sync(initialize_hashrnd); 6378 late_initcall_sync(initialize_hashrnd);
6377 6379
1 /* 1 /*
2 * net/sched/sch_api.c Packet scheduler API. 2 * net/sched/sch_api.c Packet scheduler API.
3 * 3 *
4 * This program is free software; you can redistribute it and/or 4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License 5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version. 7 * 2 of the License, or (at your option) any later version.
8 * 8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10 * 10 *
11 * Fixes: 11 * Fixes:
12 * 12 *
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired. 13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support 15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
16 */ 16 */
17 17
18 #include <linux/module.h> 18 #include <linux/module.h>
19 #include <linux/types.h> 19 #include <linux/types.h>
20 #include <linux/kernel.h> 20 #include <linux/kernel.h>
21 #include <linux/string.h> 21 #include <linux/string.h>
22 #include <linux/errno.h> 22 #include <linux/errno.h>
23 #include <linux/skbuff.h> 23 #include <linux/skbuff.h>
24 #include <linux/init.h> 24 #include <linux/init.h>
25 #include <linux/proc_fs.h> 25 #include <linux/proc_fs.h>
26 #include <linux/seq_file.h> 26 #include <linux/seq_file.h>
27 #include <linux/kmod.h> 27 #include <linux/kmod.h>
28 #include <linux/list.h> 28 #include <linux/list.h>
29 #include <linux/hrtimer.h> 29 #include <linux/hrtimer.h>
30 #include <linux/lockdep.h> 30 #include <linux/lockdep.h>
31 #include <linux/slab.h> 31 #include <linux/slab.h>
32 32
33 #include <net/net_namespace.h> 33 #include <net/net_namespace.h>
34 #include <net/sock.h> 34 #include <net/sock.h>
35 #include <net/netlink.h> 35 #include <net/netlink.h>
36 #include <net/pkt_sched.h> 36 #include <net/pkt_sched.h>
37 37
38 static int qdisc_notify(struct net *net, struct sk_buff *oskb, 38 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
39 struct nlmsghdr *n, u32 clid, 39 struct nlmsghdr *n, u32 clid,
40 struct Qdisc *old, struct Qdisc *new); 40 struct Qdisc *old, struct Qdisc *new);
41 static int tclass_notify(struct net *net, struct sk_buff *oskb, 41 static int tclass_notify(struct net *net, struct sk_buff *oskb,
42 struct nlmsghdr *n, struct Qdisc *q, 42 struct nlmsghdr *n, struct Qdisc *q,
43 unsigned long cl, int event); 43 unsigned long cl, int event);
44 44
45 /* 45 /*
46 46
47 Short review. 47 Short review.
48 ------------- 48 -------------
49 49
50 This file consists of two interrelated parts: 50 This file consists of two interrelated parts:
51 51
52 1. queueing disciplines manager frontend. 52 1. queueing disciplines manager frontend.
53 2. traffic classes manager frontend. 53 2. traffic classes manager frontend.
54 54
55 Generally, queueing discipline ("qdisc") is a black box, 55 Generally, queueing discipline ("qdisc") is a black box,
56 which is able to enqueue packets and to dequeue them (when 56 which is able to enqueue packets and to dequeue them (when
57 device is ready to send something) in order and at times 57 device is ready to send something) in order and at times
58 determined by algorithm hidden in it. 58 determined by algorithm hidden in it.
59 59
60 qdisc's are divided to two categories: 60 qdisc's are divided to two categories:
61 - "queues", which have no internal structure visible from outside. 61 - "queues", which have no internal structure visible from outside.
62 - "schedulers", which split all the packets to "traffic classes", 62 - "schedulers", which split all the packets to "traffic classes",
63 using "packet classifiers" (look at cls_api.c) 63 using "packet classifiers" (look at cls_api.c)
64 64
65 In turn, classes may have child qdiscs (as rule, queues) 65 In turn, classes may have child qdiscs (as rule, queues)
66 attached to them etc. etc. etc. 66 attached to them etc. etc. etc.
67 67
68 The goal of the routines in this file is to translate 68 The goal of the routines in this file is to translate
69 information supplied by user in the form of handles 69 information supplied by user in the form of handles
70 to more intelligible for kernel form, to make some sanity 70 to more intelligible for kernel form, to make some sanity
71 checks and part of work, which is common to all qdiscs 71 checks and part of work, which is common to all qdiscs
72 and to provide rtnetlink notifications. 72 and to provide rtnetlink notifications.
73 73
74 All real intelligent work is done inside qdisc modules. 74 All real intelligent work is done inside qdisc modules.
75 75
76 76
77 77
78 Every discipline has two major routines: enqueue and dequeue. 78 Every discipline has two major routines: enqueue and dequeue.
79 79
80 ---dequeue 80 ---dequeue
81 81
82 dequeue usually returns a skb to send. It is allowed to return NULL, 82 dequeue usually returns a skb to send. It is allowed to return NULL,
83 but it does not mean that queue is empty, it just means that 83 but it does not mean that queue is empty, it just means that
84 discipline does not want to send anything this time. 84 discipline does not want to send anything this time.
85 Queue is really empty if q->q.qlen == 0. 85 Queue is really empty if q->q.qlen == 0.
86 For complicated disciplines with multiple queues q->q is not 86 For complicated disciplines with multiple queues q->q is not
87 real packet queue, but however q->q.qlen must be valid. 87 real packet queue, but however q->q.qlen must be valid.
88 88
89 ---enqueue 89 ---enqueue
90 90
91 enqueue returns 0, if packet was enqueued successfully. 91 enqueue returns 0, if packet was enqueued successfully.
92 If packet (this one or another one) was dropped, it returns 92 If packet (this one or another one) was dropped, it returns
93 not zero error code. 93 not zero error code.
94 NET_XMIT_DROP - this packet dropped 94 NET_XMIT_DROP - this packet dropped
95 Expected action: do not backoff, but wait until queue will clear. 95 Expected action: do not backoff, but wait until queue will clear.
96 NET_XMIT_CN - probably this packet enqueued, but another one dropped. 96 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
97 Expected action: backoff or ignore 97 Expected action: backoff or ignore
98 NET_XMIT_POLICED - dropped by police. 98 NET_XMIT_POLICED - dropped by police.
99 Expected action: backoff or error to real-time apps. 99 Expected action: backoff or error to real-time apps.
100 100
101 Auxiliary routines: 101 Auxiliary routines:
102 102
103 ---peek 103 ---peek
104 104
105 like dequeue but without removing a packet from the queue 105 like dequeue but without removing a packet from the queue
106 106
107 ---reset 107 ---reset
108 108
109 returns qdisc to initial state: purge all buffers, clear all 109 returns qdisc to initial state: purge all buffers, clear all
110 timers, counters (except for statistics) etc. 110 timers, counters (except for statistics) etc.
111 111
112 ---init 112 ---init
113 113
114 initializes newly created qdisc. 114 initializes newly created qdisc.
115 115
116 ---destroy 116 ---destroy
117 117
118 destroys resources allocated by init and during lifetime of qdisc. 118 destroys resources allocated by init and during lifetime of qdisc.
119 119
120 ---change 120 ---change
121 121
122 changes qdisc parameters. 122 changes qdisc parameters.
123 */ 123 */
124 124
125 /* Protects list of registered TC modules. It is pure SMP lock. */ 125 /* Protects list of registered TC modules. It is pure SMP lock. */
126 static DEFINE_RWLOCK(qdisc_mod_lock); 126 static DEFINE_RWLOCK(qdisc_mod_lock);
127 127
128 128
129 /************************************************ 129 /************************************************
130 * Queueing disciplines manipulation. * 130 * Queueing disciplines manipulation. *
131 ************************************************/ 131 ************************************************/
132 132
133 133
134 /* The list of all installed queueing disciplines. */ 134 /* The list of all installed queueing disciplines. */
135 135
136 static struct Qdisc_ops *qdisc_base; 136 static struct Qdisc_ops *qdisc_base;
137 137
138 /* Register/uregister queueing discipline */ 138 /* Register/uregister queueing discipline */
139 139
140 int register_qdisc(struct Qdisc_ops *qops) 140 int register_qdisc(struct Qdisc_ops *qops)
141 { 141 {
142 struct Qdisc_ops *q, **qp; 142 struct Qdisc_ops *q, **qp;
143 int rc = -EEXIST; 143 int rc = -EEXIST;
144 144
145 write_lock(&qdisc_mod_lock); 145 write_lock(&qdisc_mod_lock);
146 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 146 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
147 if (!strcmp(qops->id, q->id)) 147 if (!strcmp(qops->id, q->id))
148 goto out; 148 goto out;
149 149
150 if (qops->enqueue == NULL) 150 if (qops->enqueue == NULL)
151 qops->enqueue = noop_qdisc_ops.enqueue; 151 qops->enqueue = noop_qdisc_ops.enqueue;
152 if (qops->peek == NULL) { 152 if (qops->peek == NULL) {
153 if (qops->dequeue == NULL) 153 if (qops->dequeue == NULL)
154 qops->peek = noop_qdisc_ops.peek; 154 qops->peek = noop_qdisc_ops.peek;
155 else 155 else
156 goto out_einval; 156 goto out_einval;
157 } 157 }
158 if (qops->dequeue == NULL) 158 if (qops->dequeue == NULL)
159 qops->dequeue = noop_qdisc_ops.dequeue; 159 qops->dequeue = noop_qdisc_ops.dequeue;
160 160
161 if (qops->cl_ops) { 161 if (qops->cl_ops) {
162 const struct Qdisc_class_ops *cops = qops->cl_ops; 162 const struct Qdisc_class_ops *cops = qops->cl_ops;
163 163
164 if (!(cops->get && cops->put && cops->walk && cops->leaf)) 164 if (!(cops->get && cops->put && cops->walk && cops->leaf))
165 goto out_einval; 165 goto out_einval;
166 166
167 if (cops->tcf_chain && !(cops->bind_tcf && cops->unbind_tcf)) 167 if (cops->tcf_chain && !(cops->bind_tcf && cops->unbind_tcf))
168 goto out_einval; 168 goto out_einval;
169 } 169 }
170 170
171 qops->next = NULL; 171 qops->next = NULL;
172 *qp = qops; 172 *qp = qops;
173 rc = 0; 173 rc = 0;
174 out: 174 out:
175 write_unlock(&qdisc_mod_lock); 175 write_unlock(&qdisc_mod_lock);
176 return rc; 176 return rc;
177 177
178 out_einval: 178 out_einval:
179 rc = -EINVAL; 179 rc = -EINVAL;
180 goto out; 180 goto out;
181 } 181 }
182 EXPORT_SYMBOL(register_qdisc); 182 EXPORT_SYMBOL(register_qdisc);
183 183
184 int unregister_qdisc(struct Qdisc_ops *qops) 184 int unregister_qdisc(struct Qdisc_ops *qops)
185 { 185 {
186 struct Qdisc_ops *q, **qp; 186 struct Qdisc_ops *q, **qp;
187 int err = -ENOENT; 187 int err = -ENOENT;
188 188
189 write_lock(&qdisc_mod_lock); 189 write_lock(&qdisc_mod_lock);
190 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next) 190 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
191 if (q == qops) 191 if (q == qops)
192 break; 192 break;
193 if (q) { 193 if (q) {
194 *qp = q->next; 194 *qp = q->next;
195 q->next = NULL; 195 q->next = NULL;
196 err = 0; 196 err = 0;
197 } 197 }
198 write_unlock(&qdisc_mod_lock); 198 write_unlock(&qdisc_mod_lock);
199 return err; 199 return err;
200 } 200 }
201 EXPORT_SYMBOL(unregister_qdisc); 201 EXPORT_SYMBOL(unregister_qdisc);
202 202
203 /* We know handle. Find qdisc among all qdisc's attached to device 203 /* We know handle. Find qdisc among all qdisc's attached to device
204 (root qdisc, all its children, children of children etc.) 204 (root qdisc, all its children, children of children etc.)
205 */ 205 */
206 206
207 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle) 207 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
208 { 208 {
209 struct Qdisc *q; 209 struct Qdisc *q;
210 210
211 if (!(root->flags & TCQ_F_BUILTIN) && 211 if (!(root->flags & TCQ_F_BUILTIN) &&
212 root->handle == handle) 212 root->handle == handle)
213 return root; 213 return root;
214 214
215 list_for_each_entry(q, &root->list, list) { 215 list_for_each_entry(q, &root->list, list) {
216 if (q->handle == handle) 216 if (q->handle == handle)
217 return q; 217 return q;
218 } 218 }
219 return NULL; 219 return NULL;
220 } 220 }
221 221
222 static void qdisc_list_add(struct Qdisc *q) 222 static void qdisc_list_add(struct Qdisc *q)
223 { 223 {
224 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) 224 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS))
225 list_add_tail(&q->list, &qdisc_dev(q)->qdisc->list); 225 list_add_tail(&q->list, &qdisc_dev(q)->qdisc->list);
226 } 226 }
227 227
228 void qdisc_list_del(struct Qdisc *q) 228 void qdisc_list_del(struct Qdisc *q)
229 { 229 {
230 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) 230 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS))
231 list_del(&q->list); 231 list_del(&q->list);
232 } 232 }
233 EXPORT_SYMBOL(qdisc_list_del); 233 EXPORT_SYMBOL(qdisc_list_del);
234 234
235 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle) 235 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
236 { 236 {
237 struct Qdisc *q; 237 struct Qdisc *q;
238 238
239 q = qdisc_match_from_root(dev->qdisc, handle); 239 q = qdisc_match_from_root(dev->qdisc, handle);
240 if (q) 240 if (q)
241 goto out; 241 goto out;
242 242
243 if (dev_ingress_queue(dev)) 243 if (dev_ingress_queue(dev))
244 q = qdisc_match_from_root( 244 q = qdisc_match_from_root(
245 dev_ingress_queue(dev)->qdisc_sleeping, 245 dev_ingress_queue(dev)->qdisc_sleeping,
246 handle); 246 handle);
247 out: 247 out:
248 return q; 248 return q;
249 } 249 }
250 250
251 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid) 251 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
252 { 252 {
253 unsigned long cl; 253 unsigned long cl;
254 struct Qdisc *leaf; 254 struct Qdisc *leaf;
255 const struct Qdisc_class_ops *cops = p->ops->cl_ops; 255 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
256 256
257 if (cops == NULL) 257 if (cops == NULL)
258 return NULL; 258 return NULL;
259 cl = cops->get(p, classid); 259 cl = cops->get(p, classid);
260 260
261 if (cl == 0) 261 if (cl == 0)
262 return NULL; 262 return NULL;
263 leaf = cops->leaf(p, cl); 263 leaf = cops->leaf(p, cl);
264 cops->put(p, cl); 264 cops->put(p, cl);
265 return leaf; 265 return leaf;
266 } 266 }
267 267
268 /* Find queueing discipline by name */ 268 /* Find queueing discipline by name */
269 269
270 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind) 270 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
271 { 271 {
272 struct Qdisc_ops *q = NULL; 272 struct Qdisc_ops *q = NULL;
273 273
274 if (kind) { 274 if (kind) {
275 read_lock(&qdisc_mod_lock); 275 read_lock(&qdisc_mod_lock);
276 for (q = qdisc_base; q; q = q->next) { 276 for (q = qdisc_base; q; q = q->next) {
277 if (nla_strcmp(kind, q->id) == 0) { 277 if (nla_strcmp(kind, q->id) == 0) {
278 if (!try_module_get(q->owner)) 278 if (!try_module_get(q->owner))
279 q = NULL; 279 q = NULL;
280 break; 280 break;
281 } 281 }
282 } 282 }
283 read_unlock(&qdisc_mod_lock); 283 read_unlock(&qdisc_mod_lock);
284 } 284 }
285 return q; 285 return q;
286 } 286 }
287 287
288 static struct qdisc_rate_table *qdisc_rtab_list; 288 static struct qdisc_rate_table *qdisc_rtab_list;
289 289
290 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab) 290 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
291 { 291 {
292 struct qdisc_rate_table *rtab; 292 struct qdisc_rate_table *rtab;
293 293
294 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) { 294 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
295 if (memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) == 0) { 295 if (memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) == 0) {
296 rtab->refcnt++; 296 rtab->refcnt++;
297 return rtab; 297 return rtab;
298 } 298 }
299 } 299 }
300 300
301 if (tab == NULL || r->rate == 0 || r->cell_log == 0 || 301 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
302 nla_len(tab) != TC_RTAB_SIZE) 302 nla_len(tab) != TC_RTAB_SIZE)
303 return NULL; 303 return NULL;
304 304
305 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL); 305 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
306 if (rtab) { 306 if (rtab) {
307 rtab->rate = *r; 307 rtab->rate = *r;
308 rtab->refcnt = 1; 308 rtab->refcnt = 1;
309 memcpy(rtab->data, nla_data(tab), 1024); 309 memcpy(rtab->data, nla_data(tab), 1024);
310 rtab->next = qdisc_rtab_list; 310 rtab->next = qdisc_rtab_list;
311 qdisc_rtab_list = rtab; 311 qdisc_rtab_list = rtab;
312 } 312 }
313 return rtab; 313 return rtab;
314 } 314 }
315 EXPORT_SYMBOL(qdisc_get_rtab); 315 EXPORT_SYMBOL(qdisc_get_rtab);
316 316
317 void qdisc_put_rtab(struct qdisc_rate_table *tab) 317 void qdisc_put_rtab(struct qdisc_rate_table *tab)
318 { 318 {
319 struct qdisc_rate_table *rtab, **rtabp; 319 struct qdisc_rate_table *rtab, **rtabp;
320 320
321 if (!tab || --tab->refcnt) 321 if (!tab || --tab->refcnt)
322 return; 322 return;
323 323
324 for (rtabp = &qdisc_rtab_list; 324 for (rtabp = &qdisc_rtab_list;
325 (rtab = *rtabp) != NULL; 325 (rtab = *rtabp) != NULL;
326 rtabp = &rtab->next) { 326 rtabp = &rtab->next) {
327 if (rtab == tab) { 327 if (rtab == tab) {
328 *rtabp = rtab->next; 328 *rtabp = rtab->next;
329 kfree(rtab); 329 kfree(rtab);
330 return; 330 return;
331 } 331 }
332 } 332 }
333 } 333 }
334 EXPORT_SYMBOL(qdisc_put_rtab); 334 EXPORT_SYMBOL(qdisc_put_rtab);
335 335
336 static LIST_HEAD(qdisc_stab_list); 336 static LIST_HEAD(qdisc_stab_list);
337 static DEFINE_SPINLOCK(qdisc_stab_lock); 337 static DEFINE_SPINLOCK(qdisc_stab_lock);
338 338
339 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = { 339 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
340 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) }, 340 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
341 [TCA_STAB_DATA] = { .type = NLA_BINARY }, 341 [TCA_STAB_DATA] = { .type = NLA_BINARY },
342 }; 342 };
343 343
344 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt) 344 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt)
345 { 345 {
346 struct nlattr *tb[TCA_STAB_MAX + 1]; 346 struct nlattr *tb[TCA_STAB_MAX + 1];
347 struct qdisc_size_table *stab; 347 struct qdisc_size_table *stab;
348 struct tc_sizespec *s; 348 struct tc_sizespec *s;
349 unsigned int tsize = 0; 349 unsigned int tsize = 0;
350 u16 *tab = NULL; 350 u16 *tab = NULL;
351 int err; 351 int err;
352 352
353 err = nla_parse_nested(tb, TCA_STAB_MAX, opt, stab_policy); 353 err = nla_parse_nested(tb, TCA_STAB_MAX, opt, stab_policy);
354 if (err < 0) 354 if (err < 0)
355 return ERR_PTR(err); 355 return ERR_PTR(err);
356 if (!tb[TCA_STAB_BASE]) 356 if (!tb[TCA_STAB_BASE])
357 return ERR_PTR(-EINVAL); 357 return ERR_PTR(-EINVAL);
358 358
359 s = nla_data(tb[TCA_STAB_BASE]); 359 s = nla_data(tb[TCA_STAB_BASE]);
360 360
361 if (s->tsize > 0) { 361 if (s->tsize > 0) {
362 if (!tb[TCA_STAB_DATA]) 362 if (!tb[TCA_STAB_DATA])
363 return ERR_PTR(-EINVAL); 363 return ERR_PTR(-EINVAL);
364 tab = nla_data(tb[TCA_STAB_DATA]); 364 tab = nla_data(tb[TCA_STAB_DATA]);
365 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16); 365 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
366 } 366 }
367 367
368 if (tsize != s->tsize || (!tab && tsize > 0)) 368 if (tsize != s->tsize || (!tab && tsize > 0))
369 return ERR_PTR(-EINVAL); 369 return ERR_PTR(-EINVAL);
370 370
371 spin_lock(&qdisc_stab_lock); 371 spin_lock(&qdisc_stab_lock);
372 372
373 list_for_each_entry(stab, &qdisc_stab_list, list) { 373 list_for_each_entry(stab, &qdisc_stab_list, list) {
374 if (memcmp(&stab->szopts, s, sizeof(*s))) 374 if (memcmp(&stab->szopts, s, sizeof(*s)))
375 continue; 375 continue;
376 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16))) 376 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
377 continue; 377 continue;
378 stab->refcnt++; 378 stab->refcnt++;
379 spin_unlock(&qdisc_stab_lock); 379 spin_unlock(&qdisc_stab_lock);
380 return stab; 380 return stab;
381 } 381 }
382 382
383 spin_unlock(&qdisc_stab_lock); 383 spin_unlock(&qdisc_stab_lock);
384 384
385 stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL); 385 stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
386 if (!stab) 386 if (!stab)
387 return ERR_PTR(-ENOMEM); 387 return ERR_PTR(-ENOMEM);
388 388
389 stab->refcnt = 1; 389 stab->refcnt = 1;
390 stab->szopts = *s; 390 stab->szopts = *s;
391 if (tsize > 0) 391 if (tsize > 0)
392 memcpy(stab->data, tab, tsize * sizeof(u16)); 392 memcpy(stab->data, tab, tsize * sizeof(u16));
393 393
394 spin_lock(&qdisc_stab_lock); 394 spin_lock(&qdisc_stab_lock);
395 list_add_tail(&stab->list, &qdisc_stab_list); 395 list_add_tail(&stab->list, &qdisc_stab_list);
396 spin_unlock(&qdisc_stab_lock); 396 spin_unlock(&qdisc_stab_lock);
397 397
398 return stab; 398 return stab;
399 } 399 }
400 400
401 static void stab_kfree_rcu(struct rcu_head *head)
402 {
403 kfree(container_of(head, struct qdisc_size_table, rcu));
404 }
405
401 void qdisc_put_stab(struct qdisc_size_table *tab) 406 void qdisc_put_stab(struct qdisc_size_table *tab)
402 { 407 {
403 if (!tab) 408 if (!tab)
404 return; 409 return;
405 410
406 spin_lock(&qdisc_stab_lock); 411 spin_lock(&qdisc_stab_lock);
407 412
408 if (--tab->refcnt == 0) { 413 if (--tab->refcnt == 0) {
409 list_del(&tab->list); 414 list_del(&tab->list);
410 kfree(tab); 415 call_rcu_bh(&tab->rcu, stab_kfree_rcu);
411 } 416 }
412 417
413 spin_unlock(&qdisc_stab_lock); 418 spin_unlock(&qdisc_stab_lock);
414 } 419 }
415 EXPORT_SYMBOL(qdisc_put_stab); 420 EXPORT_SYMBOL(qdisc_put_stab);
416 421
417 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab) 422 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
418 { 423 {
419 struct nlattr *nest; 424 struct nlattr *nest;
420 425
421 nest = nla_nest_start(skb, TCA_STAB); 426 nest = nla_nest_start(skb, TCA_STAB);
422 if (nest == NULL) 427 if (nest == NULL)
423 goto nla_put_failure; 428 goto nla_put_failure;
424 NLA_PUT(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts); 429 NLA_PUT(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts);
425 nla_nest_end(skb, nest); 430 nla_nest_end(skb, nest);
426 431
427 return skb->len; 432 return skb->len;
428 433
429 nla_put_failure: 434 nla_put_failure:
430 return -1; 435 return -1;
431 } 436 }
432 437
433 void qdisc_calculate_pkt_len(struct sk_buff *skb, struct qdisc_size_table *stab) 438 void __qdisc_calculate_pkt_len(struct sk_buff *skb, const struct qdisc_size_table *stab)
434 { 439 {
435 int pkt_len, slot; 440 int pkt_len, slot;
436 441
437 pkt_len = skb->len + stab->szopts.overhead; 442 pkt_len = skb->len + stab->szopts.overhead;
438 if (unlikely(!stab->szopts.tsize)) 443 if (unlikely(!stab->szopts.tsize))
439 goto out; 444 goto out;
440 445
441 slot = pkt_len + stab->szopts.cell_align; 446 slot = pkt_len + stab->szopts.cell_align;
442 if (unlikely(slot < 0)) 447 if (unlikely(slot < 0))
443 slot = 0; 448 slot = 0;
444 449
445 slot >>= stab->szopts.cell_log; 450 slot >>= stab->szopts.cell_log;
446 if (likely(slot < stab->szopts.tsize)) 451 if (likely(slot < stab->szopts.tsize))
447 pkt_len = stab->data[slot]; 452 pkt_len = stab->data[slot];
448 else 453 else
449 pkt_len = stab->data[stab->szopts.tsize - 1] * 454 pkt_len = stab->data[stab->szopts.tsize - 1] *
450 (slot / stab->szopts.tsize) + 455 (slot / stab->szopts.tsize) +
451 stab->data[slot % stab->szopts.tsize]; 456 stab->data[slot % stab->szopts.tsize];
452 457
453 pkt_len <<= stab->szopts.size_log; 458 pkt_len <<= stab->szopts.size_log;
454 out: 459 out:
455 if (unlikely(pkt_len < 1)) 460 if (unlikely(pkt_len < 1))
456 pkt_len = 1; 461 pkt_len = 1;
457 qdisc_skb_cb(skb)->pkt_len = pkt_len; 462 qdisc_skb_cb(skb)->pkt_len = pkt_len;
458 } 463 }
459 EXPORT_SYMBOL(qdisc_calculate_pkt_len); 464 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
460 465
461 void qdisc_warn_nonwc(char *txt, struct Qdisc *qdisc) 466 void qdisc_warn_nonwc(char *txt, struct Qdisc *qdisc)
462 { 467 {
463 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) { 468 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
464 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n", 469 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
465 txt, qdisc->ops->id, qdisc->handle >> 16); 470 txt, qdisc->ops->id, qdisc->handle >> 16);
466 qdisc->flags |= TCQ_F_WARN_NONWC; 471 qdisc->flags |= TCQ_F_WARN_NONWC;
467 } 472 }
468 } 473 }
469 EXPORT_SYMBOL(qdisc_warn_nonwc); 474 EXPORT_SYMBOL(qdisc_warn_nonwc);
470 475
471 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer) 476 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
472 { 477 {
473 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog, 478 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
474 timer); 479 timer);
475 480
476 qdisc_unthrottled(wd->qdisc); 481 qdisc_unthrottled(wd->qdisc);
477 __netif_schedule(qdisc_root(wd->qdisc)); 482 __netif_schedule(qdisc_root(wd->qdisc));
478 483
479 return HRTIMER_NORESTART; 484 return HRTIMER_NORESTART;
480 } 485 }
481 486
482 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc) 487 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
483 { 488 {
484 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 489 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
485 wd->timer.function = qdisc_watchdog; 490 wd->timer.function = qdisc_watchdog;
486 wd->qdisc = qdisc; 491 wd->qdisc = qdisc;
487 } 492 }
488 EXPORT_SYMBOL(qdisc_watchdog_init); 493 EXPORT_SYMBOL(qdisc_watchdog_init);
489 494
490 void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, psched_time_t expires) 495 void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, psched_time_t expires)
491 { 496 {
492 ktime_t time; 497 ktime_t time;
493 498
494 if (test_bit(__QDISC_STATE_DEACTIVATED, 499 if (test_bit(__QDISC_STATE_DEACTIVATED,
495 &qdisc_root_sleeping(wd->qdisc)->state)) 500 &qdisc_root_sleeping(wd->qdisc)->state))
496 return; 501 return;
497 502
498 qdisc_throttled(wd->qdisc); 503 qdisc_throttled(wd->qdisc);
499 time = ktime_set(0, 0); 504 time = ktime_set(0, 0);
500 time = ktime_add_ns(time, PSCHED_TICKS2NS(expires)); 505 time = ktime_add_ns(time, PSCHED_TICKS2NS(expires));
501 hrtimer_start(&wd->timer, time, HRTIMER_MODE_ABS); 506 hrtimer_start(&wd->timer, time, HRTIMER_MODE_ABS);
502 } 507 }
503 EXPORT_SYMBOL(qdisc_watchdog_schedule); 508 EXPORT_SYMBOL(qdisc_watchdog_schedule);
504 509
505 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd) 510 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
506 { 511 {
507 hrtimer_cancel(&wd->timer); 512 hrtimer_cancel(&wd->timer);
508 qdisc_unthrottled(wd->qdisc); 513 qdisc_unthrottled(wd->qdisc);
509 } 514 }
510 EXPORT_SYMBOL(qdisc_watchdog_cancel); 515 EXPORT_SYMBOL(qdisc_watchdog_cancel);
511 516
512 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n) 517 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
513 { 518 {
514 unsigned int size = n * sizeof(struct hlist_head), i; 519 unsigned int size = n * sizeof(struct hlist_head), i;
515 struct hlist_head *h; 520 struct hlist_head *h;
516 521
517 if (size <= PAGE_SIZE) 522 if (size <= PAGE_SIZE)
518 h = kmalloc(size, GFP_KERNEL); 523 h = kmalloc(size, GFP_KERNEL);
519 else 524 else
520 h = (struct hlist_head *) 525 h = (struct hlist_head *)
521 __get_free_pages(GFP_KERNEL, get_order(size)); 526 __get_free_pages(GFP_KERNEL, get_order(size));
522 527
523 if (h != NULL) { 528 if (h != NULL) {
524 for (i = 0; i < n; i++) 529 for (i = 0; i < n; i++)
525 INIT_HLIST_HEAD(&h[i]); 530 INIT_HLIST_HEAD(&h[i]);
526 } 531 }
527 return h; 532 return h;
528 } 533 }
529 534
530 static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n) 535 static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
531 { 536 {
532 unsigned int size = n * sizeof(struct hlist_head); 537 unsigned int size = n * sizeof(struct hlist_head);
533 538
534 if (size <= PAGE_SIZE) 539 if (size <= PAGE_SIZE)
535 kfree(h); 540 kfree(h);
536 else 541 else
537 free_pages((unsigned long)h, get_order(size)); 542 free_pages((unsigned long)h, get_order(size));
538 } 543 }
539 544
540 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash) 545 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
541 { 546 {
542 struct Qdisc_class_common *cl; 547 struct Qdisc_class_common *cl;
543 struct hlist_node *n, *next; 548 struct hlist_node *n, *next;
544 struct hlist_head *nhash, *ohash; 549 struct hlist_head *nhash, *ohash;
545 unsigned int nsize, nmask, osize; 550 unsigned int nsize, nmask, osize;
546 unsigned int i, h; 551 unsigned int i, h;
547 552
548 /* Rehash when load factor exceeds 0.75 */ 553 /* Rehash when load factor exceeds 0.75 */
549 if (clhash->hashelems * 4 <= clhash->hashsize * 3) 554 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
550 return; 555 return;
551 nsize = clhash->hashsize * 2; 556 nsize = clhash->hashsize * 2;
552 nmask = nsize - 1; 557 nmask = nsize - 1;
553 nhash = qdisc_class_hash_alloc(nsize); 558 nhash = qdisc_class_hash_alloc(nsize);
554 if (nhash == NULL) 559 if (nhash == NULL)
555 return; 560 return;
556 561
557 ohash = clhash->hash; 562 ohash = clhash->hash;
558 osize = clhash->hashsize; 563 osize = clhash->hashsize;
559 564
560 sch_tree_lock(sch); 565 sch_tree_lock(sch);
561 for (i = 0; i < osize; i++) { 566 for (i = 0; i < osize; i++) {
562 hlist_for_each_entry_safe(cl, n, next, &ohash[i], hnode) { 567 hlist_for_each_entry_safe(cl, n, next, &ohash[i], hnode) {
563 h = qdisc_class_hash(cl->classid, nmask); 568 h = qdisc_class_hash(cl->classid, nmask);
564 hlist_add_head(&cl->hnode, &nhash[h]); 569 hlist_add_head(&cl->hnode, &nhash[h]);
565 } 570 }
566 } 571 }
567 clhash->hash = nhash; 572 clhash->hash = nhash;
568 clhash->hashsize = nsize; 573 clhash->hashsize = nsize;
569 clhash->hashmask = nmask; 574 clhash->hashmask = nmask;
570 sch_tree_unlock(sch); 575 sch_tree_unlock(sch);
571 576
572 qdisc_class_hash_free(ohash, osize); 577 qdisc_class_hash_free(ohash, osize);
573 } 578 }
574 EXPORT_SYMBOL(qdisc_class_hash_grow); 579 EXPORT_SYMBOL(qdisc_class_hash_grow);
575 580
576 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash) 581 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
577 { 582 {
578 unsigned int size = 4; 583 unsigned int size = 4;
579 584
580 clhash->hash = qdisc_class_hash_alloc(size); 585 clhash->hash = qdisc_class_hash_alloc(size);
581 if (clhash->hash == NULL) 586 if (clhash->hash == NULL)
582 return -ENOMEM; 587 return -ENOMEM;
583 clhash->hashsize = size; 588 clhash->hashsize = size;
584 clhash->hashmask = size - 1; 589 clhash->hashmask = size - 1;
585 clhash->hashelems = 0; 590 clhash->hashelems = 0;
586 return 0; 591 return 0;
587 } 592 }
588 EXPORT_SYMBOL(qdisc_class_hash_init); 593 EXPORT_SYMBOL(qdisc_class_hash_init);
589 594
590 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash) 595 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
591 { 596 {
592 qdisc_class_hash_free(clhash->hash, clhash->hashsize); 597 qdisc_class_hash_free(clhash->hash, clhash->hashsize);
593 } 598 }
594 EXPORT_SYMBOL(qdisc_class_hash_destroy); 599 EXPORT_SYMBOL(qdisc_class_hash_destroy);
595 600
596 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash, 601 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
597 struct Qdisc_class_common *cl) 602 struct Qdisc_class_common *cl)
598 { 603 {
599 unsigned int h; 604 unsigned int h;
600 605
601 INIT_HLIST_NODE(&cl->hnode); 606 INIT_HLIST_NODE(&cl->hnode);
602 h = qdisc_class_hash(cl->classid, clhash->hashmask); 607 h = qdisc_class_hash(cl->classid, clhash->hashmask);
603 hlist_add_head(&cl->hnode, &clhash->hash[h]); 608 hlist_add_head(&cl->hnode, &clhash->hash[h]);
604 clhash->hashelems++; 609 clhash->hashelems++;
605 } 610 }
606 EXPORT_SYMBOL(qdisc_class_hash_insert); 611 EXPORT_SYMBOL(qdisc_class_hash_insert);
607 612
608 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash, 613 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
609 struct Qdisc_class_common *cl) 614 struct Qdisc_class_common *cl)
610 { 615 {
611 hlist_del(&cl->hnode); 616 hlist_del(&cl->hnode);
612 clhash->hashelems--; 617 clhash->hashelems--;
613 } 618 }
614 EXPORT_SYMBOL(qdisc_class_hash_remove); 619 EXPORT_SYMBOL(qdisc_class_hash_remove);
615 620
616 /* Allocate an unique handle from space managed by kernel */ 621 /* Allocate an unique handle from space managed by kernel */
617 622
618 static u32 qdisc_alloc_handle(struct net_device *dev) 623 static u32 qdisc_alloc_handle(struct net_device *dev)
619 { 624 {
620 int i = 0x10000; 625 int i = 0x10000;
621 static u32 autohandle = TC_H_MAKE(0x80000000U, 0); 626 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
622 627
623 do { 628 do {
624 autohandle += TC_H_MAKE(0x10000U, 0); 629 autohandle += TC_H_MAKE(0x10000U, 0);
625 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0)) 630 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
626 autohandle = TC_H_MAKE(0x80000000U, 0); 631 autohandle = TC_H_MAKE(0x80000000U, 0);
627 } while (qdisc_lookup(dev, autohandle) && --i > 0); 632 } while (qdisc_lookup(dev, autohandle) && --i > 0);
628 633
629 return i > 0 ? autohandle : 0; 634 return i > 0 ? autohandle : 0;
630 } 635 }
631 636
632 void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n) 637 void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n)
633 { 638 {
634 const struct Qdisc_class_ops *cops; 639 const struct Qdisc_class_ops *cops;
635 unsigned long cl; 640 unsigned long cl;
636 u32 parentid; 641 u32 parentid;
637 642
638 if (n == 0) 643 if (n == 0)
639 return; 644 return;
640 while ((parentid = sch->parent)) { 645 while ((parentid = sch->parent)) {
641 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS)) 646 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
642 return; 647 return;
643 648
644 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid)); 649 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
645 if (sch == NULL) { 650 if (sch == NULL) {
646 WARN_ON(parentid != TC_H_ROOT); 651 WARN_ON(parentid != TC_H_ROOT);
647 return; 652 return;
648 } 653 }
649 cops = sch->ops->cl_ops; 654 cops = sch->ops->cl_ops;
650 if (cops->qlen_notify) { 655 if (cops->qlen_notify) {
651 cl = cops->get(sch, parentid); 656 cl = cops->get(sch, parentid);
652 cops->qlen_notify(sch, cl); 657 cops->qlen_notify(sch, cl);
653 cops->put(sch, cl); 658 cops->put(sch, cl);
654 } 659 }
655 sch->q.qlen -= n; 660 sch->q.qlen -= n;
656 } 661 }
657 } 662 }
658 EXPORT_SYMBOL(qdisc_tree_decrease_qlen); 663 EXPORT_SYMBOL(qdisc_tree_decrease_qlen);
659 664
660 static void notify_and_destroy(struct net *net, struct sk_buff *skb, 665 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
661 struct nlmsghdr *n, u32 clid, 666 struct nlmsghdr *n, u32 clid,
662 struct Qdisc *old, struct Qdisc *new) 667 struct Qdisc *old, struct Qdisc *new)
663 { 668 {
664 if (new || old) 669 if (new || old)
665 qdisc_notify(net, skb, n, clid, old, new); 670 qdisc_notify(net, skb, n, clid, old, new);
666 671
667 if (old) 672 if (old)
668 qdisc_destroy(old); 673 qdisc_destroy(old);
669 } 674 }
670 675
671 /* Graft qdisc "new" to class "classid" of qdisc "parent" or 676 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
672 * to device "dev". 677 * to device "dev".
673 * 678 *
674 * When appropriate send a netlink notification using 'skb' 679 * When appropriate send a netlink notification using 'skb'
675 * and "n". 680 * and "n".
676 * 681 *
677 * On success, destroy old qdisc. 682 * On success, destroy old qdisc.
678 */ 683 */
679 684
680 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, 685 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
681 struct sk_buff *skb, struct nlmsghdr *n, u32 classid, 686 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
682 struct Qdisc *new, struct Qdisc *old) 687 struct Qdisc *new, struct Qdisc *old)
683 { 688 {
684 struct Qdisc *q = old; 689 struct Qdisc *q = old;
685 struct net *net = dev_net(dev); 690 struct net *net = dev_net(dev);
686 int err = 0; 691 int err = 0;
687 692
688 if (parent == NULL) { 693 if (parent == NULL) {
689 unsigned int i, num_q, ingress; 694 unsigned int i, num_q, ingress;
690 695
691 ingress = 0; 696 ingress = 0;
692 num_q = dev->num_tx_queues; 697 num_q = dev->num_tx_queues;
693 if ((q && q->flags & TCQ_F_INGRESS) || 698 if ((q && q->flags & TCQ_F_INGRESS) ||
694 (new && new->flags & TCQ_F_INGRESS)) { 699 (new && new->flags & TCQ_F_INGRESS)) {
695 num_q = 1; 700 num_q = 1;
696 ingress = 1; 701 ingress = 1;
697 if (!dev_ingress_queue(dev)) 702 if (!dev_ingress_queue(dev))
698 return -ENOENT; 703 return -ENOENT;
699 } 704 }
700 705
701 if (dev->flags & IFF_UP) 706 if (dev->flags & IFF_UP)
702 dev_deactivate(dev); 707 dev_deactivate(dev);
703 708
704 if (new && new->ops->attach) { 709 if (new && new->ops->attach) {
705 new->ops->attach(new); 710 new->ops->attach(new);
706 num_q = 0; 711 num_q = 0;
707 } 712 }
708 713
709 for (i = 0; i < num_q; i++) { 714 for (i = 0; i < num_q; i++) {
710 struct netdev_queue *dev_queue = dev_ingress_queue(dev); 715 struct netdev_queue *dev_queue = dev_ingress_queue(dev);
711 716
712 if (!ingress) 717 if (!ingress)
713 dev_queue = netdev_get_tx_queue(dev, i); 718 dev_queue = netdev_get_tx_queue(dev, i);
714 719
715 old = dev_graft_qdisc(dev_queue, new); 720 old = dev_graft_qdisc(dev_queue, new);
716 if (new && i > 0) 721 if (new && i > 0)
717 atomic_inc(&new->refcnt); 722 atomic_inc(&new->refcnt);
718 723
719 if (!ingress) 724 if (!ingress)
720 qdisc_destroy(old); 725 qdisc_destroy(old);
721 } 726 }
722 727
723 if (!ingress) { 728 if (!ingress) {
724 notify_and_destroy(net, skb, n, classid, 729 notify_and_destroy(net, skb, n, classid,
725 dev->qdisc, new); 730 dev->qdisc, new);
726 if (new && !new->ops->attach) 731 if (new && !new->ops->attach)
727 atomic_inc(&new->refcnt); 732 atomic_inc(&new->refcnt);
728 dev->qdisc = new ? : &noop_qdisc; 733 dev->qdisc = new ? : &noop_qdisc;
729 } else { 734 } else {
730 notify_and_destroy(net, skb, n, classid, old, new); 735 notify_and_destroy(net, skb, n, classid, old, new);
731 } 736 }
732 737
733 if (dev->flags & IFF_UP) 738 if (dev->flags & IFF_UP)
734 dev_activate(dev); 739 dev_activate(dev);
735 } else { 740 } else {
736 const struct Qdisc_class_ops *cops = parent->ops->cl_ops; 741 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
737 742
738 err = -EOPNOTSUPP; 743 err = -EOPNOTSUPP;
739 if (cops && cops->graft) { 744 if (cops && cops->graft) {
740 unsigned long cl = cops->get(parent, classid); 745 unsigned long cl = cops->get(parent, classid);
741 if (cl) { 746 if (cl) {
742 err = cops->graft(parent, cl, new, &old); 747 err = cops->graft(parent, cl, new, &old);
743 cops->put(parent, cl); 748 cops->put(parent, cl);
744 } else 749 } else
745 err = -ENOENT; 750 err = -ENOENT;
746 } 751 }
747 if (!err) 752 if (!err)
748 notify_and_destroy(net, skb, n, classid, old, new); 753 notify_and_destroy(net, skb, n, classid, old, new);
749 } 754 }
750 return err; 755 return err;
751 } 756 }
752 757
753 /* lockdep annotation is needed for ingress; egress gets it only for name */ 758 /* lockdep annotation is needed for ingress; egress gets it only for name */
754 static struct lock_class_key qdisc_tx_lock; 759 static struct lock_class_key qdisc_tx_lock;
755 static struct lock_class_key qdisc_rx_lock; 760 static struct lock_class_key qdisc_rx_lock;
756 761
757 /* 762 /*
758 Allocate and initialize new qdisc. 763 Allocate and initialize new qdisc.
759 764
760 Parameters are passed via opt. 765 Parameters are passed via opt.
761 */ 766 */
762 767
763 static struct Qdisc * 768 static struct Qdisc *
764 qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, 769 qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
765 struct Qdisc *p, u32 parent, u32 handle, 770 struct Qdisc *p, u32 parent, u32 handle,
766 struct nlattr **tca, int *errp) 771 struct nlattr **tca, int *errp)
767 { 772 {
768 int err; 773 int err;
769 struct nlattr *kind = tca[TCA_KIND]; 774 struct nlattr *kind = tca[TCA_KIND];
770 struct Qdisc *sch; 775 struct Qdisc *sch;
771 struct Qdisc_ops *ops; 776 struct Qdisc_ops *ops;
772 struct qdisc_size_table *stab; 777 struct qdisc_size_table *stab;
773 778
774 ops = qdisc_lookup_ops(kind); 779 ops = qdisc_lookup_ops(kind);
775 #ifdef CONFIG_MODULES 780 #ifdef CONFIG_MODULES
776 if (ops == NULL && kind != NULL) { 781 if (ops == NULL && kind != NULL) {
777 char name[IFNAMSIZ]; 782 char name[IFNAMSIZ];
778 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) { 783 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
779 /* We dropped the RTNL semaphore in order to 784 /* We dropped the RTNL semaphore in order to
780 * perform the module load. So, even if we 785 * perform the module load. So, even if we
781 * succeeded in loading the module we have to 786 * succeeded in loading the module we have to
782 * tell the caller to replay the request. We 787 * tell the caller to replay the request. We
783 * indicate this using -EAGAIN. 788 * indicate this using -EAGAIN.
784 * We replay the request because the device may 789 * We replay the request because the device may
785 * go away in the mean time. 790 * go away in the mean time.
786 */ 791 */
787 rtnl_unlock(); 792 rtnl_unlock();
788 request_module("sch_%s", name); 793 request_module("sch_%s", name);
789 rtnl_lock(); 794 rtnl_lock();
790 ops = qdisc_lookup_ops(kind); 795 ops = qdisc_lookup_ops(kind);
791 if (ops != NULL) { 796 if (ops != NULL) {
792 /* We will try again qdisc_lookup_ops, 797 /* We will try again qdisc_lookup_ops,
793 * so don't keep a reference. 798 * so don't keep a reference.
794 */ 799 */
795 module_put(ops->owner); 800 module_put(ops->owner);
796 err = -EAGAIN; 801 err = -EAGAIN;
797 goto err_out; 802 goto err_out;
798 } 803 }
799 } 804 }
800 } 805 }
801 #endif 806 #endif
802 807
803 err = -ENOENT; 808 err = -ENOENT;
804 if (ops == NULL) 809 if (ops == NULL)
805 goto err_out; 810 goto err_out;
806 811
807 sch = qdisc_alloc(dev_queue, ops); 812 sch = qdisc_alloc(dev_queue, ops);
808 if (IS_ERR(sch)) { 813 if (IS_ERR(sch)) {
809 err = PTR_ERR(sch); 814 err = PTR_ERR(sch);
810 goto err_out2; 815 goto err_out2;
811 } 816 }
812 817
813 sch->parent = parent; 818 sch->parent = parent;
814 819
815 if (handle == TC_H_INGRESS) { 820 if (handle == TC_H_INGRESS) {
816 sch->flags |= TCQ_F_INGRESS; 821 sch->flags |= TCQ_F_INGRESS;
817 handle = TC_H_MAKE(TC_H_INGRESS, 0); 822 handle = TC_H_MAKE(TC_H_INGRESS, 0);
818 lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock); 823 lockdep_set_class(qdisc_lock(sch), &qdisc_rx_lock);
819 } else { 824 } else {
820 if (handle == 0) { 825 if (handle == 0) {
821 handle = qdisc_alloc_handle(dev); 826 handle = qdisc_alloc_handle(dev);
822 err = -ENOMEM; 827 err = -ENOMEM;
823 if (handle == 0) 828 if (handle == 0)
824 goto err_out3; 829 goto err_out3;
825 } 830 }
826 lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock); 831 lockdep_set_class(qdisc_lock(sch), &qdisc_tx_lock);
827 } 832 }
828 833
829 sch->handle = handle; 834 sch->handle = handle;
830 835
831 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) { 836 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
832 if (tca[TCA_STAB]) { 837 if (tca[TCA_STAB]) {
833 stab = qdisc_get_stab(tca[TCA_STAB]); 838 stab = qdisc_get_stab(tca[TCA_STAB]);
834 if (IS_ERR(stab)) { 839 if (IS_ERR(stab)) {
835 err = PTR_ERR(stab); 840 err = PTR_ERR(stab);
836 goto err_out4; 841 goto err_out4;
837 } 842 }
838 sch->stab = stab; 843 rcu_assign_pointer(sch->stab, stab);
839 } 844 }
840 if (tca[TCA_RATE]) { 845 if (tca[TCA_RATE]) {
841 spinlock_t *root_lock; 846 spinlock_t *root_lock;
842 847
843 err = -EOPNOTSUPP; 848 err = -EOPNOTSUPP;
844 if (sch->flags & TCQ_F_MQROOT) 849 if (sch->flags & TCQ_F_MQROOT)
845 goto err_out4; 850 goto err_out4;
846 851
847 if ((sch->parent != TC_H_ROOT) && 852 if ((sch->parent != TC_H_ROOT) &&
848 !(sch->flags & TCQ_F_INGRESS) && 853 !(sch->flags & TCQ_F_INGRESS) &&
849 (!p || !(p->flags & TCQ_F_MQROOT))) 854 (!p || !(p->flags & TCQ_F_MQROOT)))
850 root_lock = qdisc_root_sleeping_lock(sch); 855 root_lock = qdisc_root_sleeping_lock(sch);
851 else 856 else
852 root_lock = qdisc_lock(sch); 857 root_lock = qdisc_lock(sch);
853 858
854 err = gen_new_estimator(&sch->bstats, &sch->rate_est, 859 err = gen_new_estimator(&sch->bstats, &sch->rate_est,
855 root_lock, tca[TCA_RATE]); 860 root_lock, tca[TCA_RATE]);
856 if (err) 861 if (err)
857 goto err_out4; 862 goto err_out4;
858 } 863 }
859 864
860 qdisc_list_add(sch); 865 qdisc_list_add(sch);
861 866
862 return sch; 867 return sch;
863 } 868 }
864 err_out3: 869 err_out3:
865 dev_put(dev); 870 dev_put(dev);
866 kfree((char *) sch - sch->padded); 871 kfree((char *) sch - sch->padded);
867 err_out2: 872 err_out2:
868 module_put(ops->owner); 873 module_put(ops->owner);
869 err_out: 874 err_out:
870 *errp = err; 875 *errp = err;
871 return NULL; 876 return NULL;
872 877
873 err_out4: 878 err_out4:
874 /* 879 /*
875 * Any broken qdiscs that would require a ops->reset() here? 880 * Any broken qdiscs that would require a ops->reset() here?
876 * The qdisc was never in action so it shouldn't be necessary. 881 * The qdisc was never in action so it shouldn't be necessary.
877 */ 882 */
878 qdisc_put_stab(sch->stab); 883 qdisc_put_stab(rtnl_dereference(sch->stab));
879 if (ops->destroy) 884 if (ops->destroy)
880 ops->destroy(sch); 885 ops->destroy(sch);
881 goto err_out3; 886 goto err_out3;
882 } 887 }
883 888
884 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca) 889 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
885 { 890 {
886 struct qdisc_size_table *stab = NULL; 891 struct qdisc_size_table *ostab, *stab = NULL;
887 int err = 0; 892 int err = 0;
888 893
889 if (tca[TCA_OPTIONS]) { 894 if (tca[TCA_OPTIONS]) {
890 if (sch->ops->change == NULL) 895 if (sch->ops->change == NULL)
891 return -EINVAL; 896 return -EINVAL;
892 err = sch->ops->change(sch, tca[TCA_OPTIONS]); 897 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
893 if (err) 898 if (err)
894 return err; 899 return err;
895 } 900 }
896 901
897 if (tca[TCA_STAB]) { 902 if (tca[TCA_STAB]) {
898 stab = qdisc_get_stab(tca[TCA_STAB]); 903 stab = qdisc_get_stab(tca[TCA_STAB]);
899 if (IS_ERR(stab)) 904 if (IS_ERR(stab))
900 return PTR_ERR(stab); 905 return PTR_ERR(stab);
901 } 906 }
902 907
903 qdisc_put_stab(sch->stab); 908 ostab = rtnl_dereference(sch->stab);
904 sch->stab = stab; 909 rcu_assign_pointer(sch->stab, stab);
910 qdisc_put_stab(ostab);
905 911
906 if (tca[TCA_RATE]) { 912 if (tca[TCA_RATE]) {
907 /* NB: ignores errors from replace_estimator 913 /* NB: ignores errors from replace_estimator
908 because change can't be undone. */ 914 because change can't be undone. */
909 if (sch->flags & TCQ_F_MQROOT) 915 if (sch->flags & TCQ_F_MQROOT)
910 goto out; 916 goto out;
911 gen_replace_estimator(&sch->bstats, &sch->rate_est, 917 gen_replace_estimator(&sch->bstats, &sch->rate_est,
912 qdisc_root_sleeping_lock(sch), 918 qdisc_root_sleeping_lock(sch),
913 tca[TCA_RATE]); 919 tca[TCA_RATE]);
914 } 920 }
915 out: 921 out:
916 return 0; 922 return 0;
917 } 923 }
918 924
919 struct check_loop_arg { 925 struct check_loop_arg {
920 struct qdisc_walker w; 926 struct qdisc_walker w;
921 struct Qdisc *p; 927 struct Qdisc *p;
922 int depth; 928 int depth;
923 }; 929 };
924 930
925 static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w); 931 static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
926 932
927 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth) 933 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
928 { 934 {
929 struct check_loop_arg arg; 935 struct check_loop_arg arg;
930 936
931 if (q->ops->cl_ops == NULL) 937 if (q->ops->cl_ops == NULL)
932 return 0; 938 return 0;
933 939
934 arg.w.stop = arg.w.skip = arg.w.count = 0; 940 arg.w.stop = arg.w.skip = arg.w.count = 0;
935 arg.w.fn = check_loop_fn; 941 arg.w.fn = check_loop_fn;
936 arg.depth = depth; 942 arg.depth = depth;
937 arg.p = p; 943 arg.p = p;
938 q->ops->cl_ops->walk(q, &arg.w); 944 q->ops->cl_ops->walk(q, &arg.w);
939 return arg.w.stop ? -ELOOP : 0; 945 return arg.w.stop ? -ELOOP : 0;
940 } 946 }
941 947
942 static int 948 static int
943 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w) 949 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
944 { 950 {
945 struct Qdisc *leaf; 951 struct Qdisc *leaf;
946 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 952 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
947 struct check_loop_arg *arg = (struct check_loop_arg *)w; 953 struct check_loop_arg *arg = (struct check_loop_arg *)w;
948 954
949 leaf = cops->leaf(q, cl); 955 leaf = cops->leaf(q, cl);
950 if (leaf) { 956 if (leaf) {
951 if (leaf == arg->p || arg->depth > 7) 957 if (leaf == arg->p || arg->depth > 7)
952 return -ELOOP; 958 return -ELOOP;
953 return check_loop(leaf, arg->p, arg->depth + 1); 959 return check_loop(leaf, arg->p, arg->depth + 1);
954 } 960 }
955 return 0; 961 return 0;
956 } 962 }
957 963
958 /* 964 /*
959 * Delete/get qdisc. 965 * Delete/get qdisc.
960 */ 966 */
961 967
962 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg) 968 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
963 { 969 {
964 struct net *net = sock_net(skb->sk); 970 struct net *net = sock_net(skb->sk);
965 struct tcmsg *tcm = NLMSG_DATA(n); 971 struct tcmsg *tcm = NLMSG_DATA(n);
966 struct nlattr *tca[TCA_MAX + 1]; 972 struct nlattr *tca[TCA_MAX + 1];
967 struct net_device *dev; 973 struct net_device *dev;
968 u32 clid = tcm->tcm_parent; 974 u32 clid = tcm->tcm_parent;
969 struct Qdisc *q = NULL; 975 struct Qdisc *q = NULL;
970 struct Qdisc *p = NULL; 976 struct Qdisc *p = NULL;
971 int err; 977 int err;
972 978
973 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 979 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
974 if (!dev) 980 if (!dev)
975 return -ENODEV; 981 return -ENODEV;
976 982
977 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL); 983 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
978 if (err < 0) 984 if (err < 0)
979 return err; 985 return err;
980 986
981 if (clid) { 987 if (clid) {
982 if (clid != TC_H_ROOT) { 988 if (clid != TC_H_ROOT) {
983 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) { 989 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
984 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 990 p = qdisc_lookup(dev, TC_H_MAJ(clid));
985 if (!p) 991 if (!p)
986 return -ENOENT; 992 return -ENOENT;
987 q = qdisc_leaf(p, clid); 993 q = qdisc_leaf(p, clid);
988 } else if (dev_ingress_queue(dev)) { 994 } else if (dev_ingress_queue(dev)) {
989 q = dev_ingress_queue(dev)->qdisc_sleeping; 995 q = dev_ingress_queue(dev)->qdisc_sleeping;
990 } 996 }
991 } else { 997 } else {
992 q = dev->qdisc; 998 q = dev->qdisc;
993 } 999 }
994 if (!q) 1000 if (!q)
995 return -ENOENT; 1001 return -ENOENT;
996 1002
997 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) 1003 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
998 return -EINVAL; 1004 return -EINVAL;
999 } else { 1005 } else {
1000 q = qdisc_lookup(dev, tcm->tcm_handle); 1006 q = qdisc_lookup(dev, tcm->tcm_handle);
1001 if (!q) 1007 if (!q)
1002 return -ENOENT; 1008 return -ENOENT;
1003 } 1009 }
1004 1010
1005 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) 1011 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1006 return -EINVAL; 1012 return -EINVAL;
1007 1013
1008 if (n->nlmsg_type == RTM_DELQDISC) { 1014 if (n->nlmsg_type == RTM_DELQDISC) {
1009 if (!clid) 1015 if (!clid)
1010 return -EINVAL; 1016 return -EINVAL;
1011 if (q->handle == 0) 1017 if (q->handle == 0)
1012 return -ENOENT; 1018 return -ENOENT;
1013 err = qdisc_graft(dev, p, skb, n, clid, NULL, q); 1019 err = qdisc_graft(dev, p, skb, n, clid, NULL, q);
1014 if (err != 0) 1020 if (err != 0)
1015 return err; 1021 return err;
1016 } else { 1022 } else {
1017 qdisc_notify(net, skb, n, clid, NULL, q); 1023 qdisc_notify(net, skb, n, clid, NULL, q);
1018 } 1024 }
1019 return 0; 1025 return 0;
1020 } 1026 }
1021 1027
1022 /* 1028 /*
1023 * Create/change qdisc. 1029 * Create/change qdisc.
1024 */ 1030 */
1025 1031
1026 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg) 1032 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1027 { 1033 {
1028 struct net *net = sock_net(skb->sk); 1034 struct net *net = sock_net(skb->sk);
1029 struct tcmsg *tcm; 1035 struct tcmsg *tcm;
1030 struct nlattr *tca[TCA_MAX + 1]; 1036 struct nlattr *tca[TCA_MAX + 1];
1031 struct net_device *dev; 1037 struct net_device *dev;
1032 u32 clid; 1038 u32 clid;
1033 struct Qdisc *q, *p; 1039 struct Qdisc *q, *p;
1034 int err; 1040 int err;
1035 1041
1036 replay: 1042 replay:
1037 /* Reinit, just in case something touches this. */ 1043 /* Reinit, just in case something touches this. */
1038 tcm = NLMSG_DATA(n); 1044 tcm = NLMSG_DATA(n);
1039 clid = tcm->tcm_parent; 1045 clid = tcm->tcm_parent;
1040 q = p = NULL; 1046 q = p = NULL;
1041 1047
1042 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1048 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1043 if (!dev) 1049 if (!dev)
1044 return -ENODEV; 1050 return -ENODEV;
1045 1051
1046 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL); 1052 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1047 if (err < 0) 1053 if (err < 0)
1048 return err; 1054 return err;
1049 1055
1050 if (clid) { 1056 if (clid) {
1051 if (clid != TC_H_ROOT) { 1057 if (clid != TC_H_ROOT) {
1052 if (clid != TC_H_INGRESS) { 1058 if (clid != TC_H_INGRESS) {
1053 p = qdisc_lookup(dev, TC_H_MAJ(clid)); 1059 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1054 if (!p) 1060 if (!p)
1055 return -ENOENT; 1061 return -ENOENT;
1056 q = qdisc_leaf(p, clid); 1062 q = qdisc_leaf(p, clid);
1057 } else if (dev_ingress_queue_create(dev)) { 1063 } else if (dev_ingress_queue_create(dev)) {
1058 q = dev_ingress_queue(dev)->qdisc_sleeping; 1064 q = dev_ingress_queue(dev)->qdisc_sleeping;
1059 } 1065 }
1060 } else { 1066 } else {
1061 q = dev->qdisc; 1067 q = dev->qdisc;
1062 } 1068 }
1063 1069
1064 /* It may be default qdisc, ignore it */ 1070 /* It may be default qdisc, ignore it */
1065 if (q && q->handle == 0) 1071 if (q && q->handle == 0)
1066 q = NULL; 1072 q = NULL;
1067 1073
1068 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) { 1074 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1069 if (tcm->tcm_handle) { 1075 if (tcm->tcm_handle) {
1070 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) 1076 if (q && !(n->nlmsg_flags & NLM_F_REPLACE))
1071 return -EEXIST; 1077 return -EEXIST;
1072 if (TC_H_MIN(tcm->tcm_handle)) 1078 if (TC_H_MIN(tcm->tcm_handle))
1073 return -EINVAL; 1079 return -EINVAL;
1074 q = qdisc_lookup(dev, tcm->tcm_handle); 1080 q = qdisc_lookup(dev, tcm->tcm_handle);
1075 if (!q) 1081 if (!q)
1076 goto create_n_graft; 1082 goto create_n_graft;
1077 if (n->nlmsg_flags & NLM_F_EXCL) 1083 if (n->nlmsg_flags & NLM_F_EXCL)
1078 return -EEXIST; 1084 return -EEXIST;
1079 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) 1085 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1080 return -EINVAL; 1086 return -EINVAL;
1081 if (q == p || 1087 if (q == p ||
1082 (p && check_loop(q, p, 0))) 1088 (p && check_loop(q, p, 0)))
1083 return -ELOOP; 1089 return -ELOOP;
1084 atomic_inc(&q->refcnt); 1090 atomic_inc(&q->refcnt);
1085 goto graft; 1091 goto graft;
1086 } else { 1092 } else {
1087 if (!q) 1093 if (!q)
1088 goto create_n_graft; 1094 goto create_n_graft;
1089 1095
1090 /* This magic test requires explanation. 1096 /* This magic test requires explanation.
1091 * 1097 *
1092 * We know, that some child q is already 1098 * We know, that some child q is already
1093 * attached to this parent and have choice: 1099 * attached to this parent and have choice:
1094 * either to change it or to create/graft new one. 1100 * either to change it or to create/graft new one.
1095 * 1101 *
1096 * 1. We are allowed to create/graft only 1102 * 1. We are allowed to create/graft only
1097 * if CREATE and REPLACE flags are set. 1103 * if CREATE and REPLACE flags are set.
1098 * 1104 *
1099 * 2. If EXCL is set, requestor wanted to say, 1105 * 2. If EXCL is set, requestor wanted to say,
1100 * that qdisc tcm_handle is not expected 1106 * that qdisc tcm_handle is not expected
1101 * to exist, so that we choose create/graft too. 1107 * to exist, so that we choose create/graft too.
1102 * 1108 *
1103 * 3. The last case is when no flags are set. 1109 * 3. The last case is when no flags are set.
1104 * Alas, it is sort of hole in API, we 1110 * Alas, it is sort of hole in API, we
1105 * cannot decide what to do unambiguously. 1111 * cannot decide what to do unambiguously.
1106 * For now we select create/graft, if 1112 * For now we select create/graft, if
1107 * user gave KIND, which does not match existing. 1113 * user gave KIND, which does not match existing.
1108 */ 1114 */
1109 if ((n->nlmsg_flags & NLM_F_CREATE) && 1115 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1110 (n->nlmsg_flags & NLM_F_REPLACE) && 1116 (n->nlmsg_flags & NLM_F_REPLACE) &&
1111 ((n->nlmsg_flags & NLM_F_EXCL) || 1117 ((n->nlmsg_flags & NLM_F_EXCL) ||
1112 (tca[TCA_KIND] && 1118 (tca[TCA_KIND] &&
1113 nla_strcmp(tca[TCA_KIND], q->ops->id)))) 1119 nla_strcmp(tca[TCA_KIND], q->ops->id))))
1114 goto create_n_graft; 1120 goto create_n_graft;
1115 } 1121 }
1116 } 1122 }
1117 } else { 1123 } else {
1118 if (!tcm->tcm_handle) 1124 if (!tcm->tcm_handle)
1119 return -EINVAL; 1125 return -EINVAL;
1120 q = qdisc_lookup(dev, tcm->tcm_handle); 1126 q = qdisc_lookup(dev, tcm->tcm_handle);
1121 } 1127 }
1122 1128
1123 /* Change qdisc parameters */ 1129 /* Change qdisc parameters */
1124 if (q == NULL) 1130 if (q == NULL)
1125 return -ENOENT; 1131 return -ENOENT;
1126 if (n->nlmsg_flags & NLM_F_EXCL) 1132 if (n->nlmsg_flags & NLM_F_EXCL)
1127 return -EEXIST; 1133 return -EEXIST;
1128 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) 1134 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
1129 return -EINVAL; 1135 return -EINVAL;
1130 err = qdisc_change(q, tca); 1136 err = qdisc_change(q, tca);
1131 if (err == 0) 1137 if (err == 0)
1132 qdisc_notify(net, skb, n, clid, NULL, q); 1138 qdisc_notify(net, skb, n, clid, NULL, q);
1133 return err; 1139 return err;
1134 1140
1135 create_n_graft: 1141 create_n_graft:
1136 if (!(n->nlmsg_flags & NLM_F_CREATE)) 1142 if (!(n->nlmsg_flags & NLM_F_CREATE))
1137 return -ENOENT; 1143 return -ENOENT;
1138 if (clid == TC_H_INGRESS) { 1144 if (clid == TC_H_INGRESS) {
1139 if (dev_ingress_queue(dev)) 1145 if (dev_ingress_queue(dev))
1140 q = qdisc_create(dev, dev_ingress_queue(dev), p, 1146 q = qdisc_create(dev, dev_ingress_queue(dev), p,
1141 tcm->tcm_parent, tcm->tcm_parent, 1147 tcm->tcm_parent, tcm->tcm_parent,
1142 tca, &err); 1148 tca, &err);
1143 else 1149 else
1144 err = -ENOENT; 1150 err = -ENOENT;
1145 } else { 1151 } else {
1146 struct netdev_queue *dev_queue; 1152 struct netdev_queue *dev_queue;
1147 1153
1148 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue) 1154 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1149 dev_queue = p->ops->cl_ops->select_queue(p, tcm); 1155 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1150 else if (p) 1156 else if (p)
1151 dev_queue = p->dev_queue; 1157 dev_queue = p->dev_queue;
1152 else 1158 else
1153 dev_queue = netdev_get_tx_queue(dev, 0); 1159 dev_queue = netdev_get_tx_queue(dev, 0);
1154 1160
1155 q = qdisc_create(dev, dev_queue, p, 1161 q = qdisc_create(dev, dev_queue, p,
1156 tcm->tcm_parent, tcm->tcm_handle, 1162 tcm->tcm_parent, tcm->tcm_handle,
1157 tca, &err); 1163 tca, &err);
1158 } 1164 }
1159 if (q == NULL) { 1165 if (q == NULL) {
1160 if (err == -EAGAIN) 1166 if (err == -EAGAIN)
1161 goto replay; 1167 goto replay;
1162 return err; 1168 return err;
1163 } 1169 }
1164 1170
1165 graft: 1171 graft:
1166 err = qdisc_graft(dev, p, skb, n, clid, q, NULL); 1172 err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
1167 if (err) { 1173 if (err) {
1168 if (q) 1174 if (q)
1169 qdisc_destroy(q); 1175 qdisc_destroy(q);
1170 return err; 1176 return err;
1171 } 1177 }
1172 1178
1173 return 0; 1179 return 0;
1174 } 1180 }
1175 1181
1176 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, 1182 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
1177 u32 pid, u32 seq, u16 flags, int event) 1183 u32 pid, u32 seq, u16 flags, int event)
1178 { 1184 {
1179 struct tcmsg *tcm; 1185 struct tcmsg *tcm;
1180 struct nlmsghdr *nlh; 1186 struct nlmsghdr *nlh;
1181 unsigned char *b = skb_tail_pointer(skb); 1187 unsigned char *b = skb_tail_pointer(skb);
1182 struct gnet_dump d; 1188 struct gnet_dump d;
1189 struct qdisc_size_table *stab;
1183 1190
1184 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags); 1191 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
1185 tcm = NLMSG_DATA(nlh); 1192 tcm = NLMSG_DATA(nlh);
1186 tcm->tcm_family = AF_UNSPEC; 1193 tcm->tcm_family = AF_UNSPEC;
1187 tcm->tcm__pad1 = 0; 1194 tcm->tcm__pad1 = 0;
1188 tcm->tcm__pad2 = 0; 1195 tcm->tcm__pad2 = 0;
1189 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1196 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1190 tcm->tcm_parent = clid; 1197 tcm->tcm_parent = clid;
1191 tcm->tcm_handle = q->handle; 1198 tcm->tcm_handle = q->handle;
1192 tcm->tcm_info = atomic_read(&q->refcnt); 1199 tcm->tcm_info = atomic_read(&q->refcnt);
1193 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id); 1200 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
1194 if (q->ops->dump && q->ops->dump(q, skb) < 0) 1201 if (q->ops->dump && q->ops->dump(q, skb) < 0)
1195 goto nla_put_failure; 1202 goto nla_put_failure;
1196 q->qstats.qlen = q->q.qlen; 1203 q->qstats.qlen = q->q.qlen;
1197 1204
1198 if (q->stab && qdisc_dump_stab(skb, q->stab) < 0) 1205 stab = rtnl_dereference(q->stab);
1206 if (stab && qdisc_dump_stab(skb, stab) < 0)
1199 goto nla_put_failure; 1207 goto nla_put_failure;
1200 1208
1201 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 1209 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1202 qdisc_root_sleeping_lock(q), &d) < 0) 1210 qdisc_root_sleeping_lock(q), &d) < 0)
1203 goto nla_put_failure; 1211 goto nla_put_failure;
1204 1212
1205 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0) 1213 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
1206 goto nla_put_failure; 1214 goto nla_put_failure;
1207 1215
1208 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 || 1216 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
1209 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 || 1217 gnet_stats_copy_rate_est(&d, &q->bstats, &q->rate_est) < 0 ||
1210 gnet_stats_copy_queue(&d, &q->qstats) < 0) 1218 gnet_stats_copy_queue(&d, &q->qstats) < 0)
1211 goto nla_put_failure; 1219 goto nla_put_failure;
1212 1220
1213 if (gnet_stats_finish_copy(&d) < 0) 1221 if (gnet_stats_finish_copy(&d) < 0)
1214 goto nla_put_failure; 1222 goto nla_put_failure;
1215 1223
1216 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1224 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1217 return skb->len; 1225 return skb->len;
1218 1226
1219 nlmsg_failure: 1227 nlmsg_failure:
1220 nla_put_failure: 1228 nla_put_failure:
1221 nlmsg_trim(skb, b); 1229 nlmsg_trim(skb, b);
1222 return -1; 1230 return -1;
1223 } 1231 }
1224 1232
1225 static bool tc_qdisc_dump_ignore(struct Qdisc *q) 1233 static bool tc_qdisc_dump_ignore(struct Qdisc *q)
1226 { 1234 {
1227 return (q->flags & TCQ_F_BUILTIN) ? true : false; 1235 return (q->flags & TCQ_F_BUILTIN) ? true : false;
1228 } 1236 }
1229 1237
1230 static int qdisc_notify(struct net *net, struct sk_buff *oskb, 1238 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
1231 struct nlmsghdr *n, u32 clid, 1239 struct nlmsghdr *n, u32 clid,
1232 struct Qdisc *old, struct Qdisc *new) 1240 struct Qdisc *old, struct Qdisc *new)
1233 { 1241 {
1234 struct sk_buff *skb; 1242 struct sk_buff *skb;
1235 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0; 1243 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1236 1244
1237 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1245 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1238 if (!skb) 1246 if (!skb)
1239 return -ENOBUFS; 1247 return -ENOBUFS;
1240 1248
1241 if (old && !tc_qdisc_dump_ignore(old)) { 1249 if (old && !tc_qdisc_dump_ignore(old)) {
1242 if (tc_fill_qdisc(skb, old, clid, pid, n->nlmsg_seq, 1250 if (tc_fill_qdisc(skb, old, clid, pid, n->nlmsg_seq,
1243 0, RTM_DELQDISC) < 0) 1251 0, RTM_DELQDISC) < 0)
1244 goto err_out; 1252 goto err_out;
1245 } 1253 }
1246 if (new && !tc_qdisc_dump_ignore(new)) { 1254 if (new && !tc_qdisc_dump_ignore(new)) {
1247 if (tc_fill_qdisc(skb, new, clid, pid, n->nlmsg_seq, 1255 if (tc_fill_qdisc(skb, new, clid, pid, n->nlmsg_seq,
1248 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0) 1256 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1249 goto err_out; 1257 goto err_out;
1250 } 1258 }
1251 1259
1252 if (skb->len) 1260 if (skb->len)
1253 return rtnetlink_send(skb, net, pid, RTNLGRP_TC, 1261 return rtnetlink_send(skb, net, pid, RTNLGRP_TC,
1254 n->nlmsg_flags & NLM_F_ECHO); 1262 n->nlmsg_flags & NLM_F_ECHO);
1255 1263
1256 err_out: 1264 err_out:
1257 kfree_skb(skb); 1265 kfree_skb(skb);
1258 return -EINVAL; 1266 return -EINVAL;
1259 } 1267 }
1260 1268
1261 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb, 1269 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1262 struct netlink_callback *cb, 1270 struct netlink_callback *cb,
1263 int *q_idx_p, int s_q_idx) 1271 int *q_idx_p, int s_q_idx)
1264 { 1272 {
1265 int ret = 0, q_idx = *q_idx_p; 1273 int ret = 0, q_idx = *q_idx_p;
1266 struct Qdisc *q; 1274 struct Qdisc *q;
1267 1275
1268 if (!root) 1276 if (!root)
1269 return 0; 1277 return 0;
1270 1278
1271 q = root; 1279 q = root;
1272 if (q_idx < s_q_idx) { 1280 if (q_idx < s_q_idx) {
1273 q_idx++; 1281 q_idx++;
1274 } else { 1282 } else {
1275 if (!tc_qdisc_dump_ignore(q) && 1283 if (!tc_qdisc_dump_ignore(q) &&
1276 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid, 1284 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
1277 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0) 1285 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1278 goto done; 1286 goto done;
1279 q_idx++; 1287 q_idx++;
1280 } 1288 }
1281 list_for_each_entry(q, &root->list, list) { 1289 list_for_each_entry(q, &root->list, list) {
1282 if (q_idx < s_q_idx) { 1290 if (q_idx < s_q_idx) {
1283 q_idx++; 1291 q_idx++;
1284 continue; 1292 continue;
1285 } 1293 }
1286 if (!tc_qdisc_dump_ignore(q) && 1294 if (!tc_qdisc_dump_ignore(q) &&
1287 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid, 1295 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
1288 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0) 1296 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1289 goto done; 1297 goto done;
1290 q_idx++; 1298 q_idx++;
1291 } 1299 }
1292 1300
1293 out: 1301 out:
1294 *q_idx_p = q_idx; 1302 *q_idx_p = q_idx;
1295 return ret; 1303 return ret;
1296 done: 1304 done:
1297 ret = -1; 1305 ret = -1;
1298 goto out; 1306 goto out;
1299 } 1307 }
1300 1308
1301 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb) 1309 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1302 { 1310 {
1303 struct net *net = sock_net(skb->sk); 1311 struct net *net = sock_net(skb->sk);
1304 int idx, q_idx; 1312 int idx, q_idx;
1305 int s_idx, s_q_idx; 1313 int s_idx, s_q_idx;
1306 struct net_device *dev; 1314 struct net_device *dev;
1307 1315
1308 s_idx = cb->args[0]; 1316 s_idx = cb->args[0];
1309 s_q_idx = q_idx = cb->args[1]; 1317 s_q_idx = q_idx = cb->args[1];
1310 1318
1311 rcu_read_lock(); 1319 rcu_read_lock();
1312 idx = 0; 1320 idx = 0;
1313 for_each_netdev_rcu(net, dev) { 1321 for_each_netdev_rcu(net, dev) {
1314 struct netdev_queue *dev_queue; 1322 struct netdev_queue *dev_queue;
1315 1323
1316 if (idx < s_idx) 1324 if (idx < s_idx)
1317 goto cont; 1325 goto cont;
1318 if (idx > s_idx) 1326 if (idx > s_idx)
1319 s_q_idx = 0; 1327 s_q_idx = 0;
1320 q_idx = 0; 1328 q_idx = 0;
1321 1329
1322 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx) < 0) 1330 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx) < 0)
1323 goto done; 1331 goto done;
1324 1332
1325 dev_queue = dev_ingress_queue(dev); 1333 dev_queue = dev_ingress_queue(dev);
1326 if (dev_queue && 1334 if (dev_queue &&
1327 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb, 1335 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1328 &q_idx, s_q_idx) < 0) 1336 &q_idx, s_q_idx) < 0)
1329 goto done; 1337 goto done;
1330 1338
1331 cont: 1339 cont:
1332 idx++; 1340 idx++;
1333 } 1341 }
1334 1342
1335 done: 1343 done:
1336 rcu_read_unlock(); 1344 rcu_read_unlock();
1337 1345
1338 cb->args[0] = idx; 1346 cb->args[0] = idx;
1339 cb->args[1] = q_idx; 1347 cb->args[1] = q_idx;
1340 1348
1341 return skb->len; 1349 return skb->len;
1342 } 1350 }
1343 1351
1344 1352
1345 1353
1346 /************************************************ 1354 /************************************************
1347 * Traffic classes manipulation. * 1355 * Traffic classes manipulation. *
1348 ************************************************/ 1356 ************************************************/
1349 1357
1350 1358
1351 1359
1352 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg) 1360 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1353 { 1361 {
1354 struct net *net = sock_net(skb->sk); 1362 struct net *net = sock_net(skb->sk);
1355 struct tcmsg *tcm = NLMSG_DATA(n); 1363 struct tcmsg *tcm = NLMSG_DATA(n);
1356 struct nlattr *tca[TCA_MAX + 1]; 1364 struct nlattr *tca[TCA_MAX + 1];
1357 struct net_device *dev; 1365 struct net_device *dev;
1358 struct Qdisc *q = NULL; 1366 struct Qdisc *q = NULL;
1359 const struct Qdisc_class_ops *cops; 1367 const struct Qdisc_class_ops *cops;
1360 unsigned long cl = 0; 1368 unsigned long cl = 0;
1361 unsigned long new_cl; 1369 unsigned long new_cl;
1362 u32 pid = tcm->tcm_parent; 1370 u32 pid = tcm->tcm_parent;
1363 u32 clid = tcm->tcm_handle; 1371 u32 clid = tcm->tcm_handle;
1364 u32 qid = TC_H_MAJ(clid); 1372 u32 qid = TC_H_MAJ(clid);
1365 int err; 1373 int err;
1366 1374
1367 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 1375 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1368 if (!dev) 1376 if (!dev)
1369 return -ENODEV; 1377 return -ENODEV;
1370 1378
1371 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL); 1379 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1372 if (err < 0) 1380 if (err < 0)
1373 return err; 1381 return err;
1374 1382
1375 /* 1383 /*
1376 parent == TC_H_UNSPEC - unspecified parent. 1384 parent == TC_H_UNSPEC - unspecified parent.
1377 parent == TC_H_ROOT - class is root, which has no parent. 1385 parent == TC_H_ROOT - class is root, which has no parent.
1378 parent == X:0 - parent is root class. 1386 parent == X:0 - parent is root class.
1379 parent == X:Y - parent is a node in hierarchy. 1387 parent == X:Y - parent is a node in hierarchy.
1380 parent == 0:Y - parent is X:Y, where X:0 is qdisc. 1388 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1381 1389
1382 handle == 0:0 - generate handle from kernel pool. 1390 handle == 0:0 - generate handle from kernel pool.
1383 handle == 0:Y - class is X:Y, where X:0 is qdisc. 1391 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1384 handle == X:Y - clear. 1392 handle == X:Y - clear.
1385 handle == X:0 - root class. 1393 handle == X:0 - root class.
1386 */ 1394 */
1387 1395
1388 /* Step 1. Determine qdisc handle X:0 */ 1396 /* Step 1. Determine qdisc handle X:0 */
1389 1397
1390 if (pid != TC_H_ROOT) { 1398 if (pid != TC_H_ROOT) {
1391 u32 qid1 = TC_H_MAJ(pid); 1399 u32 qid1 = TC_H_MAJ(pid);
1392 1400
1393 if (qid && qid1) { 1401 if (qid && qid1) {
1394 /* If both majors are known, they must be identical. */ 1402 /* If both majors are known, they must be identical. */
1395 if (qid != qid1) 1403 if (qid != qid1)
1396 return -EINVAL; 1404 return -EINVAL;
1397 } else if (qid1) { 1405 } else if (qid1) {
1398 qid = qid1; 1406 qid = qid1;
1399 } else if (qid == 0) 1407 } else if (qid == 0)
1400 qid = dev->qdisc->handle; 1408 qid = dev->qdisc->handle;
1401 1409
1402 /* Now qid is genuine qdisc handle consistent 1410 /* Now qid is genuine qdisc handle consistent
1403 * both with parent and child. 1411 * both with parent and child.
1404 * 1412 *
1405 * TC_H_MAJ(pid) still may be unspecified, complete it now. 1413 * TC_H_MAJ(pid) still may be unspecified, complete it now.
1406 */ 1414 */
1407 if (pid) 1415 if (pid)
1408 pid = TC_H_MAKE(qid, pid); 1416 pid = TC_H_MAKE(qid, pid);
1409 } else { 1417 } else {
1410 if (qid == 0) 1418 if (qid == 0)
1411 qid = dev->qdisc->handle; 1419 qid = dev->qdisc->handle;
1412 } 1420 }
1413 1421
1414 /* OK. Locate qdisc */ 1422 /* OK. Locate qdisc */
1415 q = qdisc_lookup(dev, qid); 1423 q = qdisc_lookup(dev, qid);
1416 if (!q) 1424 if (!q)
1417 return -ENOENT; 1425 return -ENOENT;
1418 1426
1419 /* An check that it supports classes */ 1427 /* An check that it supports classes */
1420 cops = q->ops->cl_ops; 1428 cops = q->ops->cl_ops;
1421 if (cops == NULL) 1429 if (cops == NULL)
1422 return -EINVAL; 1430 return -EINVAL;
1423 1431
1424 /* Now try to get class */ 1432 /* Now try to get class */
1425 if (clid == 0) { 1433 if (clid == 0) {
1426 if (pid == TC_H_ROOT) 1434 if (pid == TC_H_ROOT)
1427 clid = qid; 1435 clid = qid;
1428 } else 1436 } else
1429 clid = TC_H_MAKE(qid, clid); 1437 clid = TC_H_MAKE(qid, clid);
1430 1438
1431 if (clid) 1439 if (clid)
1432 cl = cops->get(q, clid); 1440 cl = cops->get(q, clid);
1433 1441
1434 if (cl == 0) { 1442 if (cl == 0) {
1435 err = -ENOENT; 1443 err = -ENOENT;
1436 if (n->nlmsg_type != RTM_NEWTCLASS || 1444 if (n->nlmsg_type != RTM_NEWTCLASS ||
1437 !(n->nlmsg_flags & NLM_F_CREATE)) 1445 !(n->nlmsg_flags & NLM_F_CREATE))
1438 goto out; 1446 goto out;
1439 } else { 1447 } else {
1440 switch (n->nlmsg_type) { 1448 switch (n->nlmsg_type) {
1441 case RTM_NEWTCLASS: 1449 case RTM_NEWTCLASS:
1442 err = -EEXIST; 1450 err = -EEXIST;
1443 if (n->nlmsg_flags & NLM_F_EXCL) 1451 if (n->nlmsg_flags & NLM_F_EXCL)
1444 goto out; 1452 goto out;
1445 break; 1453 break;
1446 case RTM_DELTCLASS: 1454 case RTM_DELTCLASS:
1447 err = -EOPNOTSUPP; 1455 err = -EOPNOTSUPP;
1448 if (cops->delete) 1456 if (cops->delete)
1449 err = cops->delete(q, cl); 1457 err = cops->delete(q, cl);
1450 if (err == 0) 1458 if (err == 0)
1451 tclass_notify(net, skb, n, q, cl, RTM_DELTCLASS); 1459 tclass_notify(net, skb, n, q, cl, RTM_DELTCLASS);
1452 goto out; 1460 goto out;
1453 case RTM_GETTCLASS: 1461 case RTM_GETTCLASS:
1454 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS); 1462 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
1455 goto out; 1463 goto out;
1456 default: 1464 default:
1457 err = -EINVAL; 1465 err = -EINVAL;
1458 goto out; 1466 goto out;
1459 } 1467 }
1460 } 1468 }
1461 1469
1462 new_cl = cl; 1470 new_cl = cl;
1463 err = -EOPNOTSUPP; 1471 err = -EOPNOTSUPP;
1464 if (cops->change) 1472 if (cops->change)
1465 err = cops->change(q, clid, pid, tca, &new_cl); 1473 err = cops->change(q, clid, pid, tca, &new_cl);
1466 if (err == 0) 1474 if (err == 0)
1467 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS); 1475 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
1468 1476
1469 out: 1477 out:
1470 if (cl) 1478 if (cl)
1471 cops->put(q, cl); 1479 cops->put(q, cl);
1472 1480
1473 return err; 1481 return err;
1474 } 1482 }
1475 1483
1476 1484
1477 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q, 1485 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1478 unsigned long cl, 1486 unsigned long cl,
1479 u32 pid, u32 seq, u16 flags, int event) 1487 u32 pid, u32 seq, u16 flags, int event)
1480 { 1488 {
1481 struct tcmsg *tcm; 1489 struct tcmsg *tcm;
1482 struct nlmsghdr *nlh; 1490 struct nlmsghdr *nlh;
1483 unsigned char *b = skb_tail_pointer(skb); 1491 unsigned char *b = skb_tail_pointer(skb);
1484 struct gnet_dump d; 1492 struct gnet_dump d;
1485 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops; 1493 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1486 1494
1487 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags); 1495 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
1488 tcm = NLMSG_DATA(nlh); 1496 tcm = NLMSG_DATA(nlh);
1489 tcm->tcm_family = AF_UNSPEC; 1497 tcm->tcm_family = AF_UNSPEC;
1490 tcm->tcm__pad1 = 0; 1498 tcm->tcm__pad1 = 0;
1491 tcm->tcm__pad2 = 0; 1499 tcm->tcm__pad2 = 0;
1492 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1500 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1493 tcm->tcm_parent = q->handle; 1501 tcm->tcm_parent = q->handle;
1494 tcm->tcm_handle = q->handle; 1502 tcm->tcm_handle = q->handle;
1495 tcm->tcm_info = 0; 1503 tcm->tcm_info = 0;
1496 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id); 1504 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
1497 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0) 1505 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1498 goto nla_put_failure; 1506 goto nla_put_failure;
1499 1507
1500 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS, 1508 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1501 qdisc_root_sleeping_lock(q), &d) < 0) 1509 qdisc_root_sleeping_lock(q), &d) < 0)
1502 goto nla_put_failure; 1510 goto nla_put_failure;
1503 1511
1504 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0) 1512 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1505 goto nla_put_failure; 1513 goto nla_put_failure;
1506 1514
1507 if (gnet_stats_finish_copy(&d) < 0) 1515 if (gnet_stats_finish_copy(&d) < 0)
1508 goto nla_put_failure; 1516 goto nla_put_failure;
1509 1517
1510 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1518 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1511 return skb->len; 1519 return skb->len;
1512 1520
1513 nlmsg_failure: 1521 nlmsg_failure:
1514 nla_put_failure: 1522 nla_put_failure:
1515 nlmsg_trim(skb, b); 1523 nlmsg_trim(skb, b);
1516 return -1; 1524 return -1;
1517 } 1525 }
1518 1526
1519 static int tclass_notify(struct net *net, struct sk_buff *oskb, 1527 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1520 struct nlmsghdr *n, struct Qdisc *q, 1528 struct nlmsghdr *n, struct Qdisc *q,
1521 unsigned long cl, int event) 1529 unsigned long cl, int event)
1522 { 1530 {
1523 struct sk_buff *skb; 1531 struct sk_buff *skb;
1524 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0; 1532 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1525 1533
1526 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1534 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1527 if (!skb) 1535 if (!skb)
1528 return -ENOBUFS; 1536 return -ENOBUFS;
1529 1537
1530 if (tc_fill_tclass(skb, q, cl, pid, n->nlmsg_seq, 0, event) < 0) { 1538 if (tc_fill_tclass(skb, q, cl, pid, n->nlmsg_seq, 0, event) < 0) {
1531 kfree_skb(skb); 1539 kfree_skb(skb);
1532 return -EINVAL; 1540 return -EINVAL;
1533 } 1541 }
1534 1542
1535 return rtnetlink_send(skb, net, pid, RTNLGRP_TC, 1543 return rtnetlink_send(skb, net, pid, RTNLGRP_TC,
1536 n->nlmsg_flags & NLM_F_ECHO); 1544 n->nlmsg_flags & NLM_F_ECHO);
1537 } 1545 }
1538 1546
1539 struct qdisc_dump_args { 1547 struct qdisc_dump_args {
1540 struct qdisc_walker w; 1548 struct qdisc_walker w;
1541 struct sk_buff *skb; 1549 struct sk_buff *skb;
1542 struct netlink_callback *cb; 1550 struct netlink_callback *cb;
1543 }; 1551 };
1544 1552
1545 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg) 1553 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1546 { 1554 {
1547 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg; 1555 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1548 1556
1549 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).pid, 1557 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).pid,
1550 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS); 1558 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1551 } 1559 }
1552 1560
1553 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb, 1561 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
1554 struct tcmsg *tcm, struct netlink_callback *cb, 1562 struct tcmsg *tcm, struct netlink_callback *cb,
1555 int *t_p, int s_t) 1563 int *t_p, int s_t)
1556 { 1564 {
1557 struct qdisc_dump_args arg; 1565 struct qdisc_dump_args arg;
1558 1566
1559 if (tc_qdisc_dump_ignore(q) || 1567 if (tc_qdisc_dump_ignore(q) ||
1560 *t_p < s_t || !q->ops->cl_ops || 1568 *t_p < s_t || !q->ops->cl_ops ||
1561 (tcm->tcm_parent && 1569 (tcm->tcm_parent &&
1562 TC_H_MAJ(tcm->tcm_parent) != q->handle)) { 1570 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1563 (*t_p)++; 1571 (*t_p)++;
1564 return 0; 1572 return 0;
1565 } 1573 }
1566 if (*t_p > s_t) 1574 if (*t_p > s_t)
1567 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0])); 1575 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1568 arg.w.fn = qdisc_class_dump; 1576 arg.w.fn = qdisc_class_dump;
1569 arg.skb = skb; 1577 arg.skb = skb;
1570 arg.cb = cb; 1578 arg.cb = cb;
1571 arg.w.stop = 0; 1579 arg.w.stop = 0;
1572 arg.w.skip = cb->args[1]; 1580 arg.w.skip = cb->args[1];
1573 arg.w.count = 0; 1581 arg.w.count = 0;
1574 q->ops->cl_ops->walk(q, &arg.w); 1582 q->ops->cl_ops->walk(q, &arg.w);
1575 cb->args[1] = arg.w.count; 1583 cb->args[1] = arg.w.count;
1576 if (arg.w.stop) 1584 if (arg.w.stop)
1577 return -1; 1585 return -1;
1578 (*t_p)++; 1586 (*t_p)++;
1579 return 0; 1587 return 0;
1580 } 1588 }
1581 1589
1582 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb, 1590 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
1583 struct tcmsg *tcm, struct netlink_callback *cb, 1591 struct tcmsg *tcm, struct netlink_callback *cb,
1584 int *t_p, int s_t) 1592 int *t_p, int s_t)
1585 { 1593 {
1586 struct Qdisc *q; 1594 struct Qdisc *q;
1587 1595
1588 if (!root) 1596 if (!root)
1589 return 0; 1597 return 0;
1590 1598
1591 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0) 1599 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
1592 return -1; 1600 return -1;
1593 1601
1594 list_for_each_entry(q, &root->list, list) { 1602 list_for_each_entry(q, &root->list, list) {
1595 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0) 1603 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
1596 return -1; 1604 return -1;
1597 } 1605 }
1598 1606
1599 return 0; 1607 return 0;
1600 } 1608 }
1601 1609
1602 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb) 1610 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1603 { 1611 {
1604 struct tcmsg *tcm = (struct tcmsg *)NLMSG_DATA(cb->nlh); 1612 struct tcmsg *tcm = (struct tcmsg *)NLMSG_DATA(cb->nlh);
1605 struct net *net = sock_net(skb->sk); 1613 struct net *net = sock_net(skb->sk);
1606 struct netdev_queue *dev_queue; 1614 struct netdev_queue *dev_queue;
1607 struct net_device *dev; 1615 struct net_device *dev;
1608 int t, s_t; 1616 int t, s_t;
1609 1617
1610 if (cb->nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*tcm))) 1618 if (cb->nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*tcm)))
1611 return 0; 1619 return 0;
1612 dev = dev_get_by_index(net, tcm->tcm_ifindex); 1620 dev = dev_get_by_index(net, tcm->tcm_ifindex);
1613 if (!dev) 1621 if (!dev)
1614 return 0; 1622 return 0;
1615 1623
1616 s_t = cb->args[0]; 1624 s_t = cb->args[0];
1617 t = 0; 1625 t = 0;
1618 1626
1619 if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0) 1627 if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0)
1620 goto done; 1628 goto done;
1621 1629
1622 dev_queue = dev_ingress_queue(dev); 1630 dev_queue = dev_ingress_queue(dev);
1623 if (dev_queue && 1631 if (dev_queue &&
1624 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb, 1632 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
1625 &t, s_t) < 0) 1633 &t, s_t) < 0)
1626 goto done; 1634 goto done;
1627 1635
1628 done: 1636 done:
1629 cb->args[0] = t; 1637 cb->args[0] = t;
1630 1638
1631 dev_put(dev); 1639 dev_put(dev);
1632 return skb->len; 1640 return skb->len;
1633 } 1641 }
1634 1642
1635 /* Main classifier routine: scans classifier chain attached 1643 /* Main classifier routine: scans classifier chain attached
1636 * to this qdisc, (optionally) tests for protocol and asks 1644 * to this qdisc, (optionally) tests for protocol and asks
1637 * specific classifiers. 1645 * specific classifiers.
1638 */ 1646 */
1639 int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp, 1647 int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp,
1640 struct tcf_result *res) 1648 struct tcf_result *res)
1641 { 1649 {
1642 __be16 protocol = skb->protocol; 1650 __be16 protocol = skb->protocol;
1643 int err; 1651 int err;
1644 1652
1645 for (; tp; tp = tp->next) { 1653 for (; tp; tp = tp->next) {
1646 if (tp->protocol != protocol && 1654 if (tp->protocol != protocol &&
1647 tp->protocol != htons(ETH_P_ALL)) 1655 tp->protocol != htons(ETH_P_ALL))
1648 continue; 1656 continue;
1649 err = tp->classify(skb, tp, res); 1657 err = tp->classify(skb, tp, res);
1650 1658
1651 if (err >= 0) { 1659 if (err >= 0) {
1652 #ifdef CONFIG_NET_CLS_ACT 1660 #ifdef CONFIG_NET_CLS_ACT
1653 if (err != TC_ACT_RECLASSIFY && skb->tc_verd) 1661 if (err != TC_ACT_RECLASSIFY && skb->tc_verd)
1654 skb->tc_verd = SET_TC_VERD(skb->tc_verd, 0); 1662 skb->tc_verd = SET_TC_VERD(skb->tc_verd, 0);
1655 #endif 1663 #endif
1656 return err; 1664 return err;
1657 } 1665 }
1658 } 1666 }
1659 return -1; 1667 return -1;
1660 } 1668 }
1661 EXPORT_SYMBOL(tc_classify_compat); 1669 EXPORT_SYMBOL(tc_classify_compat);
1662 1670
1663 int tc_classify(struct sk_buff *skb, struct tcf_proto *tp, 1671 int tc_classify(struct sk_buff *skb, struct tcf_proto *tp,
1664 struct tcf_result *res) 1672 struct tcf_result *res)
1665 { 1673 {
1666 int err = 0; 1674 int err = 0;
1667 __be16 protocol; 1675 __be16 protocol;
1668 #ifdef CONFIG_NET_CLS_ACT 1676 #ifdef CONFIG_NET_CLS_ACT
1669 struct tcf_proto *otp = tp; 1677 struct tcf_proto *otp = tp;
1670 reclassify: 1678 reclassify:
1671 #endif 1679 #endif
1672 protocol = skb->protocol; 1680 protocol = skb->protocol;
1673 1681
1674 err = tc_classify_compat(skb, tp, res); 1682 err = tc_classify_compat(skb, tp, res);
1675 #ifdef CONFIG_NET_CLS_ACT 1683 #ifdef CONFIG_NET_CLS_ACT
1676 if (err == TC_ACT_RECLASSIFY) { 1684 if (err == TC_ACT_RECLASSIFY) {
1677 u32 verd = G_TC_VERD(skb->tc_verd); 1685 u32 verd = G_TC_VERD(skb->tc_verd);
1678 tp = otp; 1686 tp = otp;
1679 1687
1680 if (verd++ >= MAX_REC_LOOP) { 1688 if (verd++ >= MAX_REC_LOOP) {
1681 if (net_ratelimit()) 1689 if (net_ratelimit())
1682 pr_notice("%s: packet reclassify loop" 1690 pr_notice("%s: packet reclassify loop"
1683 " rule prio %u protocol %02x\n", 1691 " rule prio %u protocol %02x\n",
1684 tp->q->ops->id, 1692 tp->q->ops->id,
1685 tp->prio & 0xffff, 1693 tp->prio & 0xffff,
1686 ntohs(tp->protocol)); 1694 ntohs(tp->protocol));
1687 return TC_ACT_SHOT; 1695 return TC_ACT_SHOT;
1688 } 1696 }
1689 skb->tc_verd = SET_TC_VERD(skb->tc_verd, verd); 1697 skb->tc_verd = SET_TC_VERD(skb->tc_verd, verd);
1690 goto reclassify; 1698 goto reclassify;
1691 } 1699 }
1692 #endif 1700 #endif
1693 return err; 1701 return err;
1694 } 1702 }
1695 EXPORT_SYMBOL(tc_classify); 1703 EXPORT_SYMBOL(tc_classify);
1696 1704
1697 void tcf_destroy(struct tcf_proto *tp) 1705 void tcf_destroy(struct tcf_proto *tp)
1698 { 1706 {
1699 tp->ops->destroy(tp); 1707 tp->ops->destroy(tp);
1700 module_put(tp->ops->owner); 1708 module_put(tp->ops->owner);
1701 kfree(tp); 1709 kfree(tp);
1702 } 1710 }
1703 1711
1704 void tcf_destroy_chain(struct tcf_proto **fl) 1712 void tcf_destroy_chain(struct tcf_proto **fl)
1705 { 1713 {
1706 struct tcf_proto *tp; 1714 struct tcf_proto *tp;
1707 1715
1708 while ((tp = *fl) != NULL) { 1716 while ((tp = *fl) != NULL) {
1709 *fl = tp->next; 1717 *fl = tp->next;
1710 tcf_destroy(tp); 1718 tcf_destroy(tp);
1711 } 1719 }
1712 } 1720 }
1713 EXPORT_SYMBOL(tcf_destroy_chain); 1721 EXPORT_SYMBOL(tcf_destroy_chain);
1714 1722
1715 #ifdef CONFIG_PROC_FS 1723 #ifdef CONFIG_PROC_FS
1716 static int psched_show(struct seq_file *seq, void *v) 1724 static int psched_show(struct seq_file *seq, void *v)
1717 { 1725 {
1718 struct timespec ts; 1726 struct timespec ts;
1719 1727
1720 hrtimer_get_res(CLOCK_MONOTONIC, &ts); 1728 hrtimer_get_res(CLOCK_MONOTONIC, &ts);
1721 seq_printf(seq, "%08x %08x %08x %08x\n", 1729 seq_printf(seq, "%08x %08x %08x %08x\n",
1722 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1), 1730 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
1723 1000000, 1731 1000000,
1724 (u32)NSEC_PER_SEC/(u32)ktime_to_ns(timespec_to_ktime(ts))); 1732 (u32)NSEC_PER_SEC/(u32)ktime_to_ns(timespec_to_ktime(ts)));
1725 1733
1726 return 0; 1734 return 0;
1727 } 1735 }
1728 1736
1729 static int psched_open(struct inode *inode, struct file *file) 1737 static int psched_open(struct inode *inode, struct file *file)
1730 { 1738 {
1731 return single_open(file, psched_show, NULL); 1739 return single_open(file, psched_show, NULL);
1732 } 1740 }
1733 1741
1734 static const struct file_operations psched_fops = { 1742 static const struct file_operations psched_fops = {
1735 .owner = THIS_MODULE, 1743 .owner = THIS_MODULE,
1736 .open = psched_open, 1744 .open = psched_open,
1737 .read = seq_read, 1745 .read = seq_read,
1738 .llseek = seq_lseek, 1746 .llseek = seq_lseek,
1739 .release = single_release, 1747 .release = single_release,
1740 }; 1748 };
1741 1749
1742 static int __net_init psched_net_init(struct net *net) 1750 static int __net_init psched_net_init(struct net *net)
1743 { 1751 {
1744 struct proc_dir_entry *e; 1752 struct proc_dir_entry *e;
1745 1753
1746 e = proc_net_fops_create(net, "psched", 0, &psched_fops); 1754 e = proc_net_fops_create(net, "psched", 0, &psched_fops);
1747 if (e == NULL) 1755 if (e == NULL)
1748 return -ENOMEM; 1756 return -ENOMEM;
1749 1757
1750 return 0; 1758 return 0;
1751 } 1759 }
1752 1760
1753 static void __net_exit psched_net_exit(struct net *net) 1761 static void __net_exit psched_net_exit(struct net *net)
1754 { 1762 {
1755 proc_net_remove(net, "psched"); 1763 proc_net_remove(net, "psched");
1756 } 1764 }
1757 #else 1765 #else
1758 static int __net_init psched_net_init(struct net *net) 1766 static int __net_init psched_net_init(struct net *net)
1759 { 1767 {
1760 return 0; 1768 return 0;
1761 } 1769 }
1762 1770
1763 static void __net_exit psched_net_exit(struct net *net) 1771 static void __net_exit psched_net_exit(struct net *net)
1764 { 1772 {
1765 } 1773 }
1766 #endif 1774 #endif
1767 1775
1768 static struct pernet_operations psched_net_ops = { 1776 static struct pernet_operations psched_net_ops = {
1769 .init = psched_net_init, 1777 .init = psched_net_init,
1770 .exit = psched_net_exit, 1778 .exit = psched_net_exit,
1771 }; 1779 };
1772 1780
1773 static int __init pktsched_init(void) 1781 static int __init pktsched_init(void)
1774 { 1782 {
1775 int err; 1783 int err;
1776 1784
1777 err = register_pernet_subsys(&psched_net_ops); 1785 err = register_pernet_subsys(&psched_net_ops);
1778 if (err) { 1786 if (err) {
1779 pr_err("pktsched_init: " 1787 pr_err("pktsched_init: "
1780 "cannot initialize per netns operations\n"); 1788 "cannot initialize per netns operations\n");
1781 return err; 1789 return err;
1782 } 1790 }
1783 1791
1784 register_qdisc(&pfifo_qdisc_ops); 1792 register_qdisc(&pfifo_qdisc_ops);
1785 register_qdisc(&bfifo_qdisc_ops); 1793 register_qdisc(&bfifo_qdisc_ops);
1786 register_qdisc(&pfifo_head_drop_qdisc_ops); 1794 register_qdisc(&pfifo_head_drop_qdisc_ops);
1787 register_qdisc(&mq_qdisc_ops); 1795 register_qdisc(&mq_qdisc_ops);
1788 1796
1789 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL); 1797 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL);
1790 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL); 1798 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL);
1791 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc); 1799 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc);
1792 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL); 1800 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL);
1793 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL); 1801 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL);
1794 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass); 1802 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass);
1795 1803
1796 return 0; 1804 return 0;
1797 } 1805 }
1798 1806
1799 subsys_initcall(pktsched_init); 1807 subsys_initcall(pktsched_init);
1800 1808
net/sched/sch_generic.c
Diff comments   View file @ a2da570
1 /* 1 /*
2 * net/sched/sch_generic.c Generic packet scheduler routines. 2 * net/sched/sch_generic.c Generic packet scheduler routines.
3 * 3 *
4 * This program is free software; you can redistribute it and/or 4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License 5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version. 7 * 2 of the License, or (at your option) any later version.
8 * 8 *
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
10 * Jamal Hadi Salim, <hadi@cyberus.ca> 990601 10 * Jamal Hadi Salim, <hadi@cyberus.ca> 990601
11 * - Ingress support 11 * - Ingress support
12 */ 12 */
13 13
14 #include <linux/bitops.h> 14 #include <linux/bitops.h>
15 #include <linux/module.h> 15 #include <linux/module.h>
16 #include <linux/types.h> 16 #include <linux/types.h>
17 #include <linux/kernel.h> 17 #include <linux/kernel.h>
18 #include <linux/sched.h> 18 #include <linux/sched.h>
19 #include <linux/string.h> 19 #include <linux/string.h>
20 #include <linux/errno.h> 20 #include <linux/errno.h>
21 #include <linux/netdevice.h> 21 #include <linux/netdevice.h>
22 #include <linux/skbuff.h> 22 #include <linux/skbuff.h>
23 #include <linux/rtnetlink.h> 23 #include <linux/rtnetlink.h>
24 #include <linux/init.h> 24 #include <linux/init.h>
25 #include <linux/rcupdate.h> 25 #include <linux/rcupdate.h>
26 #include <linux/list.h> 26 #include <linux/list.h>
27 #include <linux/slab.h> 27 #include <linux/slab.h>
28 #include <net/pkt_sched.h> 28 #include <net/pkt_sched.h>
29 #include <net/dst.h> 29 #include <net/dst.h>
30 30
31 /* Main transmission queue. */ 31 /* Main transmission queue. */
32 32
33 /* Modifications to data participating in scheduling must be protected with 33 /* Modifications to data participating in scheduling must be protected with
34 * qdisc_lock(qdisc) spinlock. 34 * qdisc_lock(qdisc) spinlock.
35 * 35 *
36 * The idea is the following: 36 * The idea is the following:
37 * - enqueue, dequeue are serialized via qdisc root lock 37 * - enqueue, dequeue are serialized via qdisc root lock
38 * - ingress filtering is also serialized via qdisc root lock 38 * - ingress filtering is also serialized via qdisc root lock
39 * - updates to tree and tree walking are only done under the rtnl mutex. 39 * - updates to tree and tree walking are only done under the rtnl mutex.
40 */ 40 */
41 41
42 static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) 42 static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
43 { 43 {
44 skb_dst_force(skb); 44 skb_dst_force(skb);
45 q->gso_skb = skb; 45 q->gso_skb = skb;
46 q->qstats.requeues++; 46 q->qstats.requeues++;
47 q->q.qlen++; /* it's still part of the queue */ 47 q->q.qlen++; /* it's still part of the queue */
48 __netif_schedule(q); 48 __netif_schedule(q);
49 49
50 return 0; 50 return 0;
51 } 51 }
52 52
53 static inline struct sk_buff *dequeue_skb(struct Qdisc *q) 53 static inline struct sk_buff *dequeue_skb(struct Qdisc *q)
54 { 54 {
55 struct sk_buff *skb = q->gso_skb; 55 struct sk_buff *skb = q->gso_skb;
56 56
57 if (unlikely(skb)) { 57 if (unlikely(skb)) {
58 struct net_device *dev = qdisc_dev(q); 58 struct net_device *dev = qdisc_dev(q);
59 struct netdev_queue *txq; 59 struct netdev_queue *txq;
60 60
61 /* check the reason of requeuing without tx lock first */ 61 /* check the reason of requeuing without tx lock first */
62 txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); 62 txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
63 if (!netif_tx_queue_frozen_or_stopped(txq)) { 63 if (!netif_tx_queue_frozen_or_stopped(txq)) {
64 q->gso_skb = NULL; 64 q->gso_skb = NULL;
65 q->q.qlen--; 65 q->q.qlen--;
66 } else 66 } else
67 skb = NULL; 67 skb = NULL;
68 } else { 68 } else {
69 skb = q->dequeue(q); 69 skb = q->dequeue(q);
70 } 70 }
71 71
72 return skb; 72 return skb;
73 } 73 }
74 74
75 static inline int handle_dev_cpu_collision(struct sk_buff *skb, 75 static inline int handle_dev_cpu_collision(struct sk_buff *skb,
76 struct netdev_queue *dev_queue, 76 struct netdev_queue *dev_queue,
77 struct Qdisc *q) 77 struct Qdisc *q)
78 { 78 {
79 int ret; 79 int ret;
80 80
81 if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) { 81 if (unlikely(dev_queue->xmit_lock_owner == smp_processor_id())) {
82 /* 82 /*
83 * Same CPU holding the lock. It may be a transient 83 * Same CPU holding the lock. It may be a transient
84 * configuration error, when hard_start_xmit() recurses. We 84 * configuration error, when hard_start_xmit() recurses. We
85 * detect it by checking xmit owner and drop the packet when 85 * detect it by checking xmit owner and drop the packet when
86 * deadloop is detected. Return OK to try the next skb. 86 * deadloop is detected. Return OK to try the next skb.
87 */ 87 */
88 kfree_skb(skb); 88 kfree_skb(skb);
89 if (net_ratelimit()) 89 if (net_ratelimit())
90 pr_warning("Dead loop on netdevice %s, fix it urgently!\n", 90 pr_warning("Dead loop on netdevice %s, fix it urgently!\n",
91 dev_queue->dev->name); 91 dev_queue->dev->name);
92 ret = qdisc_qlen(q); 92 ret = qdisc_qlen(q);
93 } else { 93 } else {
94 /* 94 /*
95 * Another cpu is holding lock, requeue & delay xmits for 95 * Another cpu is holding lock, requeue & delay xmits for
96 * some time. 96 * some time.
97 */ 97 */
98 __this_cpu_inc(softnet_data.cpu_collision); 98 __this_cpu_inc(softnet_data.cpu_collision);
99 ret = dev_requeue_skb(skb, q); 99 ret = dev_requeue_skb(skb, q);
100 } 100 }
101 101
102 return ret; 102 return ret;
103 } 103 }
104 104
105 /* 105 /*
106 * Transmit one skb, and handle the return status as required. Holding the 106 * Transmit one skb, and handle the return status as required. Holding the
107 * __QDISC_STATE_RUNNING bit guarantees that only one CPU can execute this 107 * __QDISC_STATE_RUNNING bit guarantees that only one CPU can execute this
108 * function. 108 * function.
109 * 109 *
110 * Returns to the caller: 110 * Returns to the caller:
111 * 0 - queue is empty or throttled. 111 * 0 - queue is empty or throttled.
112 * >0 - queue is not empty. 112 * >0 - queue is not empty.
113 */ 113 */
114 int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q, 114 int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
115 struct net_device *dev, struct netdev_queue *txq, 115 struct net_device *dev, struct netdev_queue *txq,
116 spinlock_t *root_lock) 116 spinlock_t *root_lock)
117 { 117 {
118 int ret = NETDEV_TX_BUSY; 118 int ret = NETDEV_TX_BUSY;
119 119
120 /* And release qdisc */ 120 /* And release qdisc */
121 spin_unlock(root_lock); 121 spin_unlock(root_lock);
122 122
123 HARD_TX_LOCK(dev, txq, smp_processor_id()); 123 HARD_TX_LOCK(dev, txq, smp_processor_id());
124 if (!netif_tx_queue_frozen_or_stopped(txq)) 124 if (!netif_tx_queue_frozen_or_stopped(txq))
125 ret = dev_hard_start_xmit(skb, dev, txq); 125 ret = dev_hard_start_xmit(skb, dev, txq);
126 126
127 HARD_TX_UNLOCK(dev, txq); 127 HARD_TX_UNLOCK(dev, txq);
128 128
129 spin_lock(root_lock); 129 spin_lock(root_lock);
130 130
131 if (dev_xmit_complete(ret)) { 131 if (dev_xmit_complete(ret)) {
132 /* Driver sent out skb successfully or skb was consumed */ 132 /* Driver sent out skb successfully or skb was consumed */
133 ret = qdisc_qlen(q); 133 ret = qdisc_qlen(q);
134 } else if (ret == NETDEV_TX_LOCKED) { 134 } else if (ret == NETDEV_TX_LOCKED) {
135 /* Driver try lock failed */ 135 /* Driver try lock failed */
136 ret = handle_dev_cpu_collision(skb, txq, q); 136 ret = handle_dev_cpu_collision(skb, txq, q);
137 } else { 137 } else {
138 /* Driver returned NETDEV_TX_BUSY - requeue skb */ 138 /* Driver returned NETDEV_TX_BUSY - requeue skb */
139 if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit())) 139 if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit()))
140 pr_warning("BUG %s code %d qlen %d\n", 140 pr_warning("BUG %s code %d qlen %d\n",
141 dev->name, ret, q->q.qlen); 141 dev->name, ret, q->q.qlen);
142 142
143 ret = dev_requeue_skb(skb, q); 143 ret = dev_requeue_skb(skb, q);
144 } 144 }
145 145
146 if (ret && netif_tx_queue_frozen_or_stopped(txq)) 146 if (ret && netif_tx_queue_frozen_or_stopped(txq))
147 ret = 0; 147 ret = 0;
148 148
149 return ret; 149 return ret;
150 } 150 }
151 151
152 /* 152 /*
153 * NOTE: Called under qdisc_lock(q) with locally disabled BH. 153 * NOTE: Called under qdisc_lock(q) with locally disabled BH.
154 * 154 *
155 * __QDISC_STATE_RUNNING guarantees only one CPU can process 155 * __QDISC_STATE_RUNNING guarantees only one CPU can process
156 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for 156 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
157 * this queue. 157 * this queue.
158 * 158 *
159 * netif_tx_lock serializes accesses to device driver. 159 * netif_tx_lock serializes accesses to device driver.
160 * 160 *
161 * qdisc_lock(q) and netif_tx_lock are mutually exclusive, 161 * qdisc_lock(q) and netif_tx_lock are mutually exclusive,
162 * if one is grabbed, another must be free. 162 * if one is grabbed, another must be free.
163 * 163 *
164 * Note, that this procedure can be called by a watchdog timer 164 * Note, that this procedure can be called by a watchdog timer
165 * 165 *
166 * Returns to the caller: 166 * Returns to the caller:
167 * 0 - queue is empty or throttled. 167 * 0 - queue is empty or throttled.
168 * >0 - queue is not empty. 168 * >0 - queue is not empty.
169 * 169 *
170 */ 170 */
171 static inline int qdisc_restart(struct Qdisc *q) 171 static inline int qdisc_restart(struct Qdisc *q)
172 { 172 {
173 struct netdev_queue *txq; 173 struct netdev_queue *txq;
174 struct net_device *dev; 174 struct net_device *dev;
175 spinlock_t *root_lock; 175 spinlock_t *root_lock;
176 struct sk_buff *skb; 176 struct sk_buff *skb;
177 177
178 /* Dequeue packet */ 178 /* Dequeue packet */
179 skb = dequeue_skb(q); 179 skb = dequeue_skb(q);
180 if (unlikely(!skb)) 180 if (unlikely(!skb))
181 return 0; 181 return 0;
182 WARN_ON_ONCE(skb_dst_is_noref(skb)); 182 WARN_ON_ONCE(skb_dst_is_noref(skb));
183 root_lock = qdisc_lock(q); 183 root_lock = qdisc_lock(q);
184 dev = qdisc_dev(q); 184 dev = qdisc_dev(q);
185 txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); 185 txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
186 186
187 return sch_direct_xmit(skb, q, dev, txq, root_lock); 187 return sch_direct_xmit(skb, q, dev, txq, root_lock);
188 } 188 }
189 189
190 void __qdisc_run(struct Qdisc *q) 190 void __qdisc_run(struct Qdisc *q)
191 { 191 {
192 unsigned long start_time = jiffies; 192 unsigned long start_time = jiffies;
193 193
194 while (qdisc_restart(q)) { 194 while (qdisc_restart(q)) {
195 /* 195 /*
196 * Postpone processing if 196 * Postpone processing if
197 * 1. another process needs the CPU; 197 * 1. another process needs the CPU;
198 * 2. we've been doing it for too long. 198 * 2. we've been doing it for too long.
199 */ 199 */
200 if (need_resched() || jiffies != start_time) { 200 if (need_resched() || jiffies != start_time) {
201 __netif_schedule(q); 201 __netif_schedule(q);
202 break; 202 break;
203 } 203 }
204 } 204 }
205 205
206 qdisc_run_end(q); 206 qdisc_run_end(q);
207 } 207 }
208 208
209 unsigned long dev_trans_start(struct net_device *dev) 209 unsigned long dev_trans_start(struct net_device *dev)
210 { 210 {
211 unsigned long val, res = dev->trans_start; 211 unsigned long val, res = dev->trans_start;
212 unsigned int i; 212 unsigned int i;
213 213
214 for (i = 0; i < dev->num_tx_queues; i++) { 214 for (i = 0; i < dev->num_tx_queues; i++) {
215 val = netdev_get_tx_queue(dev, i)->trans_start; 215 val = netdev_get_tx_queue(dev, i)->trans_start;
216 if (val && time_after(val, res)) 216 if (val && time_after(val, res))
217 res = val; 217 res = val;
218 } 218 }
219 dev->trans_start = res; 219 dev->trans_start = res;
220 return res; 220 return res;
221 } 221 }
222 EXPORT_SYMBOL(dev_trans_start); 222 EXPORT_SYMBOL(dev_trans_start);
223 223
224 static void dev_watchdog(unsigned long arg) 224 static void dev_watchdog(unsigned long arg)
225 { 225 {
226 struct net_device *dev = (struct net_device *)arg; 226 struct net_device *dev = (struct net_device *)arg;
227 227
228 netif_tx_lock(dev); 228 netif_tx_lock(dev);
229 if (!qdisc_tx_is_noop(dev)) { 229 if (!qdisc_tx_is_noop(dev)) {
230 if (netif_device_present(dev) && 230 if (netif_device_present(dev) &&
231 netif_running(dev) && 231 netif_running(dev) &&
232 netif_carrier_ok(dev)) { 232 netif_carrier_ok(dev)) {
233 int some_queue_timedout = 0; 233 int some_queue_timedout = 0;
234 unsigned int i; 234 unsigned int i;
235 unsigned long trans_start; 235 unsigned long trans_start;
236 236
237 for (i = 0; i < dev->num_tx_queues; i++) { 237 for (i = 0; i < dev->num_tx_queues; i++) {
238 struct netdev_queue *txq; 238 struct netdev_queue *txq;
239 239
240 txq = netdev_get_tx_queue(dev, i); 240 txq = netdev_get_tx_queue(dev, i);
241 /* 241 /*
242 * old device drivers set dev->trans_start 242 * old device drivers set dev->trans_start
243 */ 243 */
244 trans_start = txq->trans_start ? : dev->trans_start; 244 trans_start = txq->trans_start ? : dev->trans_start;
245 if (netif_tx_queue_stopped(txq) && 245 if (netif_tx_queue_stopped(txq) &&
246 time_after(jiffies, (trans_start + 246 time_after(jiffies, (trans_start +
247 dev->watchdog_timeo))) { 247 dev->watchdog_timeo))) {
248 some_queue_timedout = 1; 248 some_queue_timedout = 1;
249 break; 249 break;
250 } 250 }
251 } 251 }
252 252
253 if (some_queue_timedout) { 253 if (some_queue_timedout) {
254 char drivername[64]; 254 char drivername[64];
255 WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n", 255 WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
256 dev->name, netdev_drivername(dev, drivername, 64), i); 256 dev->name, netdev_drivername(dev, drivername, 64), i);
257 dev->netdev_ops->ndo_tx_timeout(dev); 257 dev->netdev_ops->ndo_tx_timeout(dev);
258 } 258 }
259 if (!mod_timer(&dev->watchdog_timer, 259 if (!mod_timer(&dev->watchdog_timer,
260 round_jiffies(jiffies + 260 round_jiffies(jiffies +
261 dev->watchdog_timeo))) 261 dev->watchdog_timeo)))
262 dev_hold(dev); 262 dev_hold(dev);
263 } 263 }
264 } 264 }
265 netif_tx_unlock(dev); 265 netif_tx_unlock(dev);
266 266
267 dev_put(dev); 267 dev_put(dev);
268 } 268 }
269 269
270 void __netdev_watchdog_up(struct net_device *dev) 270 void __netdev_watchdog_up(struct net_device *dev)
271 { 271 {
272 if (dev->netdev_ops->ndo_tx_timeout) { 272 if (dev->netdev_ops->ndo_tx_timeout) {
273 if (dev->watchdog_timeo <= 0) 273 if (dev->watchdog_timeo <= 0)
274 dev->watchdog_timeo = 5*HZ; 274 dev->watchdog_timeo = 5*HZ;
275 if (!mod_timer(&dev->watchdog_timer, 275 if (!mod_timer(&dev->watchdog_timer,
276 round_jiffies(jiffies + dev->watchdog_timeo))) 276 round_jiffies(jiffies + dev->watchdog_timeo)))
277 dev_hold(dev); 277 dev_hold(dev);
278 } 278 }
279 } 279 }
280 280
281 static void dev_watchdog_up(struct net_device *dev) 281 static void dev_watchdog_up(struct net_device *dev)
282 { 282 {
283 __netdev_watchdog_up(dev); 283 __netdev_watchdog_up(dev);
284 } 284 }
285 285
286 static void dev_watchdog_down(struct net_device *dev) 286 static void dev_watchdog_down(struct net_device *dev)
287 { 287 {
288 netif_tx_lock_bh(dev); 288 netif_tx_lock_bh(dev);
289 if (del_timer(&dev->watchdog_timer)) 289 if (del_timer(&dev->watchdog_timer))
290 dev_put(dev); 290 dev_put(dev);
291 netif_tx_unlock_bh(dev); 291 netif_tx_unlock_bh(dev);
292 } 292 }
293 293
294 /** 294 /**
295 * netif_carrier_on - set carrier 295 * netif_carrier_on - set carrier
296 * @dev: network device 296 * @dev: network device
297 * 297 *
298 * Device has detected that carrier. 298 * Device has detected that carrier.
299 */ 299 */
300 void netif_carrier_on(struct net_device *dev) 300 void netif_carrier_on(struct net_device *dev)
301 { 301 {
302 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) { 302 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
303 if (dev->reg_state == NETREG_UNINITIALIZED) 303 if (dev->reg_state == NETREG_UNINITIALIZED)
304 return; 304 return;
305 linkwatch_fire_event(dev); 305 linkwatch_fire_event(dev);
306 if (netif_running(dev)) 306 if (netif_running(dev))
307 __netdev_watchdog_up(dev); 307 __netdev_watchdog_up(dev);
308 } 308 }
309 } 309 }
310 EXPORT_SYMBOL(netif_carrier_on); 310 EXPORT_SYMBOL(netif_carrier_on);
311 311
312 /** 312 /**
313 * netif_carrier_off - clear carrier 313 * netif_carrier_off - clear carrier
314 * @dev: network device 314 * @dev: network device
315 * 315 *
316 * Device has detected loss of carrier. 316 * Device has detected loss of carrier.
317 */ 317 */
318 void netif_carrier_off(struct net_device *dev) 318 void netif_carrier_off(struct net_device *dev)
319 { 319 {
320 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) { 320 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
321 if (dev->reg_state == NETREG_UNINITIALIZED) 321 if (dev->reg_state == NETREG_UNINITIALIZED)
322 return; 322 return;
323 linkwatch_fire_event(dev); 323 linkwatch_fire_event(dev);
324 } 324 }
325 } 325 }
326 EXPORT_SYMBOL(netif_carrier_off); 326 EXPORT_SYMBOL(netif_carrier_off);
327 327
328 /** 328 /**
329 * netif_notify_peers - notify network peers about existence of @dev 329 * netif_notify_peers - notify network peers about existence of @dev
330 * @dev: network device 330 * @dev: network device
331 * 331 *
332 * Generate traffic such that interested network peers are aware of 332 * Generate traffic such that interested network peers are aware of
333 * @dev, such as by generating a gratuitous ARP. This may be used when 333 * @dev, such as by generating a gratuitous ARP. This may be used when
334 * a device wants to inform the rest of the network about some sort of 334 * a device wants to inform the rest of the network about some sort of
335 * reconfiguration such as a failover event or virtual machine 335 * reconfiguration such as a failover event or virtual machine
336 * migration. 336 * migration.
337 */ 337 */
338 void netif_notify_peers(struct net_device *dev) 338 void netif_notify_peers(struct net_device *dev)
339 { 339 {
340 rtnl_lock(); 340 rtnl_lock();
341 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 341 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
342 rtnl_unlock(); 342 rtnl_unlock();
343 } 343 }
344 EXPORT_SYMBOL(netif_notify_peers); 344 EXPORT_SYMBOL(netif_notify_peers);
345 345
346 /* "NOOP" scheduler: the best scheduler, recommended for all interfaces 346 /* "NOOP" scheduler: the best scheduler, recommended for all interfaces
347 under all circumstances. It is difficult to invent anything faster or 347 under all circumstances. It is difficult to invent anything faster or
348 cheaper. 348 cheaper.
349 */ 349 */
350 350
351 static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc) 351 static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
352 { 352 {
353 kfree_skb(skb); 353 kfree_skb(skb);
354 return NET_XMIT_CN; 354 return NET_XMIT_CN;
355 } 355 }
356 356
357 static struct sk_buff *noop_dequeue(struct Qdisc * qdisc) 357 static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
358 { 358 {
359 return NULL; 359 return NULL;
360 } 360 }
361 361
362 struct Qdisc_ops noop_qdisc_ops __read_mostly = { 362 struct Qdisc_ops noop_qdisc_ops __read_mostly = {
363 .id = "noop", 363 .id = "noop",
364 .priv_size = 0, 364 .priv_size = 0,
365 .enqueue = noop_enqueue, 365 .enqueue = noop_enqueue,
366 .dequeue = noop_dequeue, 366 .dequeue = noop_dequeue,
367 .peek = noop_dequeue, 367 .peek = noop_dequeue,
368 .owner = THIS_MODULE, 368 .owner = THIS_MODULE,
369 }; 369 };
370 370
371 static struct netdev_queue noop_netdev_queue = { 371 static struct netdev_queue noop_netdev_queue = {
372 .qdisc = &noop_qdisc, 372 .qdisc = &noop_qdisc,
373 .qdisc_sleeping = &noop_qdisc, 373 .qdisc_sleeping = &noop_qdisc,
374 }; 374 };
375 375
376 struct Qdisc noop_qdisc = { 376 struct Qdisc noop_qdisc = {
377 .enqueue = noop_enqueue, 377 .enqueue = noop_enqueue,
378 .dequeue = noop_dequeue, 378 .dequeue = noop_dequeue,
379 .flags = TCQ_F_BUILTIN, 379 .flags = TCQ_F_BUILTIN,
380 .ops = &noop_qdisc_ops, 380 .ops = &noop_qdisc_ops,
381 .list = LIST_HEAD_INIT(noop_qdisc.list), 381 .list = LIST_HEAD_INIT(noop_qdisc.list),
382 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), 382 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
383 .dev_queue = &noop_netdev_queue, 383 .dev_queue = &noop_netdev_queue,
384 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock), 384 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
385 }; 385 };
386 EXPORT_SYMBOL(noop_qdisc); 386 EXPORT_SYMBOL(noop_qdisc);
387 387
388 static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = { 388 static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
389 .id = "noqueue", 389 .id = "noqueue",
390 .priv_size = 0, 390 .priv_size = 0,
391 .enqueue = noop_enqueue, 391 .enqueue = noop_enqueue,
392 .dequeue = noop_dequeue, 392 .dequeue = noop_dequeue,
393 .peek = noop_dequeue, 393 .peek = noop_dequeue,
394 .owner = THIS_MODULE, 394 .owner = THIS_MODULE,
395 }; 395 };
396 396
397 static struct Qdisc noqueue_qdisc; 397 static struct Qdisc noqueue_qdisc;
398 static struct netdev_queue noqueue_netdev_queue = { 398 static struct netdev_queue noqueue_netdev_queue = {
399 .qdisc = &noqueue_qdisc, 399 .qdisc = &noqueue_qdisc,
400 .qdisc_sleeping = &noqueue_qdisc, 400 .qdisc_sleeping = &noqueue_qdisc,
401 }; 401 };
402 402
403 static struct Qdisc noqueue_qdisc = { 403 static struct Qdisc noqueue_qdisc = {
404 .enqueue = NULL, 404 .enqueue = NULL,
405 .dequeue = noop_dequeue, 405 .dequeue = noop_dequeue,
406 .flags = TCQ_F_BUILTIN, 406 .flags = TCQ_F_BUILTIN,
407 .ops = &noqueue_qdisc_ops, 407 .ops = &noqueue_qdisc_ops,
408 .list = LIST_HEAD_INIT(noqueue_qdisc.list), 408 .list = LIST_HEAD_INIT(noqueue_qdisc.list),
409 .q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock), 409 .q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock),
410 .dev_queue = &noqueue_netdev_queue, 410 .dev_queue = &noqueue_netdev_queue,
411 .busylock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.busylock), 411 .busylock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.busylock),
412 }; 412 };
413 413
414 414
415 static const u8 prio2band[TC_PRIO_MAX + 1] = { 415 static const u8 prio2band[TC_PRIO_MAX + 1] = {
416 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 416 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1
417 }; 417 };
418 418
419 /* 3-band FIFO queue: old style, but should be a bit faster than 419 /* 3-band FIFO queue: old style, but should be a bit faster than
420 generic prio+fifo combination. 420 generic prio+fifo combination.
421 */ 421 */
422 422
423 #define PFIFO_FAST_BANDS 3 423 #define PFIFO_FAST_BANDS 3
424 424
425 /* 425 /*
426 * Private data for a pfifo_fast scheduler containing: 426 * Private data for a pfifo_fast scheduler containing:
427 * - queues for the three band 427 * - queues for the three band
428 * - bitmap indicating which of the bands contain skbs 428 * - bitmap indicating which of the bands contain skbs
429 */ 429 */
430 struct pfifo_fast_priv { 430 struct pfifo_fast_priv {
431 u32 bitmap; 431 u32 bitmap;
432 struct sk_buff_head q[PFIFO_FAST_BANDS]; 432 struct sk_buff_head q[PFIFO_FAST_BANDS];
433 }; 433 };
434 434
435 /* 435 /*
436 * Convert a bitmap to the first band number where an skb is queued, where: 436 * Convert a bitmap to the first band number where an skb is queued, where:
437 * bitmap=0 means there are no skbs on any band. 437 * bitmap=0 means there are no skbs on any band.
438 * bitmap=1 means there is an skb on band 0. 438 * bitmap=1 means there is an skb on band 0.
439 * bitmap=7 means there are skbs on all 3 bands, etc. 439 * bitmap=7 means there are skbs on all 3 bands, etc.
440 */ 440 */
441 static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0}; 441 static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
442 442
443 static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv, 443 static inline struct sk_buff_head *band2list(struct pfifo_fast_priv *priv,
444 int band) 444 int band)
445 { 445 {
446 return priv->q + band; 446 return priv->q + band;
447 } 447 }
448 448
449 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc) 449 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc)
450 { 450 {
451 if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) { 451 if (skb_queue_len(&qdisc->q) < qdisc_dev(qdisc)->tx_queue_len) {
452 int band = prio2band[skb->priority & TC_PRIO_MAX]; 452 int band = prio2band[skb->priority & TC_PRIO_MAX];
453 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 453 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
454 struct sk_buff_head *list = band2list(priv, band); 454 struct sk_buff_head *list = band2list(priv, band);
455 455
456 priv->bitmap |= (1 << band); 456 priv->bitmap |= (1 << band);
457 qdisc->q.qlen++; 457 qdisc->q.qlen++;
458 return __qdisc_enqueue_tail(skb, qdisc, list); 458 return __qdisc_enqueue_tail(skb, qdisc, list);
459 } 459 }
460 460
461 return qdisc_drop(skb, qdisc); 461 return qdisc_drop(skb, qdisc);
462 } 462 }
463 463
464 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc) 464 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
465 { 465 {
466 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 466 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
467 int band = bitmap2band[priv->bitmap]; 467 int band = bitmap2band[priv->bitmap];
468 468
469 if (likely(band >= 0)) { 469 if (likely(band >= 0)) {
470 struct sk_buff_head *list = band2list(priv, band); 470 struct sk_buff_head *list = band2list(priv, band);
471 struct sk_buff *skb = __qdisc_dequeue_head(qdisc, list); 471 struct sk_buff *skb = __qdisc_dequeue_head(qdisc, list);
472 472
473 qdisc->q.qlen--; 473 qdisc->q.qlen--;
474 if (skb_queue_empty(list)) 474 if (skb_queue_empty(list))
475 priv->bitmap &= ~(1 << band); 475 priv->bitmap &= ~(1 << band);
476 476
477 return skb; 477 return skb;
478 } 478 }
479 479
480 return NULL; 480 return NULL;
481 } 481 }
482 482
483 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc) 483 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
484 { 484 {
485 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 485 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
486 int band = bitmap2band[priv->bitmap]; 486 int band = bitmap2band[priv->bitmap];
487 487
488 if (band >= 0) { 488 if (band >= 0) {
489 struct sk_buff_head *list = band2list(priv, band); 489 struct sk_buff_head *list = band2list(priv, band);
490 490
491 return skb_peek(list); 491 return skb_peek(list);
492 } 492 }
493 493
494 return NULL; 494 return NULL;
495 } 495 }
496 496
497 static void pfifo_fast_reset(struct Qdisc *qdisc) 497 static void pfifo_fast_reset(struct Qdisc *qdisc)
498 { 498 {
499 int prio; 499 int prio;
500 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 500 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
501 501
502 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) 502 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
503 __qdisc_reset_queue(qdisc, band2list(priv, prio)); 503 __qdisc_reset_queue(qdisc, band2list(priv, prio));
504 504
505 priv->bitmap = 0; 505 priv->bitmap = 0;
506 qdisc->qstats.backlog = 0; 506 qdisc->qstats.backlog = 0;
507 qdisc->q.qlen = 0; 507 qdisc->q.qlen = 0;
508 } 508 }
509 509
510 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb) 510 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
511 { 511 {
512 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS }; 512 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
513 513
514 memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1); 514 memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
515 NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt); 515 NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
516 return skb->len; 516 return skb->len;
517 517
518 nla_put_failure: 518 nla_put_failure:
519 return -1; 519 return -1;
520 } 520 }
521 521
522 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt) 522 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
523 { 523 {
524 int prio; 524 int prio;
525 struct pfifo_fast_priv *priv = qdisc_priv(qdisc); 525 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
526 526
527 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) 527 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
528 skb_queue_head_init(band2list(priv, prio)); 528 skb_queue_head_init(band2list(priv, prio));
529 529
530 return 0; 530 return 0;
531 } 531 }
532 532
533 struct Qdisc_ops pfifo_fast_ops __read_mostly = { 533 struct Qdisc_ops pfifo_fast_ops __read_mostly = {
534 .id = "pfifo_fast", 534 .id = "pfifo_fast",
535 .priv_size = sizeof(struct pfifo_fast_priv), 535 .priv_size = sizeof(struct pfifo_fast_priv),
536 .enqueue = pfifo_fast_enqueue, 536 .enqueue = pfifo_fast_enqueue,
537 .dequeue = pfifo_fast_dequeue, 537 .dequeue = pfifo_fast_dequeue,
538 .peek = pfifo_fast_peek, 538 .peek = pfifo_fast_peek,
539 .init = pfifo_fast_init, 539 .init = pfifo_fast_init,
540 .reset = pfifo_fast_reset, 540 .reset = pfifo_fast_reset,
541 .dump = pfifo_fast_dump, 541 .dump = pfifo_fast_dump,
542 .owner = THIS_MODULE, 542 .owner = THIS_MODULE,
543 }; 543 };
544 EXPORT_SYMBOL(pfifo_fast_ops); 544 EXPORT_SYMBOL(pfifo_fast_ops);
545 545
546 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, 546 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
547 struct Qdisc_ops *ops) 547 struct Qdisc_ops *ops)
548 { 548 {
549 void *p; 549 void *p;
550 struct Qdisc *sch; 550 struct Qdisc *sch;
551 unsigned int size; 551 unsigned int size;
552 int err = -ENOBUFS; 552 int err = -ENOBUFS;
553 553
554 /* ensure that the Qdisc and the private data are 64-byte aligned */ 554 /* ensure that the Qdisc and the private data are 64-byte aligned */
555 size = QDISC_ALIGN(sizeof(*sch)); 555 size = QDISC_ALIGN(sizeof(*sch));
556 size += ops->priv_size + (QDISC_ALIGNTO - 1); 556 size += ops->priv_size + (QDISC_ALIGNTO - 1);
557 557
558 p = kzalloc_node(size, GFP_KERNEL, 558 p = kzalloc_node(size, GFP_KERNEL,
559 netdev_queue_numa_node_read(dev_queue)); 559 netdev_queue_numa_node_read(dev_queue));
560 560
561 if (!p) 561 if (!p)
562 goto errout; 562 goto errout;
563 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p); 563 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
564 sch->padded = (char *) sch - (char *) p; 564 sch->padded = (char *) sch - (char *) p;
565 565
566 INIT_LIST_HEAD(&sch->list); 566 INIT_LIST_HEAD(&sch->list);
567 skb_queue_head_init(&sch->q); 567 skb_queue_head_init(&sch->q);
568 spin_lock_init(&sch->busylock); 568 spin_lock_init(&sch->busylock);
569 sch->ops = ops; 569 sch->ops = ops;
570 sch->enqueue = ops->enqueue; 570 sch->enqueue = ops->enqueue;
571 sch->dequeue = ops->dequeue; 571 sch->dequeue = ops->dequeue;
572 sch->dev_queue = dev_queue; 572 sch->dev_queue = dev_queue;
573 dev_hold(qdisc_dev(sch)); 573 dev_hold(qdisc_dev(sch));
574 atomic_set(&sch->refcnt, 1); 574 atomic_set(&sch->refcnt, 1);
575 575
576 return sch; 576 return sch;
577 errout: 577 errout:
578 return ERR_PTR(err); 578 return ERR_PTR(err);
579 } 579 }
580 580
581 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, 581 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
582 struct Qdisc_ops *ops, unsigned int parentid) 582 struct Qdisc_ops *ops, unsigned int parentid)
583 { 583 {
584 struct Qdisc *sch; 584 struct Qdisc *sch;
585 585
586 sch = qdisc_alloc(dev_queue, ops); 586 sch = qdisc_alloc(dev_queue, ops);
587 if (IS_ERR(sch)) 587 if (IS_ERR(sch))
588 goto errout; 588 goto errout;
589 sch->parent = parentid; 589 sch->parent = parentid;
590 590
591 if (!ops->init || ops->init(sch, NULL) == 0) 591 if (!ops->init || ops->init(sch, NULL) == 0)
592 return sch; 592 return sch;
593 593
594 qdisc_destroy(sch); 594 qdisc_destroy(sch);
595 errout: 595 errout:
596 return NULL; 596 return NULL;
597 } 597 }
598 EXPORT_SYMBOL(qdisc_create_dflt); 598 EXPORT_SYMBOL(qdisc_create_dflt);
599 599
600 /* Under qdisc_lock(qdisc) and BH! */ 600 /* Under qdisc_lock(qdisc) and BH! */
601 601
602 void qdisc_reset(struct Qdisc *qdisc) 602 void qdisc_reset(struct Qdisc *qdisc)
603 { 603 {
604 const struct Qdisc_ops *ops = qdisc->ops; 604 const struct Qdisc_ops *ops = qdisc->ops;
605 605
606 if (ops->reset) 606 if (ops->reset)
607 ops->reset(qdisc); 607 ops->reset(qdisc);
608 608
609 if (qdisc->gso_skb) { 609 if (qdisc->gso_skb) {
610 kfree_skb(qdisc->gso_skb); 610 kfree_skb(qdisc->gso_skb);
611 qdisc->gso_skb = NULL; 611 qdisc->gso_skb = NULL;
612 qdisc->q.qlen = 0; 612 qdisc->q.qlen = 0;
613 } 613 }
614 } 614 }
615 EXPORT_SYMBOL(qdisc_reset); 615 EXPORT_SYMBOL(qdisc_reset);
616 616
617 static void qdisc_rcu_free(struct rcu_head *head) 617 static void qdisc_rcu_free(struct rcu_head *head)
618 { 618 {
619 struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head); 619 struct Qdisc *qdisc = container_of(head, struct Qdisc, rcu_head);
620 620
621 kfree((char *) qdisc - qdisc->padded); 621 kfree((char *) qdisc - qdisc->padded);
622 } 622 }
623 623
624 void qdisc_destroy(struct Qdisc *qdisc) 624 void qdisc_destroy(struct Qdisc *qdisc)
625 { 625 {
626 const struct Qdisc_ops *ops = qdisc->ops; 626 const struct Qdisc_ops *ops = qdisc->ops;
627 627
628 if (qdisc->flags & TCQ_F_BUILTIN || 628 if (qdisc->flags & TCQ_F_BUILTIN ||
629 !atomic_dec_and_test(&qdisc->refcnt)) 629 !atomic_dec_and_test(&qdisc->refcnt))
630 return; 630 return;
631 631
632 #ifdef CONFIG_NET_SCHED 632 #ifdef CONFIG_NET_SCHED
633 qdisc_list_del(qdisc); 633 qdisc_list_del(qdisc);
634 634
635 qdisc_put_stab(qdisc->stab); 635 qdisc_put_stab(rtnl_dereference(qdisc->stab));
636 #endif 636 #endif
637 gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est); 637 gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
638 if (ops->reset) 638 if (ops->reset)
639 ops->reset(qdisc); 639 ops->reset(qdisc);
640 if (ops->destroy) 640 if (ops->destroy)
641 ops->destroy(qdisc); 641 ops->destroy(qdisc);
642 642
643 module_put(ops->owner); 643 module_put(ops->owner);
644 dev_put(qdisc_dev(qdisc)); 644 dev_put(qdisc_dev(qdisc));
645 645
646 kfree_skb(qdisc->gso_skb); 646 kfree_skb(qdisc->gso_skb);
647 /* 647 /*
648 * gen_estimator est_timer() might access qdisc->q.lock, 648 * gen_estimator est_timer() might access qdisc->q.lock,
649 * wait a RCU grace period before freeing qdisc. 649 * wait a RCU grace period before freeing qdisc.
650 */ 650 */
651 call_rcu(&qdisc->rcu_head, qdisc_rcu_free); 651 call_rcu(&qdisc->rcu_head, qdisc_rcu_free);
652 } 652 }
653 EXPORT_SYMBOL(qdisc_destroy); 653 EXPORT_SYMBOL(qdisc_destroy);
654 654
655 /* Attach toplevel qdisc to device queue. */ 655 /* Attach toplevel qdisc to device queue. */
656 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, 656 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
657 struct Qdisc *qdisc) 657 struct Qdisc *qdisc)
658 { 658 {
659 struct Qdisc *oqdisc = dev_queue->qdisc_sleeping; 659 struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
660 spinlock_t *root_lock; 660 spinlock_t *root_lock;
661 661
662 root_lock = qdisc_lock(oqdisc); 662 root_lock = qdisc_lock(oqdisc);
663 spin_lock_bh(root_lock); 663 spin_lock_bh(root_lock);
664 664
665 /* Prune old scheduler */ 665 /* Prune old scheduler */
666 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) 666 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
667 qdisc_reset(oqdisc); 667 qdisc_reset(oqdisc);
668 668
669 /* ... and graft new one */ 669 /* ... and graft new one */
670 if (qdisc == NULL) 670 if (qdisc == NULL)
671 qdisc = &noop_qdisc; 671 qdisc = &noop_qdisc;
672 dev_queue->qdisc_sleeping = qdisc; 672 dev_queue->qdisc_sleeping = qdisc;
673 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc); 673 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
674 674
675 spin_unlock_bh(root_lock); 675 spin_unlock_bh(root_lock);
676 676
677 return oqdisc; 677 return oqdisc;
678 } 678 }
679 EXPORT_SYMBOL(dev_graft_qdisc); 679 EXPORT_SYMBOL(dev_graft_qdisc);
680 680
681 static void attach_one_default_qdisc(struct net_device *dev, 681 static void attach_one_default_qdisc(struct net_device *dev,
682 struct netdev_queue *dev_queue, 682 struct netdev_queue *dev_queue,
683 void *_unused) 683 void *_unused)
684 { 684 {
685 struct Qdisc *qdisc = &noqueue_qdisc; 685 struct Qdisc *qdisc = &noqueue_qdisc;
686 686
687 if (dev->tx_queue_len) { 687 if (dev->tx_queue_len) {
688 qdisc = qdisc_create_dflt(dev_queue, 688 qdisc = qdisc_create_dflt(dev_queue,
689 &pfifo_fast_ops, TC_H_ROOT); 689 &pfifo_fast_ops, TC_H_ROOT);
690 if (!qdisc) { 690 if (!qdisc) {
691 netdev_info(dev, "activation failed\n"); 691 netdev_info(dev, "activation failed\n");
692 return; 692 return;
693 } 693 }
694 694
695 /* Can by-pass the queue discipline for default qdisc */ 695 /* Can by-pass the queue discipline for default qdisc */
696 qdisc->flags |= TCQ_F_CAN_BYPASS; 696 qdisc->flags |= TCQ_F_CAN_BYPASS;
697 } 697 }
698 dev_queue->qdisc_sleeping = qdisc; 698 dev_queue->qdisc_sleeping = qdisc;
699 } 699 }
700 700
701 static void attach_default_qdiscs(struct net_device *dev) 701 static void attach_default_qdiscs(struct net_device *dev)
702 { 702 {
703 struct netdev_queue *txq; 703 struct netdev_queue *txq;
704 struct Qdisc *qdisc; 704 struct Qdisc *qdisc;
705 705
706 txq = netdev_get_tx_queue(dev, 0); 706 txq = netdev_get_tx_queue(dev, 0);
707 707
708 if (!netif_is_multiqueue(dev) || dev->tx_queue_len == 0) { 708 if (!netif_is_multiqueue(dev) || dev->tx_queue_len == 0) {
709 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL); 709 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
710 dev->qdisc = txq->qdisc_sleeping; 710 dev->qdisc = txq->qdisc_sleeping;
711 atomic_inc(&dev->qdisc->refcnt); 711 atomic_inc(&dev->qdisc->refcnt);
712 } else { 712 } else {
713 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT); 713 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT);
714 if (qdisc) { 714 if (qdisc) {
715 qdisc->ops->attach(qdisc); 715 qdisc->ops->attach(qdisc);
716 dev->qdisc = qdisc; 716 dev->qdisc = qdisc;
717 } 717 }
718 } 718 }
719 } 719 }
720 720
721 static void transition_one_qdisc(struct net_device *dev, 721 static void transition_one_qdisc(struct net_device *dev,
722 struct netdev_queue *dev_queue, 722 struct netdev_queue *dev_queue,
723 void *_need_watchdog) 723 void *_need_watchdog)
724 { 724 {
725 struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping; 725 struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
726 int *need_watchdog_p = _need_watchdog; 726 int *need_watchdog_p = _need_watchdog;
727 727
728 if (!(new_qdisc->flags & TCQ_F_BUILTIN)) 728 if (!(new_qdisc->flags & TCQ_F_BUILTIN))
729 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state); 729 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
730 730
731 rcu_assign_pointer(dev_queue->qdisc, new_qdisc); 731 rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
732 if (need_watchdog_p && new_qdisc != &noqueue_qdisc) { 732 if (need_watchdog_p && new_qdisc != &noqueue_qdisc) {
733 dev_queue->trans_start = 0; 733 dev_queue->trans_start = 0;
734 *need_watchdog_p = 1; 734 *need_watchdog_p = 1;
735 } 735 }
736 } 736 }
737 737
738 void dev_activate(struct net_device *dev) 738 void dev_activate(struct net_device *dev)
739 { 739 {
740 int need_watchdog; 740 int need_watchdog;
741 741
742 /* No queueing discipline is attached to device; 742 /* No queueing discipline is attached to device;
743 create default one i.e. pfifo_fast for devices, 743 create default one i.e. pfifo_fast for devices,
744 which need queueing and noqueue_qdisc for 744 which need queueing and noqueue_qdisc for
745 virtual interfaces 745 virtual interfaces
746 */ 746 */
747 747
748 if (dev->qdisc == &noop_qdisc) 748 if (dev->qdisc == &noop_qdisc)
749 attach_default_qdiscs(dev); 749 attach_default_qdiscs(dev);
750 750
751 if (!netif_carrier_ok(dev)) 751 if (!netif_carrier_ok(dev))
752 /* Delay activation until next carrier-on event */ 752 /* Delay activation until next carrier-on event */
753 return; 753 return;
754 754
755 need_watchdog = 0; 755 need_watchdog = 0;
756 netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog); 756 netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
757 if (dev_ingress_queue(dev)) 757 if (dev_ingress_queue(dev))
758 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL); 758 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
759 759
760 if (need_watchdog) { 760 if (need_watchdog) {
761 dev->trans_start = jiffies; 761 dev->trans_start = jiffies;
762 dev_watchdog_up(dev); 762 dev_watchdog_up(dev);
763 } 763 }
764 } 764 }
765 EXPORT_SYMBOL(dev_activate); 765 EXPORT_SYMBOL(dev_activate);
766 766
767 static void dev_deactivate_queue(struct net_device *dev, 767 static void dev_deactivate_queue(struct net_device *dev,
768 struct netdev_queue *dev_queue, 768 struct netdev_queue *dev_queue,
769 void *_qdisc_default) 769 void *_qdisc_default)
770 { 770 {
771 struct Qdisc *qdisc_default = _qdisc_default; 771 struct Qdisc *qdisc_default = _qdisc_default;
772 struct Qdisc *qdisc; 772 struct Qdisc *qdisc;
773 773
774 qdisc = dev_queue->qdisc; 774 qdisc = dev_queue->qdisc;
775 if (qdisc) { 775 if (qdisc) {
776 spin_lock_bh(qdisc_lock(qdisc)); 776 spin_lock_bh(qdisc_lock(qdisc));
777 777
778 if (!(qdisc->flags & TCQ_F_BUILTIN)) 778 if (!(qdisc->flags & TCQ_F_BUILTIN))
779 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state); 779 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
780 780
781 rcu_assign_pointer(dev_queue->qdisc, qdisc_default); 781 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
782 qdisc_reset(qdisc); 782 qdisc_reset(qdisc);
783 783
784 spin_unlock_bh(qdisc_lock(qdisc)); 784 spin_unlock_bh(qdisc_lock(qdisc));
785 } 785 }
786 } 786 }
787 787
788 static bool some_qdisc_is_busy(struct net_device *dev) 788 static bool some_qdisc_is_busy(struct net_device *dev)
789 { 789 {
790 unsigned int i; 790 unsigned int i;
791 791
792 for (i = 0; i < dev->num_tx_queues; i++) { 792 for (i = 0; i < dev->num_tx_queues; i++) {
793 struct netdev_queue *dev_queue; 793 struct netdev_queue *dev_queue;
794 spinlock_t *root_lock; 794 spinlock_t *root_lock;
795 struct Qdisc *q; 795 struct Qdisc *q;
796 int val; 796 int val;
797 797
798 dev_queue = netdev_get_tx_queue(dev, i); 798 dev_queue = netdev_get_tx_queue(dev, i);
799 q = dev_queue->qdisc_sleeping; 799 q = dev_queue->qdisc_sleeping;
800 root_lock = qdisc_lock(q); 800 root_lock = qdisc_lock(q);
801 801
802 spin_lock_bh(root_lock); 802 spin_lock_bh(root_lock);
803 803
804 val = (qdisc_is_running(q) || 804 val = (qdisc_is_running(q) ||
805 test_bit(__QDISC_STATE_SCHED, &q->state)); 805 test_bit(__QDISC_STATE_SCHED, &q->state));
806 806
807 spin_unlock_bh(root_lock); 807 spin_unlock_bh(root_lock);
808 808
809 if (val) 809 if (val)
810 return true; 810 return true;
811 } 811 }
812 return false; 812 return false;
813 } 813 }
814 814
815 void dev_deactivate_many(struct list_head *head) 815 void dev_deactivate_many(struct list_head *head)
816 { 816 {
817 struct net_device *dev; 817 struct net_device *dev;
818 818
819 list_for_each_entry(dev, head, unreg_list) { 819 list_for_each_entry(dev, head, unreg_list) {
820 netdev_for_each_tx_queue(dev, dev_deactivate_queue, 820 netdev_for_each_tx_queue(dev, dev_deactivate_queue,
821 &noop_qdisc); 821 &noop_qdisc);
822 if (dev_ingress_queue(dev)) 822 if (dev_ingress_queue(dev))
823 dev_deactivate_queue(dev, dev_ingress_queue(dev), 823 dev_deactivate_queue(dev, dev_ingress_queue(dev),
824 &noop_qdisc); 824 &noop_qdisc);
825 825
826 dev_watchdog_down(dev); 826 dev_watchdog_down(dev);
827 } 827 }
828 828
829 /* Wait for outstanding qdisc-less dev_queue_xmit calls. */ 829 /* Wait for outstanding qdisc-less dev_queue_xmit calls. */
830 synchronize_rcu(); 830 synchronize_rcu();
831 831
832 /* Wait for outstanding qdisc_run calls. */ 832 /* Wait for outstanding qdisc_run calls. */
833 list_for_each_entry(dev, head, unreg_list) 833 list_for_each_entry(dev, head, unreg_list)
834 while (some_qdisc_is_busy(dev)) 834 while (some_qdisc_is_busy(dev))
835 yield(); 835 yield();
836 } 836 }
837 837
838 void dev_deactivate(struct net_device *dev) 838 void dev_deactivate(struct net_device *dev)
839 { 839 {
840 LIST_HEAD(single); 840 LIST_HEAD(single);
841 841
842 list_add(&dev->unreg_list, &single); 842 list_add(&dev->unreg_list, &single);
843 dev_deactivate_many(&single); 843 dev_deactivate_many(&single);
844 } 844 }
845 EXPORT_SYMBOL(dev_deactivate); 845 EXPORT_SYMBOL(dev_deactivate);
846 846
847 static void dev_init_scheduler_queue(struct net_device *dev, 847 static void dev_init_scheduler_queue(struct net_device *dev,
848 struct netdev_queue *dev_queue, 848 struct netdev_queue *dev_queue,
849 void *_qdisc) 849 void *_qdisc)
850 { 850 {
851 struct Qdisc *qdisc = _qdisc; 851 struct Qdisc *qdisc = _qdisc;
852 852
853 dev_queue->qdisc = qdisc; 853 dev_queue->qdisc = qdisc;
854 dev_queue->qdisc_sleeping = qdisc; 854 dev_queue->qdisc_sleeping = qdisc;
855 } 855 }
856 856
857 void dev_init_scheduler(struct net_device *dev) 857 void dev_init_scheduler(struct net_device *dev)
858 { 858 {
859 dev->qdisc = &noop_qdisc; 859 dev->qdisc = &noop_qdisc;
860 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc); 860 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
861 if (dev_ingress_queue(dev)) 861 if (dev_ingress_queue(dev))
862 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc); 862 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
863 863
864 setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev); 864 setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
865 } 865 }
866 866
867 static void shutdown_scheduler_queue(struct net_device *dev, 867 static void shutdown_scheduler_queue(struct net_device *dev,
868 struct netdev_queue *dev_queue, 868 struct netdev_queue *dev_queue,
869 void *_qdisc_default) 869 void *_qdisc_default)
870 { 870 {
871 struct Qdisc *qdisc = dev_queue->qdisc_sleeping; 871 struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
872 struct Qdisc *qdisc_default = _qdisc_default; 872 struct Qdisc *qdisc_default = _qdisc_default;
873 873
874 if (qdisc) { 874 if (qdisc) {
875 rcu_assign_pointer(dev_queue->qdisc, qdisc_default); 875 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
876 dev_queue->qdisc_sleeping = qdisc_default; 876 dev_queue->qdisc_sleeping = qdisc_default;
877 877
878 qdisc_destroy(qdisc); 878 qdisc_destroy(qdisc);
879 } 879 }
880 } 880 }
881 881
882 void dev_shutdown(struct net_device *dev) 882 void dev_shutdown(struct net_device *dev)
883 { 883 {
884 netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); 884 netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
885 if (dev_ingress_queue(dev)) 885 if (dev_ingress_queue(dev))
886 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc); 886 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
887 qdisc_destroy(dev->qdisc); 887 qdisc_destroy(dev->qdisc);
888 dev->qdisc = &noop_qdisc; 888 dev->qdisc = &noop_qdisc;
889 889
890 WARN_ON(timer_pending(&dev->watchdog_timer)); 890 WARN_ON(timer_pending(&dev->watchdog_timer));
891 } 891 }
892 892