net: sched: RCU cls_route

RCUify the route classifier. For now however spinlock's are used to protect fastmap cache. The issue here is the fastmap may be read by one CPU while the cache is being updated by another. An array of pointers could be one possible solution. Signed-off-by: John Fastabend <john.r.fastabend@intel.com> Acked-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>

net: sched: RCU cls_route
RCUify the route classifier. For now however spinlock's are used to protect fastmap cache. The issue here is the fastmap may be read by one CPU while the cache is being updated by another. An array of pointers could be one possible solution. Signed-off-by: John Fastabend <john.r.fastabend@intel.com> Acked-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
John Fastabend · David S. Miller
1 parent e35a8ee599
Showing 1 changed file with 132 additions and 94 deletions Side-by-side Diff
net/sched/cls_route.c
@@ -29,25 +29,26 @@
  *    are mutually  exclusive.
  * 3. "to TAG from ANY" has higher priority, than "to ANY from XXX"
  */
-
 struct route4_fastmap {
-	struct route4_filter	*filter;
-	u32			id;
-	int			iif;
+	struct route4_filter		*filter;
+	u32				id;
+	int				iif;
 };
  
 struct route4_head {
-	struct route4_fastmap	fastmap[16];
-	struct route4_bucket	*table[256 + 1];
+	struct route4_fastmap		fastmap[16];
+	struct route4_bucket __rcu	*table[256 + 1];
+	struct rcu_head			rcu;
 };
  
 struct route4_bucket {
 	/* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */
-	struct route4_filter	*ht[16 + 16 + 1];
+	struct route4_filter __rcu	*ht[16 + 16 + 1];
+	struct rcu_head			rcu;
 };
  
 struct route4_filter {
-	struct route4_filter	*next;
+	struct route4_filter __rcu	*next;
 	u32			id;
 	int			iif;
  
@@ -55,6 +56,8 @@
 	struct tcf_exts		exts;
 	u32			handle;
 	struct route4_bucket	*bkt;
+	struct tcf_proto	*tp;
+	struct rcu_head		rcu;
 };
  
 #define ROUTE4_FAILURE ((struct route4_filter *)(-1L))
  
  
  
@@ -64,14 +67,13 @@
 	return id & 0xF;
 }
  
+static DEFINE_SPINLOCK(fastmap_lock);
 static void
-route4_reset_fastmap(struct Qdisc *q, struct route4_head *head, u32 id)
+route4_reset_fastmap(struct route4_head *head)
 {
-	spinlock_t *root_lock = qdisc_root_sleeping_lock(q);
-
-	spin_lock_bh(root_lock);
+	spin_lock_bh(&fastmap_lock);
 	memset(head->fastmap, 0, sizeof(head->fastmap));
-	spin_unlock_bh(root_lock);
+	spin_unlock_bh(&fastmap_lock);
 }
  
 static void
  
@@ -80,9 +82,12 @@
 {
 	int h = route4_fastmap_hash(id, iif);
  
+	/* fastmap updates must look atomic to aling id, iff, filter */
+	spin_lock_bh(&fastmap_lock);
 	head->fastmap[h].id = id;
 	head->fastmap[h].iif = iif;
 	head->fastmap[h].filter = f;
+	spin_unlock_bh(&fastmap_lock);
 }
  
 static inline int route4_hash_to(u32 id)
@@ -123,7 +128,7 @@
 static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp,
 			   struct tcf_result *res)
 {
-	struct route4_head *head = tp->root;
+	struct route4_head *head = rcu_dereference_bh(tp->root);
 	struct dst_entry *dst;
 	struct route4_bucket *b;
 	struct route4_filter *f;
  
  
  
  
  
  
  
  
  
@@ -141,32 +146,43 @@
 	iif = inet_iif(skb);
  
 	h = route4_fastmap_hash(id, iif);
+
+	spin_lock(&fastmap_lock);
 	if (id == head->fastmap[h].id &&
 	    iif == head->fastmap[h].iif &&
 	    (f = head->fastmap[h].filter) != NULL) {
-		if (f == ROUTE4_FAILURE)
+		if (f == ROUTE4_FAILURE) {
+			spin_unlock(&fastmap_lock);
 			goto failure;
+		}
  
 		*res = f->res;
+		spin_unlock(&fastmap_lock);
 		return 0;
 	}
+	spin_unlock(&fastmap_lock);
  
 	h = route4_hash_to(id);
  
 restart:
-	b = head->table[h];
+	b = rcu_dereference_bh(head->table[h]);
 	if (b) {
-		for (f = b->ht[route4_hash_from(id)]; f; f = f->next)
+		for (f = rcu_dereference_bh(b->ht[route4_hash_from(id)]);
+		     f;
+		     f = rcu_dereference_bh(f->next))
 			if (f->id == id)
 				ROUTE4_APPLY_RESULT();
  
-		for (f = b->ht[route4_hash_iif(iif)]; f; f = f->next)
+		for (f = rcu_dereference_bh(b->ht[route4_hash_iif(iif)]);
+		     f;
+		     f = rcu_dereference_bh(f->next))
 			if (f->iif == iif)
 				ROUTE4_APPLY_RESULT();
  
-		for (f = b->ht[route4_hash_wild()]; f; f = f->next)
+		for (f = rcu_dereference_bh(b->ht[route4_hash_wild()]);
+		     f;
+		     f = rcu_dereference_bh(f->next))
 			ROUTE4_APPLY_RESULT();
-
 	}
 	if (h < 256) {
 		h = 256;
@@ -213,7 +229,7 @@
  
 static unsigned long route4_get(struct tcf_proto *tp, u32 handle)
 {
-	struct route4_head *head = tp->root;
+	struct route4_head *head = rtnl_dereference(tp->root);
 	struct route4_bucket *b;
 	struct route4_filter *f;
 	unsigned int h1, h2;
  
@@ -229,9 +245,11 @@
 	if (h2 > 32)
 		return 0;
  
-	b = head->table[h1];
+	b = rtnl_dereference(head->table[h1]);
 	if (b) {
-		for (f = b->ht[h2]; f; f = f->next)
+		for (f = rtnl_dereference(b->ht[h2]);
+		     f;
+		     f = rtnl_dereference(f->next))
 			if (f->handle == handle)
 				return (unsigned long)f;
 	}
  
@@ -248,8 +266,11 @@
 }
  
 static void
-route4_delete_filter(struct tcf_proto *tp, struct route4_filter *f)
+route4_delete_filter(struct rcu_head *head)
 {
+	struct route4_filter *f = container_of(head, struct route4_filter, rcu);
+	struct tcf_proto *tp = f->tp;
+
 	tcf_unbind_filter(tp, &f->res);
 	tcf_exts_destroy(tp, &f->exts);
 	kfree(f);
@@ -257,7 +278,7 @@
  
 static void route4_destroy(struct tcf_proto *tp)
 {
-	struct route4_head *head = tp->root;
+	struct route4_head *head = rtnl_dereference(tp->root);
 	int h1, h2;
  
 	if (head == NULL)
  
  
  
  
  
@@ -266,28 +287,35 @@
 	for (h1 = 0; h1 <= 256; h1++) {
 		struct route4_bucket *b;
  
-		b = head->table[h1];
+		b = rtnl_dereference(head->table[h1]);
 		if (b) {
 			for (h2 = 0; h2 <= 32; h2++) {
 				struct route4_filter *f;
  
-				while ((f = b->ht[h2]) != NULL) {
-					b->ht[h2] = f->next;
-					route4_delete_filter(tp, f);
+				while ((f = rtnl_dereference(b->ht[h2])) != NULL) {
+					struct route4_filter *next;
+
+					next = rtnl_dereference(f->next);
+					RCU_INIT_POINTER(b->ht[h2], next);
+					call_rcu(&f->rcu, route4_delete_filter);
 				}
 			}
-			kfree(b);
+			RCU_INIT_POINTER(head->table[h1], NULL);
+			kfree_rcu(b, rcu);
 		}
 	}
-	kfree(head);
+	RCU_INIT_POINTER(tp->root, NULL);
+	kfree_rcu(head, rcu);
 }
  
 static int route4_delete(struct tcf_proto *tp, unsigned long arg)
 {
-	struct route4_head *head = tp->root;
-	struct route4_filter **fp, *f = (struct route4_filter *)arg;
-	unsigned int h = 0;
+	struct route4_head *head = rtnl_dereference(tp->root);
+	struct route4_filter *f = (struct route4_filter *)arg;
+	struct route4_filter __rcu **fp;
+	struct route4_filter *nf;
 	struct route4_bucket *b;
+	unsigned int h = 0;
 	int i;
  
 	if (!head || !f)
  
  
  
  
  
  
@@ -296,27 +324,35 @@
 	h = f->handle;
 	b = f->bkt;
  
-	for (fp = &b->ht[from_hash(h >> 16)]; *fp; fp = &(*fp)->next) {
-		if (*fp == f) {
-			tcf_tree_lock(tp);
-			*fp = f->next;
-			tcf_tree_unlock(tp);
+	fp = &b->ht[from_hash(h >> 16)];
+	for (nf = rtnl_dereference(*fp); nf;
+	     fp = &nf->next, nf = rtnl_dereference(*fp)) {
+		if (nf == f) {
+			/* unlink it */
+			RCU_INIT_POINTER(*fp, rtnl_dereference(f->next));
  
-			route4_reset_fastmap(tp->q, head, f->id);
-			route4_delete_filter(tp, f);
+			/* Remove any fastmap lookups that might ref filter
+			 * notice we unlink'd the filter so we can't get it
+			 * back in the fastmap.
+			 */
+			route4_reset_fastmap(head);
  
-			/* Strip tree */
+			/* Delete it */
+			call_rcu(&f->rcu, route4_delete_filter);
  
-			for (i = 0; i <= 32; i++)
-				if (b->ht[i])
+			/* Strip RTNL protected tree */
+			for (i = 0; i <= 32; i++) {
+				struct route4_filter *rt;
+
+				rt = rtnl_dereference(b->ht[i]);
+				if (rt)
 					return 0;
+			}
  
 			/* OK, session has no flows */
-			tcf_tree_lock(tp);
-			head->table[to_hash(h)] = NULL;
-			tcf_tree_unlock(tp);
+			RCU_INIT_POINTER(head->table[to_hash(h)], NULL);
+			kfree_rcu(b, rcu);
  
-			kfree(b);
 			return 0;
 		}
 	}
  
  
  
@@ -380,26 +416,25 @@
 	}
  
 	h1 = to_hash(nhandle);
-	b = head->table[h1];
+	b = rtnl_dereference(head->table[h1]);
 	if (!b) {
 		err = -ENOBUFS;
 		b = kzalloc(sizeof(struct route4_bucket), GFP_KERNEL);
 		if (b == NULL)
 			goto errout;
  
-		tcf_tree_lock(tp);
-		head->table[h1] = b;
-		tcf_tree_unlock(tp);
+		rcu_assign_pointer(head->table[h1], b);
 	} else {
 		unsigned int h2 = from_hash(nhandle >> 16);
  
 		err = -EEXIST;
-		for (fp = b->ht[h2]; fp; fp = fp->next)
+		for (fp = rtnl_dereference(b->ht[h2]);
+		     fp;
+		     fp = rtnl_dereference(fp->next))
 			if (fp->handle == f->handle)
 				goto errout;
 	}
  
-	tcf_tree_lock(tp);
 	if (tb[TCA_ROUTE4_TO])
 		f->id = to;
  
@@ -410,7 +445,7 @@
  
 	f->handle = nhandle;
 	f->bkt = b;
-	tcf_tree_unlock(tp);
+	f->tp = tp;
  
 	if (tb[TCA_ROUTE4_CLASSID]) {
 		f->res.classid = nla_get_u32(tb[TCA_ROUTE4_CLASSID]);
  
  
@@ -431,14 +466,15 @@
 		       struct nlattr **tca,
 		       unsigned long *arg, bool ovr)
 {
-	struct route4_head *head = tp->root;
-	struct route4_filter *f, *f1, **fp;
+	struct route4_head *head = rtnl_dereference(tp->root);
+	struct route4_filter __rcu **fp;
+	struct route4_filter *fold, *f1, *pfp, *f = NULL;
 	struct route4_bucket *b;
 	struct nlattr *opt = tca[TCA_OPTIONS];
 	struct nlattr *tb[TCA_ROUTE4_MAX + 1];
 	unsigned int h, th;
-	u32 old_handle = 0;
 	int err;
+	bool new = true;
  
 	if (opt == NULL)
 		return handle ? -EINVAL : 0;
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -447,70 +483,70 @@
 	if (err < 0)
 		return err;
  
-	f = (struct route4_filter *)*arg;
-	if (f) {
-		if (f->handle != handle && handle)
+	fold = (struct route4_filter *)*arg;
+	if (fold && handle && fold->handle != handle)
 			return -EINVAL;
  
-		if (f->bkt)
-			old_handle = f->handle;
-
-		err = route4_set_parms(net, tp, base, f, handle, head, tb,
-			tca[TCA_RATE], 0, ovr);
-		if (err < 0)
-			return err;
-
-		goto reinsert;
-	}
-
 	err = -ENOBUFS;
 	if (head == NULL) {
 		head = kzalloc(sizeof(struct route4_head), GFP_KERNEL);
 		if (head == NULL)
 			goto errout;
-
-		tcf_tree_lock(tp);
-		tp->root = head;
-		tcf_tree_unlock(tp);
+		rcu_assign_pointer(tp->root, head);
 	}
  
 	f = kzalloc(sizeof(struct route4_filter), GFP_KERNEL);
-	if (f == NULL)
+	if (!f)
 		goto errout;
  
 	tcf_exts_init(&f->exts, TCA_ROUTE4_ACT, TCA_ROUTE4_POLICE);
+	if (fold) {
+		f->id = fold->id;
+		f->iif = fold->iif;
+		f->res = fold->res;
+		f->handle = fold->handle;
+
+		f->tp = fold->tp;
+		f->bkt = fold->bkt;
+		new = false;
+	}
+
 	err = route4_set_parms(net, tp, base, f, handle, head, tb,
-		tca[TCA_RATE], 1, ovr);
+			       tca[TCA_RATE], new, ovr);
 	if (err < 0)
 		goto errout;
  
-reinsert:
 	h = from_hash(f->handle >> 16);
-	for (fp = &f->bkt->ht[h]; (f1 = *fp) != NULL; fp = &f1->next)
+	fp = &f->bkt->ht[h];
+	for (pfp = rtnl_dereference(*fp);
+	     (f1 = rtnl_dereference(*fp)) != NULL;
+	     fp = &f1->next)
 		if (f->handle < f1->handle)
 			break;
  
-	f->next = f1;
-	tcf_tree_lock(tp);
-	*fp = f;
+	rcu_assign_pointer(f->next, f1);
+	rcu_assign_pointer(*fp, f);
  
-	if (old_handle && f->handle != old_handle) {
-		th = to_hash(old_handle);
-		h = from_hash(old_handle >> 16);
-		b = head->table[th];
+	if (fold && fold->handle && f->handle != fold->handle) {
+		th = to_hash(fold->handle);
+		h = from_hash(fold->handle >> 16);
+		b = rtnl_dereference(head->table[th]);
 		if (b) {
-			for (fp = &b->ht[h]; *fp; fp = &(*fp)->next) {
-				if (*fp == f) {
+			fp = &b->ht[h];
+			for (pfp = rtnl_dereference(*fp); pfp;
+			     fp = &pfp->next, pfp = rtnl_dereference(*fp)) {
+				if (pfp == f) {
 					*fp = f->next;
 					break;
 				}
 			}
 		}
 	}
-	tcf_tree_unlock(tp);
  
-	route4_reset_fastmap(tp->q, head, f->id);
+	route4_reset_fastmap(head);
 	*arg = (unsigned long)f;
+	if (fold)
+		call_rcu(&fold->rcu, route4_delete_filter);
 	return 0;
  
 errout:
@@ -520,7 +556,7 @@
  
 static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg)
 {
-	struct route4_head *head = tp->root;
+	struct route4_head *head = rtnl_dereference(tp->root);
 	unsigned int h, h1;
  
 	if (head == NULL)
  
@@ -530,13 +566,15 @@
 		return;
  
 	for (h = 0; h <= 256; h++) {
-		struct route4_bucket *b = head->table[h];
+		struct route4_bucket *b = rtnl_dereference(head->table[h]);
  
 		if (b) {
 			for (h1 = 0; h1 <= 32; h1++) {
 				struct route4_filter *f;
  
-				for (f = b->ht[h1]; f; f = f->next) {
+				for (f = rtnl_dereference(b->ht[h1]);
+				     f;
+				     f = rtnl_dereference(f->next)) {
 					if (arg->count < arg->skip) {
 						arg->count++;
 						continue;
...	...	@@ -29,25 +29,26 @@
29	29	* are mutually exclusive.
30	30	* 3. "to TAG from ANY" has higher priority, than "to ANY from XXX"
31	31	*/
32		-
33	32	struct route4_fastmap {
34		- struct route4_filter *filter;
35		- u32 id;
36		- int iif;
	33	+ struct route4_filter *filter;
	34	+ u32 id;
	35	+ int iif;
37	36	};
38	37
39	38	struct route4_head {
40		- struct route4_fastmap fastmap[16];
41		- struct route4_bucket *table[256 + 1];
	39	+ struct route4_fastmap fastmap[16];
	40	+ struct route4_bucket __rcu *table[256 + 1];
	41	+ struct rcu_head rcu;
42	42	};
43	43
44	44	struct route4_bucket {
45	45	/* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */
46		- struct route4_filter *ht[16 + 16 + 1];
	46	+ struct route4_filter __rcu *ht[16 + 16 + 1];
	47	+ struct rcu_head rcu;
47	48	};
48	49
49	50	struct route4_filter {
50		- struct route4_filter *next;
	51	+ struct route4_filter __rcu *next;
51	52	u32 id;
52	53	int iif;
53	54
...	...	@@ -55,6 +56,8 @@
55	56	struct tcf_exts exts;
56	57	u32 handle;
57	58	struct route4_bucket *bkt;
	59	+ struct tcf_proto *tp;
	60	+ struct rcu_head rcu;
58	61	};
59	62
60	63	#define ROUTE4_FAILURE ((struct route4_filter *)(-1L))
61	64
62	65
63	66
...	...	@@ -64,14 +67,13 @@
64	67	return id & 0xF;
65	68	}
66	69
	70	+static DEFINE_SPINLOCK(fastmap_lock);
67	71	static void
68		-route4_reset_fastmap(struct Qdisc q, struct route4_head head, u32 id)
	72	+route4_reset_fastmap(struct route4_head *head)
69	73	{
70		- spinlock_t *root_lock = qdisc_root_sleeping_lock(q);
71		-
72		- spin_lock_bh(root_lock);
	74	+ spin_lock_bh(&fastmap_lock);
73	75	memset(head->fastmap, 0, sizeof(head->fastmap));
74		- spin_unlock_bh(root_lock);
	76	+ spin_unlock_bh(&fastmap_lock);
75	77	}
76	78
77	79	static void
78	80
...	...	@@ -80,9 +82,12 @@
80	82	{
81	83	int h = route4_fastmap_hash(id, iif);
82	84
	85	+ /* fastmap updates must look atomic to aling id, iff, filter */
	86	+ spin_lock_bh(&fastmap_lock);
83	87	head->fastmap[h].id = id;
84	88	head->fastmap[h].iif = iif;
85	89	head->fastmap[h].filter = f;
	90	+ spin_unlock_bh(&fastmap_lock);
86	91	}
87	92
88	93	static inline int route4_hash_to(u32 id)
...	...	@@ -123,7 +128,7 @@
123	128	static int route4_classify(struct sk_buff skb, const struct tcf_proto tp,
124	129	struct tcf_result *res)
125	130	{
126		- struct route4_head *head = tp->root;
	131	+ struct route4_head *head = rcu_dereference_bh(tp->root);
127	132	struct dst_entry *dst;
128	133	struct route4_bucket *b;
129	134	struct route4_filter *f;
130	135
131	136
132	137
133	138
134	139
135	140
136	141
137	142
138	143
...	...	@@ -141,32 +146,43 @@
141	146	iif = inet_iif(skb);
142	147
143	148	h = route4_fastmap_hash(id, iif);
	149	+
	150	+ spin_lock(&fastmap_lock);
144	151	if (id == head->fastmap[h].id &&
145	152	iif == head->fastmap[h].iif &&
146	153	(f = head->fastmap[h].filter) != NULL) {
147		- if (f == ROUTE4_FAILURE)
	154	+ if (f == ROUTE4_FAILURE) {
	155	+ spin_unlock(&fastmap_lock);
148	156	goto failure;
	157	+ }
149	158
150	159	*res = f->res;
	160	+ spin_unlock(&fastmap_lock);
151	161	return 0;
152	162	}
	163	+ spin_unlock(&fastmap_lock);
153	164
154	165	h = route4_hash_to(id);
155	166
156	167	restart:
157		- b = head->table[h];
	168	+ b = rcu_dereference_bh(head->table[h]);
158	169	if (b) {
159		- for (f = b->ht[route4_hash_from(id)]; f; f = f->next)
	170	+ for (f = rcu_dereference_bh(b->ht[route4_hash_from(id)]);
	171	+ f;
	172	+ f = rcu_dereference_bh(f->next))
160	173	if (f->id == id)
161	174	ROUTE4_APPLY_RESULT();
162	175
163		- for (f = b->ht[route4_hash_iif(iif)]; f; f = f->next)
	176	+ for (f = rcu_dereference_bh(b->ht[route4_hash_iif(iif)]);
	177	+ f;
	178	+ f = rcu_dereference_bh(f->next))
164	179	if (f->iif == iif)
165	180	ROUTE4_APPLY_RESULT();
166	181
167		- for (f = b->ht[route4_hash_wild()]; f; f = f->next)
	182	+ for (f = rcu_dereference_bh(b->ht[route4_hash_wild()]);
	183	+ f;
	184	+ f = rcu_dereference_bh(f->next))
168	185	ROUTE4_APPLY_RESULT();
169		-
170	186	}
171	187	if (h < 256) {
172	188	h = 256;
...	...	@@ -213,7 +229,7 @@
213	229
214	230	static unsigned long route4_get(struct tcf_proto *tp, u32 handle)
215	231	{
216		- struct route4_head *head = tp->root;
	232	+ struct route4_head *head = rtnl_dereference(tp->root);
217	233	struct route4_bucket *b;
218	234	struct route4_filter *f;
219	235	unsigned int h1, h2;
220	236
...	...	@@ -229,9 +245,11 @@
229	245	if (h2 > 32)
230	246	return 0;
231	247
232		- b = head->table[h1];
	248	+ b = rtnl_dereference(head->table[h1]);
233	249	if (b) {
234		- for (f = b->ht[h2]; f; f = f->next)
	250	+ for (f = rtnl_dereference(b->ht[h2]);
	251	+ f;
	252	+ f = rtnl_dereference(f->next))
235	253	if (f->handle == handle)
236	254	return (unsigned long)f;
237	255	}
238	256
...	...	@@ -248,8 +266,11 @@
248	266	}
249	267
250	268	static void
251		-route4_delete_filter(struct tcf_proto tp, struct route4_filter f)
	269	+route4_delete_filter(struct rcu_head *head)
252	270	{
	271	+ struct route4_filter *f = container_of(head, struct route4_filter, rcu);
	272	+ struct tcf_proto *tp = f->tp;
	273	+
253	274	tcf_unbind_filter(tp, &f->res);
254	275	tcf_exts_destroy(tp, &f->exts);
255	276	kfree(f);
...	...	@@ -257,7 +278,7 @@
257	278
258	279	static void route4_destroy(struct tcf_proto *tp)
259	280	{
260		- struct route4_head *head = tp->root;
	281	+ struct route4_head *head = rtnl_dereference(tp->root);
261	282	int h1, h2;
262	283
263	284	if (head == NULL)
264	285
265	286
266	287
267	288
268	289
...	...	@@ -266,28 +287,35 @@
266	287	for (h1 = 0; h1 <= 256; h1++) {
267	288	struct route4_bucket *b;
268	289
269		- b = head->table[h1];
	290	+ b = rtnl_dereference(head->table[h1]);
270	291	if (b) {
271	292	for (h2 = 0; h2 <= 32; h2++) {
272	293	struct route4_filter *f;
273	294
274		- while ((f = b->ht[h2]) != NULL) {
275		- b->ht[h2] = f->next;
276		- route4_delete_filter(tp, f);
	295	+ while ((f = rtnl_dereference(b->ht[h2])) != NULL) {
	296	+ struct route4_filter *next;
	297	+
	298	+ next = rtnl_dereference(f->next);
	299	+ RCU_INIT_POINTER(b->ht[h2], next);
	300	+ call_rcu(&f->rcu, route4_delete_filter);
277	301	}
278	302	}
279		- kfree(b);
	303	+ RCU_INIT_POINTER(head->table[h1], NULL);
	304	+ kfree_rcu(b, rcu);
280	305	}
281	306	}
282		- kfree(head);
	307	+ RCU_INIT_POINTER(tp->root, NULL);
	308	+ kfree_rcu(head, rcu);
283	309	}
284	310
285	311	static int route4_delete(struct tcf_proto *tp, unsigned long arg)
286	312	{
287		- struct route4_head *head = tp->root;
288		- struct route4_filter *fp, f = (struct route4_filter *)arg;
289		- unsigned int h = 0;
	313	+ struct route4_head *head = rtnl_dereference(tp->root);
	314	+ struct route4_filter f = (struct route4_filter )arg;
	315	+ struct route4_filter __rcu **fp;
	316	+ struct route4_filter *nf;
290	317	struct route4_bucket *b;
	318	+ unsigned int h = 0;
291	319	int i;
292	320
293	321	if (!head \|\| !f)
294	322
295	323
296	324
297	325
298	326
299	327
...	...	@@ -296,27 +324,35 @@
296	324	h = f->handle;
297	325	b = f->bkt;
298	326
299		- for (fp = &b->ht[from_hash(h >> 16)]; fp; fp = &(fp)->next) {
300		- if (*fp == f) {
301		- tcf_tree_lock(tp);
302		- *fp = f->next;
303		- tcf_tree_unlock(tp);
	327	+ fp = &b->ht[from_hash(h >> 16)];
	328	+ for (nf = rtnl_dereference(*fp); nf;
	329	+ fp = &nf->next, nf = rtnl_dereference(*fp)) {
	330	+ if (nf == f) {
	331	+ /* unlink it */
	332	+ RCU_INIT_POINTER(*fp, rtnl_dereference(f->next));
304	333
305		- route4_reset_fastmap(tp->q, head, f->id);
306		- route4_delete_filter(tp, f);
	334	+ /* Remove any fastmap lookups that might ref filter
	335	+ * notice we unlink'd the filter so we can't get it
	336	+ * back in the fastmap.
	337	+ */
	338	+ route4_reset_fastmap(head);
307	339
308		- /* Strip tree */
	340	+ /* Delete it */
	341	+ call_rcu(&f->rcu, route4_delete_filter);
309	342
310		- for (i = 0; i <= 32; i++)
311		- if (b->ht[i])
	343	+ /* Strip RTNL protected tree */
	344	+ for (i = 0; i <= 32; i++) {
	345	+ struct route4_filter *rt;
	346	+
	347	+ rt = rtnl_dereference(b->ht[i]);
	348	+ if (rt)
312	349	return 0;
	350	+ }
313	351
314	352	/* OK, session has no flows */
315		- tcf_tree_lock(tp);
316		- head->table[to_hash(h)] = NULL;
317		- tcf_tree_unlock(tp);
	353	+ RCU_INIT_POINTER(head->table[to_hash(h)], NULL);
	354	+ kfree_rcu(b, rcu);
318	355
319		- kfree(b);
320	356	return 0;
321	357	}
322	358	}
323	359
324	360
325	361
...	...	@@ -380,26 +416,25 @@
380	416	}
381	417
382	418	h1 = to_hash(nhandle);
383		- b = head->table[h1];
	419	+ b = rtnl_dereference(head->table[h1]);
384	420	if (!b) {
385	421	err = -ENOBUFS;
386	422	b = kzalloc(sizeof(struct route4_bucket), GFP_KERNEL);
387	423	if (b == NULL)
388	424	goto errout;
389	425
390		- tcf_tree_lock(tp);
391		- head->table[h1] = b;
392		- tcf_tree_unlock(tp);
	426	+ rcu_assign_pointer(head->table[h1], b);
393	427	} else {
394	428	unsigned int h2 = from_hash(nhandle >> 16);
395	429
396	430	err = -EEXIST;
397		- for (fp = b->ht[h2]; fp; fp = fp->next)
	431	+ for (fp = rtnl_dereference(b->ht[h2]);
	432	+ fp;
	433	+ fp = rtnl_dereference(fp->next))
398	434	if (fp->handle == f->handle)
399	435	goto errout;
400	436	}
401	437
402		- tcf_tree_lock(tp);
403	438	if (tb[TCA_ROUTE4_TO])
404	439	f->id = to;
405	440
...	...	@@ -410,7 +445,7 @@
410	445
411	446	f->handle = nhandle;
412	447	f->bkt = b;
413		- tcf_tree_unlock(tp);
	448	+ f->tp = tp;
414	449
415	450	if (tb[TCA_ROUTE4_CLASSID]) {
416	451	f->res.classid = nla_get_u32(tb[TCA_ROUTE4_CLASSID]);
417	452
418	453
...	...	@@ -431,14 +466,15 @@
431	466	struct nlattr **tca,
432	467	unsigned long *arg, bool ovr)
433	468	{
434		- struct route4_head *head = tp->root;
435		- struct route4_filter f, f1, **fp;
	469	+ struct route4_head *head = rtnl_dereference(tp->root);
	470	+ struct route4_filter __rcu **fp;
	471	+ struct route4_filter fold, f1, pfp, f = NULL;
436	472	struct route4_bucket *b;
437	473	struct nlattr *opt = tca[TCA_OPTIONS];
438	474	struct nlattr *tb[TCA_ROUTE4_MAX + 1];
439	475	unsigned int h, th;
440		- u32 old_handle = 0;
441	476	int err;
	477	+ bool new = true;
442	478
443	479	if (opt == NULL)
444	480	return handle ? -EINVAL : 0;
445	481
446	482
447	483
448	484
449	485
450	486
451	487
452	488
453	489
454	490
455	491
456	492
457	493
...	...	@@ -447,70 +483,70 @@
447	483	if (err < 0)
448	484	return err;
449	485
450		- f = (struct route4_filter )arg;
451		- if (f) {
452		- if (f->handle != handle && handle)
	486	+ fold = (struct route4_filter )arg;
	487	+ if (fold && handle && fold->handle != handle)
453	488	return -EINVAL;
454	489
455		- if (f->bkt)
456		- old_handle = f->handle;
457		-
458		- err = route4_set_parms(net, tp, base, f, handle, head, tb,
459		- tca[TCA_RATE], 0, ovr);
460		- if (err < 0)
461		- return err;
462		-
463		- goto reinsert;
464		- }
465		-
466	490	err = -ENOBUFS;
467	491	if (head == NULL) {
468	492	head = kzalloc(sizeof(struct route4_head), GFP_KERNEL);
469	493	if (head == NULL)
470	494	goto errout;
471		-
472		- tcf_tree_lock(tp);
473		- tp->root = head;
474		- tcf_tree_unlock(tp);
	495	+ rcu_assign_pointer(tp->root, head);
475	496	}
476	497
477	498	f = kzalloc(sizeof(struct route4_filter), GFP_KERNEL);
478		- if (f == NULL)
	499	+ if (!f)
479	500	goto errout;
480	501
481	502	tcf_exts_init(&f->exts, TCA_ROUTE4_ACT, TCA_ROUTE4_POLICE);
	503	+ if (fold) {
	504	+ f->id = fold->id;
	505	+ f->iif = fold->iif;
	506	+ f->res = fold->res;
	507	+ f->handle = fold->handle;
	508	+
	509	+ f->tp = fold->tp;
	510	+ f->bkt = fold->bkt;
	511	+ new = false;
	512	+ }
	513	+
482	514	err = route4_set_parms(net, tp, base, f, handle, head, tb,
483		- tca[TCA_RATE], 1, ovr);
	515	+ tca[TCA_RATE], new, ovr);
484	516	if (err < 0)
485	517	goto errout;
486	518
487		-reinsert:
488	519	h = from_hash(f->handle >> 16);
489		- for (fp = &f->bkt->ht[h]; (f1 = *fp) != NULL; fp = &f1->next)
	520	+ fp = &f->bkt->ht[h];
	521	+ for (pfp = rtnl_dereference(*fp);
	522	+ (f1 = rtnl_dereference(*fp)) != NULL;
	523	+ fp = &f1->next)
490	524	if (f->handle < f1->handle)
491	525	break;
492	526
493		- f->next = f1;
494		- tcf_tree_lock(tp);
495		- *fp = f;
	527	+ rcu_assign_pointer(f->next, f1);
	528	+ rcu_assign_pointer(*fp, f);
496	529
497		- if (old_handle && f->handle != old_handle) {
498		- th = to_hash(old_handle);
499		- h = from_hash(old_handle >> 16);
500		- b = head->table[th];
	530	+ if (fold && fold->handle && f->handle != fold->handle) {
	531	+ th = to_hash(fold->handle);
	532	+ h = from_hash(fold->handle >> 16);
	533	+ b = rtnl_dereference(head->table[th]);
501	534	if (b) {
502		- for (fp = &b->ht[h]; fp; fp = &(fp)->next) {
503		- if (*fp == f) {
	535	+ fp = &b->ht[h];
	536	+ for (pfp = rtnl_dereference(*fp); pfp;
	537	+ fp = &pfp->next, pfp = rtnl_dereference(*fp)) {
	538	+ if (pfp == f) {
504	539	*fp = f->next;
505	540	break;
506	541	}
507	542	}
508	543	}
509	544	}
510		- tcf_tree_unlock(tp);
511	545
512		- route4_reset_fastmap(tp->q, head, f->id);
	546	+ route4_reset_fastmap(head);
513	547	*arg = (unsigned long)f;
	548	+ if (fold)
	549	+ call_rcu(&fold->rcu, route4_delete_filter);
514	550	return 0;
515	551
516	552	errout:
...	...	@@ -520,7 +556,7 @@
520	556
521	557	static void route4_walk(struct tcf_proto tp, struct tcf_walker arg)
522	558	{
523		- struct route4_head *head = tp->root;
	559	+ struct route4_head *head = rtnl_dereference(tp->root);
524	560	unsigned int h, h1;
525	561
526	562	if (head == NULL)
527	563
...	...	@@ -530,13 +566,15 @@
530	566	return;
531	567
532	568	for (h = 0; h <= 256; h++) {
533		- struct route4_bucket *b = head->table[h];
	569	+ struct route4_bucket *b = rtnl_dereference(head->table[h]);
534	570
535	571	if (b) {
536	572	for (h1 = 0; h1 <= 32; h1++) {
537	573	struct route4_filter *f;
538	574
539		- for (f = b->ht[h1]; f; f = f->next) {
	575	+ for (f = rtnl_dereference(b->ht[h1]);
	576	+ f;
	577	+ f = rtnl_dereference(f->next)) {
540	578	if (arg->count < arg->skip) {
541	579	arg->count++;
542	580	continue;