ipv6 sit: RCU conversion phase I

SIT tunnels use one rwlock to protect their prl entries. This first patch adds RCU locking for prl management, with standard call_rcu() calls. Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>

ipv6 sit: RCU conversion phase I
SIT tunnels use one rwlock to protect their prl entries. This first patch adds RCU locking for prl management, with standard call_rcu() calls. Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Eric Dumazet · David S. Miller
1 parent b37b62fea1
Showing 2 changed files with 51 additions and 23 deletions Side-by-side Diff
include/net/ipip.h
net/ipv6/sit.c
@@ -45,6 +45,7 @@
 	struct ip_tunnel_prl_entry	*next;
 	__be32				addr;
 	u16				flags;
+	struct rcu_head			rcu_head;
 };
  
 #define IPTUNNEL_XMIT() do {						\
@@ -240,15 +240,22 @@
 	return NULL;
 }
  
+static DEFINE_SPINLOCK(ipip6_prl_lock);
+
+#define for_each_prl_rcu(start)			\
+	for (prl = rcu_dereference(start);	\
+	     prl;				\
+	     prl = rcu_dereference(prl->next))
+
 static struct ip_tunnel_prl_entry *
 __ipip6_tunnel_locate_prl(struct ip_tunnel *t, __be32 addr)
 {
-	struct ip_tunnel_prl_entry *p = (struct ip_tunnel_prl_entry *)NULL;
+	struct ip_tunnel_prl_entry *prl;
  
-	for (p = t->prl; p; p = p->next)
-		if (p->addr == addr)
+	for_each_prl_rcu(t->prl)
+		if (prl->addr == addr)
 			break;
-	return p;
+	return prl;
  
 }
  
@@ -273,7 +280,7 @@
 		kcalloc(cmax, sizeof(*kp), GFP_KERNEL) :
 		NULL;
  
-	read_lock(&ipip6_lock);
+	rcu_read_lock();
  
 	ca = t->prl_count < cmax ? t->prl_count : cmax;
  
@@ -291,7 +298,7 @@
 	}
  
 	c = 0;
-	for (prl = t->prl; prl; prl = prl->next) {
+	for_each_prl_rcu(t->prl) {
 		if (c >= cmax)
 			break;
 		if (kprl.addr != htonl(INADDR_ANY) && prl->addr != kprl.addr)
@@ -303,7 +310,7 @@
 			break;
 	}
 out:
-	read_unlock(&ipip6_lock);
+	rcu_read_unlock();
  
 	len = sizeof(*kp) * c;
 	ret = 0;
  
@@ -324,12 +331,14 @@
 	if (a->addr == htonl(INADDR_ANY))
 		return -EINVAL;
  
-	write_lock(&ipip6_lock);
+	spin_lock(&ipip6_prl_lock);
  
 	for (p = t->prl; p; p = p->next) {
 		if (p->addr == a->addr) {
-			if (chg)
-				goto update;
+			if (chg) {
+				p->flags = a->flags;
+				goto out;
+			}
 			err = -EEXIST;
 			goto out;
 		}
  
  
  
  
  
  
  
  
  
@@ -346,46 +355,63 @@
 		goto out;
 	}
  
+	INIT_RCU_HEAD(&p->rcu_head);
 	p->next = t->prl;
-	t->prl = p;
-	t->prl_count++;
-update:
 	p->addr = a->addr;
 	p->flags = a->flags;
+	t->prl_count++;
+	rcu_assign_pointer(t->prl, p);
 out:
-	write_unlock(&ipip6_lock);
+	spin_unlock(&ipip6_prl_lock);
 	return err;
 }
  
+static void prl_entry_destroy_rcu(struct rcu_head *head)
+{
+	kfree(container_of(head, struct ip_tunnel_prl_entry, rcu_head));
+}
+
+static void prl_list_destroy_rcu(struct rcu_head *head)
+{
+	struct ip_tunnel_prl_entry *p, *n;
+
+	p = container_of(head, struct ip_tunnel_prl_entry, rcu_head);
+	do {
+		n = p->next;
+		kfree(p);
+		p = n;
+	} while (p);
+}
+
 static int
 ipip6_tunnel_del_prl(struct ip_tunnel *t, struct ip_tunnel_prl *a)
 {
 	struct ip_tunnel_prl_entry *x, **p;
 	int err = 0;
  
-	write_lock(&ipip6_lock);
+	spin_lock(&ipip6_prl_lock);
  
 	if (a && a->addr != htonl(INADDR_ANY)) {
 		for (p = &t->prl; *p; p = &(*p)->next) {
 			if ((*p)->addr == a->addr) {
 				x = *p;
 				*p = x->next;
-				kfree(x);
+				call_rcu(&x->rcu_head, prl_entry_destroy_rcu);
 				t->prl_count--;
 				goto out;
 			}
 		}
 		err = -ENXIO;
 	} else {
-		while (t->prl) {
+		if (t->prl) {
+			t->prl_count = 0;
 			x = t->prl;
-			t->prl = t->prl->next;
-			kfree(x);
-			t->prl_count--;
+			call_rcu(&x->rcu_head, prl_list_destroy_rcu);
+			t->prl = NULL;
 		}
 	}
 out:
-	write_unlock(&ipip6_lock);
+	spin_unlock(&ipip6_prl_lock);
 	return err;
 }
  
@@ -395,7 +421,7 @@
 	struct ip_tunnel_prl_entry *p;
 	int ok = 1;
  
-	read_lock(&ipip6_lock);
+	rcu_read_lock();
 	p = __ipip6_tunnel_locate_prl(t, iph->saddr);
 	if (p) {
 		if (p->flags & PRL_DEFAULT)
@@ -411,7 +437,7 @@
 		else
 			ok = 0;
 	}
-	read_unlock(&ipip6_lock);
+	rcu_read_unlock();
 	return ok;
 }
  
@@ -1192,6 +1218,7 @@
 	xfrm4_tunnel_deregister(&sit_handler, AF_INET6);
  
 	unregister_pernet_gen_device(sit_net_id, &sit_net_ops);
+	rcu_barrier(); /* Wait for completion of call_rcu()'s */
 }
  
 static int __init sit_init(void)
...	...	@@ -45,6 +45,7 @@
45	45	struct ip_tunnel_prl_entry *next;
46	46	__be32 addr;
47	47	u16 flags;
	48	+ struct rcu_head rcu_head;
48	49	};
49	50
50	51	#define IPTUNNEL_XMIT() do { \
...	...	@@ -240,15 +240,22 @@
240	240	return NULL;
241	241	}
242	242
	243	+static DEFINE_SPINLOCK(ipip6_prl_lock);
	244	+
	245	+#define for_each_prl_rcu(start) \
	246	+ for (prl = rcu_dereference(start); \
	247	+ prl; \
	248	+ prl = rcu_dereference(prl->next))
	249	+
243	250	static struct ip_tunnel_prl_entry *
244	251	__ipip6_tunnel_locate_prl(struct ip_tunnel *t, __be32 addr)
245	252	{
246		- struct ip_tunnel_prl_entry p = (struct ip_tunnel_prl_entry )NULL;
	253	+ struct ip_tunnel_prl_entry *prl;
247	254
248		- for (p = t->prl; p; p = p->next)
249		- if (p->addr == addr)
	255	+ for_each_prl_rcu(t->prl)
	256	+ if (prl->addr == addr)
250	257	break;
251		- return p;
	258	+ return prl;
252	259
253	260	}
254	261
...	...	@@ -273,7 +280,7 @@
273	280	kcalloc(cmax, sizeof(*kp), GFP_KERNEL) :
274	281	NULL;
275	282
276		- read_lock(&ipip6_lock);
	283	+ rcu_read_lock();
277	284
278	285	ca = t->prl_count < cmax ? t->prl_count : cmax;
279	286
...	...	@@ -291,7 +298,7 @@
291	298	}
292	299
293	300	c = 0;
294		- for (prl = t->prl; prl; prl = prl->next) {
	301	+ for_each_prl_rcu(t->prl) {
295	302	if (c >= cmax)
296	303	break;
297	304	if (kprl.addr != htonl(INADDR_ANY) && prl->addr != kprl.addr)
...	...	@@ -303,7 +310,7 @@
303	310	break;
304	311	}
305	312	out:
306		- read_unlock(&ipip6_lock);
	313	+ rcu_read_unlock();
307	314
308	315	len = sizeof(kp) c;
309	316	ret = 0;
310	317
...	...	@@ -324,12 +331,14 @@
324	331	if (a->addr == htonl(INADDR_ANY))
325	332	return -EINVAL;
326	333
327		- write_lock(&ipip6_lock);
	334	+ spin_lock(&ipip6_prl_lock);
328	335
329	336	for (p = t->prl; p; p = p->next) {
330	337	if (p->addr == a->addr) {
331		- if (chg)
332		- goto update;
	338	+ if (chg) {
	339	+ p->flags = a->flags;
	340	+ goto out;
	341	+ }
333	342	err = -EEXIST;
334	343	goto out;
335	344	}
336	345
337	346
338	347
339	348
340	349
341	350
342	351
343	352
344	353
...	...	@@ -346,46 +355,63 @@
346	355	goto out;
347	356	}
348	357
	358	+ INIT_RCU_HEAD(&p->rcu_head);
349	359	p->next = t->prl;
350		- t->prl = p;
351		- t->prl_count++;
352		-update:
353	360	p->addr = a->addr;
354	361	p->flags = a->flags;
	362	+ t->prl_count++;
	363	+ rcu_assign_pointer(t->prl, p);
355	364	out:
356		- write_unlock(&ipip6_lock);
	365	+ spin_unlock(&ipip6_prl_lock);
357	366	return err;
358	367	}
359	368
	369	+static void prl_entry_destroy_rcu(struct rcu_head *head)
	370	+{
	371	+ kfree(container_of(head, struct ip_tunnel_prl_entry, rcu_head));
	372	+}
	373	+
	374	+static void prl_list_destroy_rcu(struct rcu_head *head)
	375	+{
	376	+ struct ip_tunnel_prl_entry p, n;
	377	+
	378	+ p = container_of(head, struct ip_tunnel_prl_entry, rcu_head);
	379	+ do {
	380	+ n = p->next;
	381	+ kfree(p);
	382	+ p = n;
	383	+ } while (p);
	384	+}
	385	+
360	386	static int
361	387	ipip6_tunnel_del_prl(struct ip_tunnel t, struct ip_tunnel_prl a)
362	388	{
363	389	struct ip_tunnel_prl_entry x, *p;
364	390	int err = 0;
365	391
366		- write_lock(&ipip6_lock);
	392	+ spin_lock(&ipip6_prl_lock);
367	393
368	394	if (a && a->addr != htonl(INADDR_ANY)) {
369	395	for (p = &t->prl; p; p = &(p)->next) {
370	396	if ((*p)->addr == a->addr) {
371	397	x = *p;
372	398	*p = x->next;
373		- kfree(x);
	399	+ call_rcu(&x->rcu_head, prl_entry_destroy_rcu);
374	400	t->prl_count--;
375	401	goto out;
376	402	}
377	403	}
378	404	err = -ENXIO;
379	405	} else {
380		- while (t->prl) {
	406	+ if (t->prl) {
	407	+ t->prl_count = 0;
381	408	x = t->prl;
382		- t->prl = t->prl->next;
383		- kfree(x);
384		- t->prl_count--;
	409	+ call_rcu(&x->rcu_head, prl_list_destroy_rcu);
	410	+ t->prl = NULL;
385	411	}
386	412	}
387	413	out:
388		- write_unlock(&ipip6_lock);
	414	+ spin_unlock(&ipip6_prl_lock);
389	415	return err;
390	416	}
391	417
...	...	@@ -395,7 +421,7 @@
395	421	struct ip_tunnel_prl_entry *p;
396	422	int ok = 1;
397	423
398		- read_lock(&ipip6_lock);
	424	+ rcu_read_lock();
399	425	p = __ipip6_tunnel_locate_prl(t, iph->saddr);
400	426	if (p) {
401	427	if (p->flags & PRL_DEFAULT)
...	...	@@ -411,7 +437,7 @@
411	437	else
412	438	ok = 0;
413	439	}
414		- read_unlock(&ipip6_lock);
	440	+ rcu_read_unlock();
415	441	return ok;
416	442	}
417	443
...	...	@@ -1192,6 +1218,7 @@
1192	1218	xfrm4_tunnel_deregister(&sit_handler, AF_INET6);
1193	1219
1194	1220	unregister_pernet_gen_device(sit_net_id, &sit_net_ops);
	1221	+ rcu_barrier(); /* Wait for completion of call_rcu()'s */
1195	1222	}
1196	1223
1197	1224	static int __init sit_init(void)