once: make helper generic for calling functions once

Make the get_random_once() helper generic enough, so that functions in general would only be called once, where one user of this is then net_get_random_once(). The only implementation specific call is to get_random_bytes(), all the rest of this *_once() facility would be duplicated among different subsystems otherwise. The new DO_ONCE() helper will be used by prandom() later on, but might also be useful for other scenarios/subsystems as well where a one-time initialization in often-called, possibly fast path code could occur. Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Acked-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: David S. Miller <davem@davemloft.net>

once: make helper generic for calling functions once
Make the get_random_once() helper generic enough, so that functions in general would only be called once, where one user of this is then net_get_random_once(). The only implementation specific call is to get_random_bytes(), all the rest of this *_once() facility would be duplicated among different subsystems otherwise. The new DO_ONCE() helper will be used by prandom() later on, but might also be useful for other scenarios/subsystems as well where a one-time initialization in often-called, possibly fast path code could occur. Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Acked-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: David S. Miller <davem@davemloft.net>
Hannes Frederic Sowa · David S. Miller
1 parent 46234253b9
Showing 2 changed files with 76 additions and 35 deletions Side-by-side Diff
include/linux/once.h
lib/once.c
@@ -4,22 +4,55 @@
 #include <linux/types.h>
 #include <linux/jump_label.h>
  
-bool __get_random_once(void *buf, int nbytes, bool *done,
-		       struct static_key *once_key);
+bool __do_once_start(bool *done, unsigned long *flags);
+void __do_once_done(bool *done, struct static_key *once_key,
+		    unsigned long *flags);
  
-#define get_random_once(buf, nbytes)					\
-	({								\
-		bool ___ret = false;					\
-		static bool ___done = false;				\
-		static struct static_key ___once_key =			\
-			STATIC_KEY_INIT_TRUE;				\
-		if (static_key_true(&___once_key))			\
-			___ret = __get_random_once((buf),		\
-						   (nbytes),		\
-						   &___done,		\
-						   &___once_key);	\
-		___ret;							\
+/* Call a function exactly once. The idea of DO_ONCE() is to perform
+ * a function call such as initialization of random seeds, etc, only
+ * once, where DO_ONCE() can live in the fast-path. After @func has
+ * been called with the passed arguments, the static key will patch
+ * out the condition into a nop. DO_ONCE() guarantees type safety of
+ * arguments!
+ *
+ * Not that the following is not equivalent ...
+ *
+ *   DO_ONCE(func, arg);
+ *   DO_ONCE(func, arg);
+ *
+ * ... to this version:
+ *
+ *   void foo(void)
+ *   {
+ *     DO_ONCE(func, arg);
+ *   }
+ *
+ *   foo();
+ *   foo();
+ *
+ * In case the one-time invocation could be triggered from multiple
+ * places, then a common helper function must be defined, so that only
+ * a single static key will be placed there!
+ */
+#define DO_ONCE(func, ...)						     \
+	({								     \
+		bool ___ret = false;					     \
+		static bool ___done = false;				     \
+		static struct static_key ___once_key = STATIC_KEY_INIT_TRUE; \
+		if (static_key_true(&___once_key)) {			     \
+			unsigned long ___flags;				     \
+			___ret = __do_once_start(&___done, &___flags);	     \
+			if (unlikely(___ret)) {				     \
+				func(__VA_ARGS__);			     \
+				__do_once_done(&___done, &___once_key,	     \
+					       &___flags);		     \
+			}						     \
+		}							     \
+		___ret;							     \
 	})
+
+#define get_random_once(buf, nbytes)					     \
+	DO_ONCE(get_random_bytes, (buf), (nbytes))
  
 #endif /* _LINUX_ONCE_H */
@@ -3,53 +3,61 @@
 #include <linux/once.h>
 #include <linux/random.h>
  
-struct __random_once_work {
+struct once_work {
 	struct work_struct work;
 	struct static_key *key;
 };
  
-static void __random_once_deferred(struct work_struct *w)
+static void once_deferred(struct work_struct *w)
 {
-	struct __random_once_work *work;
+	struct once_work *work;
  
-	work = container_of(w, struct __random_once_work, work);
+	work = container_of(w, struct once_work, work);
 	BUG_ON(!static_key_enabled(work->key));
 	static_key_slow_dec(work->key);
 	kfree(work);
 }
  
-static void __random_once_disable_jump(struct static_key *key)
+static void once_disable_jump(struct static_key *key)
 {
-	struct __random_once_work *w;
+	struct once_work *w;
  
 	w = kmalloc(sizeof(*w), GFP_ATOMIC);
 	if (!w)
 		return;
  
-	INIT_WORK(&w->work, __random_once_deferred);
+	INIT_WORK(&w->work, once_deferred);
 	w->key = key;
 	schedule_work(&w->work);
 }
  
-bool __get_random_once(void *buf, int nbytes, bool *done,
-		       struct static_key *once_key)
-{
-	static DEFINE_SPINLOCK(lock);
-	unsigned long flags;
+static DEFINE_SPINLOCK(once_lock);
  
-	spin_lock_irqsave(&lock, flags);
+bool __do_once_start(bool *done, unsigned long *flags)
+	__acquires(once_lock)
+{
+	spin_lock_irqsave(&once_lock, *flags);
 	if (*done) {
-		spin_unlock_irqrestore(&lock, flags);
+		spin_unlock_irqrestore(&once_lock, *flags);
+		/* Keep sparse happy by restoring an even lock count on
+		 * this lock. In case we return here, we don't call into
+		 * __do_once_done but return early in the DO_ONCE() macro.
+		 */
+		__acquire(once_lock);
 		return false;
 	}
  
-	get_random_bytes(buf, nbytes);
-	*done = true;
-	spin_unlock_irqrestore(&lock, flags);
-
-	__random_once_disable_jump(once_key);
-
 	return true;
 }
-EXPORT_SYMBOL(__get_random_once);
+EXPORT_SYMBOL(__do_once_start);
+
+void __do_once_done(bool *done, struct static_key *once_key,
+		    unsigned long *flags)
+	__releases(once_lock)
+{
+	*done = true;
+	spin_unlock_irqrestore(&once_lock, *flags);
+	once_disable_jump(once_key);
+}
+EXPORT_SYMBOL(__do_once_done);
...	...	@@ -4,22 +4,55 @@
4	4	#include <linux/types.h>
5	5	#include <linux/jump_label.h>
6	6
7		-bool __get_random_once(void buf, int nbytes, bool done,
8		- struct static_key *once_key);
	7	+bool __do_once_start(bool done, unsigned long flags);
	8	+void __do_once_done(bool done, struct static_key once_key,
	9	+ unsigned long *flags);
9	10
10		-#define get_random_once(buf, nbytes) \
11		- ({ \
12		- bool ___ret = false; \
13		- static bool ___done = false; \
14		- static struct static_key ___once_key = \
15		- STATIC_KEY_INIT_TRUE; \
16		- if (static_key_true(&___once_key)) \
17		- ___ret = __get_random_once((buf), \
18		- (nbytes), \
19		- &___done, \
20		- &___once_key); \
21		- ___ret; \
	11	+/* Call a function exactly once. The idea of DO_ONCE() is to perform
	12	+ * a function call such as initialization of random seeds, etc, only
	13	+ * once, where DO_ONCE() can live in the fast-path. After @func has
	14	+ * been called with the passed arguments, the static key will patch
	15	+ * out the condition into a nop. DO_ONCE() guarantees type safety of
	16	+ * arguments!
	17	+ *
	18	+ * Not that the following is not equivalent ...
	19	+ *
	20	+ * DO_ONCE(func, arg);
	21	+ * DO_ONCE(func, arg);
	22	+ *
	23	+ * ... to this version:
	24	+ *
	25	+ * void foo(void)
	26	+ * {
	27	+ * DO_ONCE(func, arg);
	28	+ * }
	29	+ *
	30	+ * foo();
	31	+ * foo();
	32	+ *
	33	+ * In case the one-time invocation could be triggered from multiple
	34	+ * places, then a common helper function must be defined, so that only
	35	+ * a single static key will be placed there!
	36	+ */
	37	+#define DO_ONCE(func, ...) \
	38	+ ({ \
	39	+ bool ___ret = false; \
	40	+ static bool ___done = false; \
	41	+ static struct static_key ___once_key = STATIC_KEY_INIT_TRUE; \
	42	+ if (static_key_true(&___once_key)) { \
	43	+ unsigned long ___flags; \
	44	+ ___ret = __do_once_start(&___done, &___flags); \
	45	+ if (unlikely(___ret)) { \
	46	+ func(__VA_ARGS__); \
	47	+ __do_once_done(&___done, &___once_key, \
	48	+ &___flags); \
	49	+ } \
	50	+ } \
	51	+ ___ret; \
22	52	})
	53	+
	54	+#define get_random_once(buf, nbytes) \
	55	+ DO_ONCE(get_random_bytes, (buf), (nbytes))
23	56
24	57	#endif /* _LINUX_ONCE_H */
...	...	@@ -3,53 +3,61 @@
3	3	#include <linux/once.h>
4	4	#include <linux/random.h>
5	5
6		-struct __random_once_work {
	6	+struct once_work {
7	7	struct work_struct work;
8	8	struct static_key *key;
9	9	};
10	10
11		-static void __random_once_deferred(struct work_struct *w)
	11	+static void once_deferred(struct work_struct *w)
12	12	{
13		- struct __random_once_work *work;
	13	+ struct once_work *work;
14	14
15		- work = container_of(w, struct __random_once_work, work);
	15	+ work = container_of(w, struct once_work, work);
16	16	BUG_ON(!static_key_enabled(work->key));
17	17	static_key_slow_dec(work->key);
18	18	kfree(work);
19	19	}
20	20
21		-static void __random_once_disable_jump(struct static_key *key)
	21	+static void once_disable_jump(struct static_key *key)
22	22	{
23		- struct __random_once_work *w;
	23	+ struct once_work *w;
24	24
25	25	w = kmalloc(sizeof(*w), GFP_ATOMIC);
26	26	if (!w)
27	27	return;
28	28
29		- INIT_WORK(&w->work, __random_once_deferred);
	29	+ INIT_WORK(&w->work, once_deferred);
30	30	w->key = key;
31	31	schedule_work(&w->work);
32	32	}
33	33
34		-bool __get_random_once(void buf, int nbytes, bool done,
35		- struct static_key *once_key)
36		-{
37		- static DEFINE_SPINLOCK(lock);
38		- unsigned long flags;
	34	+static DEFINE_SPINLOCK(once_lock);
39	35
40		- spin_lock_irqsave(&lock, flags);
	36	+bool __do_once_start(bool done, unsigned long flags)
	37	+ __acquires(once_lock)
	38	+{
	39	+ spin_lock_irqsave(&once_lock, *flags);
41	40	if (*done) {
42		- spin_unlock_irqrestore(&lock, flags);
	41	+ spin_unlock_irqrestore(&once_lock, *flags);
	42	+ /* Keep sparse happy by restoring an even lock count on
	43	+ * this lock. In case we return here, we don't call into
	44	+ * __do_once_done but return early in the DO_ONCE() macro.
	45	+ */
	46	+ __acquire(once_lock);
43	47	return false;
44	48	}
45	49
46		- get_random_bytes(buf, nbytes);
47		- *done = true;
48		- spin_unlock_irqrestore(&lock, flags);
49		-
50		- __random_once_disable_jump(once_key);
51		-
52	50	return true;
53	51	}
54		-EXPORT_SYMBOL(__get_random_once);
	52	+EXPORT_SYMBOL(__do_once_start);
	53	+
	54	+void __do_once_done(bool done, struct static_key once_key,
	55	+ unsigned long *flags)
	56	+ __releases(once_lock)
	57	+{
	58	+ *done = true;
	59	+ spin_unlock_irqrestore(&once_lock, *flags);
	60	+ once_disable_jump(once_key);
	61	+}
	62	+EXPORT_SYMBOL(__do_once_done);