net: Move napi polling code out of net_rx_action

This patch creates a new function napi_poll and moves the napi polling code from net_rx_action into it. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>

net: Move napi polling code out of net_rx_action
This patch creates a new function napi_poll and moves the napi polling code from net_rx_action into it. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
Herbert Xu · David S. Miller
1 parent 0d16449195
Showing 1 changed file with 54 additions and 44 deletions Side-by-side Diff
net/core/dev.c
@@ -4557,6 +4557,59 @@
 }
 EXPORT_SYMBOL(netif_napi_del);
  
+static int napi_poll(struct napi_struct *n, struct list_head *repoll)
+{
+	void *have;
+	int work, weight;
+
+	list_del_init(&n->poll_list);
+
+	have = netpoll_poll_lock(n);
+
+	weight = n->weight;
+
+	/* This NAPI_STATE_SCHED test is for avoiding a race
+	 * with netpoll's poll_napi().  Only the entity which
+	 * obtains the lock and sees NAPI_STATE_SCHED set will
+	 * actually make the ->poll() call.  Therefore we avoid
+	 * accidentally calling ->poll() when NAPI is not scheduled.
+	 */
+	work = 0;
+	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
+		work = n->poll(n, weight);
+		trace_napi_poll(n);
+	}
+
+	WARN_ON_ONCE(work > weight);
+
+	if (likely(work < weight))
+		goto out_unlock;
+
+	/* Drivers must not modify the NAPI state if they
+	 * consume the entire weight.  In such cases this code
+	 * still "owns" the NAPI instance and therefore can
+	 * move the instance around on the list at-will.
+	 */
+	if (unlikely(napi_disable_pending(n))) {
+		napi_complete(n);
+		goto out_unlock;
+	}
+
+	if (n->gro_list) {
+		/* flush too old packets
+		 * If HZ < 1000, flush all packets.
+		 */
+		napi_gro_flush(n, HZ >= 1000);
+	}
+
+	list_add_tail(&n->poll_list, repoll);
+
+out_unlock:
+	netpoll_poll_unlock(have);
+
+	return work;
+}
+
 static void net_rx_action(struct softirq_action *h)
 {
 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
@@ -4564,7 +4617,6 @@
 	int budget = netdev_budget;
 	LIST_HEAD(list);
 	LIST_HEAD(repoll);
-	void *have;
  
 	local_irq_disable();
 	list_splice_init(&sd->poll_list, &list);
@@ -4572,7 +4624,6 @@
  
 	while (!list_empty(&list)) {
 		struct napi_struct *n;
-		int work, weight;
  
 		/* If softirq window is exhausted then punt.
 		 * Allow this to run for 2 jiffies since which will allow
@@ -4583,48 +4634,7 @@
  
  
 		n = list_first_entry(&list, struct napi_struct, poll_list);
-		list_del_init(&n->poll_list);
-
-		have = netpoll_poll_lock(n);
-
-		weight = n->weight;
-
-		/* This NAPI_STATE_SCHED test is for avoiding a race
-		 * with netpoll's poll_napi().  Only the entity which
-		 * obtains the lock and sees NAPI_STATE_SCHED set will
-		 * actually make the ->poll() call.  Therefore we avoid
-		 * accidentally calling ->poll() when NAPI is not scheduled.
-		 */
-		work = 0;
-		if (test_bit(NAPI_STATE_SCHED, &n->state)) {
-			work = n->poll(n, weight);
-			trace_napi_poll(n);
-		}
-
-		WARN_ON_ONCE(work > weight);
-
-		budget -= work;
-
-		/* Drivers must not modify the NAPI state if they
-		 * consume the entire weight.  In such cases this code
-		 * still "owns" the NAPI instance and therefore can
-		 * move the instance around on the list at-will.
-		 */
-		if (unlikely(work == weight)) {
-			if (unlikely(napi_disable_pending(n))) {
-				napi_complete(n);
-			} else {
-				if (n->gro_list) {
-					/* flush too old packets
-					 * If HZ < 1000, flush all packets.
-					 */
-					napi_gro_flush(n, HZ >= 1000);
-				}
-				list_add_tail(&n->poll_list, &repoll);
-			}
-		}
-
-		netpoll_poll_unlock(have);
+		budget -= napi_poll(n, &repoll);
 	}
  
 	if (!sd_has_rps_ipi_waiting(sd) &&
...	...	@@ -4557,6 +4557,59 @@
4557	4557	}
4558	4558	EXPORT_SYMBOL(netif_napi_del);
4559	4559
	4560	+static int napi_poll(struct napi_struct n, struct list_head repoll)
	4561	+{
	4562	+ void *have;
	4563	+ int work, weight;
	4564	+
	4565	+ list_del_init(&n->poll_list);
	4566	+
	4567	+ have = netpoll_poll_lock(n);
	4568	+
	4569	+ weight = n->weight;
	4570	+
	4571	+ /* This NAPI_STATE_SCHED test is for avoiding a race
	4572	+ * with netpoll's poll_napi(). Only the entity which
	4573	+ * obtains the lock and sees NAPI_STATE_SCHED set will
	4574	+ * actually make the ->poll() call. Therefore we avoid
	4575	+ * accidentally calling ->poll() when NAPI is not scheduled.
	4576	+ */
	4577	+ work = 0;
	4578	+ if (test_bit(NAPI_STATE_SCHED, &n->state)) {
	4579	+ work = n->poll(n, weight);
	4580	+ trace_napi_poll(n);
	4581	+ }
	4582	+
	4583	+ WARN_ON_ONCE(work > weight);
	4584	+
	4585	+ if (likely(work < weight))
	4586	+ goto out_unlock;
	4587	+
	4588	+ /* Drivers must not modify the NAPI state if they
	4589	+ * consume the entire weight. In such cases this code
	4590	+ * still "owns" the NAPI instance and therefore can
	4591	+ * move the instance around on the list at-will.
	4592	+ */
	4593	+ if (unlikely(napi_disable_pending(n))) {
	4594	+ napi_complete(n);
	4595	+ goto out_unlock;
	4596	+ }
	4597	+
	4598	+ if (n->gro_list) {
	4599	+ /* flush too old packets
	4600	+ * If HZ < 1000, flush all packets.
	4601	+ */
	4602	+ napi_gro_flush(n, HZ >= 1000);
	4603	+ }
	4604	+
	4605	+ list_add_tail(&n->poll_list, repoll);
	4606	+
	4607	+out_unlock:
	4608	+ netpoll_poll_unlock(have);
	4609	+
	4610	+ return work;
	4611	+}
	4612	+
4560	4613	static void net_rx_action(struct softirq_action *h)
4561	4614	{
4562	4615	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
...	...	@@ -4564,7 +4617,6 @@
4564	4617	int budget = netdev_budget;
4565	4618	LIST_HEAD(list);
4566	4619	LIST_HEAD(repoll);
4567		- void *have;
4568	4620
4569	4621	local_irq_disable();
4570	4622	list_splice_init(&sd->poll_list, &list);
...	...	@@ -4572,7 +4624,6 @@
4572	4624
4573	4625	while (!list_empty(&list)) {
4574	4626	struct napi_struct *n;
4575		- int work, weight;
4576	4627
4577	4628	/* If softirq window is exhausted then punt.
4578	4629	* Allow this to run for 2 jiffies since which will allow
...	...	@@ -4583,48 +4634,7 @@
4583	4634
4584	4635
4585	4636	n = list_first_entry(&list, struct napi_struct, poll_list);
4586		- list_del_init(&n->poll_list);
4587		-
4588		- have = netpoll_poll_lock(n);
4589		-
4590		- weight = n->weight;
4591		-
4592		- /* This NAPI_STATE_SCHED test is for avoiding a race
4593		- * with netpoll's poll_napi(). Only the entity which
4594		- * obtains the lock and sees NAPI_STATE_SCHED set will
4595		- * actually make the ->poll() call. Therefore we avoid
4596		- * accidentally calling ->poll() when NAPI is not scheduled.
4597		- */
4598		- work = 0;
4599		- if (test_bit(NAPI_STATE_SCHED, &n->state)) {
4600		- work = n->poll(n, weight);
4601		- trace_napi_poll(n);
4602		- }
4603		-
4604		- WARN_ON_ONCE(work > weight);
4605		-
4606		- budget -= work;
4607		-
4608		- /* Drivers must not modify the NAPI state if they
4609		- * consume the entire weight. In such cases this code
4610		- * still "owns" the NAPI instance and therefore can
4611		- * move the instance around on the list at-will.
4612		- */
4613		- if (unlikely(work == weight)) {
4614		- if (unlikely(napi_disable_pending(n))) {
4615		- napi_complete(n);
4616		- } else {
4617		- if (n->gro_list) {
4618		- /* flush too old packets
4619		- * If HZ < 1000, flush all packets.
4620		- */
4621		- napi_gro_flush(n, HZ >= 1000);
4622		- }
4623		- list_add_tail(&n->poll_list, &repoll);
4624		- }
4625		- }
4626		-
4627		- netpoll_poll_unlock(have);
	4637	+ budget -= napi_poll(n, &repoll);
4628	4638	}
4629	4639
4630	4640	if (!sd_has_rps_ipi_waiting(sd) &&