SLUB: Alternate fast paths using cmpxchg_local

Provide an alternate implementation of the SLUB fast paths for alloc and free using cmpxchg_local. The cmpxchg_local fast path is selected for arches that have CONFIG_FAST_CMPXCHG_LOCAL set. An arch should only set CONFIG_FAST_CMPXCHG_LOCAL if the cmpxchg_local is faster than an interrupt enable/disable sequence. This is known to be true for both x86 platforms so set FAST_CMPXCHG_LOCAL for both arches. Currently another requirement for the fastpath is that the kernel is compiled without preemption. The restriction will go away with the introduction of a new per cpu allocator and new per cpu operations. The advantages of a cmpxchg_local based fast path are: 1. Potentially lower cycle count (30%-60% faster) 2. There is no need to disable and enable interrupts on the fast path. Currently interrupts have to be disabled and enabled on every slab operation. This is likely avoiding a significant percentage of interrupt off / on sequences in the kernel. 3. The disposal of freed slabs can occur with interrupts enabled. The alternate path is realized using #ifdef's. Several attempts to do the same with macros and inline functions resulted in a mess (in particular due to the strange way that local_interrupt_save() handles its argument and due to the need to define macros/functions that sometimes disable interrupts and sometimes do something else). [clameter: Stripped preempt bits and disabled fastpath if preempt is enabled] Signed-off-by: Christoph Lameter <clameter@sgi.com> Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi> Cc: <linux-arch@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

SLUB: Alternate fast paths using cmpxchg_local
Provide an alternate implementation of the SLUB fast paths for alloc and free using cmpxchg_local. The cmpxchg_local fast path is selected for arches that have CONFIG_FAST_CMPXCHG_LOCAL set. An arch should only set CONFIG_FAST_CMPXCHG_LOCAL if the cmpxchg_local is faster than an interrupt enable/disable sequence. This is known to be true for both x86 platforms so set FAST_CMPXCHG_LOCAL for both arches. Currently another requirement for the fastpath is that the kernel is compiled without preemption. The restriction will go away with the introduction of a new per cpu allocator and new per cpu operations. The advantages of a cmpxchg_local based fast path are: 1. Potentially lower cycle count (30%-60% faster) 2. There is no need to disable and enable interrupts on the fast path. Currently interrupts have to be disabled and enabled on every slab operation. This is likely avoiding a significant percentage of interrupt off / on sequences in the kernel. 3. The disposal of freed slabs can occur with interrupts enabled. The alternate path is realized using #ifdef's. Several attempts to do the same with macros and inline functions resulted in a mess (in particular due to the strange way that local_interrupt_save() handles its argument and due to the need to define macros/functions that sometimes disable interrupts and sometimes do something else). [clameter: Stripped preempt bits and disabled fastpath if preempt is enabled] Signed-off-by: Christoph Lameter <clameter@sgi.com> Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi> Cc: <linux-arch@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Christoph Lameter · Christoph Lameter
1 parent 683d0baad3
Showing 2 changed files with 92 additions and 5 deletions Side-by-side Diff
arch/x86/Kconfig
mm/slub.c
@@ -52,6 +52,10 @@
 config SEMAPHORE_SLEEPERS
 	def_bool y
  
+config FAST_CMPXCHG_LOCAL
+	bool
+	default y
+
 config MMU
 	def_bool y
  
@@ -149,6 +149,13 @@
 /* Enable to test recovery from slab corruption on boot */
 #undef SLUB_RESILIENCY_TEST
  
+/*
+ * Currently fastpath is not supported if preemption is enabled.
+ */
+#if defined(CONFIG_FAST_CMPXCHG_LOCAL) && !defined(CONFIG_PREEMPT)
+#define SLUB_FASTPATH
+#endif
+
 #if PAGE_SHIFT <= 12
  
 /*
  
@@ -1493,7 +1500,11 @@
 {
 	void **object;
 	struct page *new;
+#ifdef SLUB_FASTPATH
+	unsigned long flags;
  
+	local_irq_save(flags);
+#endif
 	if (!c->page)
 		goto new_slab;
  
  
@@ -1512,7 +1523,12 @@
 	c->page->inuse = s->objects;
 	c->page->freelist = c->page->end;
 	c->node = page_to_nid(c->page);
+unlock_out:
 	slab_unlock(c->page);
+out:
+#ifdef SLUB_FASTPATH
+	local_irq_restore(flags);
+#endif
 	return object;
  
 another_slab:
@@ -1542,7 +1558,8 @@
 		c->page = new;
 		goto load_freelist;
 	}
-	return NULL;
+	object = NULL;
+	goto out;
 debug:
 	object = c->page->freelist;
 	if (!alloc_debug_processing(s, c->page, object, addr))
@@ -1551,8 +1568,7 @@
 	c->page->inuse++;
 	c->page->freelist = object[c->offset];
 	c->node = -1;
-	slab_unlock(c->page);
-	return object;
+	goto unlock_out;
 }
  
 /*
  
@@ -1569,9 +1585,36 @@
 		gfp_t gfpflags, int node, void *addr)
 {
 	void **object;
-	unsigned long flags;
 	struct kmem_cache_cpu *c;
  
+/*
+ * The SLUB_FASTPATH path is provisional and is currently disabled if the
+ * kernel is compiled with preemption or if the arch does not support
+ * fast cmpxchg operations. There are a couple of coming changes that will
+ * simplify matters and allow preemption. Ultimately we may end up making
+ * SLUB_FASTPATH the default.
+ *
+ * 1. The introduction of the per cpu allocator will avoid array lookups
+ *    through get_cpu_slab(). A special register can be used instead.
+ *
+ * 2. The introduction of per cpu atomic operations (cpu_ops) means that
+ *    we can realize the logic here entirely with per cpu atomics. The
+ *    per cpu atomic ops will take care of the preemption issues.
+ */
+
+#ifdef SLUB_FASTPATH
+	c = get_cpu_slab(s, raw_smp_processor_id());
+	do {
+		object = c->freelist;
+		if (unlikely(is_end(object) || !node_match(c, node))) {
+			object = __slab_alloc(s, gfpflags, node, addr, c);
+			break;
+		}
+	} while (cmpxchg_local(&c->freelist, object, object[c->offset])
+								!= object);
+#else
+	unsigned long flags;
+
 	local_irq_save(flags);
 	c = get_cpu_slab(s, smp_processor_id());
 	if (unlikely(is_end(c->freelist) || !node_match(c, node)))
@@ -1583,6 +1626,7 @@
 		c->freelist = object[c->offset];
 	}
 	local_irq_restore(flags);
+#endif
  
 	if (unlikely((gfpflags & __GFP_ZERO) && object))
 		memset(object, 0, c->objsize);
@@ -1618,6 +1662,11 @@
 	void *prior;
 	void **object = (void *)x;
  
+#ifdef SLUB_FASTPATH
+	unsigned long flags;
+
+	local_irq_save(flags);
+#endif
 	slab_lock(page);
  
 	if (unlikely(SlabDebug(page)))
@@ -1643,6 +1692,9 @@
  
 out_unlock:
 	slab_unlock(page);
+#ifdef SLUB_FASTPATH
+	local_irq_restore(flags);
+#endif
 	return;
  
 slab_empty:
@@ -1653,6 +1705,9 @@
 		remove_partial(s, page);
  
 	slab_unlock(page);
+#ifdef SLUB_FASTPATH
+	local_irq_restore(flags);
+#endif
 	discard_slab(s, page);
 	return;
  
  
@@ -1677,9 +1732,36 @@
 			struct page *page, void *x, void *addr)
 {
 	void **object = (void *)x;
-	unsigned long flags;
 	struct kmem_cache_cpu *c;
  
+#ifdef SLUB_FASTPATH
+	void **freelist;
+
+	c = get_cpu_slab(s, raw_smp_processor_id());
+	debug_check_no_locks_freed(object, s->objsize);
+	do {
+		freelist = c->freelist;
+		barrier();
+		/*
+		 * If the compiler would reorder the retrieval of c->page to
+		 * come before c->freelist then an interrupt could
+		 * change the cpu slab before we retrieve c->freelist. We
+		 * could be matching on a page no longer active and put the
+		 * object onto the freelist of the wrong slab.
+		 *
+		 * On the other hand: If we already have the freelist pointer
+		 * then any change of cpu_slab will cause the cmpxchg to fail
+		 * since the freelist pointers are unique per slab.
+		 */
+		if (unlikely(page != c->page || c->node < 0)) {
+			__slab_free(s, page, x, addr, c->offset);
+			break;
+		}
+		object[c->offset] = freelist;
+	} while (cmpxchg_local(&c->freelist, freelist, object) != freelist);
+#else
+	unsigned long flags;
+
 	local_irq_save(flags);
 	debug_check_no_locks_freed(object, s->objsize);
 	c = get_cpu_slab(s, smp_processor_id());
@@ -1690,6 +1772,7 @@
 		__slab_free(s, page, x, addr, c->offset);
  
 	local_irq_restore(flags);
+#endif
 }
  
 void kmem_cache_free(struct kmem_cache *s, void *x)
...	...	@@ -52,6 +52,10 @@
52	52	config SEMAPHORE_SLEEPERS
53	53	def_bool y
54	54
	55	+config FAST_CMPXCHG_LOCAL
	56	+ bool
	57	+ default y
	58	+
55	59	config MMU
56	60	def_bool y
57	61
...	...	@@ -149,6 +149,13 @@
149	149	/* Enable to test recovery from slab corruption on boot */
150	150	#undef SLUB_RESILIENCY_TEST
151	151
	152	+/*
	153	+ * Currently fastpath is not supported if preemption is enabled.
	154	+ */
	155	+#if defined(CONFIG_FAST_CMPXCHG_LOCAL) && !defined(CONFIG_PREEMPT)
	156	+#define SLUB_FASTPATH
	157	+#endif
	158	+
152	159	#if PAGE_SHIFT <= 12
153	160
154	161	/*
155	162
...	...	@@ -1493,7 +1500,11 @@
1493	1500	{
1494	1501	void **object;
1495	1502	struct page *new;
	1503	+#ifdef SLUB_FASTPATH
	1504	+ unsigned long flags;
1496	1505
	1506	+ local_irq_save(flags);
	1507	+#endif
1497	1508	if (!c->page)
1498	1509	goto new_slab;
1499	1510
1500	1511
...	...	@@ -1512,7 +1523,12 @@
1512	1523	c->page->inuse = s->objects;
1513	1524	c->page->freelist = c->page->end;
1514	1525	c->node = page_to_nid(c->page);
	1526	+unlock_out:
1515	1527	slab_unlock(c->page);
	1528	+out:
	1529	+#ifdef SLUB_FASTPATH
	1530	+ local_irq_restore(flags);
	1531	+#endif
1516	1532	return object;
1517	1533
1518	1534	another_slab:
...	...	@@ -1542,7 +1558,8 @@
1542	1558	c->page = new;
1543	1559	goto load_freelist;
1544	1560	}
1545		- return NULL;
	1561	+ object = NULL;
	1562	+ goto out;
1546	1563	debug:
1547	1564	object = c->page->freelist;
1548	1565	if (!alloc_debug_processing(s, c->page, object, addr))
...	...	@@ -1551,8 +1568,7 @@
1551	1568	c->page->inuse++;
1552	1569	c->page->freelist = object[c->offset];
1553	1570	c->node = -1;
1554		- slab_unlock(c->page);
1555		- return object;
	1571	+ goto unlock_out;
1556	1572	}
1557	1573
1558	1574	/*
1559	1575
...	...	@@ -1569,9 +1585,36 @@
1569	1585	gfp_t gfpflags, int node, void *addr)
1570	1586	{
1571	1587	void **object;
1572		- unsigned long flags;
1573	1588	struct kmem_cache_cpu *c;
1574	1589
	1590	+/*
	1591	+ * The SLUB_FASTPATH path is provisional and is currently disabled if the
	1592	+ * kernel is compiled with preemption or if the arch does not support
	1593	+ * fast cmpxchg operations. There are a couple of coming changes that will
	1594	+ * simplify matters and allow preemption. Ultimately we may end up making
	1595	+ * SLUB_FASTPATH the default.
	1596	+ *
	1597	+ * 1. The introduction of the per cpu allocator will avoid array lookups
	1598	+ * through get_cpu_slab(). A special register can be used instead.
	1599	+ *
	1600	+ * 2. The introduction of per cpu atomic operations (cpu_ops) means that
	1601	+ * we can realize the logic here entirely with per cpu atomics. The
	1602	+ * per cpu atomic ops will take care of the preemption issues.
	1603	+ */
	1604	+
	1605	+#ifdef SLUB_FASTPATH
	1606	+ c = get_cpu_slab(s, raw_smp_processor_id());
	1607	+ do {
	1608	+ object = c->freelist;
	1609	+ if (unlikely(is_end(object) \|\| !node_match(c, node))) {
	1610	+ object = __slab_alloc(s, gfpflags, node, addr, c);
	1611	+ break;
	1612	+ }
	1613	+ } while (cmpxchg_local(&c->freelist, object, object[c->offset])
	1614	+ != object);
	1615	+#else
	1616	+ unsigned long flags;
	1617	+
1575	1618	local_irq_save(flags);
1576	1619	c = get_cpu_slab(s, smp_processor_id());
1577	1620	if (unlikely(is_end(c->freelist) \|\| !node_match(c, node)))
...	...	@@ -1583,6 +1626,7 @@
1583	1626	c->freelist = object[c->offset];
1584	1627	}
1585	1628	local_irq_restore(flags);
	1629	+#endif
1586	1630
1587	1631	if (unlikely((gfpflags & __GFP_ZERO) && object))
1588	1632	memset(object, 0, c->objsize);
...	...	@@ -1618,6 +1662,11 @@
1618	1662	void *prior;
1619	1663	void *object = (void )x;
1620	1664
	1665	+#ifdef SLUB_FASTPATH
	1666	+ unsigned long flags;
	1667	+
	1668	+ local_irq_save(flags);
	1669	+#endif
1621	1670	slab_lock(page);
1622	1671
1623	1672	if (unlikely(SlabDebug(page)))
...	...	@@ -1643,6 +1692,9 @@
1643	1692
1644	1693	out_unlock:
1645	1694	slab_unlock(page);
	1695	+#ifdef SLUB_FASTPATH
	1696	+ local_irq_restore(flags);
	1697	+#endif
1646	1698	return;
1647	1699
1648	1700	slab_empty:
...	...	@@ -1653,6 +1705,9 @@
1653	1705	remove_partial(s, page);
1654	1706
1655	1707	slab_unlock(page);
	1708	+#ifdef SLUB_FASTPATH
	1709	+ local_irq_restore(flags);
	1710	+#endif
1656	1711	discard_slab(s, page);
1657	1712	return;
1658	1713
1659	1714
...	...	@@ -1677,9 +1732,36 @@
1677	1732	struct page page, void x, void *addr)
1678	1733	{
1679	1734	void *object = (void )x;
1680		- unsigned long flags;
1681	1735	struct kmem_cache_cpu *c;
1682	1736
	1737	+#ifdef SLUB_FASTPATH
	1738	+ void **freelist;
	1739	+
	1740	+ c = get_cpu_slab(s, raw_smp_processor_id());
	1741	+ debug_check_no_locks_freed(object, s->objsize);
	1742	+ do {
	1743	+ freelist = c->freelist;
	1744	+ barrier();
	1745	+ /*
	1746	+ * If the compiler would reorder the retrieval of c->page to
	1747	+ * come before c->freelist then an interrupt could
	1748	+ * change the cpu slab before we retrieve c->freelist. We
	1749	+ * could be matching on a page no longer active and put the
	1750	+ * object onto the freelist of the wrong slab.
	1751	+ *
	1752	+ * On the other hand: If we already have the freelist pointer
	1753	+ * then any change of cpu_slab will cause the cmpxchg to fail
	1754	+ * since the freelist pointers are unique per slab.
	1755	+ */
	1756	+ if (unlikely(page != c->page \|\| c->node < 0)) {
	1757	+ __slab_free(s, page, x, addr, c->offset);
	1758	+ break;
	1759	+ }
	1760	+ object[c->offset] = freelist;
	1761	+ } while (cmpxchg_local(&c->freelist, freelist, object) != freelist);
	1762	+#else
	1763	+ unsigned long flags;
	1764	+
1683	1765	local_irq_save(flags);
1684	1766	debug_check_no_locks_freed(object, s->objsize);
1685	1767	c = get_cpu_slab(s, smp_processor_id());
...	...	@@ -1690,6 +1772,7 @@
1690	1772	__slab_free(s, page, x, addr, c->offset);
1691	1773
1692	1774	local_irq_restore(flags);
	1775	+#endif
1693	1776	}
1694	1777
1695	1778	void kmem_cache_free(struct kmem_cache s, void x)