[PATCH] slab: cache_estimate cleanup

Clean up cache_estimate() in mm/slab.c and improves the algorithm from O(n) to O(1). We first calculate the maximum number of objects a slab can hold after struct slab and kmem_bufctl_t for each object has been given enough space. After that, to respect alignment rules, we decrease the number of objects if necessary. As required padding is at most align-1 and memory of obj_size is at least align, it is always enough to decrease number of objects by one. The optimization was originally made by Balbir Singh with more improvements from Steven Rostedt. Manfred Spraul provider further modifications: no loop at all for the off-slab case and added comments to explain the background. Acked-by: Balbir Singh <bsingharora@gmail.com> Signed-off-by: Manfred Spraul <manfred@colorfullife.com> Signed-off-by: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] slab: cache_estimate cleanup
Clean up cache_estimate() in mm/slab.c and improves the algorithm from O(n) to O(1). We first calculate the maximum number of objects a slab can hold after struct slab and kmem_bufctl_t for each object has been given enough space. After that, to respect alignment rules, we decrease the number of objects if necessary. As required padding is at most align-1 and memory of obj_size is at least align, it is always enough to decrease number of objects by one. The optimization was originally made by Balbir Singh with more improvements from Steven Rostedt. Manfred Spraul provider further modifications: no loop at all for the off-slab case and added comments to explain the background. Acked-by: Balbir Singh <bsingharora@gmail.com> Signed-off-by: Manfred Spraul <manfred@colorfullife.com> Signed-off-by: Steven Rostedt <rostedt@goodmis.org> Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Steven Rostedt · Linus Torvalds
1 parent 5ec8a847bb
Showing 1 changed file with 59 additions and 22 deletions Side-by-side Diff
mm/slab.c
@@ -702,32 +702,69 @@
 }
 EXPORT_SYMBOL(kmem_find_general_cachep);
  
-/* Cal the num objs, wastage, and bytes left over for a given slab size. */
-static void cache_estimate(unsigned long gfporder, size_t size, size_t align,
-			   int flags, size_t *left_over, unsigned int *num)
+static size_t slab_mgmt_size(size_t nr_objs, size_t align)
 {
-	int i;
-	size_t wastage = PAGE_SIZE << gfporder;
-	size_t extra = 0;
-	size_t base = 0;
+	return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
+}
  
-	if (!(flags & CFLGS_OFF_SLAB)) {
-		base = sizeof(struct slab);
-		extra = sizeof(kmem_bufctl_t);
-	}
-	i = 0;
-	while (i * size + ALIGN(base + i * extra, align) <= wastage)
-		i++;
-	if (i > 0)
-		i--;
+/* Calculate the number of objects and left-over bytes for a given
+   buffer size. */
+static void cache_estimate(unsigned long gfporder, size_t buffer_size,
+			   size_t align, int flags, size_t *left_over,
+			   unsigned int *num)
+{
+	int nr_objs;
+	size_t mgmt_size;
+	size_t slab_size = PAGE_SIZE << gfporder;
  
-	if (i > SLAB_LIMIT)
-		i = SLAB_LIMIT;
+	/*
+	 * The slab management structure can be either off the slab or
+	 * on it. For the latter case, the memory allocated for a
+	 * slab is used for:
+	 *
+	 * - The struct slab
+	 * - One kmem_bufctl_t for each object
+	 * - Padding to respect alignment of @align
+	 * - @buffer_size bytes for each object
+	 *
+	 * If the slab management structure is off the slab, then the
+	 * alignment will already be calculated into the size. Because
+	 * the slabs are all pages aligned, the objects will be at the
+	 * correct alignment when allocated.
+	 */
+	if (flags & CFLGS_OFF_SLAB) {
+		mgmt_size = 0;
+		nr_objs = slab_size / buffer_size;
  
-	*num = i;
-	wastage -= i * size;
-	wastage -= ALIGN(base + i * extra, align);
-	*left_over = wastage;
+		if (nr_objs > SLAB_LIMIT)
+			nr_objs = SLAB_LIMIT;
+	} else {
+		/*
+		 * Ignore padding for the initial guess. The padding
+		 * is at most @align-1 bytes, and @buffer_size is at
+		 * least @align. In the worst case, this result will
+		 * be one greater than the number of objects that fit
+		 * into the memory allocation when taking the padding
+		 * into account.
+		 */
+		nr_objs = (slab_size - sizeof(struct slab)) /
+			  (buffer_size + sizeof(kmem_bufctl_t));
+
+		/*
+		 * This calculated number will be either the right
+		 * amount, or one greater than what we want.
+		 */
+		if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
+		       > slab_size)
+			nr_objs--;
+
+		if (nr_objs > SLAB_LIMIT)
+			nr_objs = SLAB_LIMIT;
+
+		mgmt_size = slab_mgmt_size(nr_objs, align);
+	}
+	*num = nr_objs;
+	*left_over = slab_size - nr_objs*buffer_size - mgmt_size;
 }
  
 #define slab_error(cachep, msg) __slab_error(__FUNCTION__, cachep, msg)
...	...	@@ -702,32 +702,69 @@
702	702	}
703	703	EXPORT_SYMBOL(kmem_find_general_cachep);
704	704
705		-/* Cal the num objs, wastage, and bytes left over for a given slab size. */
706		-static void cache_estimate(unsigned long gfporder, size_t size, size_t align,
707		- int flags, size_t left_over, unsigned int num)
	705	+static size_t slab_mgmt_size(size_t nr_objs, size_t align)
708	706	{
709		- int i;
710		- size_t wastage = PAGE_SIZE << gfporder;
711		- size_t extra = 0;
712		- size_t base = 0;
	707	+ return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
	708	+}
713	709
714		- if (!(flags & CFLGS_OFF_SLAB)) {
715		- base = sizeof(struct slab);
716		- extra = sizeof(kmem_bufctl_t);
717		- }
718		- i = 0;
719		- while (i * size + ALIGN(base + i * extra, align) <= wastage)
720		- i++;
721		- if (i > 0)
722		- i--;
	710	+/* Calculate the number of objects and left-over bytes for a given
	711	+ buffer size. */
	712	+static void cache_estimate(unsigned long gfporder, size_t buffer_size,
	713	+ size_t align, int flags, size_t *left_over,
	714	+ unsigned int *num)
	715	+{
	716	+ int nr_objs;
	717	+ size_t mgmt_size;
	718	+ size_t slab_size = PAGE_SIZE << gfporder;
723	719
724		- if (i > SLAB_LIMIT)
725		- i = SLAB_LIMIT;
	720	+ /*
	721	+ * The slab management structure can be either off the slab or
	722	+ * on it. For the latter case, the memory allocated for a
	723	+ * slab is used for:
	724	+ *
	725	+ * - The struct slab
	726	+ * - One kmem_bufctl_t for each object
	727	+ * - Padding to respect alignment of @align
	728	+ * - @buffer_size bytes for each object
	729	+ *
	730	+ * If the slab management structure is off the slab, then the
	731	+ * alignment will already be calculated into the size. Because
	732	+ * the slabs are all pages aligned, the objects will be at the
	733	+ * correct alignment when allocated.
	734	+ */
	735	+ if (flags & CFLGS_OFF_SLAB) {
	736	+ mgmt_size = 0;
	737	+ nr_objs = slab_size / buffer_size;
726	738
727		- *num = i;
728		- wastage -= i * size;
729		- wastage -= ALIGN(base + i * extra, align);
730		- *left_over = wastage;
	739	+ if (nr_objs > SLAB_LIMIT)
	740	+ nr_objs = SLAB_LIMIT;
	741	+ } else {
	742	+ /*
	743	+ * Ignore padding for the initial guess. The padding
	744	+ * is at most @align-1 bytes, and @buffer_size is at
	745	+ * least @align. In the worst case, this result will
	746	+ * be one greater than the number of objects that fit
	747	+ * into the memory allocation when taking the padding
	748	+ * into account.
	749	+ */
	750	+ nr_objs = (slab_size - sizeof(struct slab)) /
	751	+ (buffer_size + sizeof(kmem_bufctl_t));
	752	+
	753	+ /*
	754	+ * This calculated number will be either the right
	755	+ * amount, or one greater than what we want.
	756	+ */
	757	+ if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
	758	+ > slab_size)
	759	+ nr_objs--;
	760	+
	761	+ if (nr_objs > SLAB_LIMIT)
	762	+ nr_objs = SLAB_LIMIT;
	763	+
	764	+ mgmt_size = slab_mgmt_size(nr_objs, align);
	765	+ }
	766	+ *num = nr_objs;
	767	+ left_over = slab_size - nr_objsbuffer_size - mgmt_size;
731	768	}
732	769
733	770	#define slab_error(cachep, msg) __slab_error(__FUNCTION__, cachep, msg)