/*
   * linux/mm/slab.c
   * Written by Mark Hemment, 1996/97.
   * (markhe@nextd.demon.co.uk)
   *
   * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
   *
   * Major cleanup, different bufctl logic, per-cpu arrays
   *	(c) 2000 Manfred Spraul
   *
   * Cleanup, make the head arrays unconditional, preparation for NUMA
   * 	(c) 2002 Manfred Spraul
   *
   * An implementation of the Slab Allocator as described in outline in;
   *	UNIX Internals: The New Frontiers by Uresh Vahalia
   *	Pub: Prentice Hall	ISBN 0-13-101908-2
   * or with a little more detail in;
   *	The Slab Allocator: An Object-Caching Kernel Memory Allocator
   *	Jeff Bonwick (Sun Microsystems).
   *	Presented at: USENIX Summer 1994 Technical Conference
   *
   * The memory is organized in caches, one cache for each object type.
   * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
   * Each cache consists out of many slabs (they are small (usually one
   * page long) and always contiguous), and each slab contains multiple
   * initialized objects.
   *
   * This means, that your constructor is used only for newly allocated

   * slabs and you must pass objects with the same initializations to

   * kmem_cache_free.
   *
   * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
   * normal). If you need a special memory type, then must create a new
   * cache for that memory type.
   *
   * In order to reduce fragmentation, the slabs are sorted in 3 groups:
   *   full slabs with 0 free objects
   *   partial slabs
   *   empty slabs with no allocated objects
   *
   * If partial slabs exist, then new allocations come from these slabs,
   * otherwise from empty slabs or new slabs are allocated.
   *
   * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
   * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
   *
   * Each cache has a short per-cpu head array, most allocs
   * and frees go into that array, and if that array overflows, then 1/2
   * of the entries in the array are given back into the global cache.
   * The head array is strictly LIFO and should improve the cache hit rates.
   * On SMP, it additionally reduces the spinlock operations.
   *

   * The c_cpuarray may not be read with enabled local interrupts -

   * it's changed with a smp_call_function().
   *
   * SMP synchronization:
   *  constructors and destructors are called without any locking.

   *  Several members in struct kmem_cache and struct slab never change, they

   *	are accessed without any locking.
   *  The per-cpu arrays are never accessed from the wrong cpu, no locking,
   *  	and local interrupts are disabled so slab code is preempt-safe.
   *  The non-constant members are protected with a per-cache irq spinlock.
   *
   * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
   * in 2000 - many ideas in the current implementation are derived from
   * his patch.
   *
   * Further notes from the original documentation:
   *
   * 11 April '97.  Started multi-threading - markhe

   *	The global cache-chain is protected by the mutex 'slab_mutex'.

   *	The sem is only needed when accessing/extending the cache-chain, which
   *	can never happen inside an interrupt (kmem_cache_create(),
   *	kmem_cache_shrink() and kmem_cache_reap()).
   *
   *	At present, each engine can be growing a cache.  This should be blocked.
   *

   * 15 March 2005. NUMA slab allocator.
   *	Shai Fultheim <shai@scalex86.org>.
   *	Shobhit Dayal <shobhit@calsoftinc.com>
   *	Alok N Kataria <alokk@calsoftinc.com>
   *	Christoph Lameter <christoph@lameter.com>
   *
   *	Modified the slab allocator to be node aware on NUMA systems.
   *	Each node has its own list of partial, free and full slabs.
   *	All object allocations for a node occur from node specific slab lists.

   */

  #include	<linux/slab.h>
  #include	<linux/mm.h>

  #include	<linux/poison.h>

  #include	<linux/swap.h>
  #include	<linux/cache.h>
  #include	<linux/interrupt.h>
  #include	<linux/init.h>
  #include	<linux/compiler.h>

  #include	<linux/cpuset.h>

  #include	<linux/proc_fs.h>

  #include	<linux/seq_file.h>
  #include	<linux/notifier.h>
  #include	<linux/kallsyms.h>
  #include	<linux/cpu.h>
  #include	<linux/sysctl.h>
  #include	<linux/module.h>
  #include	<linux/rcupdate.h>

  #include	<linux/string.h>

  #include	<linux/uaccess.h>

  #include	<linux/nodemask.h>

  #include	<linux/kmemleak.h>

  #include	<linux/mempolicy.h>

  #include	<linux/mutex.h>

  #include	<linux/fault-inject.h>

  #include	<linux/rtmutex.h>

  #include	<linux/reciprocal_div.h>

  #include	<linux/debugobjects.h>

  #include	<linux/kmemcheck.h>

  #include	<linux/memory.h>

  #include	<linux/prefetch.h>

  #include	<net/sock.h>

  #include	<asm/cacheflush.h>
  #include	<asm/tlbflush.h>
  #include	<asm/page.h>

  #include <trace/events/kmem.h>

  #include	"internal.h"

  #include	"slab.h"

  /*

   * DEBUG	- 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.

   *		  0 for faster, smaller code (especially in the critical paths).
   *
   * STATS	- 1 to collect stats for /proc/slabinfo.
   *		  0 for faster, smaller code (especially in the critical paths).
   *
   * FORCED_DEBUG	- 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
   */
  
  #ifdef CONFIG_DEBUG_SLAB
  #define	DEBUG		1
  #define	STATS		1
  #define	FORCED_DEBUG	1
  #else
  #define	DEBUG		0
  #define	STATS		0
  #define	FORCED_DEBUG	0
  #endif

  /* Shouldn't this be in a header file somewhere? */
  #define	BYTES_PER_WORD		sizeof(void *)

  #define	REDZONE_ALIGN		max(BYTES_PER_WORD, __alignof__(unsigned long long))

  #ifndef ARCH_KMALLOC_FLAGS
  #define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
  #endif

  /*
   * true if a page was allocated from pfmemalloc reserves for network-based
   * swap
   */
  static bool pfmemalloc_active __read_mostly;

  /*
   * kmem_bufctl_t:
   *
   * Bufctl's are used for linking objs within a slab
   * linked offsets.
   *
   * This implementation relies on "struct page" for locating the cache &
   * slab an object belongs to.
   * This allows the bufctl structure to be small (one int), but limits
   * the number of objects a slab (not a cache) can contain when off-slab
   * bufctls are used. The limit is the size of the largest general cache
   * that does not use off-slab slabs.
   * For 32bit archs with 4 kB pages, is this 56.
   * This is not serious, as it is only for large objects, when it is unwise
   * to have too many per slab.
   * Note: This limit can be raised by introducing a general cache whose size
   * is less than 512 (PAGE_SIZE<<3), but greater than 256.
   */

  typedef unsigned int kmem_bufctl_t;

  #define BUFCTL_END	(((kmem_bufctl_t)(~0U))-0)
  #define BUFCTL_FREE	(((kmem_bufctl_t)(~0U))-1)

  #define	BUFCTL_ACTIVE	(((kmem_bufctl_t)(~0U))-2)
  #define	SLAB_LIMIT	(((kmem_bufctl_t)(~0U))-3)

  /*

   * struct slab_rcu
   *
   * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to
   * arrange for kmem_freepages to be called via RCU.  This is useful if
   * we need to approach a kernel structure obliquely, from its address
   * obtained without the usual locking.  We can lock the structure to
   * stabilize it and check it's still at the given address, only if we
   * can be sure that the memory has not been meanwhile reused for some
   * other kind of object (which our subsystem's lock might corrupt).
   *
   * rcu_read_lock before reading the address, then rcu_read_unlock after
   * taking the spinlock within the structure expected at that address.

   */
  struct slab_rcu {

  	struct rcu_head head;

  	struct kmem_cache *cachep;

  	void *addr;

  };
  
  /*

   * struct slab
   *
   * Manages the objs in a slab. Placed either at the beginning of mem allocated
   * for a slab, or allocated from an general cache.
   * Slabs are chained into three list: fully used, partial, fully free slabs.
   */
  struct slab {
  	union {
  		struct {
  			struct list_head list;
  			unsigned long colouroff;
  			void *s_mem;		/* including colour offset */
  			unsigned int inuse;	/* num of objs active in slab */
  			kmem_bufctl_t free;
  			unsigned short nodeid;
  		};
  		struct slab_rcu __slab_cover_slab_rcu;
  	};
  };
  
  /*

   * struct array_cache
   *

   * Purpose:
   * - LIFO ordering, to hand out cache-warm objects from _alloc
   * - reduce the number of linked list operations
   * - reduce spinlock operations
   *
   * The limit is stored in the per-cpu structure to reduce the data cache
   * footprint.
   *
   */
  struct array_cache {
  	unsigned int avail;
  	unsigned int limit;
  	unsigned int batchcount;
  	unsigned int touched;

  	spinlock_t lock;

  	void *entry[];	/*

  			 * Must have this definition in here for the proper
  			 * alignment of array_cache. Also simplifies accessing
  			 * the entries.

  			 *
  			 * Entries should not be directly dereferenced as
  			 * entries belonging to slabs marked pfmemalloc will
  			 * have the lower bits set SLAB_OBJ_PFMEMALLOC

  			 */

  };

  #define SLAB_OBJ_PFMEMALLOC	1
  static inline bool is_obj_pfmemalloc(void *objp)
  {
  	return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
  }
  
  static inline void set_obj_pfmemalloc(void **objp)
  {
  	*objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC);
  	return;
  }
  
  static inline void clear_obj_pfmemalloc(void **objp)
  {
  	*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
  }

  /*
   * bootstrap: The caches do not work without cpuarrays anymore, but the
   * cpuarrays are allocated from the generic caches...

   */
  #define BOOT_CPUCACHE_ENTRIES	1
  struct arraycache_init {
  	struct array_cache cache;

  	void *entries[BOOT_CPUCACHE_ENTRIES];

  };
  
  /*

   * Need this for bootstrapping a per node allocator.
   */

  #define NUM_INIT_LISTS (3 * MAX_NUMNODES)

  static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];

  #define	CACHE_CACHE 0

  #define	SIZE_AC MAX_NUMNODES

  #define	SIZE_NODE (2 * MAX_NUMNODES)

  static int drain_freelist(struct kmem_cache *cache,

  			struct kmem_cache_node *n, int tofree);

  static void free_block(struct kmem_cache *cachep, void **objpp, int len,
  			int node);

  static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);

  static void cache_reap(struct work_struct *unused);

  static int slab_early_init = 1;

  #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))

  #define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))

  static void kmem_cache_node_init(struct kmem_cache_node *parent)

  {
  	INIT_LIST_HEAD(&parent->slabs_full);
  	INIT_LIST_HEAD(&parent->slabs_partial);
  	INIT_LIST_HEAD(&parent->slabs_free);
  	parent->shared = NULL;
  	parent->alien = NULL;

  	parent->colour_next = 0;

  	spin_lock_init(&parent->list_lock);
  	parent->free_objects = 0;
  	parent->free_touched = 0;
  }

  #define MAKE_LIST(cachep, listp, slab, nodeid)				\
  	do {								\
  		INIT_LIST_HEAD(listp);					\

  		list_splice(&(cachep->node[nodeid]->slab), listp);	\

  	} while (0)

  #define	MAKE_ALL_LISTS(cachep, ptr, nodeid)				\
  	do {								\

  	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid);	\
  	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
  	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);	\
  	} while (0)

  #define CFLGS_OFF_SLAB		(0x80000000UL)
  #define	OFF_SLAB(x)	((x)->flags & CFLGS_OFF_SLAB)
  
  #define BATCHREFILL_LIMIT	16

  /*
   * Optimization question: fewer reaps means less probability for unnessary
   * cpucache drain/refill cycles.

   *

   * OTOH the cpuarrays can contain lots of objects,

   * which could lock up otherwise freeable slabs.
   */
  #define REAPTIMEOUT_CPUC	(2*HZ)
  #define REAPTIMEOUT_LIST3	(4*HZ)
  
  #if STATS
  #define	STATS_INC_ACTIVE(x)	((x)->num_active++)
  #define	STATS_DEC_ACTIVE(x)	((x)->num_active--)
  #define	STATS_INC_ALLOCED(x)	((x)->num_allocations++)
  #define	STATS_INC_GROWN(x)	((x)->grown++)

  #define	STATS_ADD_REAPED(x,y)	((x)->reaped += (y))

  #define	STATS_SET_HIGH(x)						\
  	do {								\
  		if ((x)->num_active > (x)->high_mark)			\
  			(x)->high_mark = (x)->num_active;		\
  	} while (0)

  #define	STATS_INC_ERR(x)	((x)->errors++)
  #define	STATS_INC_NODEALLOCS(x)	((x)->node_allocs++)

  #define	STATS_INC_NODEFREES(x)	((x)->node_frees++)

  #define STATS_INC_ACOVERFLOW(x)   ((x)->node_overflow++)

  #define	STATS_SET_FREEABLE(x, i)					\
  	do {								\
  		if ((x)->max_freeable < i)				\
  			(x)->max_freeable = i;				\
  	} while (0)

  #define STATS_INC_ALLOCHIT(x)	atomic_inc(&(x)->allochit)
  #define STATS_INC_ALLOCMISS(x)	atomic_inc(&(x)->allocmiss)
  #define STATS_INC_FREEHIT(x)	atomic_inc(&(x)->freehit)
  #define STATS_INC_FREEMISS(x)	atomic_inc(&(x)->freemiss)
  #else
  #define	STATS_INC_ACTIVE(x)	do { } while (0)
  #define	STATS_DEC_ACTIVE(x)	do { } while (0)
  #define	STATS_INC_ALLOCED(x)	do { } while (0)
  #define	STATS_INC_GROWN(x)	do { } while (0)

  #define	STATS_ADD_REAPED(x,y)	do { (void)(y); } while (0)

  #define	STATS_SET_HIGH(x)	do { } while (0)
  #define	STATS_INC_ERR(x)	do { } while (0)
  #define	STATS_INC_NODEALLOCS(x)	do { } while (0)

  #define	STATS_INC_NODEFREES(x)	do { } while (0)

  #define STATS_INC_ACOVERFLOW(x)   do { } while (0)

  #define	STATS_SET_FREEABLE(x, i) do { } while (0)

  #define STATS_INC_ALLOCHIT(x)	do { } while (0)
  #define STATS_INC_ALLOCMISS(x)	do { } while (0)
  #define STATS_INC_FREEHIT(x)	do { } while (0)
  #define STATS_INC_FREEMISS(x)	do { } while (0)
  #endif
  
  #if DEBUG

  /*
   * memory layout of objects:

   * 0		: objp

   * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that

   * 		the end of an object is aligned with the end of the real
   * 		allocation. Catches writes behind the end of the allocation.

   * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:

   * 		redzone word.

   * cachep->obj_offset: The real object.

   * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
   * cachep->size - 1* BYTES_PER_WORD: last caller address

   *					[BYTES_PER_WORD long]

   */

  static int obj_offset(struct kmem_cache *cachep)

  {

  	return cachep->obj_offset;

  }

  static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)

  {
  	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));

  	return (unsigned long long*) (objp + obj_offset(cachep) -
  				      sizeof(unsigned long long));

  }

  static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)

  {
  	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
  	if (cachep->flags & SLAB_STORE_USER)

  		return (unsigned long long *)(objp + cachep->size -

  					      sizeof(unsigned long long) -

  					      REDZONE_ALIGN);

  	return (unsigned long long *) (objp + cachep->size -

  				       sizeof(unsigned long long));

  }

  static void **dbg_userword(struct kmem_cache *cachep, void *objp)

  {
  	BUG_ON(!(cachep->flags & SLAB_STORE_USER));

  	return (void **)(objp + cachep->size - BYTES_PER_WORD);

  }
  
  #else

  #define obj_offset(x)			0

  #define dbg_redzone1(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
  #define dbg_redzone2(cachep, objp)	({BUG(); (unsigned long long *)NULL;})

  #define dbg_userword(cachep, objp)	({BUG(); (void **)NULL;})
  
  #endif
  
  /*

   * Do not go above this order unless 0 objects fit into the slab or
   * overridden on the command line.

   */

  #define	SLAB_MAX_ORDER_HI	1
  #define	SLAB_MAX_ORDER_LO	0
  static int slab_max_order = SLAB_MAX_ORDER_LO;

  static bool slab_max_order_set __initdata;

  static inline struct kmem_cache *virt_to_cache(const void *obj)
  {

  	struct page *page = virt_to_head_page(obj);

  	return page->slab_cache;

  }
  
  static inline struct slab *virt_to_slab(const void *obj)
  {

  	struct page *page = virt_to_head_page(obj);

  
  	VM_BUG_ON(!PageSlab(page));
  	return page->slab_page;

  }

  static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab,
  				 unsigned int idx)
  {

  	return slab->s_mem + cache->size * idx;

  }

  /*

   * We want to avoid an expensive divide : (offset / cache->size)
   *   Using the fact that size is a constant for a particular cache,
   *   we can replace (offset / cache->size) by

   *   reciprocal_divide(offset, cache->reciprocal_buffer_size)
   */
  static inline unsigned int obj_to_index(const struct kmem_cache *cache,
  					const struct slab *slab, void *obj)

  {

  	u32 offset = (obj - slab->s_mem);
  	return reciprocal_divide(offset, cache->reciprocal_buffer_size);

  }

  static struct arraycache_init initarray_generic =

      { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };

  
  /* internal cache of cache description objs */

  static struct kmem_cache kmem_cache_boot = {

  	.batchcount = 1,
  	.limit = BOOT_CPUCACHE_ENTRIES,
  	.shared = 1,

  	.size = sizeof(struct kmem_cache),

  	.name = "kmem_cache",

  };

  #define BAD_ALIEN_MAGIC 0x01020304ul

  #ifdef CONFIG_LOCKDEP
  
  /*
   * Slab sometimes uses the kmalloc slabs to store the slab headers
   * for other slabs "off slab".
   * The locking for this is tricky in that it nests within the locks
   * of all other slabs in a few places; to deal with this special
   * locking we put on-slab caches into a separate lock-class.

   *
   * We set lock class for alien array caches which are up during init.
   * The lock annotation will be lost if all cpus of a node goes down and
   * then comes back up during hotplug

   */

  static struct lock_class_key on_slab_l3_key;
  static struct lock_class_key on_slab_alc_key;

  static struct lock_class_key debugobj_l3_key;
  static struct lock_class_key debugobj_alc_key;
  
  static void slab_set_lock_classes(struct kmem_cache *cachep,
  		struct lock_class_key *l3_key, struct lock_class_key *alc_key,
  		int q)
  {
  	struct array_cache **alc;

  	struct kmem_cache_node *n;

  	int r;

  	n = cachep->node[q];
  	if (!n)

  		return;

  	lockdep_set_class(&n->list_lock, l3_key);
  	alc = n->alien;

  	/*
  	 * FIXME: This check for BAD_ALIEN_MAGIC
  	 * should go away when common slab code is taught to
  	 * work even without alien caches.
  	 * Currently, non NUMA code returns BAD_ALIEN_MAGIC
  	 * for alloc_alien_cache,
  	 */
  	if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
  		return;
  	for_each_node(r) {
  		if (alc[r])
  			lockdep_set_class(&alc[r]->lock, alc_key);
  	}
  }
  
  static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
  {
  	slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
  }
  
  static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
  {
  	int node;
  
  	for_each_online_node(node)
  		slab_set_debugobj_lock_classes_node(cachep, node);
  }

  static void init_node_lock_keys(int q)

  {

  	int i;

  	if (slab_state < UP)

  		return;

  	for (i = 1; i <= KMALLOC_SHIFT_HIGH; i++) {

  		struct kmem_cache_node *n;

  		struct kmem_cache *cache = kmalloc_caches[i];
  
  		if (!cache)
  			continue;

  		n = cache->node[q];
  		if (!n || OFF_SLAB(cache))

  			continue;

  		slab_set_lock_classes(cache, &on_slab_l3_key,

  				&on_slab_alc_key, q);

  	}
  }

  static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q)
  {

  	if (!cachep->node[q])

  		return;
  
  	slab_set_lock_classes(cachep, &on_slab_l3_key,
  			&on_slab_alc_key, q);
  }
  
  static inline void on_slab_lock_classes(struct kmem_cache *cachep)
  {
  	int node;
  
  	VM_BUG_ON(OFF_SLAB(cachep));
  	for_each_node(node)
  		on_slab_lock_classes_node(cachep, node);
  }

  static inline void init_lock_keys(void)
  {
  	int node;
  
  	for_each_node(node)
  		init_node_lock_keys(node);
  }

  #else

  static void init_node_lock_keys(int q)
  {
  }

  static inline void init_lock_keys(void)

  {
  }

  static inline void on_slab_lock_classes(struct kmem_cache *cachep)
  {
  }
  
  static inline void on_slab_lock_classes_node(struct kmem_cache *cachep, int node)
  {
  }

  static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
  {
  }
  
  static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
  {
  }

  #endif

  static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);

  static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)

  {
  	return cachep->array[smp_processor_id()];
  }

  static size_t slab_mgmt_size(size_t nr_objs, size_t align)

  {

  	return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
  }

  /*
   * Calculate the number of objects and left-over bytes for a given buffer size.
   */

  static void cache_estimate(unsigned long gfporder, size_t buffer_size,
  			   size_t align, int flags, size_t *left_over,
  			   unsigned int *num)
  {
  	int nr_objs;
  	size_t mgmt_size;
  	size_t slab_size = PAGE_SIZE << gfporder;

  	/*
  	 * The slab management structure can be either off the slab or
  	 * on it. For the latter case, the memory allocated for a
  	 * slab is used for:
  	 *
  	 * - The struct slab
  	 * - One kmem_bufctl_t for each object
  	 * - Padding to respect alignment of @align
  	 * - @buffer_size bytes for each object
  	 *
  	 * If the slab management structure is off the slab, then the
  	 * alignment will already be calculated into the size. Because
  	 * the slabs are all pages aligned, the objects will be at the
  	 * correct alignment when allocated.
  	 */
  	if (flags & CFLGS_OFF_SLAB) {
  		mgmt_size = 0;
  		nr_objs = slab_size / buffer_size;
  
  		if (nr_objs > SLAB_LIMIT)
  			nr_objs = SLAB_LIMIT;
  	} else {
  		/*
  		 * Ignore padding for the initial guess. The padding
  		 * is at most @align-1 bytes, and @buffer_size is at
  		 * least @align. In the worst case, this result will
  		 * be one greater than the number of objects that fit
  		 * into the memory allocation when taking the padding
  		 * into account.
  		 */
  		nr_objs = (slab_size - sizeof(struct slab)) /
  			  (buffer_size + sizeof(kmem_bufctl_t));
  
  		/*
  		 * This calculated number will be either the right
  		 * amount, or one greater than what we want.
  		 */
  		if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size
  		       > slab_size)
  			nr_objs--;
  
  		if (nr_objs > SLAB_LIMIT)
  			nr_objs = SLAB_LIMIT;
  
  		mgmt_size = slab_mgmt_size(nr_objs, align);
  	}
  	*num = nr_objs;
  	*left_over = slab_size - nr_objs*buffer_size - mgmt_size;

  }

  #if DEBUG

  #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)

  static void __slab_error(const char *function, struct kmem_cache *cachep,
  			char *msg)

  {
  	printk(KERN_ERR "slab error in %s(): cache `%s': %s
  ",

  	       function, cachep->name, msg);

  	dump_stack();

  	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);

  }

  #endif

  /*
   * By default on NUMA we use alien caches to stage the freeing of
   * objects allocated from other nodes. This causes massive memory
   * inefficiencies when using fake NUMA setup to split memory into a
   * large number of small nodes, so it can be disabled on the command
   * line
    */
  
  static int use_alien_caches __read_mostly = 1;
  static int __init noaliencache_setup(char *s)
  {
  	use_alien_caches = 0;
  	return 1;
  }
  __setup("noaliencache", noaliencache_setup);

  static int __init slab_max_order_setup(char *str)
  {
  	get_option(&str, &slab_max_order);
  	slab_max_order = slab_max_order < 0 ? 0 :
  				min(slab_max_order, MAX_ORDER - 1);
  	slab_max_order_set = true;
  
  	return 1;
  }
  __setup("slab_max_order=", slab_max_order_setup);

  #ifdef CONFIG_NUMA
  /*
   * Special reaping functions for NUMA systems called from cache_reap().
   * These take care of doing round robin flushing of alien caches (containing
   * objects freed on different nodes from which they were allocated) and the
   * flushing of remote pcps by calling drain_node_pages.
   */

  static DEFINE_PER_CPU(unsigned long, slab_reap_node);

  
  static void init_reap_node(int cpu)
  {
  	int node;

  	node = next_node(cpu_to_mem(cpu), node_online_map);

  	if (node == MAX_NUMNODES)

  		node = first_node(node_online_map);

  	per_cpu(slab_reap_node, cpu) = node;

  }
  
  static void next_reap_node(void)
  {

  	int node = __this_cpu_read(slab_reap_node);

  	node = next_node(node, node_online_map);
  	if (unlikely(node >= MAX_NUMNODES))
  		node = first_node(node_online_map);

  	__this_cpu_write(slab_reap_node, node);

  }
  
  #else
  #define init_reap_node(cpu) do { } while (0)
  #define next_reap_node(void) do { } while (0)
  #endif

  /*
   * Initiate the reap timer running on the target CPU.  We run at around 1 to 2Hz
   * via the workqueue/eventd.
   * Add the CPU number into the expiration time to minimize the possibility of
   * the CPUs getting into lockstep and contending for the global cache chain
   * lock.
   */

  static void __cpuinit start_cpu_timer(int cpu)

  {

  	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);

  
  	/*
  	 * When this gets called from do_initcalls via cpucache_init(),
  	 * init_workqueues() has already run, so keventd will be setup
  	 * at that time.
  	 */

  	if (keventd_up() && reap_work->work.func == NULL) {

  		init_reap_node(cpu);

  		INIT_DEFERRABLE_WORK(reap_work, cache_reap);

  		schedule_delayed_work_on(cpu, reap_work,
  					__round_jiffies_relative(HZ, cpu));

  	}
  }

  static struct array_cache *alloc_arraycache(int node, int entries,

  					    int batchcount, gfp_t gfp)

  {

  	int memsize = sizeof(void *) * entries + sizeof(struct array_cache);

  	struct array_cache *nc = NULL;

  	nc = kmalloc_node(memsize, gfp, node);

  	/*
  	 * The array_cache structures contain pointers to free object.

  	 * However, when such objects are allocated or transferred to another

  	 * cache the pointers are not cleared and they could be counted as
  	 * valid references during a kmemleak scan. Therefore, kmemleak must
  	 * not scan such objects.
  	 */
  	kmemleak_no_scan(nc);

  	if (nc) {
  		nc->avail = 0;
  		nc->limit = entries;
  		nc->batchcount = batchcount;
  		nc->touched = 0;

  		spin_lock_init(&nc->lock);

  	}
  	return nc;
  }

  static inline bool is_slab_pfmemalloc(struct slab *slabp)
  {
  	struct page *page = virt_to_page(slabp->s_mem);
  
  	return PageSlabPfmemalloc(page);
  }
  
  /* Clears pfmemalloc_active if no slabs have pfmalloc set */
  static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
  						struct array_cache *ac)
  {

  	struct kmem_cache_node *n = cachep->node[numa_mem_id()];

  	struct slab *slabp;
  	unsigned long flags;
  
  	if (!pfmemalloc_active)
  		return;

  	spin_lock_irqsave(&n->list_lock, flags);
  	list_for_each_entry(slabp, &n->slabs_full, list)

  		if (is_slab_pfmemalloc(slabp))
  			goto out;

  	list_for_each_entry(slabp, &n->slabs_partial, list)

  		if (is_slab_pfmemalloc(slabp))
  			goto out;

  	list_for_each_entry(slabp, &n->slabs_free, list)

  		if (is_slab_pfmemalloc(slabp))
  			goto out;
  
  	pfmemalloc_active = false;
  out:

  	spin_unlock_irqrestore(&n->list_lock, flags);

  }

  static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,

  						gfp_t flags, bool force_refill)
  {
  	int i;
  	void *objp = ac->entry[--ac->avail];
  
  	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
  	if (unlikely(is_obj_pfmemalloc(objp))) {

  		struct kmem_cache_node *n;

  
  		if (gfp_pfmemalloc_allowed(flags)) {
  			clear_obj_pfmemalloc(&objp);
  			return objp;
  		}
  
  		/* The caller cannot use PFMEMALLOC objects, find another one */

  		for (i = 0; i < ac->avail; i++) {

  			/* If a !PFMEMALLOC object is found, swap them */
  			if (!is_obj_pfmemalloc(ac->entry[i])) {
  				objp = ac->entry[i];
  				ac->entry[i] = ac->entry[ac->avail];
  				ac->entry[ac->avail] = objp;
  				return objp;
  			}
  		}
  
  		/*
  		 * If there are empty slabs on the slabs_free list and we are
  		 * being forced to refill the cache, mark this one !pfmemalloc.
  		 */

  		n = cachep->node[numa_mem_id()];
  		if (!list_empty(&n->slabs_free) && force_refill) {

  			struct slab *slabp = virt_to_slab(objp);

  			ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem));

  			clear_obj_pfmemalloc(&objp);
  			recheck_pfmemalloc_active(cachep, ac);
  			return objp;
  		}
  
  		/* No !PFMEMALLOC objects available */
  		ac->avail++;
  		objp = NULL;
  	}
  
  	return objp;
  }

  static inline void *ac_get_obj(struct kmem_cache *cachep,
  			struct array_cache *ac, gfp_t flags, bool force_refill)
  {
  	void *objp;
  
  	if (unlikely(sk_memalloc_socks()))
  		objp = __ac_get_obj(cachep, ac, flags, force_refill);
  	else
  		objp = ac->entry[--ac->avail];
  
  	return objp;
  }
  
  static void *__ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,

  								void *objp)
  {
  	if (unlikely(pfmemalloc_active)) {
  		/* Some pfmemalloc slabs exist, check if this is one */

  		struct page *page = virt_to_head_page(objp);

  		if (PageSlabPfmemalloc(page))
  			set_obj_pfmemalloc(&objp);
  	}

  	return objp;
  }
  
  static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
  								void *objp)
  {
  	if (unlikely(sk_memalloc_socks()))
  		objp = __ac_put_obj(cachep, ac, objp);

  	ac->entry[ac->avail++] = objp;
  }

  /*
   * Transfer objects in one arraycache to another.
   * Locking must be handled by the caller.
   *
   * Return the number of entries transferred.
   */
  static int transfer_objects(struct array_cache *to,
  		struct array_cache *from, unsigned int max)
  {
  	/* Figure out how many entries to transfer */

  	int nr = min3(from->avail, max, to->limit - to->avail);

  
  	if (!nr)
  		return 0;
  
  	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
  			sizeof(void *) *nr);
  
  	from->avail -= nr;
  	to->avail += nr;

  	return nr;
  }

  #ifndef CONFIG_NUMA
  
  #define drain_alien_cache(cachep, alien) do { } while (0)

  #define reap_alien(cachep, n) do { } while (0)

  static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)

  {
  	return (struct array_cache **)BAD_ALIEN_MAGIC;
  }
  
  static inline void free_alien_cache(struct array_cache **ac_ptr)
  {
  }
  
  static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
  {
  	return 0;
  }
  
  static inline void *alternate_node_alloc(struct kmem_cache *cachep,
  		gfp_t flags)
  {
  	return NULL;
  }

  static inline void *____cache_alloc_node(struct kmem_cache *cachep,

  		 gfp_t flags, int nodeid)
  {
  	return NULL;
  }
  
  #else	/* CONFIG_NUMA */

  static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);

  static void *alternate_node_alloc(struct kmem_cache *, gfp_t);

  static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)

  {
  	struct array_cache **ac_ptr;

  	int memsize = sizeof(void *) * nr_node_ids;

  	int i;
  
  	if (limit > 1)
  		limit = 12;

  	ac_ptr = kzalloc_node(memsize, gfp, node);

  	if (ac_ptr) {
  		for_each_node(i) {

  			if (i == node || !node_online(i))

  				continue;

  			ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);

  			if (!ac_ptr[i]) {

  				for (i--; i >= 0; i--)

  					kfree(ac_ptr[i]);
  				kfree(ac_ptr);
  				return NULL;
  			}
  		}
  	}
  	return ac_ptr;
  }

  static void free_alien_cache(struct array_cache **ac_ptr)

  {
  	int i;
  
  	if (!ac_ptr)
  		return;

  	for_each_node(i)

  	    kfree(ac_ptr[i]);

  	kfree(ac_ptr);
  }

  static void __drain_alien_cache(struct kmem_cache *cachep,

  				struct array_cache *ac, int node)

  {

  	struct kmem_cache_node *n = cachep->node[node];

  
  	if (ac->avail) {

  		spin_lock(&n->list_lock);

  		/*
  		 * Stuff objects into the remote nodes shared array first.
  		 * That way we could avoid the overhead of putting the objects
  		 * into the free lists and getting them back later.
  		 */

  		if (n->shared)
  			transfer_objects(n->shared, ac, ac->limit);

  		free_block(cachep, ac->entry, ac->avail, node);

  		ac->avail = 0;

  		spin_unlock(&n->list_lock);

  	}
  }

  /*
   * Called from cache_reap() to regularly drain alien caches round robin.
   */

  static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)

  {

  	int node = __this_cpu_read(slab_reap_node);

  	if (n->alien) {
  		struct array_cache *ac = n->alien[node];

  
  		if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {

  			__drain_alien_cache(cachep, ac, node);
  			spin_unlock_irq(&ac->lock);
  		}
  	}
  }

  static void drain_alien_cache(struct kmem_cache *cachep,
  				struct array_cache **alien)

  {

  	int i = 0;

  	struct array_cache *ac;
  	unsigned long flags;
  
  	for_each_online_node(i) {

  		ac = alien[i];

  		if (ac) {
  			spin_lock_irqsave(&ac->lock, flags);
  			__drain_alien_cache(cachep, ac, i);
  			spin_unlock_irqrestore(&ac->lock, flags);
  		}
  	}
  }

  static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)

  {
  	struct slab *slabp = virt_to_slab(objp);
  	int nodeid = slabp->nodeid;

  	struct kmem_cache_node *n;

  	struct array_cache *alien = NULL;

  	int node;

  	node = numa_mem_id();

  
  	/*
  	 * Make sure we are not freeing a object from another node to the array
  	 * cache on this cpu.
  	 */

  	if (likely(slabp->nodeid == node))

  		return 0;

  	n = cachep->node[node];

  	STATS_INC_NODEFREES(cachep);

  	if (n->alien && n->alien[nodeid]) {
  		alien = n->alien[nodeid];

  		spin_lock(&alien->lock);

  		if (unlikely(alien->avail == alien->limit)) {
  			STATS_INC_ACOVERFLOW(cachep);
  			__drain_alien_cache(cachep, alien, nodeid);
  		}

  		ac_put_obj(cachep, alien, objp);

  		spin_unlock(&alien->lock);
  	} else {

  		spin_lock(&(cachep->node[nodeid])->list_lock);

  		free_block(cachep, &objp, 1, nodeid);

  		spin_unlock(&(cachep->node[nodeid])->list_lock);

  	}
  	return 1;
  }

  #endif

  /*

   * Allocates and initializes node for a node on each slab cache, used for

   * either memory or cpu hotplug.  If memory is being hot-added, the kmem_cache_node

   * will be allocated off-node since memory is not yet online for the new node.

   * When hotplugging memory or a cpu, existing node are not replaced if

   * already in use.
   *

   * Must hold slab_mutex.

   */

  static int init_cache_node_node(int node)

  {
  	struct kmem_cache *cachep;

  	struct kmem_cache_node *n;

  	const int memsize = sizeof(struct kmem_cache_node);

  	list_for_each_entry(cachep, &slab_caches, list) {

  		/*
  		 * Set up the size64 kmemlist for cpu before we can
  		 * begin anything. Make sure some other cpu on this
  		 * node has not already allocated this
  		 */

  		if (!cachep->node[node]) {

  			n = kmalloc_node(memsize, GFP_KERNEL, node);
  			if (!n)

  				return -ENOMEM;

  			kmem_cache_node_init(n);
  			n->next_reap = jiffies + REAPTIMEOUT_LIST3 +

  			    ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
  
  			/*
  			 * The l3s don't come and go as CPUs come and

  			 * go.  slab_mutex is sufficient

  			 * protection here.
  			 */

  			cachep->node[node] = n;

  		}

  		spin_lock_irq(&cachep->node[node]->list_lock);
  		cachep->node[node]->free_limit =

  			(1 + nr_cpus_node(node)) *
  			cachep->batchcount + cachep->num;

  		spin_unlock_irq(&cachep->node[node]->list_lock);

  	}
  	return 0;
  }

  static void __cpuinit cpuup_canceled(long cpu)
  {
  	struct kmem_cache *cachep;

  	struct kmem_cache_node *n = NULL;

  	int node = cpu_to_mem(cpu);

  	const struct cpumask *mask = cpumask_of_node(node);

  	list_for_each_entry(cachep, &slab_caches, list) {

  		struct array_cache *nc;
  		struct array_cache *shared;
  		struct array_cache **alien;

  		/* cpu is dead; no one can alloc from it. */
  		nc = cachep->array[cpu];
  		cachep->array[cpu] = NULL;

  		n = cachep->node[node];

  		if (!n)

  			goto free_array_cache;

  		spin_lock_irq(&n->list_lock);

  		/* Free limit for this kmem_cache_node */
  		n->free_limit -= cachep->batchcount;

  		if (nc)
  			free_block(cachep, nc->entry, nc->avail, node);

  		if (!cpumask_empty(mask)) {

  			spin_unlock_irq(&n->list_lock);

  			goto free_array_cache;
  		}

  		shared = n->shared;

  		if (shared) {
  			free_block(cachep, shared->entry,
  				   shared->avail, node);

  			n->shared = NULL;

  		}

  		alien = n->alien;
  		n->alien = NULL;

  		spin_unlock_irq(&n->list_lock);

  
  		kfree(shared);
  		if (alien) {
  			drain_alien_cache(cachep, alien);
  			free_alien_cache(alien);
  		}
  free_array_cache:
  		kfree(nc);
  	}
  	/*
  	 * In the previous loop, all the objects were freed to
  	 * the respective cache's slabs,  now we can go ahead and
  	 * shrink each nodelist to its limit.
  	 */

  	list_for_each_entry(cachep, &slab_caches, list) {

  		n = cachep->node[node];
  		if (!n)

  			continue;

  		drain_freelist(cachep, n, n->free_objects);

  	}
  }
  
  static int __cpuinit cpuup_prepare(long cpu)

  {

  	struct kmem_cache *cachep;

  	struct kmem_cache_node *n = NULL;

  	int node = cpu_to_mem(cpu);

  	int err;

  	/*
  	 * We need to do this right in the beginning since
  	 * alloc_arraycache's are going to use this list.
  	 * kmalloc_node allows us to add the slab to the right

  	 * kmem_cache_node and not this cpu's kmem_cache_node

  	 */

  	err = init_cache_node_node(node);

  	if (err < 0)
  		goto bad;

  
  	/*
  	 * Now we can go ahead with allocating the shared arrays and
  	 * array caches
  	 */

  	list_for_each_entry(cachep, &slab_caches, list) {

  		struct array_cache *nc;
  		struct array_cache *shared = NULL;
  		struct array_cache **alien = NULL;
  
  		nc = alloc_arraycache(node, cachep->limit,

  					cachep->batchcount, GFP_KERNEL);

  		if (!nc)
  			goto bad;
  		if (cachep->shared) {
  			shared = alloc_arraycache(node,
  				cachep->shared * cachep->batchcount,

  				0xbaadf00d, GFP_KERNEL);

  			if (!shared) {
  				kfree(nc);

  				goto bad;

  			}

  		}
  		if (use_alien_caches) {

  			alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);

  			if (!alien) {
  				kfree(shared);
  				kfree(nc);

  				goto bad;

  			}

  		}
  		cachep->array[cpu] = nc;

  		n = cachep->node[node];
  		BUG_ON(!n);

  		spin_lock_irq(&n->list_lock);
  		if (!n->shared) {

  			/*
  			 * We are serialised from CPU_DEAD or
  			 * CPU_UP_CANCELLED by the cpucontrol lock
  			 */

  			n->shared = shared;

  			shared = NULL;
  		}

  #ifdef CONFIG_NUMA

  		if (!n->alien) {
  			n->alien = alien;

  			alien = NULL;

  		}

  #endif

  		spin_unlock_irq(&n->list_lock);

  		kfree(shared);
  		free_alien_cache(alien);

  		if (cachep->flags & SLAB_DEBUG_OBJECTS)
  			slab_set_debugobj_lock_classes_node(cachep, node);

  		else if (!OFF_SLAB(cachep) &&
  			 !(cachep->flags & SLAB_DESTROY_BY_RCU))
  			on_slab_lock_classes_node(cachep, node);

  	}

  	init_node_lock_keys(node);

  	return 0;
  bad:

  	cpuup_canceled(cpu);

  	return -ENOMEM;
  }
  
  static int __cpuinit cpuup_callback(struct notifier_block *nfb,
  				    unsigned long action, void *hcpu)
  {
  	long cpu = (long)hcpu;
  	int err = 0;
  
  	switch (action) {

  	case CPU_UP_PREPARE:
  	case CPU_UP_PREPARE_FROZEN:

  		mutex_lock(&slab_mutex);

  		err = cpuup_prepare(cpu);

  		mutex_unlock(&slab_mutex);

  		break;
  	case CPU_ONLINE:

  	case CPU_ONLINE_FROZEN:

  		start_cpu_timer(cpu);
  		break;
  #ifdef CONFIG_HOTPLUG_CPU

    	case CPU_DOWN_PREPARE:

    	case CPU_DOWN_PREPARE_FROZEN:

  		/*

  		 * Shutdown cache reaper. Note that the slab_mutex is

  		 * held so that if cache_reap() is invoked it cannot do
  		 * anything expensive but will only modify reap_work
  		 * and reschedule the timer.
  		*/

  		cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));

  		/* Now the cache_reaper is guaranteed to be not running. */

  		per_cpu(slab_reap_work, cpu).work.func = NULL;

    		break;
    	case CPU_DOWN_FAILED:

    	case CPU_DOWN_FAILED_FROZEN:

  		start_cpu_timer(cpu);
    		break;

  	case CPU_DEAD:

  	case CPU_DEAD_FROZEN:

  		/*
  		 * Even if all the cpus of a node are down, we don't free the

  		 * kmem_cache_node of any cache. This to avoid a race between

  		 * cpu_down, and a kmalloc allocation from another cpu for

  		 * memory from the node of the cpu going down.  The node

  		 * structure is usually allocated from kmem_cache_create() and
  		 * gets destroyed at kmem_cache_destroy().
  		 */

  		/* fall through */

  #endif

  	case CPU_UP_CANCELED:

  	case CPU_UP_CANCELED_FROZEN:

  		mutex_lock(&slab_mutex);

  		cpuup_canceled(cpu);

  		mutex_unlock(&slab_mutex);

  		break;

  	}

  	return notifier_from_errno(err);

  }

  static struct notifier_block __cpuinitdata cpucache_notifier = {
  	&cpuup_callback, NULL, 0
  };

  #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
  /*
   * Drains freelist for a node on each slab cache, used for memory hot-remove.
   * Returns -EBUSY if all objects cannot be drained so that the node is not
   * removed.
   *

   * Must hold slab_mutex.

   */

  static int __meminit drain_cache_node_node(int node)

  {
  	struct kmem_cache *cachep;
  	int ret = 0;

  	list_for_each_entry(cachep, &slab_caches, list) {

  		struct kmem_cache_node *n;

  		n = cachep->node[node];
  		if (!n)

  			continue;

  		drain_freelist(cachep, n, n->free_objects);

  		if (!list_empty(&n->slabs_full) ||
  		    !list_empty(&n->slabs_partial)) {

  			ret = -EBUSY;
  			break;
  		}
  	}
  	return ret;
  }
  
  static int __meminit slab_memory_callback(struct notifier_block *self,
  					unsigned long action, void *arg)
  {
  	struct memory_notify *mnb = arg;
  	int ret = 0;
  	int nid;
  
  	nid = mnb->status_change_nid;
  	if (nid < 0)
  		goto out;
  
  	switch (action) {
  	case MEM_GOING_ONLINE:

  		mutex_lock(&slab_mutex);

  		ret = init_cache_node_node(nid);

  		mutex_unlock(&slab_mutex);

  		break;
  	case MEM_GOING_OFFLINE:

  		mutex_lock(&slab_mutex);

  		ret = drain_cache_node_node(nid);

  		mutex_unlock(&slab_mutex);

  		break;
  	case MEM_ONLINE:
  	case MEM_OFFLINE:
  	case MEM_CANCEL_ONLINE:
  	case MEM_CANCEL_OFFLINE:
  		break;
  	}
  out:

  	return notifier_from_errno(ret);

  }
  #endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */

  /*

   * swap the static kmem_cache_node with kmalloced memory

   */

  static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,

  				int nodeid)

  {

  	struct kmem_cache_node *ptr;

  	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);

  	BUG_ON(!ptr);

  	memcpy(ptr, list, sizeof(struct kmem_cache_node));

  	/*
  	 * Do not assume that spinlocks can be initialized via memcpy:
  	 */
  	spin_lock_init(&ptr->list_lock);

  	MAKE_ALL_LISTS(cachep, ptr, nodeid);

  	cachep->node[nodeid] = ptr;

  }

  /*

   * For setting up all the kmem_cache_node for cache whose buffer_size is same as
   * size of kmem_cache_node.

   */

  static void __init set_up_node(struct kmem_cache *cachep, int index)

  {
  	int node;
  
  	for_each_online_node(node) {

  		cachep->node[node] = &init_kmem_cache_node[index + node];

  		cachep->node[node]->next_reap = jiffies +

  		    REAPTIMEOUT_LIST3 +
  		    ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
  	}
  }
  
  /*

   * The memory after the last cpu cache pointer is used for the

   * the node pointer.

   */

  static void setup_node_pointer(struct kmem_cache *cachep)

  {

  	cachep->node = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids];

  }
  
  /*

   * Initialisation.  Called after the page allocator have been initialised and
   * before smp_init().

   */
  void __init kmem_cache_init(void)
  {

  	int i;

  	kmem_cache = &kmem_cache_boot;

  	setup_node_pointer(kmem_cache);

  	if (num_possible_nodes() == 1)

  		use_alien_caches = 0;

  	for (i = 0; i < NUM_INIT_LISTS; i++)

  		kmem_cache_node_init(&init_kmem_cache_node[i]);

  	set_up_node(kmem_cache, CACHE_CACHE);

  
  	/*
  	 * Fragmentation resistance on low memory - only use bigger

  	 * page orders on machines with more than 32MB of memory if
  	 * not overridden on the command line.

  	 */

  	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)

  		slab_max_order = SLAB_MAX_ORDER_HI;

  	/* Bootstrap is tricky, because several objects are allocated
  	 * from caches that do not exist yet:

  	 * 1) initialize the kmem_cache cache: it contains the struct
  	 *    kmem_cache structures of all caches, except kmem_cache itself:
  	 *    kmem_cache is statically allocated.

  	 *    Initially an __init data area is used for the head array and the

  	 *    kmem_cache_node structures, it's replaced with a kmalloc allocated

  	 *    array at the end of the bootstrap.

  	 * 2) Create the first kmalloc cache.

  	 *    The struct kmem_cache for the new cache is allocated normally.

  	 *    An __init data area is used for the head array.
  	 * 3) Create the remaining kmalloc caches, with minimally sized
  	 *    head arrays.

  	 * 4) Replace the __init data head arrays for kmem_cache and the first

  	 *    kmalloc cache with kmalloc allocated arrays.

  	 * 5) Replace the __init data for kmem_cache_node for kmem_cache and

  	 *    the other cache's with kmalloc allocated memory.
  	 * 6) Resize the head arrays of the kmalloc caches to their final sizes.

  	 */

  	/* 1) create the kmem_cache */

  	/*

  	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids

  	 */

  	create_boot_cache(kmem_cache, "kmem_cache",
  		offsetof(struct kmem_cache, array[nr_cpu_ids]) +

  				  nr_node_ids * sizeof(struct kmem_cache_node *),

  				  SLAB_HWCACHE_ALIGN);
  	list_add(&kmem_cache->list, &slab_caches);

  
  	/* 2+3) create the kmalloc caches */

  	/*
  	 * Initialize the caches that provide memory for the array cache and the

  	 * kmem_cache_node structures first.  Without this, further allocations will

  	 * bug.

  	 */

  	kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
  					kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);

  	if (INDEX_AC != INDEX_NODE)
  		kmalloc_caches[INDEX_NODE] =
  			create_kmalloc_cache("kmalloc-node",
  				kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);

  	slab_early_init = 0;

  	/* 4) Replace the bootstrap head arrays */
  	{

  		struct array_cache *ptr;

  		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);

  		memcpy(ptr, cpu_cache_get(kmem_cache),

  		       sizeof(struct arraycache_init));

  		/*
  		 * Do not assume that spinlocks can be initialized via memcpy:
  		 */
  		spin_lock_init(&ptr->lock);

  		kmem_cache->array[smp_processor_id()] = ptr;

  		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);

  		BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC])

  		       != &initarray_generic.cache);

  		memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]),

  		       sizeof(struct arraycache_init));

  		/*
  		 * Do not assume that spinlocks can be initialized via memcpy:
  		 */
  		spin_lock_init(&ptr->lock);

  		kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr;

  	}

  	/* 5) Replace the bootstrap kmem_cache_node */

  	{

  		int nid;

  		for_each_online_node(nid) {

  			init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);

  			init_list(kmalloc_caches[INDEX_AC],