Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 1eb5ac6466d4be7b15b38ce3ab709600f1bc891f

Authored by Nick Piggin
Committed by Pekka Enberg
1 parent ce8a7424d2
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

mm: SLUB fix reclaim_state 

SLUB does not correctly account reclaim_state.reclaimed_slab, so it will
break memory reclaim. Account it like SLAB does.

Cc: stable@kernel.org
Cc: linux-mm@kvack.org
Cc: Matt Mackall <mpm@selenic.com>
Acked-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>

Showing 1 changed file with 3 additions and 0 deletions Inline Diff

1 /* 1 /*
2 * SLUB: A slab allocator that limits cache line use instead of queuing 2 * SLUB: A slab allocator that limits cache line use instead of queuing
3 * objects in per cpu and per node lists. 3 * objects in per cpu and per node lists.
4 * 4 *
5 * The allocator synchronizes using per slab locks and only 5 * The allocator synchronizes using per slab locks and only
6 * uses a centralized lock to manage a pool of partial slabs. 6 * uses a centralized lock to manage a pool of partial slabs.
7 * 7 *
8 * (C) 2007 SGI, Christoph Lameter 8 * (C) 2007 SGI, Christoph Lameter
9 */ 9 */
10 10
11 #include <linux/mm.h> 11 #include <linux/mm.h>
12 #include <linux/swap.h> /* struct reclaim_state */
12 #include <linux/module.h> 13 #include <linux/module.h>
13 #include <linux/bit_spinlock.h> 14 #include <linux/bit_spinlock.h>
14 #include <linux/interrupt.h> 15 #include <linux/interrupt.h>
15 #include <linux/bitops.h> 16 #include <linux/bitops.h>
16 #include <linux/slab.h> 17 #include <linux/slab.h>
17 #include <linux/proc_fs.h> 18 #include <linux/proc_fs.h>
18 #include <linux/seq_file.h> 19 #include <linux/seq_file.h>
19 #include <trace/kmemtrace.h> 20 #include <trace/kmemtrace.h>
20 #include <linux/cpu.h> 21 #include <linux/cpu.h>
21 #include <linux/cpuset.h> 22 #include <linux/cpuset.h>
22 #include <linux/mempolicy.h> 23 #include <linux/mempolicy.h>
23 #include <linux/ctype.h> 24 #include <linux/ctype.h>
24 #include <linux/debugobjects.h> 25 #include <linux/debugobjects.h>
25 #include <linux/kallsyms.h> 26 #include <linux/kallsyms.h>
26 #include <linux/memory.h> 27 #include <linux/memory.h>
27 #include <linux/math64.h> 28 #include <linux/math64.h>
28 #include <linux/fault-inject.h> 29 #include <linux/fault-inject.h>
29 30
30 /* 31 /*
31 * Lock order: 32 * Lock order:
32 * 1. slab_lock(page) 33 * 1. slab_lock(page)
33 * 2. slab->list_lock 34 * 2. slab->list_lock
34 * 35 *
35 * The slab_lock protects operations on the object of a particular 36 * The slab_lock protects operations on the object of a particular
36 * slab and its metadata in the page struct. If the slab lock 37 * slab and its metadata in the page struct. If the slab lock
37 * has been taken then no allocations nor frees can be performed 38 * has been taken then no allocations nor frees can be performed
38 * on the objects in the slab nor can the slab be added or removed 39 * on the objects in the slab nor can the slab be added or removed
39 * from the partial or full lists since this would mean modifying 40 * from the partial or full lists since this would mean modifying
40 * the page_struct of the slab. 41 * the page_struct of the slab.
41 * 42 *
42 * The list_lock protects the partial and full list on each node and 43 * The list_lock protects the partial and full list on each node and
43 * the partial slab counter. If taken then no new slabs may be added or 44 * the partial slab counter. If taken then no new slabs may be added or
44 * removed from the lists nor make the number of partial slabs be modified. 45 * removed from the lists nor make the number of partial slabs be modified.
45 * (Note that the total number of slabs is an atomic value that may be 46 * (Note that the total number of slabs is an atomic value that may be
46 * modified without taking the list lock). 47 * modified without taking the list lock).
47 * 48 *
48 * The list_lock is a centralized lock and thus we avoid taking it as 49 * The list_lock is a centralized lock and thus we avoid taking it as
49 * much as possible. As long as SLUB does not have to handle partial 50 * much as possible. As long as SLUB does not have to handle partial
50 * slabs, operations can continue without any centralized lock. F.e. 51 * slabs, operations can continue without any centralized lock. F.e.
51 * allocating a long series of objects that fill up slabs does not require 52 * allocating a long series of objects that fill up slabs does not require
52 * the list lock. 53 * the list lock.
53 * 54 *
54 * The lock order is sometimes inverted when we are trying to get a slab 55 * The lock order is sometimes inverted when we are trying to get a slab
55 * off a list. We take the list_lock and then look for a page on the list 56 * off a list. We take the list_lock and then look for a page on the list
56 * to use. While we do that objects in the slabs may be freed. We can 57 * to use. While we do that objects in the slabs may be freed. We can
57 * only operate on the slab if we have also taken the slab_lock. So we use 58 * only operate on the slab if we have also taken the slab_lock. So we use
58 * a slab_trylock() on the slab. If trylock was successful then no frees 59 * a slab_trylock() on the slab. If trylock was successful then no frees
59 * can occur anymore and we can use the slab for allocations etc. If the 60 * can occur anymore and we can use the slab for allocations etc. If the
60 * slab_trylock() does not succeed then frees are in progress in the slab and 61 * slab_trylock() does not succeed then frees are in progress in the slab and
61 * we must stay away from it for a while since we may cause a bouncing 62 * we must stay away from it for a while since we may cause a bouncing
62 * cacheline if we try to acquire the lock. So go onto the next slab. 63 * cacheline if we try to acquire the lock. So go onto the next slab.
63 * If all pages are busy then we may allocate a new slab instead of reusing 64 * If all pages are busy then we may allocate a new slab instead of reusing
64 * a partial slab. A new slab has noone operating on it and thus there is 65 * a partial slab. A new slab has noone operating on it and thus there is
65 * no danger of cacheline contention. 66 * no danger of cacheline contention.
66 * 67 *
67 * Interrupts are disabled during allocation and deallocation in order to 68 * Interrupts are disabled during allocation and deallocation in order to
68 * make the slab allocator safe to use in the context of an irq. In addition 69 * make the slab allocator safe to use in the context of an irq. In addition
69 * interrupts are disabled to ensure that the processor does not change 70 * interrupts are disabled to ensure that the processor does not change
70 * while handling per_cpu slabs, due to kernel preemption. 71 * while handling per_cpu slabs, due to kernel preemption.
71 * 72 *
72 * SLUB assigns one slab for allocation to each processor. 73 * SLUB assigns one slab for allocation to each processor.
73 * Allocations only occur from these slabs called cpu slabs. 74 * Allocations only occur from these slabs called cpu slabs.
74 * 75 *
75 * Slabs with free elements are kept on a partial list and during regular 76 * Slabs with free elements are kept on a partial list and during regular
76 * operations no list for full slabs is used. If an object in a full slab is 77 * operations no list for full slabs is used. If an object in a full slab is
77 * freed then the slab will show up again on the partial lists. 78 * freed then the slab will show up again on the partial lists.
78 * We track full slabs for debugging purposes though because otherwise we 79 * We track full slabs for debugging purposes though because otherwise we
79 * cannot scan all objects. 80 * cannot scan all objects.
80 * 81 *
81 * Slabs are freed when they become empty. Teardown and setup is 82 * Slabs are freed when they become empty. Teardown and setup is
82 * minimal so we rely on the page allocators per cpu caches for 83 * minimal so we rely on the page allocators per cpu caches for
83 * fast frees and allocs. 84 * fast frees and allocs.
84 * 85 *
85 * Overloading of page flags that are otherwise used for LRU management. 86 * Overloading of page flags that are otherwise used for LRU management.
86 * 87 *
87 * PageActive The slab is frozen and exempt from list processing. 88 * PageActive The slab is frozen and exempt from list processing.
88 * This means that the slab is dedicated to a purpose 89 * This means that the slab is dedicated to a purpose
89 * such as satisfying allocations for a specific 90 * such as satisfying allocations for a specific
90 * processor. Objects may be freed in the slab while 91 * processor. Objects may be freed in the slab while
91 * it is frozen but slab_free will then skip the usual 92 * it is frozen but slab_free will then skip the usual
92 * list operations. It is up to the processor holding 93 * list operations. It is up to the processor holding
93 * the slab to integrate the slab into the slab lists 94 * the slab to integrate the slab into the slab lists
94 * when the slab is no longer needed. 95 * when the slab is no longer needed.
95 * 96 *
96 * One use of this flag is to mark slabs that are 97 * One use of this flag is to mark slabs that are
97 * used for allocations. Then such a slab becomes a cpu 98 * used for allocations. Then such a slab becomes a cpu
98 * slab. The cpu slab may be equipped with an additional 99 * slab. The cpu slab may be equipped with an additional
99 * freelist that allows lockless access to 100 * freelist that allows lockless access to
100 * free objects in addition to the regular freelist 101 * free objects in addition to the regular freelist
101 * that requires the slab lock. 102 * that requires the slab lock.
102 * 103 *
103 * PageError Slab requires special handling due to debug 104 * PageError Slab requires special handling due to debug
104 * options set. This moves slab handling out of 105 * options set. This moves slab handling out of
105 * the fast path and disables lockless freelists. 106 * the fast path and disables lockless freelists.
106 */ 107 */
107 108
108 #ifdef CONFIG_SLUB_DEBUG 109 #ifdef CONFIG_SLUB_DEBUG
109 #define SLABDEBUG 1 110 #define SLABDEBUG 1
110 #else 111 #else
111 #define SLABDEBUG 0 112 #define SLABDEBUG 0
112 #endif 113 #endif
113 114
114 /* 115 /*
115 * Issues still to be resolved: 116 * Issues still to be resolved:
116 * 117 *
117 * - Support PAGE_ALLOC_DEBUG. Should be easy to do. 118 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
118 * 119 *
119 * - Variable sizing of the per node arrays 120 * - Variable sizing of the per node arrays
120 */ 121 */
121 122
122 /* Enable to test recovery from slab corruption on boot */ 123 /* Enable to test recovery from slab corruption on boot */
123 #undef SLUB_RESILIENCY_TEST 124 #undef SLUB_RESILIENCY_TEST
124 125
125 /* 126 /*
126 * Mininum number of partial slabs. These will be left on the partial 127 * Mininum number of partial slabs. These will be left on the partial
127 * lists even if they are empty. kmem_cache_shrink may reclaim them. 128 * lists even if they are empty. kmem_cache_shrink may reclaim them.
128 */ 129 */
129 #define MIN_PARTIAL 5 130 #define MIN_PARTIAL 5
130 131
131 /* 132 /*
132 * Maximum number of desirable partial slabs. 133 * Maximum number of desirable partial slabs.
133 * The existence of more partial slabs makes kmem_cache_shrink 134 * The existence of more partial slabs makes kmem_cache_shrink
134 * sort the partial list by the number of objects in the. 135 * sort the partial list by the number of objects in the.
135 */ 136 */
136 #define MAX_PARTIAL 10 137 #define MAX_PARTIAL 10
137 138
138 #define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \ 139 #define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
139 SLAB_POISON | SLAB_STORE_USER) 140 SLAB_POISON | SLAB_STORE_USER)
140 141
141 /* 142 /*
142 * Set of flags that will prevent slab merging 143 * Set of flags that will prevent slab merging
143 */ 144 */
144 #define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 145 #define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
145 SLAB_TRACE | SLAB_DESTROY_BY_RCU) 146 SLAB_TRACE | SLAB_DESTROY_BY_RCU)
146 147
147 #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \ 148 #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
148 SLAB_CACHE_DMA) 149 SLAB_CACHE_DMA)
149 150
150 #ifndef ARCH_KMALLOC_MINALIGN 151 #ifndef ARCH_KMALLOC_MINALIGN
151 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) 152 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
152 #endif 153 #endif
153 154
154 #ifndef ARCH_SLAB_MINALIGN 155 #ifndef ARCH_SLAB_MINALIGN
155 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 156 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
156 #endif 157 #endif
157 158
158 #define OO_SHIFT 16 159 #define OO_SHIFT 16
159 #define OO_MASK ((1 << OO_SHIFT) - 1) 160 #define OO_MASK ((1 << OO_SHIFT) - 1)
160 #define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */ 161 #define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
161 162
162 /* Internal SLUB flags */ 163 /* Internal SLUB flags */
163 #define __OBJECT_POISON 0x80000000 /* Poison object */ 164 #define __OBJECT_POISON 0x80000000 /* Poison object */
164 #define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */ 165 #define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */
165 166
166 static int kmem_size = sizeof(struct kmem_cache); 167 static int kmem_size = sizeof(struct kmem_cache);
167 168
168 #ifdef CONFIG_SMP 169 #ifdef CONFIG_SMP
169 static struct notifier_block slab_notifier; 170 static struct notifier_block slab_notifier;
170 #endif 171 #endif
171 172
172 static enum { 173 static enum {
173 DOWN, /* No slab functionality available */ 174 DOWN, /* No slab functionality available */
174 PARTIAL, /* kmem_cache_open() works but kmalloc does not */ 175 PARTIAL, /* kmem_cache_open() works but kmalloc does not */
175 UP, /* Everything works but does not show up in sysfs */ 176 UP, /* Everything works but does not show up in sysfs */
176 SYSFS /* Sysfs up */ 177 SYSFS /* Sysfs up */
177 } slab_state = DOWN; 178 } slab_state = DOWN;
178 179
179 /* A list of all slab caches on the system */ 180 /* A list of all slab caches on the system */
180 static DECLARE_RWSEM(slub_lock); 181 static DECLARE_RWSEM(slub_lock);
181 static LIST_HEAD(slab_caches); 182 static LIST_HEAD(slab_caches);
182 183
183 /* 184 /*
184 * Tracking user of a slab. 185 * Tracking user of a slab.
185 */ 186 */
186 struct track { 187 struct track {
187 unsigned long addr; /* Called from address */ 188 unsigned long addr; /* Called from address */
188 int cpu; /* Was running on cpu */ 189 int cpu; /* Was running on cpu */
189 int pid; /* Pid context */ 190 int pid; /* Pid context */
190 unsigned long when; /* When did the operation occur */ 191 unsigned long when; /* When did the operation occur */
191 }; 192 };
192 193
193 enum track_item { TRACK_ALLOC, TRACK_FREE }; 194 enum track_item { TRACK_ALLOC, TRACK_FREE };
194 195
195 #ifdef CONFIG_SLUB_DEBUG 196 #ifdef CONFIG_SLUB_DEBUG
196 static int sysfs_slab_add(struct kmem_cache *); 197 static int sysfs_slab_add(struct kmem_cache *);
197 static int sysfs_slab_alias(struct kmem_cache *, const char *); 198 static int sysfs_slab_alias(struct kmem_cache *, const char *);
198 static void sysfs_slab_remove(struct kmem_cache *); 199 static void sysfs_slab_remove(struct kmem_cache *);
199 200
200 #else 201 #else
201 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } 202 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
202 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) 203 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
203 { return 0; } 204 { return 0; }
204 static inline void sysfs_slab_remove(struct kmem_cache *s) 205 static inline void sysfs_slab_remove(struct kmem_cache *s)
205 { 206 {
206 kfree(s); 207 kfree(s);
207 } 208 }
208 209
209 #endif 210 #endif
210 211
211 static inline void stat(struct kmem_cache_cpu *c, enum stat_item si) 212 static inline void stat(struct kmem_cache_cpu *c, enum stat_item si)
212 { 213 {
213 #ifdef CONFIG_SLUB_STATS 214 #ifdef CONFIG_SLUB_STATS
214 c->stat[si]++; 215 c->stat[si]++;
215 #endif 216 #endif
216 } 217 }
217 218
218 /******************************************************************** 219 /********************************************************************
219 * Core slab cache functions 220 * Core slab cache functions
220 *******************************************************************/ 221 *******************************************************************/
221 222
222 int slab_is_available(void) 223 int slab_is_available(void)
223 { 224 {
224 return slab_state >= UP; 225 return slab_state >= UP;
225 } 226 }
226 227
227 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) 228 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
228 { 229 {
229 #ifdef CONFIG_NUMA 230 #ifdef CONFIG_NUMA
230 return s->node[node]; 231 return s->node[node];
231 #else 232 #else
232 return &s->local_node; 233 return &s->local_node;
233 #endif 234 #endif
234 } 235 }
235 236
236 static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu) 237 static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu)
237 { 238 {
238 #ifdef CONFIG_SMP 239 #ifdef CONFIG_SMP
239 return s->cpu_slab[cpu]; 240 return s->cpu_slab[cpu];
240 #else 241 #else
241 return &s->cpu_slab; 242 return &s->cpu_slab;
242 #endif 243 #endif
243 } 244 }
244 245
245 /* Verify that a pointer has an address that is valid within a slab page */ 246 /* Verify that a pointer has an address that is valid within a slab page */
246 static inline int check_valid_pointer(struct kmem_cache *s, 247 static inline int check_valid_pointer(struct kmem_cache *s,
247 struct page *page, const void *object) 248 struct page *page, const void *object)
248 { 249 {
249 void *base; 250 void *base;
250 251
251 if (!object) 252 if (!object)
252 return 1; 253 return 1;
253 254
254 base = page_address(page); 255 base = page_address(page);
255 if (object < base || object >= base + page->objects * s->size || 256 if (object < base || object >= base + page->objects * s->size ||
256 (object - base) % s->size) { 257 (object - base) % s->size) {
257 return 0; 258 return 0;
258 } 259 }
259 260
260 return 1; 261 return 1;
261 } 262 }
262 263
263 /* 264 /*
264 * Slow version of get and set free pointer. 265 * Slow version of get and set free pointer.
265 * 266 *
266 * This version requires touching the cache lines of kmem_cache which 267 * This version requires touching the cache lines of kmem_cache which
267 * we avoid to do in the fast alloc free paths. There we obtain the offset 268 * we avoid to do in the fast alloc free paths. There we obtain the offset
268 * from the page struct. 269 * from the page struct.
269 */ 270 */
270 static inline void *get_freepointer(struct kmem_cache *s, void *object) 271 static inline void *get_freepointer(struct kmem_cache *s, void *object)
271 { 272 {
272 return *(void **)(object + s->offset); 273 return *(void **)(object + s->offset);
273 } 274 }
274 275
275 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 276 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
276 { 277 {
277 *(void **)(object + s->offset) = fp; 278 *(void **)(object + s->offset) = fp;
278 } 279 }
279 280
280 /* Loop over all objects in a slab */ 281 /* Loop over all objects in a slab */
281 #define for_each_object(__p, __s, __addr, __objects) \ 282 #define for_each_object(__p, __s, __addr, __objects) \
282 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ 283 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\
283 __p += (__s)->size) 284 __p += (__s)->size)
284 285
285 /* Scan freelist */ 286 /* Scan freelist */
286 #define for_each_free_object(__p, __s, __free) \ 287 #define for_each_free_object(__p, __s, __free) \
287 for (__p = (__free); __p; __p = get_freepointer((__s), __p)) 288 for (__p = (__free); __p; __p = get_freepointer((__s), __p))
288 289
289 /* Determine object index from a given position */ 290 /* Determine object index from a given position */
290 static inline int slab_index(void *p, struct kmem_cache *s, void *addr) 291 static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
291 { 292 {
292 return (p - addr) / s->size; 293 return (p - addr) / s->size;
293 } 294 }
294 295
295 static inline struct kmem_cache_order_objects oo_make(int order, 296 static inline struct kmem_cache_order_objects oo_make(int order,
296 unsigned long size) 297 unsigned long size)
297 { 298 {
298 struct kmem_cache_order_objects x = { 299 struct kmem_cache_order_objects x = {
299 (order << OO_SHIFT) + (PAGE_SIZE << order) / size 300 (order << OO_SHIFT) + (PAGE_SIZE << order) / size
300 }; 301 };
301 302
302 return x; 303 return x;
303 } 304 }
304 305
305 static inline int oo_order(struct kmem_cache_order_objects x) 306 static inline int oo_order(struct kmem_cache_order_objects x)
306 { 307 {
307 return x.x >> OO_SHIFT; 308 return x.x >> OO_SHIFT;
308 } 309 }
309 310
310 static inline int oo_objects(struct kmem_cache_order_objects x) 311 static inline int oo_objects(struct kmem_cache_order_objects x)
311 { 312 {
312 return x.x & OO_MASK; 313 return x.x & OO_MASK;
313 } 314 }
314 315
315 #ifdef CONFIG_SLUB_DEBUG 316 #ifdef CONFIG_SLUB_DEBUG
316 /* 317 /*
317 * Debug settings: 318 * Debug settings:
318 */ 319 */
319 #ifdef CONFIG_SLUB_DEBUG_ON 320 #ifdef CONFIG_SLUB_DEBUG_ON
320 static int slub_debug = DEBUG_DEFAULT_FLAGS; 321 static int slub_debug = DEBUG_DEFAULT_FLAGS;
321 #else 322 #else
322 static int slub_debug; 323 static int slub_debug;
323 #endif 324 #endif
324 325
325 static char *slub_debug_slabs; 326 static char *slub_debug_slabs;
326 327
327 /* 328 /*
328 * Object debugging 329 * Object debugging
329 */ 330 */
330 static void print_section(char *text, u8 *addr, unsigned int length) 331 static void print_section(char *text, u8 *addr, unsigned int length)
331 { 332 {
332 int i, offset; 333 int i, offset;
333 int newline = 1; 334 int newline = 1;
334 char ascii[17]; 335 char ascii[17];
335 336
336 ascii[16] = 0; 337 ascii[16] = 0;
337 338
338 for (i = 0; i < length; i++) { 339 for (i = 0; i < length; i++) {
339 if (newline) { 340 if (newline) {
340 printk(KERN_ERR "%8s 0x%p: ", text, addr + i); 341 printk(KERN_ERR "%8s 0x%p: ", text, addr + i);
341 newline = 0; 342 newline = 0;
342 } 343 }
343 printk(KERN_CONT " %02x", addr[i]); 344 printk(KERN_CONT " %02x", addr[i]);
344 offset = i % 16; 345 offset = i % 16;
345 ascii[offset] = isgraph(addr[i]) ? addr[i] : '.'; 346 ascii[offset] = isgraph(addr[i]) ? addr[i] : '.';
346 if (offset == 15) { 347 if (offset == 15) {
347 printk(KERN_CONT " %s\n", ascii); 348 printk(KERN_CONT " %s\n", ascii);
348 newline = 1; 349 newline = 1;
349 } 350 }
350 } 351 }
351 if (!newline) { 352 if (!newline) {
352 i %= 16; 353 i %= 16;
353 while (i < 16) { 354 while (i < 16) {
354 printk(KERN_CONT " "); 355 printk(KERN_CONT " ");
355 ascii[i] = ' '; 356 ascii[i] = ' ';
356 i++; 357 i++;
357 } 358 }
358 printk(KERN_CONT " %s\n", ascii); 359 printk(KERN_CONT " %s\n", ascii);
359 } 360 }
360 } 361 }
361 362
362 static struct track *get_track(struct kmem_cache *s, void *object, 363 static struct track *get_track(struct kmem_cache *s, void *object,
363 enum track_item alloc) 364 enum track_item alloc)
364 { 365 {
365 struct track *p; 366 struct track *p;
366 367
367 if (s->offset) 368 if (s->offset)
368 p = object + s->offset + sizeof(void *); 369 p = object + s->offset + sizeof(void *);
369 else 370 else
370 p = object + s->inuse; 371 p = object + s->inuse;
371 372
372 return p + alloc; 373 return p + alloc;
373 } 374 }
374 375
375 static void set_track(struct kmem_cache *s, void *object, 376 static void set_track(struct kmem_cache *s, void *object,
376 enum track_item alloc, unsigned long addr) 377 enum track_item alloc, unsigned long addr)
377 { 378 {
378 struct track *p = get_track(s, object, alloc); 379 struct track *p = get_track(s, object, alloc);
379 380
380 if (addr) { 381 if (addr) {
381 p->addr = addr; 382 p->addr = addr;
382 p->cpu = smp_processor_id(); 383 p->cpu = smp_processor_id();
383 p->pid = current->pid; 384 p->pid = current->pid;
384 p->when = jiffies; 385 p->when = jiffies;
385 } else 386 } else
386 memset(p, 0, sizeof(struct track)); 387 memset(p, 0, sizeof(struct track));
387 } 388 }
388 389
389 static void init_tracking(struct kmem_cache *s, void *object) 390 static void init_tracking(struct kmem_cache *s, void *object)
390 { 391 {
391 if (!(s->flags & SLAB_STORE_USER)) 392 if (!(s->flags & SLAB_STORE_USER))
392 return; 393 return;
393 394
394 set_track(s, object, TRACK_FREE, 0UL); 395 set_track(s, object, TRACK_FREE, 0UL);
395 set_track(s, object, TRACK_ALLOC, 0UL); 396 set_track(s, object, TRACK_ALLOC, 0UL);
396 } 397 }
397 398
398 static void print_track(const char *s, struct track *t) 399 static void print_track(const char *s, struct track *t)
399 { 400 {
400 if (!t->addr) 401 if (!t->addr)
401 return; 402 return;
402 403
403 printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n", 404 printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
404 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); 405 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
405 } 406 }
406 407
407 static void print_tracking(struct kmem_cache *s, void *object) 408 static void print_tracking(struct kmem_cache *s, void *object)
408 { 409 {
409 if (!(s->flags & SLAB_STORE_USER)) 410 if (!(s->flags & SLAB_STORE_USER))
410 return; 411 return;
411 412
412 print_track("Allocated", get_track(s, object, TRACK_ALLOC)); 413 print_track("Allocated", get_track(s, object, TRACK_ALLOC));
413 print_track("Freed", get_track(s, object, TRACK_FREE)); 414 print_track("Freed", get_track(s, object, TRACK_FREE));
414 } 415 }
415 416
416 static void print_page_info(struct page *page) 417 static void print_page_info(struct page *page)
417 { 418 {
418 printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", 419 printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
419 page, page->objects, page->inuse, page->freelist, page->flags); 420 page, page->objects, page->inuse, page->freelist, page->flags);
420 421
421 } 422 }
422 423
423 static void slab_bug(struct kmem_cache *s, char *fmt, ...) 424 static void slab_bug(struct kmem_cache *s, char *fmt, ...)
424 { 425 {
425 va_list args; 426 va_list args;
426 char buf[100]; 427 char buf[100];
427 428
428 va_start(args, fmt); 429 va_start(args, fmt);
429 vsnprintf(buf, sizeof(buf), fmt, args); 430 vsnprintf(buf, sizeof(buf), fmt, args);
430 va_end(args); 431 va_end(args);
431 printk(KERN_ERR "========================================" 432 printk(KERN_ERR "========================================"
432 "=====================================\n"); 433 "=====================================\n");
433 printk(KERN_ERR "BUG %s: %s\n", s->name, buf); 434 printk(KERN_ERR "BUG %s: %s\n", s->name, buf);
434 printk(KERN_ERR "----------------------------------------" 435 printk(KERN_ERR "----------------------------------------"
435 "-------------------------------------\n\n"); 436 "-------------------------------------\n\n");
436 } 437 }
437 438
438 static void slab_fix(struct kmem_cache *s, char *fmt, ...) 439 static void slab_fix(struct kmem_cache *s, char *fmt, ...)
439 { 440 {
440 va_list args; 441 va_list args;
441 char buf[100]; 442 char buf[100];
442 443
443 va_start(args, fmt); 444 va_start(args, fmt);
444 vsnprintf(buf, sizeof(buf), fmt, args); 445 vsnprintf(buf, sizeof(buf), fmt, args);
445 va_end(args); 446 va_end(args);
446 printk(KERN_ERR "FIX %s: %s\n", s->name, buf); 447 printk(KERN_ERR "FIX %s: %s\n", s->name, buf);
447 } 448 }
448 449
449 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) 450 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
450 { 451 {
451 unsigned int off; /* Offset of last byte */ 452 unsigned int off; /* Offset of last byte */
452 u8 *addr = page_address(page); 453 u8 *addr = page_address(page);
453 454
454 print_tracking(s, p); 455 print_tracking(s, p);
455 456
456 print_page_info(page); 457 print_page_info(page);
457 458
458 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n", 459 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
459 p, p - addr, get_freepointer(s, p)); 460 p, p - addr, get_freepointer(s, p));
460 461
461 if (p > addr + 16) 462 if (p > addr + 16)
462 print_section("Bytes b4", p - 16, 16); 463 print_section("Bytes b4", p - 16, 16);
463 464
464 print_section("Object", p, min_t(unsigned long, s->objsize, PAGE_SIZE)); 465 print_section("Object", p, min_t(unsigned long, s->objsize, PAGE_SIZE));
465 466
466 if (s->flags & SLAB_RED_ZONE) 467 if (s->flags & SLAB_RED_ZONE)
467 print_section("Redzone", p + s->objsize, 468 print_section("Redzone", p + s->objsize,
468 s->inuse - s->objsize); 469 s->inuse - s->objsize);
469 470
470 if (s->offset) 471 if (s->offset)
471 off = s->offset + sizeof(void *); 472 off = s->offset + sizeof(void *);
472 else 473 else
473 off = s->inuse; 474 off = s->inuse;
474 475
475 if (s->flags & SLAB_STORE_USER) 476 if (s->flags & SLAB_STORE_USER)
476 off += 2 * sizeof(struct track); 477 off += 2 * sizeof(struct track);
477 478
478 if (off != s->size) 479 if (off != s->size)
479 /* Beginning of the filler is the free pointer */ 480 /* Beginning of the filler is the free pointer */
480 print_section("Padding", p + off, s->size - off); 481 print_section("Padding", p + off, s->size - off);
481 482
482 dump_stack(); 483 dump_stack();
483 } 484 }
484 485
485 static void object_err(struct kmem_cache *s, struct page *page, 486 static void object_err(struct kmem_cache *s, struct page *page,
486 u8 *object, char *reason) 487 u8 *object, char *reason)
487 { 488 {
488 slab_bug(s, "%s", reason); 489 slab_bug(s, "%s", reason);
489 print_trailer(s, page, object); 490 print_trailer(s, page, object);
490 } 491 }
491 492
492 static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...) 493 static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...)
493 { 494 {
494 va_list args; 495 va_list args;
495 char buf[100]; 496 char buf[100];
496 497
497 va_start(args, fmt); 498 va_start(args, fmt);
498 vsnprintf(buf, sizeof(buf), fmt, args); 499 vsnprintf(buf, sizeof(buf), fmt, args);
499 va_end(args); 500 va_end(args);
500 slab_bug(s, "%s", buf); 501 slab_bug(s, "%s", buf);
501 print_page_info(page); 502 print_page_info(page);
502 dump_stack(); 503 dump_stack();
503 } 504 }
504 505
505 static void init_object(struct kmem_cache *s, void *object, int active) 506 static void init_object(struct kmem_cache *s, void *object, int active)
506 { 507 {
507 u8 *p = object; 508 u8 *p = object;
508 509
509 if (s->flags & __OBJECT_POISON) { 510 if (s->flags & __OBJECT_POISON) {
510 memset(p, POISON_FREE, s->objsize - 1); 511 memset(p, POISON_FREE, s->objsize - 1);
511 p[s->objsize - 1] = POISON_END; 512 p[s->objsize - 1] = POISON_END;
512 } 513 }
513 514
514 if (s->flags & SLAB_RED_ZONE) 515 if (s->flags & SLAB_RED_ZONE)
515 memset(p + s->objsize, 516 memset(p + s->objsize,
516 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE, 517 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE,
517 s->inuse - s->objsize); 518 s->inuse - s->objsize);
518 } 519 }
519 520
520 static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes) 521 static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes)
521 { 522 {
522 while (bytes) { 523 while (bytes) {
523 if (*start != (u8)value) 524 if (*start != (u8)value)
524 return start; 525 return start;
525 start++; 526 start++;
526 bytes--; 527 bytes--;
527 } 528 }
528 return NULL; 529 return NULL;
529 } 530 }
530 531
531 static void restore_bytes(struct kmem_cache *s, char *message, u8 data, 532 static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
532 void *from, void *to) 533 void *from, void *to)
533 { 534 {
534 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); 535 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
535 memset(from, data, to - from); 536 memset(from, data, to - from);
536 } 537 }
537 538
538 static int check_bytes_and_report(struct kmem_cache *s, struct page *page, 539 static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
539 u8 *object, char *what, 540 u8 *object, char *what,
540 u8 *start, unsigned int value, unsigned int bytes) 541 u8 *start, unsigned int value, unsigned int bytes)
541 { 542 {
542 u8 *fault; 543 u8 *fault;
543 u8 *end; 544 u8 *end;
544 545
545 fault = check_bytes(start, value, bytes); 546 fault = check_bytes(start, value, bytes);
546 if (!fault) 547 if (!fault)
547 return 1; 548 return 1;
548 549
549 end = start + bytes; 550 end = start + bytes;
550 while (end > fault && end[-1] == value) 551 while (end > fault && end[-1] == value)
551 end--; 552 end--;
552 553
553 slab_bug(s, "%s overwritten", what); 554 slab_bug(s, "%s overwritten", what);
554 printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n", 555 printk(KERN_ERR "INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n",
555 fault, end - 1, fault[0], value); 556 fault, end - 1, fault[0], value);
556 print_trailer(s, page, object); 557 print_trailer(s, page, object);
557 558
558 restore_bytes(s, what, value, fault, end); 559 restore_bytes(s, what, value, fault, end);
559 return 0; 560 return 0;
560 } 561 }
561 562
562 /* 563 /*
563 * Object layout: 564 * Object layout:
564 * 565 *
565 * object address 566 * object address
566 * Bytes of the object to be managed. 567 * Bytes of the object to be managed.
567 * If the freepointer may overlay the object then the free 568 * If the freepointer may overlay the object then the free
568 * pointer is the first word of the object. 569 * pointer is the first word of the object.
569 * 570 *
570 * Poisoning uses 0x6b (POISON_FREE) and the last byte is 571 * Poisoning uses 0x6b (POISON_FREE) and the last byte is
571 * 0xa5 (POISON_END) 572 * 0xa5 (POISON_END)
572 * 573 *
573 * object + s->objsize 574 * object + s->objsize
574 * Padding to reach word boundary. This is also used for Redzoning. 575 * Padding to reach word boundary. This is also used for Redzoning.
575 * Padding is extended by another word if Redzoning is enabled and 576 * Padding is extended by another word if Redzoning is enabled and
576 * objsize == inuse. 577 * objsize == inuse.
577 * 578 *
578 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with 579 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
579 * 0xcc (RED_ACTIVE) for objects in use. 580 * 0xcc (RED_ACTIVE) for objects in use.
580 * 581 *
581 * object + s->inuse 582 * object + s->inuse
582 * Meta data starts here. 583 * Meta data starts here.
583 * 584 *
584 * A. Free pointer (if we cannot overwrite object on free) 585 * A. Free pointer (if we cannot overwrite object on free)
585 * B. Tracking data for SLAB_STORE_USER 586 * B. Tracking data for SLAB_STORE_USER
586 * C. Padding to reach required alignment boundary or at mininum 587 * C. Padding to reach required alignment boundary or at mininum
587 * one word if debugging is on to be able to detect writes 588 * one word if debugging is on to be able to detect writes
588 * before the word boundary. 589 * before the word boundary.
589 * 590 *
590 * Padding is done using 0x5a (POISON_INUSE) 591 * Padding is done using 0x5a (POISON_INUSE)
591 * 592 *
592 * object + s->size 593 * object + s->size
593 * Nothing is used beyond s->size. 594 * Nothing is used beyond s->size.
594 * 595 *
595 * If slabcaches are merged then the objsize and inuse boundaries are mostly 596 * If slabcaches are merged then the objsize and inuse boundaries are mostly
596 * ignored. And therefore no slab options that rely on these boundaries 597 * ignored. And therefore no slab options that rely on these boundaries
597 * may be used with merged slabcaches. 598 * may be used with merged slabcaches.
598 */ 599 */
599 600
600 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) 601 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
601 { 602 {
602 unsigned long off = s->inuse; /* The end of info */ 603 unsigned long off = s->inuse; /* The end of info */
603 604
604 if (s->offset) 605 if (s->offset)
605 /* Freepointer is placed after the object. */ 606 /* Freepointer is placed after the object. */
606 off += sizeof(void *); 607 off += sizeof(void *);
607 608
608 if (s->flags & SLAB_STORE_USER) 609 if (s->flags & SLAB_STORE_USER)
609 /* We also have user information there */ 610 /* We also have user information there */
610 off += 2 * sizeof(struct track); 611 off += 2 * sizeof(struct track);
611 612
612 if (s->size == off) 613 if (s->size == off)
613 return 1; 614 return 1;
614 615
615 return check_bytes_and_report(s, page, p, "Object padding", 616 return check_bytes_and_report(s, page, p, "Object padding",
616 p + off, POISON_INUSE, s->size - off); 617 p + off, POISON_INUSE, s->size - off);
617 } 618 }
618 619
619 /* Check the pad bytes at the end of a slab page */ 620 /* Check the pad bytes at the end of a slab page */
620 static int slab_pad_check(struct kmem_cache *s, struct page *page) 621 static int slab_pad_check(struct kmem_cache *s, struct page *page)
621 { 622 {
622 u8 *start; 623 u8 *start;
623 u8 *fault; 624 u8 *fault;
624 u8 *end; 625 u8 *end;
625 int length; 626 int length;
626 int remainder; 627 int remainder;
627 628
628 if (!(s->flags & SLAB_POISON)) 629 if (!(s->flags & SLAB_POISON))
629 return 1; 630 return 1;
630 631
631 start = page_address(page); 632 start = page_address(page);
632 length = (PAGE_SIZE << compound_order(page)); 633 length = (PAGE_SIZE << compound_order(page));
633 end = start + length; 634 end = start + length;
634 remainder = length % s->size; 635 remainder = length % s->size;
635 if (!remainder) 636 if (!remainder)
636 return 1; 637 return 1;
637 638
638 fault = check_bytes(end - remainder, POISON_INUSE, remainder); 639 fault = check_bytes(end - remainder, POISON_INUSE, remainder);
639 if (!fault) 640 if (!fault)
640 return 1; 641 return 1;
641 while (end > fault && end[-1] == POISON_INUSE) 642 while (end > fault && end[-1] == POISON_INUSE)
642 end--; 643 end--;
643 644
644 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); 645 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
645 print_section("Padding", end - remainder, remainder); 646 print_section("Padding", end - remainder, remainder);
646 647
647 restore_bytes(s, "slab padding", POISON_INUSE, start, end); 648 restore_bytes(s, "slab padding", POISON_INUSE, start, end);
648 return 0; 649 return 0;
649 } 650 }
650 651
651 static int check_object(struct kmem_cache *s, struct page *page, 652 static int check_object(struct kmem_cache *s, struct page *page,
652 void *object, int active) 653 void *object, int active)
653 { 654 {
654 u8 *p = object; 655 u8 *p = object;
655 u8 *endobject = object + s->objsize; 656 u8 *endobject = object + s->objsize;
656 657
657 if (s->flags & SLAB_RED_ZONE) { 658 if (s->flags & SLAB_RED_ZONE) {
658 unsigned int red = 659 unsigned int red =
659 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE; 660 active ? SLUB_RED_ACTIVE : SLUB_RED_INACTIVE;
660 661
661 if (!check_bytes_and_report(s, page, object, "Redzone", 662 if (!check_bytes_and_report(s, page, object, "Redzone",
662 endobject, red, s->inuse - s->objsize)) 663 endobject, red, s->inuse - s->objsize))
663 return 0; 664 return 0;
664 } else { 665 } else {
665 if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) { 666 if ((s->flags & SLAB_POISON) && s->objsize < s->inuse) {
666 check_bytes_and_report(s, page, p, "Alignment padding", 667 check_bytes_and_report(s, page, p, "Alignment padding",
667 endobject, POISON_INUSE, s->inuse - s->objsize); 668 endobject, POISON_INUSE, s->inuse - s->objsize);
668 } 669 }
669 } 670 }
670 671
671 if (s->flags & SLAB_POISON) { 672 if (s->flags & SLAB_POISON) {
672 if (!active && (s->flags & __OBJECT_POISON) && 673 if (!active && (s->flags & __OBJECT_POISON) &&
673 (!check_bytes_and_report(s, page, p, "Poison", p, 674 (!check_bytes_and_report(s, page, p, "Poison", p,
674 POISON_FREE, s->objsize - 1) || 675 POISON_FREE, s->objsize - 1) ||
675 !check_bytes_and_report(s, page, p, "Poison", 676 !check_bytes_and_report(s, page, p, "Poison",
676 p + s->objsize - 1, POISON_END, 1))) 677 p + s->objsize - 1, POISON_END, 1)))
677 return 0; 678 return 0;
678 /* 679 /*
679 * check_pad_bytes cleans up on its own. 680 * check_pad_bytes cleans up on its own.
680 */ 681 */
681 check_pad_bytes(s, page, p); 682 check_pad_bytes(s, page, p);
682 } 683 }
683 684
684 if (!s->offset && active) 685 if (!s->offset && active)
685 /* 686 /*
686 * Object and freepointer overlap. Cannot check 687 * Object and freepointer overlap. Cannot check
687 * freepointer while object is allocated. 688 * freepointer while object is allocated.
688 */ 689 */
689 return 1; 690 return 1;
690 691
691 /* Check free pointer validity */ 692 /* Check free pointer validity */
692 if (!check_valid_pointer(s, page, get_freepointer(s, p))) { 693 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
693 object_err(s, page, p, "Freepointer corrupt"); 694 object_err(s, page, p, "Freepointer corrupt");
694 /* 695 /*
695 * No choice but to zap it and thus lose the remainder 696 * No choice but to zap it and thus lose the remainder
696 * of the free objects in this slab. May cause 697 * of the free objects in this slab. May cause
697 * another error because the object count is now wrong. 698 * another error because the object count is now wrong.
698 */ 699 */
699 set_freepointer(s, p, NULL); 700 set_freepointer(s, p, NULL);
700 return 0; 701 return 0;
701 } 702 }
702 return 1; 703 return 1;
703 } 704 }
704 705
705 static int check_slab(struct kmem_cache *s, struct page *page) 706 static int check_slab(struct kmem_cache *s, struct page *page)
706 { 707 {
707 int maxobj; 708 int maxobj;
708 709
709 VM_BUG_ON(!irqs_disabled()); 710 VM_BUG_ON(!irqs_disabled());
710 711
711 if (!PageSlab(page)) { 712 if (!PageSlab(page)) {
712 slab_err(s, page, "Not a valid slab page"); 713 slab_err(s, page, "Not a valid slab page");
713 return 0; 714 return 0;
714 } 715 }
715 716
716 maxobj = (PAGE_SIZE << compound_order(page)) / s->size; 717 maxobj = (PAGE_SIZE << compound_order(page)) / s->size;
717 if (page->objects > maxobj) { 718 if (page->objects > maxobj) {
718 slab_err(s, page, "objects %u > max %u", 719 slab_err(s, page, "objects %u > max %u",
719 s->name, page->objects, maxobj); 720 s->name, page->objects, maxobj);
720 return 0; 721 return 0;
721 } 722 }
722 if (page->inuse > page->objects) { 723 if (page->inuse > page->objects) {
723 slab_err(s, page, "inuse %u > max %u", 724 slab_err(s, page, "inuse %u > max %u",
724 s->name, page->inuse, page->objects); 725 s->name, page->inuse, page->objects);
725 return 0; 726 return 0;
726 } 727 }
727 /* Slab_pad_check fixes things up after itself */ 728 /* Slab_pad_check fixes things up after itself */
728 slab_pad_check(s, page); 729 slab_pad_check(s, page);
729 return 1; 730 return 1;
730 } 731 }
731 732
732 /* 733 /*
733 * Determine if a certain object on a page is on the freelist. Must hold the 734 * Determine if a certain object on a page is on the freelist. Must hold the
734 * slab lock to guarantee that the chains are in a consistent state. 735 * slab lock to guarantee that the chains are in a consistent state.
735 */ 736 */
736 static int on_freelist(struct kmem_cache *s, struct page *page, void *search) 737 static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
737 { 738 {
738 int nr = 0; 739 int nr = 0;
739 void *fp = page->freelist; 740 void *fp = page->freelist;
740 void *object = NULL; 741 void *object = NULL;
741 unsigned long max_objects; 742 unsigned long max_objects;
742 743
743 while (fp && nr <= page->objects) { 744 while (fp && nr <= page->objects) {
744 if (fp == search) 745 if (fp == search)
745 return 1; 746 return 1;
746 if (!check_valid_pointer(s, page, fp)) { 747 if (!check_valid_pointer(s, page, fp)) {
747 if (object) { 748 if (object) {
748 object_err(s, page, object, 749 object_err(s, page, object,
749 "Freechain corrupt"); 750 "Freechain corrupt");
750 set_freepointer(s, object, NULL); 751 set_freepointer(s, object, NULL);
751 break; 752 break;
752 } else { 753 } else {
753 slab_err(s, page, "Freepointer corrupt"); 754 slab_err(s, page, "Freepointer corrupt");
754 page->freelist = NULL; 755 page->freelist = NULL;
755 page->inuse = page->objects; 756 page->inuse = page->objects;
756 slab_fix(s, "Freelist cleared"); 757 slab_fix(s, "Freelist cleared");
757 return 0; 758 return 0;
758 } 759 }
759 break; 760 break;
760 } 761 }
761 object = fp; 762 object = fp;
762 fp = get_freepointer(s, object); 763 fp = get_freepointer(s, object);
763 nr++; 764 nr++;
764 } 765 }
765 766
766 max_objects = (PAGE_SIZE << compound_order(page)) / s->size; 767 max_objects = (PAGE_SIZE << compound_order(page)) / s->size;
767 if (max_objects > MAX_OBJS_PER_PAGE) 768 if (max_objects > MAX_OBJS_PER_PAGE)
768 max_objects = MAX_OBJS_PER_PAGE; 769 max_objects = MAX_OBJS_PER_PAGE;
769 770
770 if (page->objects != max_objects) { 771 if (page->objects != max_objects) {
771 slab_err(s, page, "Wrong number of objects. Found %d but " 772 slab_err(s, page, "Wrong number of objects. Found %d but "
772 "should be %d", page->objects, max_objects); 773 "should be %d", page->objects, max_objects);
773 page->objects = max_objects; 774 page->objects = max_objects;
774 slab_fix(s, "Number of objects adjusted."); 775 slab_fix(s, "Number of objects adjusted.");
775 } 776 }
776 if (page->inuse != page->objects - nr) { 777 if (page->inuse != page->objects - nr) {
777 slab_err(s, page, "Wrong object count. Counter is %d but " 778 slab_err(s, page, "Wrong object count. Counter is %d but "
778 "counted were %d", page->inuse, page->objects - nr); 779 "counted were %d", page->inuse, page->objects - nr);
779 page->inuse = page->objects - nr; 780 page->inuse = page->objects - nr;
780 slab_fix(s, "Object count adjusted."); 781 slab_fix(s, "Object count adjusted.");
781 } 782 }
782 return search == NULL; 783 return search == NULL;
783 } 784 }
784 785
785 static void trace(struct kmem_cache *s, struct page *page, void *object, 786 static void trace(struct kmem_cache *s, struct page *page, void *object,
786 int alloc) 787 int alloc)
787 { 788 {
788 if (s->flags & SLAB_TRACE) { 789 if (s->flags & SLAB_TRACE) {
789 printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n", 790 printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
790 s->name, 791 s->name,
791 alloc ? "alloc" : "free", 792 alloc ? "alloc" : "free",
792 object, page->inuse, 793 object, page->inuse,
793 page->freelist); 794 page->freelist);
794 795
795 if (!alloc) 796 if (!alloc)
796 print_section("Object", (void *)object, s->objsize); 797 print_section("Object", (void *)object, s->objsize);
797 798
798 dump_stack(); 799 dump_stack();
799 } 800 }
800 } 801 }
801 802
802 /* 803 /*
803 * Tracking of fully allocated slabs for debugging purposes. 804 * Tracking of fully allocated slabs for debugging purposes.
804 */ 805 */
805 static void add_full(struct kmem_cache_node *n, struct page *page) 806 static void add_full(struct kmem_cache_node *n, struct page *page)
806 { 807 {
807 spin_lock(&n->list_lock); 808 spin_lock(&n->list_lock);
808 list_add(&page->lru, &n->full); 809 list_add(&page->lru, &n->full);
809 spin_unlock(&n->list_lock); 810 spin_unlock(&n->list_lock);
810 } 811 }
811 812
812 static void remove_full(struct kmem_cache *s, struct page *page) 813 static void remove_full(struct kmem_cache *s, struct page *page)
813 { 814 {
814 struct kmem_cache_node *n; 815 struct kmem_cache_node *n;
815 816
816 if (!(s->flags & SLAB_STORE_USER)) 817 if (!(s->flags & SLAB_STORE_USER))
817 return; 818 return;
818 819
819 n = get_node(s, page_to_nid(page)); 820 n = get_node(s, page_to_nid(page));
820 821
821 spin_lock(&n->list_lock); 822 spin_lock(&n->list_lock);
822 list_del(&page->lru); 823 list_del(&page->lru);
823 spin_unlock(&n->list_lock); 824 spin_unlock(&n->list_lock);
824 } 825 }
825 826
826 /* Tracking of the number of slabs for debugging purposes */ 827 /* Tracking of the number of slabs for debugging purposes */
827 static inline unsigned long slabs_node(struct kmem_cache *s, int node) 828 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
828 { 829 {
829 struct kmem_cache_node *n = get_node(s, node); 830 struct kmem_cache_node *n = get_node(s, node);
830 831
831 return atomic_long_read(&n->nr_slabs); 832 return atomic_long_read(&n->nr_slabs);
832 } 833 }
833 834
834 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) 835 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
835 { 836 {
836 struct kmem_cache_node *n = get_node(s, node); 837 struct kmem_cache_node *n = get_node(s, node);
837 838
838 /* 839 /*
839 * May be called early in order to allocate a slab for the 840 * May be called early in order to allocate a slab for the
840 * kmem_cache_node structure. Solve the chicken-egg 841 * kmem_cache_node structure. Solve the chicken-egg
841 * dilemma by deferring the increment of the count during 842 * dilemma by deferring the increment of the count during
842 * bootstrap (see early_kmem_cache_node_alloc). 843 * bootstrap (see early_kmem_cache_node_alloc).
843 */ 844 */
844 if (!NUMA_BUILD || n) { 845 if (!NUMA_BUILD || n) {
845 atomic_long_inc(&n->nr_slabs); 846 atomic_long_inc(&n->nr_slabs);
846 atomic_long_add(objects, &n->total_objects); 847 atomic_long_add(objects, &n->total_objects);
847 } 848 }
848 } 849 }
849 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) 850 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
850 { 851 {
851 struct kmem_cache_node *n = get_node(s, node); 852 struct kmem_cache_node *n = get_node(s, node);
852 853
853 atomic_long_dec(&n->nr_slabs); 854 atomic_long_dec(&n->nr_slabs);
854 atomic_long_sub(objects, &n->total_objects); 855 atomic_long_sub(objects, &n->total_objects);
855 } 856 }
856 857
857 /* Object debug checks for alloc/free paths */ 858 /* Object debug checks for alloc/free paths */
858 static void setup_object_debug(struct kmem_cache *s, struct page *page, 859 static void setup_object_debug(struct kmem_cache *s, struct page *page,
859 void *object) 860 void *object)
860 { 861 {
861 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))) 862 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON)))
862 return; 863 return;
863 864
864 init_object(s, object, 0); 865 init_object(s, object, 0);
865 init_tracking(s, object); 866 init_tracking(s, object);
866 } 867 }
867 868
868 static int alloc_debug_processing(struct kmem_cache *s, struct page *page, 869 static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
869 void *object, unsigned long addr) 870 void *object, unsigned long addr)
870 { 871 {
871 if (!check_slab(s, page)) 872 if (!check_slab(s, page))
872 goto bad; 873 goto bad;
873 874
874 if (!on_freelist(s, page, object)) { 875 if (!on_freelist(s, page, object)) {
875 object_err(s, page, object, "Object already allocated"); 876 object_err(s, page, object, "Object already allocated");
876 goto bad; 877 goto bad;
877 } 878 }
878 879
879 if (!check_valid_pointer(s, page, object)) { 880 if (!check_valid_pointer(s, page, object)) {
880 object_err(s, page, object, "Freelist Pointer check fails"); 881 object_err(s, page, object, "Freelist Pointer check fails");
881 goto bad; 882 goto bad;
882 } 883 }
883 884
884 if (!check_object(s, page, object, 0)) 885 if (!check_object(s, page, object, 0))
885 goto bad; 886 goto bad;
886 887
887 /* Success perform special debug activities for allocs */ 888 /* Success perform special debug activities for allocs */
888 if (s->flags & SLAB_STORE_USER) 889 if (s->flags & SLAB_STORE_USER)
889 set_track(s, object, TRACK_ALLOC, addr); 890 set_track(s, object, TRACK_ALLOC, addr);
890 trace(s, page, object, 1); 891 trace(s, page, object, 1);
891 init_object(s, object, 1); 892 init_object(s, object, 1);
892 return 1; 893 return 1;
893 894
894 bad: 895 bad:
895 if (PageSlab(page)) { 896 if (PageSlab(page)) {
896 /* 897 /*
897 * If this is a slab page then lets do the best we can 898 * If this is a slab page then lets do the best we can
898 * to avoid issues in the future. Marking all objects 899 * to avoid issues in the future. Marking all objects
899 * as used avoids touching the remaining objects. 900 * as used avoids touching the remaining objects.
900 */ 901 */
901 slab_fix(s, "Marking all objects used"); 902 slab_fix(s, "Marking all objects used");
902 page->inuse = page->objects; 903 page->inuse = page->objects;
903 page->freelist = NULL; 904 page->freelist = NULL;
904 } 905 }
905 return 0; 906 return 0;
906 } 907 }
907 908
908 static int free_debug_processing(struct kmem_cache *s, struct page *page, 909 static int free_debug_processing(struct kmem_cache *s, struct page *page,
909 void *object, unsigned long addr) 910 void *object, unsigned long addr)
910 { 911 {
911 if (!check_slab(s, page)) 912 if (!check_slab(s, page))
912 goto fail; 913 goto fail;
913 914
914 if (!check_valid_pointer(s, page, object)) { 915 if (!check_valid_pointer(s, page, object)) {
915 slab_err(s, page, "Invalid object pointer 0x%p", object); 916 slab_err(s, page, "Invalid object pointer 0x%p", object);
916 goto fail; 917 goto fail;
917 } 918 }
918 919
919 if (on_freelist(s, page, object)) { 920 if (on_freelist(s, page, object)) {
920 object_err(s, page, object, "Object already free"); 921 object_err(s, page, object, "Object already free");
921 goto fail; 922 goto fail;
922 } 923 }
923 924
924 if (!check_object(s, page, object, 1)) 925 if (!check_object(s, page, object, 1))
925 return 0; 926 return 0;
926 927
927 if (unlikely(s != page->slab)) { 928 if (unlikely(s != page->slab)) {
928 if (!PageSlab(page)) { 929 if (!PageSlab(page)) {
929 slab_err(s, page, "Attempt to free object(0x%p) " 930 slab_err(s, page, "Attempt to free object(0x%p) "
930 "outside of slab", object); 931 "outside of slab", object);
931 } else if (!page->slab) { 932 } else if (!page->slab) {
932 printk(KERN_ERR 933 printk(KERN_ERR
933 "SLUB <none>: no slab for object 0x%p.\n", 934 "SLUB <none>: no slab for object 0x%p.\n",
934 object); 935 object);
935 dump_stack(); 936 dump_stack();
936 } else 937 } else
937 object_err(s, page, object, 938 object_err(s, page, object,
938 "page slab pointer corrupt."); 939 "page slab pointer corrupt.");
939 goto fail; 940 goto fail;
940 } 941 }
941 942
942 /* Special debug activities for freeing objects */ 943 /* Special debug activities for freeing objects */
943 if (!PageSlubFrozen(page) && !page->freelist) 944 if (!PageSlubFrozen(page) && !page->freelist)
944 remove_full(s, page); 945 remove_full(s, page);
945 if (s->flags & SLAB_STORE_USER) 946 if (s->flags & SLAB_STORE_USER)
946 set_track(s, object, TRACK_FREE, addr); 947 set_track(s, object, TRACK_FREE, addr);
947 trace(s, page, object, 0); 948 trace(s, page, object, 0);
948 init_object(s, object, 0); 949 init_object(s, object, 0);
949 return 1; 950 return 1;
950 951
951 fail: 952 fail:
952 slab_fix(s, "Object at 0x%p not freed", object); 953 slab_fix(s, "Object at 0x%p not freed", object);
953 return 0; 954 return 0;
954 } 955 }
955 956
956 static int __init setup_slub_debug(char *str) 957 static int __init setup_slub_debug(char *str)
957 { 958 {
958 slub_debug = DEBUG_DEFAULT_FLAGS; 959 slub_debug = DEBUG_DEFAULT_FLAGS;
959 if (*str++ != '=' || !*str) 960 if (*str++ != '=' || !*str)
960 /* 961 /*
961 * No options specified. Switch on full debugging. 962 * No options specified. Switch on full debugging.
962 */ 963 */
963 goto out; 964 goto out;
964 965
965 if (*str == ',') 966 if (*str == ',')
966 /* 967 /*
967 * No options but restriction on slabs. This means full 968 * No options but restriction on slabs. This means full
968 * debugging for slabs matching a pattern. 969 * debugging for slabs matching a pattern.
969 */ 970 */
970 goto check_slabs; 971 goto check_slabs;
971 972
972 slub_debug = 0; 973 slub_debug = 0;
973 if (*str == '-') 974 if (*str == '-')
974 /* 975 /*
975 * Switch off all debugging measures. 976 * Switch off all debugging measures.
976 */ 977 */
977 goto out; 978 goto out;
978 979
979 /* 980 /*
980 * Determine which debug features should be switched on 981 * Determine which debug features should be switched on
981 */ 982 */
982 for (; *str && *str != ','; str++) { 983 for (; *str && *str != ','; str++) {
983 switch (tolower(*str)) { 984 switch (tolower(*str)) {
984 case 'f': 985 case 'f':
985 slub_debug |= SLAB_DEBUG_FREE; 986 slub_debug |= SLAB_DEBUG_FREE;
986 break; 987 break;
987 case 'z': 988 case 'z':
988 slub_debug |= SLAB_RED_ZONE; 989 slub_debug |= SLAB_RED_ZONE;
989 break; 990 break;
990 case 'p': 991 case 'p':
991 slub_debug |= SLAB_POISON; 992 slub_debug |= SLAB_POISON;
992 break; 993 break;
993 case 'u': 994 case 'u':
994 slub_debug |= SLAB_STORE_USER; 995 slub_debug |= SLAB_STORE_USER;
995 break; 996 break;
996 case 't': 997 case 't':
997 slub_debug |= SLAB_TRACE; 998 slub_debug |= SLAB_TRACE;
998 break; 999 break;
999 default: 1000 default:
1000 printk(KERN_ERR "slub_debug option '%c' " 1001 printk(KERN_ERR "slub_debug option '%c' "
1001 "unknown. skipped\n", *str); 1002 "unknown. skipped\n", *str);
1002 } 1003 }
1003 } 1004 }
1004 1005
1005 check_slabs: 1006 check_slabs:
1006 if (*str == ',') 1007 if (*str == ',')
1007 slub_debug_slabs = str + 1; 1008 slub_debug_slabs = str + 1;
1008 out: 1009 out:
1009 return 1; 1010 return 1;
1010 } 1011 }
1011 1012
1012 __setup("slub_debug", setup_slub_debug); 1013 __setup("slub_debug", setup_slub_debug);
1013 1014
1014 static unsigned long kmem_cache_flags(unsigned long objsize, 1015 static unsigned long kmem_cache_flags(unsigned long objsize,
1015 unsigned long flags, const char *name, 1016 unsigned long flags, const char *name,
1016 void (*ctor)(void *)) 1017 void (*ctor)(void *))
1017 { 1018 {
1018 /* 1019 /*
1019 * Enable debugging if selected on the kernel commandline. 1020 * Enable debugging if selected on the kernel commandline.
1020 */ 1021 */
1021 if (slub_debug && (!slub_debug_slabs || 1022 if (slub_debug && (!slub_debug_slabs ||
1022 strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)) == 0)) 1023 strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs)) == 0))
1023 flags |= slub_debug; 1024 flags |= slub_debug;
1024 1025
1025 return flags; 1026 return flags;
1026 } 1027 }
1027 #else 1028 #else
1028 static inline void setup_object_debug(struct kmem_cache *s, 1029 static inline void setup_object_debug(struct kmem_cache *s,
1029 struct page *page, void *object) {} 1030 struct page *page, void *object) {}
1030 1031
1031 static inline int alloc_debug_processing(struct kmem_cache *s, 1032 static inline int alloc_debug_processing(struct kmem_cache *s,
1032 struct page *page, void *object, unsigned long addr) { return 0; } 1033 struct page *page, void *object, unsigned long addr) { return 0; }
1033 1034
1034 static inline int free_debug_processing(struct kmem_cache *s, 1035 static inline int free_debug_processing(struct kmem_cache *s,
1035 struct page *page, void *object, unsigned long addr) { return 0; } 1036 struct page *page, void *object, unsigned long addr) { return 0; }
1036 1037
1037 static inline int slab_pad_check(struct kmem_cache *s, struct page *page) 1038 static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1038 { return 1; } 1039 { return 1; }
1039 static inline int check_object(struct kmem_cache *s, struct page *page, 1040 static inline int check_object(struct kmem_cache *s, struct page *page,
1040 void *object, int active) { return 1; } 1041 void *object, int active) { return 1; }
1041 static inline void add_full(struct kmem_cache_node *n, struct page *page) {} 1042 static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
1042 static inline unsigned long kmem_cache_flags(unsigned long objsize, 1043 static inline unsigned long kmem_cache_flags(unsigned long objsize,
1043 unsigned long flags, const char *name, 1044 unsigned long flags, const char *name,
1044 void (*ctor)(void *)) 1045 void (*ctor)(void *))
1045 { 1046 {
1046 return flags; 1047 return flags;
1047 } 1048 }
1048 #define slub_debug 0 1049 #define slub_debug 0
1049 1050
1050 static inline unsigned long slabs_node(struct kmem_cache *s, int node) 1051 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
1051 { return 0; } 1052 { return 0; }
1052 static inline void inc_slabs_node(struct kmem_cache *s, int node, 1053 static inline void inc_slabs_node(struct kmem_cache *s, int node,
1053 int objects) {} 1054 int objects) {}
1054 static inline void dec_slabs_node(struct kmem_cache *s, int node, 1055 static inline void dec_slabs_node(struct kmem_cache *s, int node,
1055 int objects) {} 1056 int objects) {}
1056 #endif 1057 #endif
1057 1058
1058 /* 1059 /*
1059 * Slab allocation and freeing 1060 * Slab allocation and freeing
1060 */ 1061 */
1061 static inline struct page *alloc_slab_page(gfp_t flags, int node, 1062 static inline struct page *alloc_slab_page(gfp_t flags, int node,
1062 struct kmem_cache_order_objects oo) 1063 struct kmem_cache_order_objects oo)
1063 { 1064 {
1064 int order = oo_order(oo); 1065 int order = oo_order(oo);
1065 1066
1066 if (node == -1) 1067 if (node == -1)
1067 return alloc_pages(flags, order); 1068 return alloc_pages(flags, order);
1068 else 1069 else
1069 return alloc_pages_node(node, flags, order); 1070 return alloc_pages_node(node, flags, order);
1070 } 1071 }
1071 1072
1072 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 1073 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
1073 { 1074 {
1074 struct page *page; 1075 struct page *page;
1075 struct kmem_cache_order_objects oo = s->oo; 1076 struct kmem_cache_order_objects oo = s->oo;
1076 1077
1077 flags |= s->allocflags; 1078 flags |= s->allocflags;
1078 1079
1079 page = alloc_slab_page(flags | __GFP_NOWARN | __GFP_NORETRY, node, 1080 page = alloc_slab_page(flags | __GFP_NOWARN | __GFP_NORETRY, node,
1080 oo); 1081 oo);
1081 if (unlikely(!page)) { 1082 if (unlikely(!page)) {
1082 oo = s->min; 1083 oo = s->min;
1083 /* 1084 /*
1084 * Allocation may have failed due to fragmentation. 1085 * Allocation may have failed due to fragmentation.
1085 * Try a lower order alloc if possible 1086 * Try a lower order alloc if possible
1086 */ 1087 */
1087 page = alloc_slab_page(flags, node, oo); 1088 page = alloc_slab_page(flags, node, oo);
1088 if (!page) 1089 if (!page)
1089 return NULL; 1090 return NULL;
1090 1091
1091 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK); 1092 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
1092 } 1093 }
1093 page->objects = oo_objects(oo); 1094 page->objects = oo_objects(oo);
1094 mod_zone_page_state(page_zone(page), 1095 mod_zone_page_state(page_zone(page),
1095 (s->flags & SLAB_RECLAIM_ACCOUNT) ? 1096 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1096 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, 1097 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1097 1 << oo_order(oo)); 1098 1 << oo_order(oo));
1098 1099
1099 return page; 1100 return page;
1100 } 1101 }
1101 1102
1102 static void setup_object(struct kmem_cache *s, struct page *page, 1103 static void setup_object(struct kmem_cache *s, struct page *page,
1103 void *object) 1104 void *object)
1104 { 1105 {
1105 setup_object_debug(s, page, object); 1106 setup_object_debug(s, page, object);
1106 if (unlikely(s->ctor)) 1107 if (unlikely(s->ctor))
1107 s->ctor(object); 1108 s->ctor(object);
1108 } 1109 }
1109 1110
1110 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) 1111 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1111 { 1112 {
1112 struct page *page; 1113 struct page *page;
1113 void *start; 1114 void *start;
1114 void *last; 1115 void *last;
1115 void *p; 1116 void *p;
1116 1117
1117 BUG_ON(flags & GFP_SLAB_BUG_MASK); 1118 BUG_ON(flags & GFP_SLAB_BUG_MASK);
1118 1119
1119 page = allocate_slab(s, 1120 page = allocate_slab(s,
1120 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 1121 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
1121 if (!page) 1122 if (!page)
1122 goto out; 1123 goto out;
1123 1124
1124 inc_slabs_node(s, page_to_nid(page), page->objects); 1125 inc_slabs_node(s, page_to_nid(page), page->objects);
1125 page->slab = s; 1126 page->slab = s;
1126 page->flags |= 1 << PG_slab; 1127 page->flags |= 1 << PG_slab;
1127 if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON | 1128 if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
1128 SLAB_STORE_USER | SLAB_TRACE)) 1129 SLAB_STORE_USER | SLAB_TRACE))
1129 __SetPageSlubDebug(page); 1130 __SetPageSlubDebug(page);
1130 1131
1131 start = page_address(page); 1132 start = page_address(page);
1132 1133
1133 if (unlikely(s->flags & SLAB_POISON)) 1134 if (unlikely(s->flags & SLAB_POISON))
1134 memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page)); 1135 memset(start, POISON_INUSE, PAGE_SIZE << compound_order(page));
1135 1136
1136 last = start; 1137 last = start;
1137 for_each_object(p, s, start, page->objects) { 1138 for_each_object(p, s, start, page->objects) {
1138 setup_object(s, page, last); 1139 setup_object(s, page, last);
1139 set_freepointer(s, last, p); 1140 set_freepointer(s, last, p);
1140 last = p; 1141 last = p;
1141 } 1142 }
1142 setup_object(s, page, last); 1143 setup_object(s, page, last);
1143 set_freepointer(s, last, NULL); 1144 set_freepointer(s, last, NULL);
1144 1145
1145 page->freelist = start; 1146 page->freelist = start;
1146 page->inuse = 0; 1147 page->inuse = 0;
1147 out: 1148 out:
1148 return page; 1149 return page;
1149 } 1150 }
1150 1151
1151 static void __free_slab(struct kmem_cache *s, struct page *page) 1152 static void __free_slab(struct kmem_cache *s, struct page *page)
1152 { 1153 {
1153 int order = compound_order(page); 1154 int order = compound_order(page);
1154 int pages = 1 << order; 1155 int pages = 1 << order;
1155 1156
1156 if (unlikely(SLABDEBUG && PageSlubDebug(page))) { 1157 if (unlikely(SLABDEBUG && PageSlubDebug(page))) {
1157 void *p; 1158 void *p;
1158 1159
1159 slab_pad_check(s, page); 1160 slab_pad_check(s, page);
1160 for_each_object(p, s, page_address(page), 1161 for_each_object(p, s, page_address(page),
1161 page->objects) 1162 page->objects)
1162 check_object(s, page, p, 0); 1163 check_object(s, page, p, 0);
1163 __ClearPageSlubDebug(page); 1164 __ClearPageSlubDebug(page);
1164 } 1165 }
1165 1166
1166 mod_zone_page_state(page_zone(page), 1167 mod_zone_page_state(page_zone(page),
1167 (s->flags & SLAB_RECLAIM_ACCOUNT) ? 1168 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
1168 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, 1169 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
1169 -pages); 1170 -pages);
1170 1171
1171 __ClearPageSlab(page); 1172 __ClearPageSlab(page);
1172 reset_page_mapcount(page); 1173 reset_page_mapcount(page);
1174 if (current->reclaim_state)
1175 current->reclaim_state->reclaimed_slab += pages;
1173 __free_pages(page, order); 1176 __free_pages(page, order);
1174 } 1177 }
1175 1178
1176 static void rcu_free_slab(struct rcu_head *h) 1179 static void rcu_free_slab(struct rcu_head *h)
1177 { 1180 {
1178 struct page *page; 1181 struct page *page;
1179 1182
1180 page = container_of((struct list_head *)h, struct page, lru); 1183 page = container_of((struct list_head *)h, struct page, lru);
1181 __free_slab(page->slab, page); 1184 __free_slab(page->slab, page);
1182 } 1185 }
1183 1186
1184 static void free_slab(struct kmem_cache *s, struct page *page) 1187 static void free_slab(struct kmem_cache *s, struct page *page)
1185 { 1188 {
1186 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { 1189 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) {
1187 /* 1190 /*
1188 * RCU free overloads the RCU head over the LRU 1191 * RCU free overloads the RCU head over the LRU
1189 */ 1192 */
1190 struct rcu_head *head = (void *)&page->lru; 1193 struct rcu_head *head = (void *)&page->lru;
1191 1194
1192 call_rcu(head, rcu_free_slab); 1195 call_rcu(head, rcu_free_slab);
1193 } else 1196 } else
1194 __free_slab(s, page); 1197 __free_slab(s, page);
1195 } 1198 }
1196 1199
1197 static void discard_slab(struct kmem_cache *s, struct page *page) 1200 static void discard_slab(struct kmem_cache *s, struct page *page)
1198 { 1201 {
1199 dec_slabs_node(s, page_to_nid(page), page->objects); 1202 dec_slabs_node(s, page_to_nid(page), page->objects);
1200 free_slab(s, page); 1203 free_slab(s, page);
1201 } 1204 }
1202 1205
1203 /* 1206 /*
1204 * Per slab locking using the pagelock 1207 * Per slab locking using the pagelock
1205 */ 1208 */
1206 static __always_inline void slab_lock(struct page *page) 1209 static __always_inline void slab_lock(struct page *page)
1207 { 1210 {
1208 bit_spin_lock(PG_locked, &page->flags); 1211 bit_spin_lock(PG_locked, &page->flags);
1209 } 1212 }
1210 1213
1211 static __always_inline void slab_unlock(struct page *page) 1214 static __always_inline void slab_unlock(struct page *page)
1212 { 1215 {
1213 __bit_spin_unlock(PG_locked, &page->flags); 1216 __bit_spin_unlock(PG_locked, &page->flags);
1214 } 1217 }
1215 1218
1216 static __always_inline int slab_trylock(struct page *page) 1219 static __always_inline int slab_trylock(struct page *page)
1217 { 1220 {
1218 int rc = 1; 1221 int rc = 1;
1219 1222
1220 rc = bit_spin_trylock(PG_locked, &page->flags); 1223 rc = bit_spin_trylock(PG_locked, &page->flags);
1221 return rc; 1224 return rc;
1222 } 1225 }
1223 1226
1224 /* 1227 /*
1225 * Management of partially allocated slabs 1228 * Management of partially allocated slabs
1226 */ 1229 */
1227 static void add_partial(struct kmem_cache_node *n, 1230 static void add_partial(struct kmem_cache_node *n,
1228 struct page *page, int tail) 1231 struct page *page, int tail)
1229 { 1232 {
1230 spin_lock(&n->list_lock); 1233 spin_lock(&n->list_lock);
1231 n->nr_partial++; 1234 n->nr_partial++;
1232 if (tail) 1235 if (tail)
1233 list_add_tail(&page->lru, &n->partial); 1236 list_add_tail(&page->lru, &n->partial);
1234 else 1237 else
1235 list_add(&page->lru, &n->partial); 1238 list_add(&page->lru, &n->partial);
1236 spin_unlock(&n->list_lock); 1239 spin_unlock(&n->list_lock);
1237 } 1240 }
1238 1241
1239 static void remove_partial(struct kmem_cache *s, struct page *page) 1242 static void remove_partial(struct kmem_cache *s, struct page *page)
1240 { 1243 {
1241 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); 1244 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1242 1245
1243 spin_lock(&n->list_lock); 1246 spin_lock(&n->list_lock);
1244 list_del(&page->lru); 1247 list_del(&page->lru);
1245 n->nr_partial--; 1248 n->nr_partial--;
1246 spin_unlock(&n->list_lock); 1249 spin_unlock(&n->list_lock);
1247 } 1250 }
1248 1251
1249 /* 1252 /*
1250 * Lock slab and remove from the partial list. 1253 * Lock slab and remove from the partial list.
1251 * 1254 *
1252 * Must hold list_lock. 1255 * Must hold list_lock.
1253 */ 1256 */
1254 static inline int lock_and_freeze_slab(struct kmem_cache_node *n, 1257 static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
1255 struct page *page) 1258 struct page *page)
1256 { 1259 {
1257 if (slab_trylock(page)) { 1260 if (slab_trylock(page)) {
1258 list_del(&page->lru); 1261 list_del(&page->lru);
1259 n->nr_partial--; 1262 n->nr_partial--;
1260 __SetPageSlubFrozen(page); 1263 __SetPageSlubFrozen(page);
1261 return 1; 1264 return 1;
1262 } 1265 }
1263 return 0; 1266 return 0;
1264 } 1267 }
1265 1268
1266 /* 1269 /*
1267 * Try to allocate a partial slab from a specific node. 1270 * Try to allocate a partial slab from a specific node.
1268 */ 1271 */
1269 static struct page *get_partial_node(struct kmem_cache_node *n) 1272 static struct page *get_partial_node(struct kmem_cache_node *n)
1270 { 1273 {
1271 struct page *page; 1274 struct page *page;
1272 1275
1273 /* 1276 /*
1274 * Racy check. If we mistakenly see no partial slabs then we 1277 * Racy check. If we mistakenly see no partial slabs then we
1275 * just allocate an empty slab. If we mistakenly try to get a 1278 * just allocate an empty slab. If we mistakenly try to get a
1276 * partial slab and there is none available then get_partials() 1279 * partial slab and there is none available then get_partials()
1277 * will return NULL. 1280 * will return NULL.
1278 */ 1281 */
1279 if (!n || !n->nr_partial) 1282 if (!n || !n->nr_partial)
1280 return NULL; 1283 return NULL;
1281 1284
1282 spin_lock(&n->list_lock); 1285 spin_lock(&n->list_lock);
1283 list_for_each_entry(page, &n->partial, lru) 1286 list_for_each_entry(page, &n->partial, lru)
1284 if (lock_and_freeze_slab(n, page)) 1287 if (lock_and_freeze_slab(n, page))
1285 goto out; 1288 goto out;
1286 page = NULL; 1289 page = NULL;
1287 out: 1290 out:
1288 spin_unlock(&n->list_lock); 1291 spin_unlock(&n->list_lock);
1289 return page; 1292 return page;
1290 } 1293 }
1291 1294
1292 /* 1295 /*
1293 * Get a page from somewhere. Search in increasing NUMA distances. 1296 * Get a page from somewhere. Search in increasing NUMA distances.
1294 */ 1297 */
1295 static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags) 1298 static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
1296 { 1299 {
1297 #ifdef CONFIG_NUMA 1300 #ifdef CONFIG_NUMA
1298 struct zonelist *zonelist; 1301 struct zonelist *zonelist;
1299 struct zoneref *z; 1302 struct zoneref *z;
1300 struct zone *zone; 1303 struct zone *zone;
1301 enum zone_type high_zoneidx = gfp_zone(flags); 1304 enum zone_type high_zoneidx = gfp_zone(flags);
1302 struct page *page; 1305 struct page *page;
1303 1306
1304 /* 1307 /*
1305 * The defrag ratio allows a configuration of the tradeoffs between 1308 * The defrag ratio allows a configuration of the tradeoffs between
1306 * inter node defragmentation and node local allocations. A lower 1309 * inter node defragmentation and node local allocations. A lower
1307 * defrag_ratio increases the tendency to do local allocations 1310 * defrag_ratio increases the tendency to do local allocations
1308 * instead of attempting to obtain partial slabs from other nodes. 1311 * instead of attempting to obtain partial slabs from other nodes.
1309 * 1312 *
1310 * If the defrag_ratio is set to 0 then kmalloc() always 1313 * If the defrag_ratio is set to 0 then kmalloc() always
1311 * returns node local objects. If the ratio is higher then kmalloc() 1314 * returns node local objects. If the ratio is higher then kmalloc()
1312 * may return off node objects because partial slabs are obtained 1315 * may return off node objects because partial slabs are obtained
1313 * from other nodes and filled up. 1316 * from other nodes and filled up.
1314 * 1317 *
1315 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes 1318 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes
1316 * defrag_ratio = 1000) then every (well almost) allocation will 1319 * defrag_ratio = 1000) then every (well almost) allocation will
1317 * first attempt to defrag slab caches on other nodes. This means 1320 * first attempt to defrag slab caches on other nodes. This means
1318 * scanning over all nodes to look for partial slabs which may be 1321 * scanning over all nodes to look for partial slabs which may be
1319 * expensive if we do it every time we are trying to find a slab 1322 * expensive if we do it every time we are trying to find a slab
1320 * with available objects. 1323 * with available objects.
1321 */ 1324 */
1322 if (!s->remote_node_defrag_ratio || 1325 if (!s->remote_node_defrag_ratio ||
1323 get_cycles() % 1024 > s->remote_node_defrag_ratio) 1326 get_cycles() % 1024 > s->remote_node_defrag_ratio)
1324 return NULL; 1327 return NULL;
1325 1328
1326 zonelist = node_zonelist(slab_node(current->mempolicy), flags); 1329 zonelist = node_zonelist(slab_node(current->mempolicy), flags);
1327 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 1330 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
1328 struct kmem_cache_node *n; 1331 struct kmem_cache_node *n;
1329 1332
1330 n = get_node(s, zone_to_nid(zone)); 1333 n = get_node(s, zone_to_nid(zone));
1331 1334
1332 if (n && cpuset_zone_allowed_hardwall(zone, flags) && 1335 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
1333 n->nr_partial > s->min_partial) { 1336 n->nr_partial > s->min_partial) {
1334 page = get_partial_node(n); 1337 page = get_partial_node(n);
1335 if (page) 1338 if (page)
1336 return page; 1339 return page;
1337 } 1340 }
1338 } 1341 }
1339 #endif 1342 #endif
1340 return NULL; 1343 return NULL;
1341 } 1344 }
1342 1345
1343 /* 1346 /*
1344 * Get a partial page, lock it and return it. 1347 * Get a partial page, lock it and return it.
1345 */ 1348 */
1346 static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) 1349 static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
1347 { 1350 {
1348 struct page *page; 1351 struct page *page;
1349 int searchnode = (node == -1) ? numa_node_id() : node; 1352 int searchnode = (node == -1) ? numa_node_id() : node;
1350 1353
1351 page = get_partial_node(get_node(s, searchnode)); 1354 page = get_partial_node(get_node(s, searchnode));
1352 if (page || (flags & __GFP_THISNODE)) 1355 if (page || (flags & __GFP_THISNODE))
1353 return page; 1356 return page;
1354 1357
1355 return get_any_partial(s, flags); 1358 return get_any_partial(s, flags);
1356 } 1359 }
1357 1360
1358 /* 1361 /*
1359 * Move a page back to the lists. 1362 * Move a page back to the lists.
1360 * 1363 *
1361 * Must be called with the slab lock held. 1364 * Must be called with the slab lock held.
1362 * 1365 *
1363 * On exit the slab lock will have been dropped. 1366 * On exit the slab lock will have been dropped.
1364 */ 1367 */
1365 static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) 1368 static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
1366 { 1369 {
1367 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); 1370 struct kmem_cache_node *n = get_node(s, page_to_nid(page));
1368 struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id()); 1371 struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id());
1369 1372
1370 __ClearPageSlubFrozen(page); 1373 __ClearPageSlubFrozen(page);
1371 if (page->inuse) { 1374 if (page->inuse) {
1372 1375
1373 if (page->freelist) { 1376 if (page->freelist) {
1374 add_partial(n, page, tail); 1377 add_partial(n, page, tail);
1375 stat(c, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD); 1378 stat(c, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
1376 } else { 1379 } else {
1377 stat(c, DEACTIVATE_FULL); 1380 stat(c, DEACTIVATE_FULL);
1378 if (SLABDEBUG && PageSlubDebug(page) && 1381 if (SLABDEBUG && PageSlubDebug(page) &&
1379 (s->flags & SLAB_STORE_USER)) 1382 (s->flags & SLAB_STORE_USER))
1380 add_full(n, page); 1383 add_full(n, page);
1381 } 1384 }
1382 slab_unlock(page); 1385 slab_unlock(page);
1383 } else { 1386 } else {
1384 stat(c, DEACTIVATE_EMPTY); 1387 stat(c, DEACTIVATE_EMPTY);
1385 if (n->nr_partial < s->min_partial) { 1388 if (n->nr_partial < s->min_partial) {
1386 /* 1389 /*
1387 * Adding an empty slab to the partial slabs in order 1390 * Adding an empty slab to the partial slabs in order
1388 * to avoid page allocator overhead. This slab needs 1391 * to avoid page allocator overhead. This slab needs
1389 * to come after the other slabs with objects in 1392 * to come after the other slabs with objects in
1390 * so that the others get filled first. That way the 1393 * so that the others get filled first. That way the
1391 * size of the partial list stays small. 1394 * size of the partial list stays small.
1392 * 1395 *
1393 * kmem_cache_shrink can reclaim any empty slabs from 1396 * kmem_cache_shrink can reclaim any empty slabs from
1394 * the partial list. 1397 * the partial list.
1395 */ 1398 */
1396 add_partial(n, page, 1); 1399 add_partial(n, page, 1);
1397 slab_unlock(page); 1400 slab_unlock(page);
1398 } else { 1401 } else {
1399 slab_unlock(page); 1402 slab_unlock(page);
1400 stat(get_cpu_slab(s, raw_smp_processor_id()), FREE_SLAB); 1403 stat(get_cpu_slab(s, raw_smp_processor_id()), FREE_SLAB);
1401 discard_slab(s, page); 1404 discard_slab(s, page);
1402 } 1405 }
1403 } 1406 }
1404 } 1407 }
1405 1408
1406 /* 1409 /*
1407 * Remove the cpu slab 1410 * Remove the cpu slab
1408 */ 1411 */
1409 static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 1412 static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
1410 { 1413 {
1411 struct page *page = c->page; 1414 struct page *page = c->page;
1412 int tail = 1; 1415 int tail = 1;
1413 1416
1414 if (page->freelist) 1417 if (page->freelist)
1415 stat(c, DEACTIVATE_REMOTE_FREES); 1418 stat(c, DEACTIVATE_REMOTE_FREES);
1416 /* 1419 /*
1417 * Merge cpu freelist into slab freelist. Typically we get here 1420 * Merge cpu freelist into slab freelist. Typically we get here
1418 * because both freelists are empty. So this is unlikely 1421 * because both freelists are empty. So this is unlikely
1419 * to occur. 1422 * to occur.
1420 */ 1423 */
1421 while (unlikely(c->freelist)) { 1424 while (unlikely(c->freelist)) {
1422 void **object; 1425 void **object;
1423 1426
1424 tail = 0; /* Hot objects. Put the slab first */ 1427 tail = 0; /* Hot objects. Put the slab first */
1425 1428
1426 /* Retrieve object from cpu_freelist */ 1429 /* Retrieve object from cpu_freelist */
1427 object = c->freelist; 1430 object = c->freelist;
1428 c->freelist = c->freelist[c->offset]; 1431 c->freelist = c->freelist[c->offset];
1429 1432
1430 /* And put onto the regular freelist */ 1433 /* And put onto the regular freelist */
1431 object[c->offset] = page->freelist; 1434 object[c->offset] = page->freelist;
1432 page->freelist = object; 1435 page->freelist = object;
1433 page->inuse--; 1436 page->inuse--;
1434 } 1437 }
1435 c->page = NULL; 1438 c->page = NULL;
1436 unfreeze_slab(s, page, tail); 1439 unfreeze_slab(s, page, tail);
1437 } 1440 }
1438 1441
1439 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 1442 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
1440 { 1443 {
1441 stat(c, CPUSLAB_FLUSH); 1444 stat(c, CPUSLAB_FLUSH);
1442 slab_lock(c->page); 1445 slab_lock(c->page);
1443 deactivate_slab(s, c); 1446 deactivate_slab(s, c);
1444 } 1447 }
1445 1448
1446 /* 1449 /*
1447 * Flush cpu slab. 1450 * Flush cpu slab.
1448 * 1451 *
1449 * Called from IPI handler with interrupts disabled. 1452 * Called from IPI handler with interrupts disabled.
1450 */ 1453 */
1451 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) 1454 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
1452 { 1455 {
1453 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); 1456 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
1454 1457
1455 if (likely(c && c->page)) 1458 if (likely(c && c->page))
1456 flush_slab(s, c); 1459 flush_slab(s, c);
1457 } 1460 }
1458 1461
1459 static void flush_cpu_slab(void *d) 1462 static void flush_cpu_slab(void *d)
1460 { 1463 {
1461 struct kmem_cache *s = d; 1464 struct kmem_cache *s = d;
1462 1465
1463 __flush_cpu_slab(s, smp_processor_id()); 1466 __flush_cpu_slab(s, smp_processor_id());
1464 } 1467 }
1465 1468
1466 static void flush_all(struct kmem_cache *s) 1469 static void flush_all(struct kmem_cache *s)
1467 { 1470 {
1468 on_each_cpu(flush_cpu_slab, s, 1); 1471 on_each_cpu(flush_cpu_slab, s, 1);
1469 } 1472 }
1470 1473
1471 /* 1474 /*
1472 * Check if the objects in a per cpu structure fit numa 1475 * Check if the objects in a per cpu structure fit numa
1473 * locality expectations. 1476 * locality expectations.
1474 */ 1477 */
1475 static inline int node_match(struct kmem_cache_cpu *c, int node) 1478 static inline int node_match(struct kmem_cache_cpu *c, int node)
1476 { 1479 {
1477 #ifdef CONFIG_NUMA 1480 #ifdef CONFIG_NUMA
1478 if (node != -1 && c->node != node) 1481 if (node != -1 && c->node != node)
1479 return 0; 1482 return 0;
1480 #endif 1483 #endif
1481 return 1; 1484 return 1;
1482 } 1485 }
1483 1486
1484 /* 1487 /*
1485 * Slow path. The lockless freelist is empty or we need to perform 1488 * Slow path. The lockless freelist is empty or we need to perform
1486 * debugging duties. 1489 * debugging duties.
1487 * 1490 *
1488 * Interrupts are disabled. 1491 * Interrupts are disabled.
1489 * 1492 *
1490 * Processing is still very fast if new objects have been freed to the 1493 * Processing is still very fast if new objects have been freed to the
1491 * regular freelist. In that case we simply take over the regular freelist 1494 * regular freelist. In that case we simply take over the regular freelist
1492 * as the lockless freelist and zap the regular freelist. 1495 * as the lockless freelist and zap the regular freelist.
1493 * 1496 *
1494 * If that is not working then we fall back to the partial lists. We take the 1497 * If that is not working then we fall back to the partial lists. We take the
1495 * first element of the freelist as the object to allocate now and move the 1498 * first element of the freelist as the object to allocate now and move the
1496 * rest of the freelist to the lockless freelist. 1499 * rest of the freelist to the lockless freelist.
1497 * 1500 *
1498 * And if we were unable to get a new slab from the partial slab lists then 1501 * And if we were unable to get a new slab from the partial slab lists then
1499 * we need to allocate a new slab. This is the slowest path since it involves 1502 * we need to allocate a new slab. This is the slowest path since it involves
1500 * a call to the page allocator and the setup of a new slab. 1503 * a call to the page allocator and the setup of a new slab.
1501 */ 1504 */
1502 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 1505 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
1503 unsigned long addr, struct kmem_cache_cpu *c) 1506 unsigned long addr, struct kmem_cache_cpu *c)
1504 { 1507 {
1505 void **object; 1508 void **object;
1506 struct page *new; 1509 struct page *new;
1507 1510
1508 /* We handle __GFP_ZERO in the caller */ 1511 /* We handle __GFP_ZERO in the caller */
1509 gfpflags &= ~__GFP_ZERO; 1512 gfpflags &= ~__GFP_ZERO;
1510 1513
1511 if (!c->page) 1514 if (!c->page)
1512 goto new_slab; 1515 goto new_slab;
1513 1516
1514 slab_lock(c->page); 1517 slab_lock(c->page);
1515 if (unlikely(!node_match(c, node))) 1518 if (unlikely(!node_match(c, node)))
1516 goto another_slab; 1519 goto another_slab;
1517 1520
1518 stat(c, ALLOC_REFILL); 1521 stat(c, ALLOC_REFILL);
1519 1522
1520 load_freelist: 1523 load_freelist:
1521 object = c->page->freelist; 1524 object = c->page->freelist;
1522 if (unlikely(!object)) 1525 if (unlikely(!object))
1523 goto another_slab; 1526 goto another_slab;
1524 if (unlikely(SLABDEBUG && PageSlubDebug(c->page))) 1527 if (unlikely(SLABDEBUG && PageSlubDebug(c->page)))
1525 goto debug; 1528 goto debug;
1526 1529
1527 c->freelist = object[c->offset]; 1530 c->freelist = object[c->offset];
1528 c->page->inuse = c->page->objects; 1531 c->page->inuse = c->page->objects;
1529 c->page->freelist = NULL; 1532 c->page->freelist = NULL;
1530 c->node = page_to_nid(c->page); 1533 c->node = page_to_nid(c->page);
1531 unlock_out: 1534 unlock_out:
1532 slab_unlock(c->page); 1535 slab_unlock(c->page);
1533 stat(c, ALLOC_SLOWPATH); 1536 stat(c, ALLOC_SLOWPATH);
1534 return object; 1537 return object;
1535 1538
1536 another_slab: 1539 another_slab:
1537 deactivate_slab(s, c); 1540 deactivate_slab(s, c);
1538 1541
1539 new_slab: 1542 new_slab:
1540 new = get_partial(s, gfpflags, node); 1543 new = get_partial(s, gfpflags, node);
1541 if (new) { 1544 if (new) {
1542 c->page = new; 1545 c->page = new;
1543 stat(c, ALLOC_FROM_PARTIAL); 1546 stat(c, ALLOC_FROM_PARTIAL);
1544 goto load_freelist; 1547 goto load_freelist;
1545 } 1548 }
1546 1549
1547 if (gfpflags & __GFP_WAIT) 1550 if (gfpflags & __GFP_WAIT)
1548 local_irq_enable(); 1551 local_irq_enable();
1549 1552
1550 new = new_slab(s, gfpflags, node); 1553 new = new_slab(s, gfpflags, node);
1551 1554
1552 if (gfpflags & __GFP_WAIT) 1555 if (gfpflags & __GFP_WAIT)
1553 local_irq_disable(); 1556 local_irq_disable();
1554 1557
1555 if (new) { 1558 if (new) {
1556 c = get_cpu_slab(s, smp_processor_id()); 1559 c = get_cpu_slab(s, smp_processor_id());
1557 stat(c, ALLOC_SLAB); 1560 stat(c, ALLOC_SLAB);
1558 if (c->page) 1561 if (c->page)
1559 flush_slab(s, c); 1562 flush_slab(s, c);
1560 slab_lock(new); 1563 slab_lock(new);
1561 __SetPageSlubFrozen(new); 1564 __SetPageSlubFrozen(new);
1562 c->page = new; 1565 c->page = new;
1563 goto load_freelist; 1566 goto load_freelist;
1564 } 1567 }
1565 return NULL; 1568 return NULL;
1566 debug: 1569 debug:
1567 if (!alloc_debug_processing(s, c->page, object, addr)) 1570 if (!alloc_debug_processing(s, c->page, object, addr))
1568 goto another_slab; 1571 goto another_slab;
1569 1572
1570 c->page->inuse++; 1573 c->page->inuse++;
1571 c->page->freelist = object[c->offset]; 1574 c->page->freelist = object[c->offset];
1572 c->node = -1; 1575 c->node = -1;
1573 goto unlock_out; 1576 goto unlock_out;
1574 } 1577 }
1575 1578
1576 /* 1579 /*
1577 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) 1580 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
1578 * have the fastpath folded into their functions. So no function call 1581 * have the fastpath folded into their functions. So no function call
1579 * overhead for requests that can be satisfied on the fastpath. 1582 * overhead for requests that can be satisfied on the fastpath.
1580 * 1583 *
1581 * The fastpath works by first checking if the lockless freelist can be used. 1584 * The fastpath works by first checking if the lockless freelist can be used.
1582 * If not then __slab_alloc is called for slow processing. 1585 * If not then __slab_alloc is called for slow processing.
1583 * 1586 *
1584 * Otherwise we can simply pick the next object from the lockless free list. 1587 * Otherwise we can simply pick the next object from the lockless free list.
1585 */ 1588 */
1586 static __always_inline void *slab_alloc(struct kmem_cache *s, 1589 static __always_inline void *slab_alloc(struct kmem_cache *s,
1587 gfp_t gfpflags, int node, unsigned long addr) 1590 gfp_t gfpflags, int node, unsigned long addr)
1588 { 1591 {
1589 void **object; 1592 void **object;
1590 struct kmem_cache_cpu *c; 1593 struct kmem_cache_cpu *c;
1591 unsigned long flags; 1594 unsigned long flags;
1592 unsigned int objsize; 1595 unsigned int objsize;
1593 1596
1594 lockdep_trace_alloc(gfpflags); 1597 lockdep_trace_alloc(gfpflags);
1595 might_sleep_if(gfpflags & __GFP_WAIT); 1598 might_sleep_if(gfpflags & __GFP_WAIT);
1596 1599
1597 if (should_failslab(s->objsize, gfpflags)) 1600 if (should_failslab(s->objsize, gfpflags))
1598 return NULL; 1601 return NULL;
1599 1602
1600 local_irq_save(flags); 1603 local_irq_save(flags);
1601 c = get_cpu_slab(s, smp_processor_id()); 1604 c = get_cpu_slab(s, smp_processor_id());
1602 objsize = c->objsize; 1605 objsize = c->objsize;
1603 if (unlikely(!c->freelist || !node_match(c, node))) 1606 if (unlikely(!c->freelist || !node_match(c, node)))
1604 1607
1605 object = __slab_alloc(s, gfpflags, node, addr, c); 1608 object = __slab_alloc(s, gfpflags, node, addr, c);
1606 1609
1607 else { 1610 else {
1608 object = c->freelist; 1611 object = c->freelist;
1609 c->freelist = object[c->offset]; 1612 c->freelist = object[c->offset];
1610 stat(c, ALLOC_FASTPATH); 1613 stat(c, ALLOC_FASTPATH);
1611 } 1614 }
1612 local_irq_restore(flags); 1615 local_irq_restore(flags);
1613 1616
1614 if (unlikely((gfpflags & __GFP_ZERO) && object)) 1617 if (unlikely((gfpflags & __GFP_ZERO) && object))
1615 memset(object, 0, objsize); 1618 memset(object, 0, objsize);
1616 1619
1617 return object; 1620 return object;
1618 } 1621 }
1619 1622
1620 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) 1623 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
1621 { 1624 {
1622 void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_); 1625 void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
1623 1626
1624 trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags); 1627 trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
1625 1628
1626 return ret; 1629 return ret;
1627 } 1630 }
1628 EXPORT_SYMBOL(kmem_cache_alloc); 1631 EXPORT_SYMBOL(kmem_cache_alloc);
1629 1632
1630 #ifdef CONFIG_KMEMTRACE 1633 #ifdef CONFIG_KMEMTRACE
1631 void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags) 1634 void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
1632 { 1635 {
1633 return slab_alloc(s, gfpflags, -1, _RET_IP_); 1636 return slab_alloc(s, gfpflags, -1, _RET_IP_);
1634 } 1637 }
1635 EXPORT_SYMBOL(kmem_cache_alloc_notrace); 1638 EXPORT_SYMBOL(kmem_cache_alloc_notrace);
1636 #endif 1639 #endif
1637 1640
1638 #ifdef CONFIG_NUMA 1641 #ifdef CONFIG_NUMA
1639 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) 1642 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
1640 { 1643 {
1641 void *ret = slab_alloc(s, gfpflags, node, _RET_IP_); 1644 void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
1642 1645
1643 trace_kmem_cache_alloc_node(_RET_IP_, ret, 1646 trace_kmem_cache_alloc_node(_RET_IP_, ret,
1644 s->objsize, s->size, gfpflags, node); 1647 s->objsize, s->size, gfpflags, node);
1645 1648
1646 return ret; 1649 return ret;
1647 } 1650 }
1648 EXPORT_SYMBOL(kmem_cache_alloc_node); 1651 EXPORT_SYMBOL(kmem_cache_alloc_node);
1649 #endif 1652 #endif
1650 1653
1651 #ifdef CONFIG_KMEMTRACE 1654 #ifdef CONFIG_KMEMTRACE
1652 void *kmem_cache_alloc_node_notrace(struct kmem_cache *s, 1655 void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
1653 gfp_t gfpflags, 1656 gfp_t gfpflags,
1654 int node) 1657 int node)
1655 { 1658 {
1656 return slab_alloc(s, gfpflags, node, _RET_IP_); 1659 return slab_alloc(s, gfpflags, node, _RET_IP_);
1657 } 1660 }
1658 EXPORT_SYMBOL(kmem_cache_alloc_node_notrace); 1661 EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
1659 #endif 1662 #endif
1660 1663
1661 /* 1664 /*
1662 * Slow patch handling. This may still be called frequently since objects 1665 * Slow patch handling. This may still be called frequently since objects
1663 * have a longer lifetime than the cpu slabs in most processing loads. 1666 * have a longer lifetime than the cpu slabs in most processing loads.
1664 * 1667 *
1665 * So we still attempt to reduce cache line usage. Just take the slab 1668 * So we still attempt to reduce cache line usage. Just take the slab
1666 * lock and free the item. If there is no additional partial page 1669 * lock and free the item. If there is no additional partial page
1667 * handling required then we can return immediately. 1670 * handling required then we can return immediately.
1668 */ 1671 */
1669 static void __slab_free(struct kmem_cache *s, struct page *page, 1672 static void __slab_free(struct kmem_cache *s, struct page *page,
1670 void *x, unsigned long addr, unsigned int offset) 1673 void *x, unsigned long addr, unsigned int offset)
1671 { 1674 {
1672 void *prior; 1675 void *prior;
1673 void **object = (void *)x; 1676 void **object = (void *)x;
1674 struct kmem_cache_cpu *c; 1677 struct kmem_cache_cpu *c;
1675 1678
1676 c = get_cpu_slab(s, raw_smp_processor_id()); 1679 c = get_cpu_slab(s, raw_smp_processor_id());
1677 stat(c, FREE_SLOWPATH); 1680 stat(c, FREE_SLOWPATH);
1678 slab_lock(page); 1681 slab_lock(page);
1679 1682
1680 if (unlikely(SLABDEBUG && PageSlubDebug(page))) 1683 if (unlikely(SLABDEBUG && PageSlubDebug(page)))
1681 goto debug; 1684 goto debug;
1682 1685
1683 checks_ok: 1686 checks_ok:
1684 prior = object[offset] = page->freelist; 1687 prior = object[offset] = page->freelist;
1685 page->freelist = object; 1688 page->freelist = object;
1686 page->inuse--; 1689 page->inuse--;
1687 1690
1688 if (unlikely(PageSlubFrozen(page))) { 1691 if (unlikely(PageSlubFrozen(page))) {
1689 stat(c, FREE_FROZEN); 1692 stat(c, FREE_FROZEN);
1690 goto out_unlock; 1693 goto out_unlock;
1691 } 1694 }
1692 1695
1693 if (unlikely(!page->inuse)) 1696 if (unlikely(!page->inuse))
1694 goto slab_empty; 1697 goto slab_empty;
1695 1698
1696 /* 1699 /*
1697 * Objects left in the slab. If it was not on the partial list before 1700 * Objects left in the slab. If it was not on the partial list before
1698 * then add it. 1701 * then add it.
1699 */ 1702 */
1700 if (unlikely(!prior)) { 1703 if (unlikely(!prior)) {
1701 add_partial(get_node(s, page_to_nid(page)), page, 1); 1704 add_partial(get_node(s, page_to_nid(page)), page, 1);
1702 stat(c, FREE_ADD_PARTIAL); 1705 stat(c, FREE_ADD_PARTIAL);
1703 } 1706 }
1704 1707
1705 out_unlock: 1708 out_unlock:
1706 slab_unlock(page); 1709 slab_unlock(page);
1707 return; 1710 return;
1708 1711
1709 slab_empty: 1712 slab_empty:
1710 if (prior) { 1713 if (prior) {
1711 /* 1714 /*
1712 * Slab still on the partial list. 1715 * Slab still on the partial list.
1713 */ 1716 */
1714 remove_partial(s, page); 1717 remove_partial(s, page);
1715 stat(c, FREE_REMOVE_PARTIAL); 1718 stat(c, FREE_REMOVE_PARTIAL);
1716 } 1719 }
1717 slab_unlock(page); 1720 slab_unlock(page);
1718 stat(c, FREE_SLAB); 1721 stat(c, FREE_SLAB);
1719 discard_slab(s, page); 1722 discard_slab(s, page);
1720 return; 1723 return;
1721 1724
1722 debug: 1725 debug:
1723 if (!free_debug_processing(s, page, x, addr)) 1726 if (!free_debug_processing(s, page, x, addr))
1724 goto out_unlock; 1727 goto out_unlock;
1725 goto checks_ok; 1728 goto checks_ok;
1726 } 1729 }
1727 1730
1728 /* 1731 /*
1729 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 1732 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
1730 * can perform fastpath freeing without additional function calls. 1733 * can perform fastpath freeing without additional function calls.
1731 * 1734 *
1732 * The fastpath is only possible if we are freeing to the current cpu slab 1735 * The fastpath is only possible if we are freeing to the current cpu slab
1733 * of this processor. This typically the case if we have just allocated 1736 * of this processor. This typically the case if we have just allocated
1734 * the item before. 1737 * the item before.
1735 * 1738 *
1736 * If fastpath is not possible then fall back to __slab_free where we deal 1739 * If fastpath is not possible then fall back to __slab_free where we deal
1737 * with all sorts of special processing. 1740 * with all sorts of special processing.
1738 */ 1741 */
1739 static __always_inline void slab_free(struct kmem_cache *s, 1742 static __always_inline void slab_free(struct kmem_cache *s,
1740 struct page *page, void *x, unsigned long addr) 1743 struct page *page, void *x, unsigned long addr)
1741 { 1744 {
1742 void **object = (void *)x; 1745 void **object = (void *)x;
1743 struct kmem_cache_cpu *c; 1746 struct kmem_cache_cpu *c;
1744 unsigned long flags; 1747 unsigned long flags;
1745 1748
1746 local_irq_save(flags); 1749 local_irq_save(flags);
1747 c = get_cpu_slab(s, smp_processor_id()); 1750 c = get_cpu_slab(s, smp_processor_id());
1748 debug_check_no_locks_freed(object, c->objsize); 1751 debug_check_no_locks_freed(object, c->objsize);
1749 if (!(s->flags & SLAB_DEBUG_OBJECTS)) 1752 if (!(s->flags & SLAB_DEBUG_OBJECTS))
1750 debug_check_no_obj_freed(object, c->objsize); 1753 debug_check_no_obj_freed(object, c->objsize);
1751 if (likely(page == c->page && c->node >= 0)) { 1754 if (likely(page == c->page && c->node >= 0)) {
1752 object[c->offset] = c->freelist; 1755 object[c->offset] = c->freelist;
1753 c->freelist = object; 1756 c->freelist = object;
1754 stat(c, FREE_FASTPATH); 1757 stat(c, FREE_FASTPATH);
1755 } else 1758 } else
1756 __slab_free(s, page, x, addr, c->offset); 1759 __slab_free(s, page, x, addr, c->offset);
1757 1760
1758 local_irq_restore(flags); 1761 local_irq_restore(flags);
1759 } 1762 }
1760 1763
1761 void kmem_cache_free(struct kmem_cache *s, void *x) 1764 void kmem_cache_free(struct kmem_cache *s, void *x)
1762 { 1765 {
1763 struct page *page; 1766 struct page *page;
1764 1767
1765 page = virt_to_head_page(x); 1768 page = virt_to_head_page(x);
1766 1769
1767 slab_free(s, page, x, _RET_IP_); 1770 slab_free(s, page, x, _RET_IP_);
1768 1771
1769 trace_kmem_cache_free(_RET_IP_, x); 1772 trace_kmem_cache_free(_RET_IP_, x);
1770 } 1773 }
1771 EXPORT_SYMBOL(kmem_cache_free); 1774 EXPORT_SYMBOL(kmem_cache_free);
1772 1775
1773 /* Figure out on which slab page the object resides */ 1776 /* Figure out on which slab page the object resides */
1774 static struct page *get_object_page(const void *x) 1777 static struct page *get_object_page(const void *x)
1775 { 1778 {
1776 struct page *page = virt_to_head_page(x); 1779 struct page *page = virt_to_head_page(x);
1777 1780
1778 if (!PageSlab(page)) 1781 if (!PageSlab(page))
1779 return NULL; 1782 return NULL;
1780 1783
1781 return page; 1784 return page;
1782 } 1785 }
1783 1786
1784 /* 1787 /*
1785 * Object placement in a slab is made very easy because we always start at 1788 * Object placement in a slab is made very easy because we always start at
1786 * offset 0. If we tune the size of the object to the alignment then we can 1789 * offset 0. If we tune the size of the object to the alignment then we can
1787 * get the required alignment by putting one properly sized object after 1790 * get the required alignment by putting one properly sized object after
1788 * another. 1791 * another.
1789 * 1792 *
1790 * Notice that the allocation order determines the sizes of the per cpu 1793 * Notice that the allocation order determines the sizes of the per cpu
1791 * caches. Each processor has always one slab available for allocations. 1794 * caches. Each processor has always one slab available for allocations.
1792 * Increasing the allocation order reduces the number of times that slabs 1795 * Increasing the allocation order reduces the number of times that slabs
1793 * must be moved on and off the partial lists and is therefore a factor in 1796 * must be moved on and off the partial lists and is therefore a factor in
1794 * locking overhead. 1797 * locking overhead.
1795 */ 1798 */
1796 1799
1797 /* 1800 /*
1798 * Mininum / Maximum order of slab pages. This influences locking overhead 1801 * Mininum / Maximum order of slab pages. This influences locking overhead
1799 * and slab fragmentation. A higher order reduces the number of partial slabs 1802 * and slab fragmentation. A higher order reduces the number of partial slabs
1800 * and increases the number of allocations possible without having to 1803 * and increases the number of allocations possible without having to
1801 * take the list_lock. 1804 * take the list_lock.
1802 */ 1805 */
1803 static int slub_min_order; 1806 static int slub_min_order;
1804 static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; 1807 static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
1805 static int slub_min_objects; 1808 static int slub_min_objects;
1806 1809
1807 /* 1810 /*
1808 * Merge control. If this is set then no merging of slab caches will occur. 1811 * Merge control. If this is set then no merging of slab caches will occur.
1809 * (Could be removed. This was introduced to pacify the merge skeptics.) 1812 * (Could be removed. This was introduced to pacify the merge skeptics.)
1810 */ 1813 */
1811 static int slub_nomerge; 1814 static int slub_nomerge;
1812 1815
1813 /* 1816 /*
1814 * Calculate the order of allocation given an slab object size. 1817 * Calculate the order of allocation given an slab object size.
1815 * 1818 *
1816 * The order of allocation has significant impact on performance and other 1819 * The order of allocation has significant impact on performance and other
1817 * system components. Generally order 0 allocations should be preferred since 1820 * system components. Generally order 0 allocations should be preferred since
1818 * order 0 does not cause fragmentation in the page allocator. Larger objects 1821 * order 0 does not cause fragmentation in the page allocator. Larger objects
1819 * be problematic to put into order 0 slabs because there may be too much 1822 * be problematic to put into order 0 slabs because there may be too much
1820 * unused space left. We go to a higher order if more than 1/16th of the slab 1823 * unused space left. We go to a higher order if more than 1/16th of the slab
1821 * would be wasted. 1824 * would be wasted.
1822 * 1825 *
1823 * In order to reach satisfactory performance we must ensure that a minimum 1826 * In order to reach satisfactory performance we must ensure that a minimum
1824 * number of objects is in one slab. Otherwise we may generate too much 1827 * number of objects is in one slab. Otherwise we may generate too much
1825 * activity on the partial lists which requires taking the list_lock. This is 1828 * activity on the partial lists which requires taking the list_lock. This is
1826 * less a concern for large slabs though which are rarely used. 1829 * less a concern for large slabs though which are rarely used.
1827 * 1830 *
1828 * slub_max_order specifies the order where we begin to stop considering the 1831 * slub_max_order specifies the order where we begin to stop considering the
1829 * number of objects in a slab as critical. If we reach slub_max_order then 1832 * number of objects in a slab as critical. If we reach slub_max_order then
1830 * we try to keep the page order as low as possible. So we accept more waste 1833 * we try to keep the page order as low as possible. So we accept more waste
1831 * of space in favor of a small page order. 1834 * of space in favor of a small page order.
1832 * 1835 *
1833 * Higher order allocations also allow the placement of more objects in a 1836 * Higher order allocations also allow the placement of more objects in a
1834 * slab and thereby reduce object handling overhead. If the user has 1837 * slab and thereby reduce object handling overhead. If the user has
1835 * requested a higher mininum order then we start with that one instead of 1838 * requested a higher mininum order then we start with that one instead of
1836 * the smallest order which will fit the object. 1839 * the smallest order which will fit the object.
1837 */ 1840 */
1838 static inline int slab_order(int size, int min_objects, 1841 static inline int slab_order(int size, int min_objects,
1839 int max_order, int fract_leftover) 1842 int max_order, int fract_leftover)
1840 { 1843 {
1841 int order; 1844 int order;
1842 int rem; 1845 int rem;
1843 int min_order = slub_min_order; 1846 int min_order = slub_min_order;
1844 1847
1845 if ((PAGE_SIZE << min_order) / size > MAX_OBJS_PER_PAGE) 1848 if ((PAGE_SIZE << min_order) / size > MAX_OBJS_PER_PAGE)
1846 return get_order(size * MAX_OBJS_PER_PAGE) - 1; 1849 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
1847 1850
1848 for (order = max(min_order, 1851 for (order = max(min_order,
1849 fls(min_objects * size - 1) - PAGE_SHIFT); 1852 fls(min_objects * size - 1) - PAGE_SHIFT);
1850 order <= max_order; order++) { 1853 order <= max_order; order++) {
1851 1854
1852 unsigned long slab_size = PAGE_SIZE << order; 1855 unsigned long slab_size = PAGE_SIZE << order;
1853 1856
1854 if (slab_size < min_objects * size) 1857 if (slab_size < min_objects * size)
1855 continue; 1858 continue;
1856 1859
1857 rem = slab_size % size; 1860 rem = slab_size % size;
1858 1861
1859 if (rem <= slab_size / fract_leftover) 1862 if (rem <= slab_size / fract_leftover)
1860 break; 1863 break;
1861 1864
1862 } 1865 }
1863 1866
1864 return order; 1867 return order;
1865 } 1868 }
1866 1869
1867 static inline int calculate_order(int size) 1870 static inline int calculate_order(int size)
1868 { 1871 {
1869 int order; 1872 int order;
1870 int min_objects; 1873 int min_objects;
1871 int fraction; 1874 int fraction;
1872 int max_objects; 1875 int max_objects;
1873 1876
1874 /* 1877 /*
1875 * Attempt to find best configuration for a slab. This 1878 * Attempt to find best configuration for a slab. This
1876 * works by first attempting to generate a layout with 1879 * works by first attempting to generate a layout with
1877 * the best configuration and backing off gradually. 1880 * the best configuration and backing off gradually.
1878 * 1881 *
1879 * First we reduce the acceptable waste in a slab. Then 1882 * First we reduce the acceptable waste in a slab. Then
1880 * we reduce the minimum objects required in a slab. 1883 * we reduce the minimum objects required in a slab.
1881 */ 1884 */
1882 min_objects = slub_min_objects; 1885 min_objects = slub_min_objects;
1883 if (!min_objects) 1886 if (!min_objects)
1884 min_objects = 4 * (fls(nr_cpu_ids) + 1); 1887 min_objects = 4 * (fls(nr_cpu_ids) + 1);
1885 max_objects = (PAGE_SIZE << slub_max_order)/size; 1888 max_objects = (PAGE_SIZE << slub_max_order)/size;
1886 min_objects = min(min_objects, max_objects); 1889 min_objects = min(min_objects, max_objects);
1887 1890
1888 while (min_objects > 1) { 1891 while (min_objects > 1) {
1889 fraction = 16; 1892 fraction = 16;
1890 while (fraction >= 4) { 1893 while (fraction >= 4) {
1891 order = slab_order(size, min_objects, 1894 order = slab_order(size, min_objects,
1892 slub_max_order, fraction); 1895 slub_max_order, fraction);
1893 if (order <= slub_max_order) 1896 if (order <= slub_max_order)
1894 return order; 1897 return order;
1895 fraction /= 2; 1898 fraction /= 2;
1896 } 1899 }
1897 min_objects --; 1900 min_objects --;
1898 } 1901 }
1899 1902
1900 /* 1903 /*
1901 * We were unable to place multiple objects in a slab. Now 1904 * We were unable to place multiple objects in a slab. Now
1902 * lets see if we can place a single object there. 1905 * lets see if we can place a single object there.
1903 */ 1906 */
1904 order = slab_order(size, 1, slub_max_order, 1); 1907 order = slab_order(size, 1, slub_max_order, 1);
1905 if (order <= slub_max_order) 1908 if (order <= slub_max_order)
1906 return order; 1909 return order;
1907 1910
1908 /* 1911 /*
1909 * Doh this slab cannot be placed using slub_max_order. 1912 * Doh this slab cannot be placed using slub_max_order.
1910 */ 1913 */
1911 order = slab_order(size, 1, MAX_ORDER, 1); 1914 order = slab_order(size, 1, MAX_ORDER, 1);
1912 if (order <= MAX_ORDER) 1915 if (order <= MAX_ORDER)
1913 return order; 1916 return order;
1914 return -ENOSYS; 1917 return -ENOSYS;
1915 } 1918 }
1916 1919
1917 /* 1920 /*
1918 * Figure out what the alignment of the objects will be. 1921 * Figure out what the alignment of the objects will be.
1919 */ 1922 */
1920 static unsigned long calculate_alignment(unsigned long flags, 1923 static unsigned long calculate_alignment(unsigned long flags,
1921 unsigned long align, unsigned long size) 1924 unsigned long align, unsigned long size)
1922 { 1925 {
1923 /* 1926 /*
1924 * If the user wants hardware cache aligned objects then follow that 1927 * If the user wants hardware cache aligned objects then follow that
1925 * suggestion if the object is sufficiently large. 1928 * suggestion if the object is sufficiently large.
1926 * 1929 *
1927 * The hardware cache alignment cannot override the specified 1930 * The hardware cache alignment cannot override the specified
1928 * alignment though. If that is greater then use it. 1931 * alignment though. If that is greater then use it.
1929 */ 1932 */
1930 if (flags & SLAB_HWCACHE_ALIGN) { 1933 if (flags & SLAB_HWCACHE_ALIGN) {
1931 unsigned long ralign = cache_line_size(); 1934 unsigned long ralign = cache_line_size();
1932 while (size <= ralign / 2) 1935 while (size <= ralign / 2)
1933 ralign /= 2; 1936 ralign /= 2;
1934 align = max(align, ralign); 1937 align = max(align, ralign);
1935 } 1938 }
1936 1939
1937 if (align < ARCH_SLAB_MINALIGN) 1940 if (align < ARCH_SLAB_MINALIGN)
1938 align = ARCH_SLAB_MINALIGN; 1941 align = ARCH_SLAB_MINALIGN;
1939 1942
1940 return ALIGN(align, sizeof(void *)); 1943 return ALIGN(align, sizeof(void *));
1941 } 1944 }
1942 1945
1943 static void init_kmem_cache_cpu(struct kmem_cache *s, 1946 static void init_kmem_cache_cpu(struct kmem_cache *s,
1944 struct kmem_cache_cpu *c) 1947 struct kmem_cache_cpu *c)
1945 { 1948 {
1946 c->page = NULL; 1949 c->page = NULL;
1947 c->freelist = NULL; 1950 c->freelist = NULL;
1948 c->node = 0; 1951 c->node = 0;
1949 c->offset = s->offset / sizeof(void *); 1952 c->offset = s->offset / sizeof(void *);
1950 c->objsize = s->objsize; 1953 c->objsize = s->objsize;
1951 #ifdef CONFIG_SLUB_STATS 1954 #ifdef CONFIG_SLUB_STATS
1952 memset(c->stat, 0, NR_SLUB_STAT_ITEMS * sizeof(unsigned)); 1955 memset(c->stat, 0, NR_SLUB_STAT_ITEMS * sizeof(unsigned));
1953 #endif 1956 #endif
1954 } 1957 }
1955 1958
1956 static void 1959 static void
1957 init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s) 1960 init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
1958 { 1961 {
1959 n->nr_partial = 0; 1962 n->nr_partial = 0;
1960 spin_lock_init(&n->list_lock); 1963 spin_lock_init(&n->list_lock);
1961 INIT_LIST_HEAD(&n->partial); 1964 INIT_LIST_HEAD(&n->partial);
1962 #ifdef CONFIG_SLUB_DEBUG 1965 #ifdef CONFIG_SLUB_DEBUG
1963 atomic_long_set(&n->nr_slabs, 0); 1966 atomic_long_set(&n->nr_slabs, 0);
1964 atomic_long_set(&n->total_objects, 0); 1967 atomic_long_set(&n->total_objects, 0);
1965 INIT_LIST_HEAD(&n->full); 1968 INIT_LIST_HEAD(&n->full);
1966 #endif 1969 #endif
1967 } 1970 }
1968 1971
1969 #ifdef CONFIG_SMP 1972 #ifdef CONFIG_SMP
1970 /* 1973 /*
1971 * Per cpu array for per cpu structures. 1974 * Per cpu array for per cpu structures.
1972 * 1975 *
1973 * The per cpu array places all kmem_cache_cpu structures from one processor 1976 * The per cpu array places all kmem_cache_cpu structures from one processor
1974 * close together meaning that it becomes possible that multiple per cpu 1977 * close together meaning that it becomes possible that multiple per cpu
1975 * structures are contained in one cacheline. This may be particularly 1978 * structures are contained in one cacheline. This may be particularly
1976 * beneficial for the kmalloc caches. 1979 * beneficial for the kmalloc caches.
1977 * 1980 *
1978 * A desktop system typically has around 60-80 slabs. With 100 here we are 1981 * A desktop system typically has around 60-80 slabs. With 100 here we are
1979 * likely able to get per cpu structures for all caches from the array defined 1982 * likely able to get per cpu structures for all caches from the array defined
1980 * here. We must be able to cover all kmalloc caches during bootstrap. 1983 * here. We must be able to cover all kmalloc caches during bootstrap.
1981 * 1984 *
1982 * If the per cpu array is exhausted then fall back to kmalloc 1985 * If the per cpu array is exhausted then fall back to kmalloc
1983 * of individual cachelines. No sharing is possible then. 1986 * of individual cachelines. No sharing is possible then.
1984 */ 1987 */
1985 #define NR_KMEM_CACHE_CPU 100 1988 #define NR_KMEM_CACHE_CPU 100
1986 1989
1987 static DEFINE_PER_CPU(struct kmem_cache_cpu, 1990 static DEFINE_PER_CPU(struct kmem_cache_cpu,
1988 kmem_cache_cpu)[NR_KMEM_CACHE_CPU]; 1991 kmem_cache_cpu)[NR_KMEM_CACHE_CPU];
1989 1992
1990 static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free); 1993 static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free);
1991 static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS); 1994 static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS);
1992 1995
1993 static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s, 1996 static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s,
1994 int cpu, gfp_t flags) 1997 int cpu, gfp_t flags)
1995 { 1998 {
1996 struct kmem_cache_cpu *c = per_cpu(kmem_cache_cpu_free, cpu); 1999 struct kmem_cache_cpu *c = per_cpu(kmem_cache_cpu_free, cpu);
1997 2000
1998 if (c) 2001 if (c)
1999 per_cpu(kmem_cache_cpu_free, cpu) = 2002 per_cpu(kmem_cache_cpu_free, cpu) =
2000 (void *)c->freelist; 2003 (void *)c->freelist;
2001 else { 2004 else {
2002 /* Table overflow: So allocate ourselves */ 2005 /* Table overflow: So allocate ourselves */
2003 c = kmalloc_node( 2006 c = kmalloc_node(
2004 ALIGN(sizeof(struct kmem_cache_cpu), cache_line_size()), 2007 ALIGN(sizeof(struct kmem_cache_cpu), cache_line_size()),
2005 flags, cpu_to_node(cpu)); 2008 flags, cpu_to_node(cpu));
2006 if (!c) 2009 if (!c)
2007 return NULL; 2010 return NULL;
2008 } 2011 }
2009 2012
2010 init_kmem_cache_cpu(s, c); 2013 init_kmem_cache_cpu(s, c);
2011 return c; 2014 return c;
2012 } 2015 }
2013 2016
2014 static void free_kmem_cache_cpu(struct kmem_cache_cpu *c, int cpu) 2017 static void free_kmem_cache_cpu(struct kmem_cache_cpu *c, int cpu)
2015 { 2018 {
2016 if (c < per_cpu(kmem_cache_cpu, cpu) || 2019 if (c < per_cpu(kmem_cache_cpu, cpu) ||
2017 c >= per_cpu(kmem_cache_cpu, cpu) + NR_KMEM_CACHE_CPU) { 2020 c >= per_cpu(kmem_cache_cpu, cpu) + NR_KMEM_CACHE_CPU) {
2018 kfree(c); 2021 kfree(c);
2019 return; 2022 return;
2020 } 2023 }
2021 c->freelist = (void *)per_cpu(kmem_cache_cpu_free, cpu); 2024 c->freelist = (void *)per_cpu(kmem_cache_cpu_free, cpu);
2022 per_cpu(kmem_cache_cpu_free, cpu) = c; 2025 per_cpu(kmem_cache_cpu_free, cpu) = c;
2023 } 2026 }
2024 2027
2025 static void free_kmem_cache_cpus(struct kmem_cache *s) 2028 static void free_kmem_cache_cpus(struct kmem_cache *s)
2026 { 2029 {
2027 int cpu; 2030 int cpu;
2028 2031
2029 for_each_online_cpu(cpu) { 2032 for_each_online_cpu(cpu) {
2030 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); 2033 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
2031 2034
2032 if (c) { 2035 if (c) {
2033 s->cpu_slab[cpu] = NULL; 2036 s->cpu_slab[cpu] = NULL;
2034 free_kmem_cache_cpu(c, cpu); 2037 free_kmem_cache_cpu(c, cpu);
2035 } 2038 }
2036 } 2039 }
2037 } 2040 }
2038 2041
2039 static int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) 2042 static int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
2040 { 2043 {
2041 int cpu; 2044 int cpu;
2042 2045
2043 for_each_online_cpu(cpu) { 2046 for_each_online_cpu(cpu) {
2044 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); 2047 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
2045 2048
2046 if (c) 2049 if (c)
2047 continue; 2050 continue;
2048 2051
2049 c = alloc_kmem_cache_cpu(s, cpu, flags); 2052 c = alloc_kmem_cache_cpu(s, cpu, flags);
2050 if (!c) { 2053 if (!c) {
2051 free_kmem_cache_cpus(s); 2054 free_kmem_cache_cpus(s);
2052 return 0; 2055 return 0;
2053 } 2056 }
2054 s->cpu_slab[cpu] = c; 2057 s->cpu_slab[cpu] = c;
2055 } 2058 }
2056 return 1; 2059 return 1;
2057 } 2060 }
2058 2061
2059 /* 2062 /*
2060 * Initialize the per cpu array. 2063 * Initialize the per cpu array.
2061 */ 2064 */
2062 static void init_alloc_cpu_cpu(int cpu) 2065 static void init_alloc_cpu_cpu(int cpu)
2063 { 2066 {
2064 int i; 2067 int i;
2065 2068
2066 if (cpumask_test_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once))) 2069 if (cpumask_test_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once)))
2067 return; 2070 return;
2068 2071
2069 for (i = NR_KMEM_CACHE_CPU - 1; i >= 0; i--) 2072 for (i = NR_KMEM_CACHE_CPU - 1; i >= 0; i--)
2070 free_kmem_cache_cpu(&per_cpu(kmem_cache_cpu, cpu)[i], cpu); 2073 free_kmem_cache_cpu(&per_cpu(kmem_cache_cpu, cpu)[i], cpu);
2071 2074
2072 cpumask_set_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once)); 2075 cpumask_set_cpu(cpu, to_cpumask(kmem_cach_cpu_free_init_once));
2073 } 2076 }
2074 2077
2075 static void __init init_alloc_cpu(void) 2078 static void __init init_alloc_cpu(void)
2076 { 2079 {
2077 int cpu; 2080 int cpu;
2078 2081
2079 for_each_online_cpu(cpu) 2082 for_each_online_cpu(cpu)
2080 init_alloc_cpu_cpu(cpu); 2083 init_alloc_cpu_cpu(cpu);
2081 } 2084 }
2082 2085
2083 #else 2086 #else
2084 static inline void free_kmem_cache_cpus(struct kmem_cache *s) {} 2087 static inline void free_kmem_cache_cpus(struct kmem_cache *s) {}
2085 static inline void init_alloc_cpu(void) {} 2088 static inline void init_alloc_cpu(void) {}
2086 2089
2087 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) 2090 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
2088 { 2091 {
2089 init_kmem_cache_cpu(s, &s->cpu_slab); 2092 init_kmem_cache_cpu(s, &s->cpu_slab);
2090 return 1; 2093 return 1;
2091 } 2094 }
2092 #endif 2095 #endif
2093 2096
2094 #ifdef CONFIG_NUMA 2097 #ifdef CONFIG_NUMA
2095 /* 2098 /*
2096 * No kmalloc_node yet so do it by hand. We know that this is the first 2099 * No kmalloc_node yet so do it by hand. We know that this is the first
2097 * slab on the node for this slabcache. There are no concurrent accesses 2100 * slab on the node for this slabcache. There are no concurrent accesses
2098 * possible. 2101 * possible.
2099 * 2102 *
2100 * Note that this function only works on the kmalloc_node_cache 2103 * Note that this function only works on the kmalloc_node_cache
2101 * when allocating for the kmalloc_node_cache. This is used for bootstrapping 2104 * when allocating for the kmalloc_node_cache. This is used for bootstrapping
2102 * memory on a fresh node that has no slab structures yet. 2105 * memory on a fresh node that has no slab structures yet.
2103 */ 2106 */
2104 static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node) 2107 static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node)
2105 { 2108 {
2106 struct page *page; 2109 struct page *page;
2107 struct kmem_cache_node *n; 2110 struct kmem_cache_node *n;
2108 unsigned long flags; 2111 unsigned long flags;
2109 2112
2110 BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node)); 2113 BUG_ON(kmalloc_caches->size < sizeof(struct kmem_cache_node));
2111 2114
2112 page = new_slab(kmalloc_caches, gfpflags, node); 2115 page = new_slab(kmalloc_caches, gfpflags, node);
2113 2116
2114 BUG_ON(!page); 2117 BUG_ON(!page);
2115 if (page_to_nid(page) != node) { 2118 if (page_to_nid(page) != node) {
2116 printk(KERN_ERR "SLUB: Unable to allocate memory from " 2119 printk(KERN_ERR "SLUB: Unable to allocate memory from "
2117 "node %d\n", node); 2120 "node %d\n", node);
2118 printk(KERN_ERR "SLUB: Allocating a useless per node structure " 2121 printk(KERN_ERR "SLUB: Allocating a useless per node structure "
2119 "in order to be able to continue\n"); 2122 "in order to be able to continue\n");
2120 } 2123 }
2121 2124
2122 n = page->freelist; 2125 n = page->freelist;
2123 BUG_ON(!n); 2126 BUG_ON(!n);
2124 page->freelist = get_freepointer(kmalloc_caches, n); 2127 page->freelist = get_freepointer(kmalloc_caches, n);
2125 page->inuse++; 2128 page->inuse++;
2126 kmalloc_caches->node[node] = n; 2129 kmalloc_caches->node[node] = n;
2127 #ifdef CONFIG_SLUB_DEBUG 2130 #ifdef CONFIG_SLUB_DEBUG
2128 init_object(kmalloc_caches, n, 1); 2131 init_object(kmalloc_caches, n, 1);
2129 init_tracking(kmalloc_caches, n); 2132 init_tracking(kmalloc_caches, n);
2130 #endif 2133 #endif
2131 init_kmem_cache_node(n, kmalloc_caches); 2134 init_kmem_cache_node(n, kmalloc_caches);
2132 inc_slabs_node(kmalloc_caches, node, page->objects); 2135 inc_slabs_node(kmalloc_caches, node, page->objects);
2133 2136
2134 /* 2137 /*
2135 * lockdep requires consistent irq usage for each lock 2138 * lockdep requires consistent irq usage for each lock
2136 * so even though there cannot be a race this early in 2139 * so even though there cannot be a race this early in
2137 * the boot sequence, we still disable irqs. 2140 * the boot sequence, we still disable irqs.
2138 */ 2141 */
2139 local_irq_save(flags); 2142 local_irq_save(flags);
2140 add_partial(n, page, 0); 2143 add_partial(n, page, 0);
2141 local_irq_restore(flags); 2144 local_irq_restore(flags);
2142 } 2145 }
2143 2146
2144 static void free_kmem_cache_nodes(struct kmem_cache *s) 2147 static void free_kmem_cache_nodes(struct kmem_cache *s)
2145 { 2148 {
2146 int node; 2149 int node;
2147 2150
2148 for_each_node_state(node, N_NORMAL_MEMORY) { 2151 for_each_node_state(node, N_NORMAL_MEMORY) {
2149 struct kmem_cache_node *n = s->node[node]; 2152 struct kmem_cache_node *n = s->node[node];
2150 if (n && n != &s->local_node) 2153 if (n && n != &s->local_node)
2151 kmem_cache_free(kmalloc_caches, n); 2154 kmem_cache_free(kmalloc_caches, n);
2152 s->node[node] = NULL; 2155 s->node[node] = NULL;
2153 } 2156 }
2154 } 2157 }
2155 2158
2156 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) 2159 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2157 { 2160 {
2158 int node; 2161 int node;
2159 int local_node; 2162 int local_node;
2160 2163
2161 if (slab_state >= UP) 2164 if (slab_state >= UP)
2162 local_node = page_to_nid(virt_to_page(s)); 2165 local_node = page_to_nid(virt_to_page(s));
2163 else 2166 else
2164 local_node = 0; 2167 local_node = 0;
2165 2168
2166 for_each_node_state(node, N_NORMAL_MEMORY) { 2169 for_each_node_state(node, N_NORMAL_MEMORY) {
2167 struct kmem_cache_node *n; 2170 struct kmem_cache_node *n;
2168 2171
2169 if (local_node == node) 2172 if (local_node == node)
2170 n = &s->local_node; 2173 n = &s->local_node;
2171 else { 2174 else {
2172 if (slab_state == DOWN) { 2175 if (slab_state == DOWN) {
2173 early_kmem_cache_node_alloc(gfpflags, node); 2176 early_kmem_cache_node_alloc(gfpflags, node);
2174 continue; 2177 continue;
2175 } 2178 }
2176 n = kmem_cache_alloc_node(kmalloc_caches, 2179 n = kmem_cache_alloc_node(kmalloc_caches,
2177 gfpflags, node); 2180 gfpflags, node);
2178 2181
2179 if (!n) { 2182 if (!n) {
2180 free_kmem_cache_nodes(s); 2183 free_kmem_cache_nodes(s);
2181 return 0; 2184 return 0;
2182 } 2185 }
2183 2186
2184 } 2187 }
2185 s->node[node] = n; 2188 s->node[node] = n;
2186 init_kmem_cache_node(n, s); 2189 init_kmem_cache_node(n, s);
2187 } 2190 }
2188 return 1; 2191 return 1;
2189 } 2192 }
2190 #else 2193 #else
2191 static void free_kmem_cache_nodes(struct kmem_cache *s) 2194 static void free_kmem_cache_nodes(struct kmem_cache *s)
2192 { 2195 {
2193 } 2196 }
2194 2197
2195 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) 2198 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2196 { 2199 {
2197 init_kmem_cache_node(&s->local_node, s); 2200 init_kmem_cache_node(&s->local_node, s);
2198 return 1; 2201 return 1;
2199 } 2202 }
2200 #endif 2203 #endif
2201 2204
2202 static void set_min_partial(struct kmem_cache *s, unsigned long min) 2205 static void set_min_partial(struct kmem_cache *s, unsigned long min)
2203 { 2206 {
2204 if (min < MIN_PARTIAL) 2207 if (min < MIN_PARTIAL)
2205 min = MIN_PARTIAL; 2208 min = MIN_PARTIAL;
2206 else if (min > MAX_PARTIAL) 2209 else if (min > MAX_PARTIAL)
2207 min = MAX_PARTIAL; 2210 min = MAX_PARTIAL;
2208 s->min_partial = min; 2211 s->min_partial = min;
2209 } 2212 }
2210 2213
2211 /* 2214 /*
2212 * calculate_sizes() determines the order and the distribution of data within 2215 * calculate_sizes() determines the order and the distribution of data within
2213 * a slab object. 2216 * a slab object.
2214 */ 2217 */
2215 static int calculate_sizes(struct kmem_cache *s, int forced_order) 2218 static int calculate_sizes(struct kmem_cache *s, int forced_order)
2216 { 2219 {
2217 unsigned long flags = s->flags; 2220 unsigned long flags = s->flags;
2218 unsigned long size = s->objsize; 2221 unsigned long size = s->objsize;
2219 unsigned long align = s->align; 2222 unsigned long align = s->align;
2220 int order; 2223 int order;
2221 2224
2222 /* 2225 /*
2223 * Round up object size to the next word boundary. We can only 2226 * Round up object size to the next word boundary. We can only
2224 * place the free pointer at word boundaries and this determines 2227 * place the free pointer at word boundaries and this determines
2225 * the possible location of the free pointer. 2228 * the possible location of the free pointer.
2226 */ 2229 */
2227 size = ALIGN(size, sizeof(void *)); 2230 size = ALIGN(size, sizeof(void *));
2228 2231
2229 #ifdef CONFIG_SLUB_DEBUG 2232 #ifdef CONFIG_SLUB_DEBUG
2230 /* 2233 /*
2231 * Determine if we can poison the object itself. If the user of 2234 * Determine if we can poison the object itself. If the user of
2232 * the slab may touch the object after free or before allocation 2235 * the slab may touch the object after free or before allocation
2233 * then we should never poison the object itself. 2236 * then we should never poison the object itself.
2234 */ 2237 */
2235 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && 2238 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) &&
2236 !s->ctor) 2239 !s->ctor)
2237 s->flags |= __OBJECT_POISON; 2240 s->flags |= __OBJECT_POISON;
2238 else 2241 else
2239 s->flags &= ~__OBJECT_POISON; 2242 s->flags &= ~__OBJECT_POISON;
2240 2243
2241 2244
2242 /* 2245 /*
2243 * If we are Redzoning then check if there is some space between the 2246 * If we are Redzoning then check if there is some space between the
2244 * end of the object and the free pointer. If not then add an 2247 * end of the object and the free pointer. If not then add an
2245 * additional word to have some bytes to store Redzone information. 2248 * additional word to have some bytes to store Redzone information.
2246 */ 2249 */
2247 if ((flags & SLAB_RED_ZONE) && size == s->objsize) 2250 if ((flags & SLAB_RED_ZONE) && size == s->objsize)
2248 size += sizeof(void *); 2251 size += sizeof(void *);
2249 #endif 2252 #endif
2250 2253
2251 /* 2254 /*
2252 * With that we have determined the number of bytes in actual use 2255 * With that we have determined the number of bytes in actual use
2253 * by the object. This is the potential offset to the free pointer. 2256 * by the object. This is the potential offset to the free pointer.
2254 */ 2257 */
2255 s->inuse = size; 2258 s->inuse = size;
2256 2259
2257 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || 2260 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) ||
2258 s->ctor)) { 2261 s->ctor)) {
2259 /* 2262 /*
2260 * Relocate free pointer after the object if it is not 2263 * Relocate free pointer after the object if it is not
2261 * permitted to overwrite the first word of the object on 2264 * permitted to overwrite the first word of the object on
2262 * kmem_cache_free. 2265 * kmem_cache_free.
2263 * 2266 *
2264 * This is the case if we do RCU, have a constructor or 2267 * This is the case if we do RCU, have a constructor or
2265 * destructor or are poisoning the objects. 2268 * destructor or are poisoning the objects.
2266 */ 2269 */
2267 s->offset = size; 2270 s->offset = size;
2268 size += sizeof(void *); 2271 size += sizeof(void *);
2269 } 2272 }
2270 2273
2271 #ifdef CONFIG_SLUB_DEBUG 2274 #ifdef CONFIG_SLUB_DEBUG
2272 if (flags & SLAB_STORE_USER) 2275 if (flags & SLAB_STORE_USER)
2273 /* 2276 /*
2274 * Need to store information about allocs and frees after 2277 * Need to store information about allocs and frees after
2275 * the object. 2278 * the object.
2276 */ 2279 */
2277 size += 2 * sizeof(struct track); 2280 size += 2 * sizeof(struct track);
2278 2281
2279 if (flags & SLAB_RED_ZONE) 2282 if (flags & SLAB_RED_ZONE)
2280 /* 2283 /*
2281 * Add some empty padding so that we can catch 2284 * Add some empty padding so that we can catch
2282 * overwrites from earlier objects rather than let 2285 * overwrites from earlier objects rather than let
2283 * tracking information or the free pointer be 2286 * tracking information or the free pointer be
2284 * corrupted if a user writes before the start 2287 * corrupted if a user writes before the start
2285 * of the object. 2288 * of the object.
2286 */ 2289 */
2287 size += sizeof(void *); 2290 size += sizeof(void *);
2288 #endif 2291 #endif
2289 2292
2290 /* 2293 /*
2291 * Determine the alignment based on various parameters that the 2294 * Determine the alignment based on various parameters that the
2292 * user specified and the dynamic determination of cache line size 2295 * user specified and the dynamic determination of cache line size
2293 * on bootup. 2296 * on bootup.
2294 */ 2297 */
2295 align = calculate_alignment(flags, align, s->objsize); 2298 align = calculate_alignment(flags, align, s->objsize);
2296 2299
2297 /* 2300 /*
2298 * SLUB stores one object immediately after another beginning from 2301 * SLUB stores one object immediately after another beginning from
2299 * offset 0. In order to align the objects we have to simply size 2302 * offset 0. In order to align the objects we have to simply size
2300 * each object to conform to the alignment. 2303 * each object to conform to the alignment.
2301 */ 2304 */
2302 size = ALIGN(size, align); 2305 size = ALIGN(size, align);
2303 s->size = size; 2306 s->size = size;
2304 if (forced_order >= 0) 2307 if (forced_order >= 0)
2305 order = forced_order; 2308 order = forced_order;
2306 else 2309 else
2307 order = calculate_order(size); 2310 order = calculate_order(size);
2308 2311
2309 if (order < 0) 2312 if (order < 0)
2310 return 0; 2313 return 0;
2311 2314
2312 s->allocflags = 0; 2315 s->allocflags = 0;
2313 if (order) 2316 if (order)
2314 s->allocflags |= __GFP_COMP; 2317 s->allocflags |= __GFP_COMP;
2315 2318
2316 if (s->flags & SLAB_CACHE_DMA) 2319 if (s->flags & SLAB_CACHE_DMA)
2317 s->allocflags |= SLUB_DMA; 2320 s->allocflags |= SLUB_DMA;
2318 2321
2319 if (s->flags & SLAB_RECLAIM_ACCOUNT) 2322 if (s->flags & SLAB_RECLAIM_ACCOUNT)
2320 s->allocflags |= __GFP_RECLAIMABLE; 2323 s->allocflags |= __GFP_RECLAIMABLE;
2321 2324
2322 /* 2325 /*
2323 * Determine the number of objects per slab 2326 * Determine the number of objects per slab
2324 */ 2327 */
2325 s->oo = oo_make(order, size); 2328 s->oo = oo_make(order, size);
2326 s->min = oo_make(get_order(size), size); 2329 s->min = oo_make(get_order(size), size);
2327 if (oo_objects(s->oo) > oo_objects(s->max)) 2330 if (oo_objects(s->oo) > oo_objects(s->max))
2328 s->max = s->oo; 2331 s->max = s->oo;
2329 2332
2330 return !!oo_objects(s->oo); 2333 return !!oo_objects(s->oo);
2331 2334
2332 } 2335 }
2333 2336
2334 static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, 2337 static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
2335 const char *name, size_t size, 2338 const char *name, size_t size,
2336 size_t align, unsigned long flags, 2339 size_t align, unsigned long flags,
2337 void (*ctor)(void *)) 2340 void (*ctor)(void *))
2338 { 2341 {
2339 memset(s, 0, kmem_size); 2342 memset(s, 0, kmem_size);
2340 s->name = name; 2343 s->name = name;
2341 s->ctor = ctor; 2344 s->ctor = ctor;
2342 s->objsize = size; 2345 s->objsize = size;
2343 s->align = align; 2346 s->align = align;
2344 s->flags = kmem_cache_flags(size, flags, name, ctor); 2347 s->flags = kmem_cache_flags(size, flags, name, ctor);
2345 2348
2346 if (!calculate_sizes(s, -1)) 2349 if (!calculate_sizes(s, -1))
2347 goto error; 2350 goto error;
2348 2351
2349 /* 2352 /*
2350 * The larger the object size is, the more pages we want on the partial 2353 * The larger the object size is, the more pages we want on the partial
2351 * list to avoid pounding the page allocator excessively. 2354 * list to avoid pounding the page allocator excessively.
2352 */ 2355 */
2353 set_min_partial(s, ilog2(s->size)); 2356 set_min_partial(s, ilog2(s->size));
2354 s->refcount = 1; 2357 s->refcount = 1;
2355 #ifdef CONFIG_NUMA 2358 #ifdef CONFIG_NUMA
2356 s->remote_node_defrag_ratio = 1000; 2359 s->remote_node_defrag_ratio = 1000;
2357 #endif 2360 #endif
2358 if (!init_kmem_cache_nodes(s, gfpflags & ~SLUB_DMA)) 2361 if (!init_kmem_cache_nodes(s, gfpflags & ~SLUB_DMA))
2359 goto error; 2362 goto error;
2360 2363
2361 if (alloc_kmem_cache_cpus(s, gfpflags & ~SLUB_DMA)) 2364 if (alloc_kmem_cache_cpus(s, gfpflags & ~SLUB_DMA))
2362 return 1; 2365 return 1;
2363 free_kmem_cache_nodes(s); 2366 free_kmem_cache_nodes(s);
2364 error: 2367 error:
2365 if (flags & SLAB_PANIC) 2368 if (flags & SLAB_PANIC)
2366 panic("Cannot create slab %s size=%lu realsize=%u " 2369 panic("Cannot create slab %s size=%lu realsize=%u "
2367 "order=%u offset=%u flags=%lx\n", 2370 "order=%u offset=%u flags=%lx\n",
2368 s->name, (unsigned long)size, s->size, oo_order(s->oo), 2371 s->name, (unsigned long)size, s->size, oo_order(s->oo),
2369 s->offset, flags); 2372 s->offset, flags);
2370 return 0; 2373 return 0;
2371 } 2374 }
2372 2375
2373 /* 2376 /*
2374 * Check if a given pointer is valid 2377 * Check if a given pointer is valid
2375 */ 2378 */
2376 int kmem_ptr_validate(struct kmem_cache *s, const void *object) 2379 int kmem_ptr_validate(struct kmem_cache *s, const void *object)
2377 { 2380 {
2378 struct page *page; 2381 struct page *page;
2379 2382
2380 page = get_object_page(object); 2383 page = get_object_page(object);
2381 2384
2382 if (!page || s != page->slab) 2385 if (!page || s != page->slab)
2383 /* No slab or wrong slab */ 2386 /* No slab or wrong slab */
2384 return 0; 2387 return 0;
2385 2388
2386 if (!check_valid_pointer(s, page, object)) 2389 if (!check_valid_pointer(s, page, object))
2387 return 0; 2390 return 0;
2388 2391
2389 /* 2392 /*
2390 * We could also check if the object is on the slabs freelist. 2393 * We could also check if the object is on the slabs freelist.
2391 * But this would be too expensive and it seems that the main 2394 * But this would be too expensive and it seems that the main
2392 * purpose of kmem_ptr_valid() is to check if the object belongs 2395 * purpose of kmem_ptr_valid() is to check if the object belongs
2393 * to a certain slab. 2396 * to a certain slab.
2394 */ 2397 */
2395 return 1; 2398 return 1;
2396 } 2399 }
2397 EXPORT_SYMBOL(kmem_ptr_validate); 2400 EXPORT_SYMBOL(kmem_ptr_validate);
2398 2401
2399 /* 2402 /*
2400 * Determine the size of a slab object 2403 * Determine the size of a slab object
2401 */ 2404 */
2402 unsigned int kmem_cache_size(struct kmem_cache *s) 2405 unsigned int kmem_cache_size(struct kmem_cache *s)
2403 { 2406 {
2404 return s->objsize; 2407 return s->objsize;
2405 } 2408 }
2406 EXPORT_SYMBOL(kmem_cache_size); 2409 EXPORT_SYMBOL(kmem_cache_size);
2407 2410
2408 const char *kmem_cache_name(struct kmem_cache *s) 2411 const char *kmem_cache_name(struct kmem_cache *s)
2409 { 2412 {
2410 return s->name; 2413 return s->name;
2411 } 2414 }
2412 EXPORT_SYMBOL(kmem_cache_name); 2415 EXPORT_SYMBOL(kmem_cache_name);
2413 2416
2414 static void list_slab_objects(struct kmem_cache *s, struct page *page, 2417 static void list_slab_objects(struct kmem_cache *s, struct page *page,
2415 const char *text) 2418 const char *text)
2416 { 2419 {
2417 #ifdef CONFIG_SLUB_DEBUG 2420 #ifdef CONFIG_SLUB_DEBUG
2418 void *addr = page_address(page); 2421 void *addr = page_address(page);
2419 void *p; 2422 void *p;
2420 DECLARE_BITMAP(map, page->objects); 2423 DECLARE_BITMAP(map, page->objects);
2421 2424
2422 bitmap_zero(map, page->objects); 2425 bitmap_zero(map, page->objects);
2423 slab_err(s, page, "%s", text); 2426 slab_err(s, page, "%s", text);
2424 slab_lock(page); 2427 slab_lock(page);
2425 for_each_free_object(p, s, page->freelist) 2428 for_each_free_object(p, s, page->freelist)
2426 set_bit(slab_index(p, s, addr), map); 2429 set_bit(slab_index(p, s, addr), map);
2427 2430
2428 for_each_object(p, s, addr, page->objects) { 2431 for_each_object(p, s, addr, page->objects) {
2429 2432
2430 if (!test_bit(slab_index(p, s, addr), map)) { 2433 if (!test_bit(slab_index(p, s, addr), map)) {
2431 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n", 2434 printk(KERN_ERR "INFO: Object 0x%p @offset=%tu\n",
2432 p, p - addr); 2435 p, p - addr);
2433 print_tracking(s, p); 2436 print_tracking(s, p);
2434 } 2437 }
2435 } 2438 }
2436 slab_unlock(page); 2439 slab_unlock(page);
2437 #endif 2440 #endif
2438 } 2441 }
2439 2442
2440 /* 2443 /*
2441 * Attempt to free all partial slabs on a node. 2444 * Attempt to free all partial slabs on a node.
2442 */ 2445 */
2443 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) 2446 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
2444 { 2447 {
2445 unsigned long flags; 2448 unsigned long flags;
2446 struct page *page, *h; 2449 struct page *page, *h;
2447 2450
2448 spin_lock_irqsave(&n->list_lock, flags); 2451 spin_lock_irqsave(&n->list_lock, flags);
2449 list_for_each_entry_safe(page, h, &n->partial, lru) { 2452 list_for_each_entry_safe(page, h, &n->partial, lru) {
2450 if (!page->inuse) { 2453 if (!page->inuse) {
2451 list_del(&page->lru); 2454 list_del(&page->lru);
2452 discard_slab(s, page); 2455 discard_slab(s, page);
2453 n->nr_partial--; 2456 n->nr_partial--;
2454 } else { 2457 } else {
2455 list_slab_objects(s, page, 2458 list_slab_objects(s, page,
2456 "Objects remaining on kmem_cache_close()"); 2459 "Objects remaining on kmem_cache_close()");
2457 } 2460 }
2458 } 2461 }
2459 spin_unlock_irqrestore(&n->list_lock, flags); 2462 spin_unlock_irqrestore(&n->list_lock, flags);
2460 } 2463 }
2461 2464
2462 /* 2465 /*
2463 * Release all resources used by a slab cache. 2466 * Release all resources used by a slab cache.
2464 */ 2467 */
2465 static inline int kmem_cache_close(struct kmem_cache *s) 2468 static inline int kmem_cache_close(struct kmem_cache *s)
2466 { 2469 {
2467 int node; 2470 int node;
2468 2471
2469 flush_all(s); 2472 flush_all(s);
2470 2473
2471 /* Attempt to free all objects */ 2474 /* Attempt to free all objects */
2472 free_kmem_cache_cpus(s); 2475 free_kmem_cache_cpus(s);
2473 for_each_node_state(node, N_NORMAL_MEMORY) { 2476 for_each_node_state(node, N_NORMAL_MEMORY) {
2474 struct kmem_cache_node *n = get_node(s, node); 2477 struct kmem_cache_node *n = get_node(s, node);
2475 2478
2476 free_partial(s, n); 2479 free_partial(s, n);
2477 if (n->nr_partial || slabs_node(s, node)) 2480 if (n->nr_partial || slabs_node(s, node))
2478 return 1; 2481 return 1;
2479 } 2482 }
2480 free_kmem_cache_nodes(s); 2483 free_kmem_cache_nodes(s);
2481 return 0; 2484 return 0;
2482 } 2485 }
2483 2486
2484 /* 2487 /*
2485 * Close a cache and release the kmem_cache structure 2488 * Close a cache and release the kmem_cache structure
2486 * (must be used for caches created using kmem_cache_create) 2489 * (must be used for caches created using kmem_cache_create)
2487 */ 2490 */
2488 void kmem_cache_destroy(struct kmem_cache *s) 2491 void kmem_cache_destroy(struct kmem_cache *s)
2489 { 2492 {
2490 down_write(&slub_lock); 2493 down_write(&slub_lock);
2491 s->refcount--; 2494 s->refcount--;
2492 if (!s->refcount) { 2495 if (!s->refcount) {
2493 list_del(&s->list); 2496 list_del(&s->list);
2494 up_write(&slub_lock); 2497 up_write(&slub_lock);
2495 if (kmem_cache_close(s)) { 2498 if (kmem_cache_close(s)) {
2496 printk(KERN_ERR "SLUB %s: %s called for cache that " 2499 printk(KERN_ERR "SLUB %s: %s called for cache that "
2497 "still has objects.\n", s->name, __func__); 2500 "still has objects.\n", s->name, __func__);
2498 dump_stack(); 2501 dump_stack();
2499 } 2502 }
2500 sysfs_slab_remove(s); 2503 sysfs_slab_remove(s);
2501 } else 2504 } else
2502 up_write(&slub_lock); 2505 up_write(&slub_lock);
2503 } 2506 }
2504 EXPORT_SYMBOL(kmem_cache_destroy); 2507 EXPORT_SYMBOL(kmem_cache_destroy);
2505 2508
2506 /******************************************************************** 2509 /********************************************************************
2507 * Kmalloc subsystem 2510 * Kmalloc subsystem
2508 *******************************************************************/ 2511 *******************************************************************/
2509 2512
2510 struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned; 2513 struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned;
2511 EXPORT_SYMBOL(kmalloc_caches); 2514 EXPORT_SYMBOL(kmalloc_caches);
2512 2515
2513 static int __init setup_slub_min_order(char *str) 2516 static int __init setup_slub_min_order(char *str)
2514 { 2517 {
2515 get_option(&str, &slub_min_order); 2518 get_option(&str, &slub_min_order);
2516 2519
2517 return 1; 2520 return 1;
2518 } 2521 }
2519 2522
2520 __setup("slub_min_order=", setup_slub_min_order); 2523 __setup("slub_min_order=", setup_slub_min_order);
2521 2524
2522 static int __init setup_slub_max_order(char *str) 2525 static int __init setup_slub_max_order(char *str)
2523 { 2526 {
2524 get_option(&str, &slub_max_order); 2527 get_option(&str, &slub_max_order);
2525 2528
2526 return 1; 2529 return 1;
2527 } 2530 }
2528 2531
2529 __setup("slub_max_order=", setup_slub_max_order); 2532 __setup("slub_max_order=", setup_slub_max_order);
2530 2533
2531 static int __init setup_slub_min_objects(char *str) 2534 static int __init setup_slub_min_objects(char *str)
2532 { 2535 {
2533 get_option(&str, &slub_min_objects); 2536 get_option(&str, &slub_min_objects);
2534 2537
2535 return 1; 2538 return 1;
2536 } 2539 }
2537 2540
2538 __setup("slub_min_objects=", setup_slub_min_objects); 2541 __setup("slub_min_objects=", setup_slub_min_objects);
2539 2542
2540 static int __init setup_slub_nomerge(char *str) 2543 static int __init setup_slub_nomerge(char *str)
2541 { 2544 {
2542 slub_nomerge = 1; 2545 slub_nomerge = 1;
2543 return 1; 2546 return 1;
2544 } 2547 }
2545 2548
2546 __setup("slub_nomerge", setup_slub_nomerge); 2549 __setup("slub_nomerge", setup_slub_nomerge);
2547 2550
2548 static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s, 2551 static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
2549 const char *name, int size, gfp_t gfp_flags) 2552 const char *name, int size, gfp_t gfp_flags)
2550 { 2553 {
2551 unsigned int flags = 0; 2554 unsigned int flags = 0;
2552 2555
2553 if (gfp_flags & SLUB_DMA) 2556 if (gfp_flags & SLUB_DMA)
2554 flags = SLAB_CACHE_DMA; 2557 flags = SLAB_CACHE_DMA;
2555 2558
2556 down_write(&slub_lock); 2559 down_write(&slub_lock);
2557 if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN, 2560 if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
2558 flags, NULL)) 2561 flags, NULL))
2559 goto panic; 2562 goto panic;
2560 2563
2561 list_add(&s->list, &slab_caches); 2564 list_add(&s->list, &slab_caches);
2562 up_write(&slub_lock); 2565 up_write(&slub_lock);
2563 if (sysfs_slab_add(s)) 2566 if (sysfs_slab_add(s))
2564 goto panic; 2567 goto panic;
2565 return s; 2568 return s;
2566 2569
2567 panic: 2570 panic:
2568 panic("Creation of kmalloc slab %s size=%d failed.\n", name, size); 2571 panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
2569 } 2572 }
2570 2573
2571 #ifdef CONFIG_ZONE_DMA 2574 #ifdef CONFIG_ZONE_DMA
2572 static struct kmem_cache *kmalloc_caches_dma[SLUB_PAGE_SHIFT]; 2575 static struct kmem_cache *kmalloc_caches_dma[SLUB_PAGE_SHIFT];
2573 2576
2574 static void sysfs_add_func(struct work_struct *w) 2577 static void sysfs_add_func(struct work_struct *w)
2575 { 2578 {
2576 struct kmem_cache *s; 2579 struct kmem_cache *s;
2577 2580
2578 down_write(&slub_lock); 2581 down_write(&slub_lock);
2579 list_for_each_entry(s, &slab_caches, list) { 2582 list_for_each_entry(s, &slab_caches, list) {
2580 if (s->flags & __SYSFS_ADD_DEFERRED) { 2583 if (s->flags & __SYSFS_ADD_DEFERRED) {
2581 s->flags &= ~__SYSFS_ADD_DEFERRED; 2584 s->flags &= ~__SYSFS_ADD_DEFERRED;
2582 sysfs_slab_add(s); 2585 sysfs_slab_add(s);
2583 } 2586 }
2584 } 2587 }
2585 up_write(&slub_lock); 2588 up_write(&slub_lock);
2586 } 2589 }
2587 2590
2588 static DECLARE_WORK(sysfs_add_work, sysfs_add_func); 2591 static DECLARE_WORK(sysfs_add_work, sysfs_add_func);
2589 2592
2590 static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) 2593 static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
2591 { 2594 {
2592 struct kmem_cache *s; 2595 struct kmem_cache *s;
2593 char *text; 2596 char *text;
2594 size_t realsize; 2597 size_t realsize;
2595 2598
2596 s = kmalloc_caches_dma[index]; 2599 s = kmalloc_caches_dma[index];
2597 if (s) 2600 if (s)
2598 return s; 2601 return s;
2599 2602
2600 /* Dynamically create dma cache */ 2603 /* Dynamically create dma cache */
2601 if (flags & __GFP_WAIT) 2604 if (flags & __GFP_WAIT)
2602 down_write(&slub_lock); 2605 down_write(&slub_lock);
2603 else { 2606 else {
2604 if (!down_write_trylock(&slub_lock)) 2607 if (!down_write_trylock(&slub_lock))
2605 goto out; 2608 goto out;
2606 } 2609 }
2607 2610
2608 if (kmalloc_caches_dma[index]) 2611 if (kmalloc_caches_dma[index])
2609 goto unlock_out; 2612 goto unlock_out;
2610 2613
2611 realsize = kmalloc_caches[index].objsize; 2614 realsize = kmalloc_caches[index].objsize;
2612 text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d", 2615 text = kasprintf(flags & ~SLUB_DMA, "kmalloc_dma-%d",
2613 (unsigned int)realsize); 2616 (unsigned int)realsize);
2614 s = kmalloc(kmem_size, flags & ~SLUB_DMA); 2617 s = kmalloc(kmem_size, flags & ~SLUB_DMA);
2615 2618
2616 if (!s || !text || !kmem_cache_open(s, flags, text, 2619 if (!s || !text || !kmem_cache_open(s, flags, text,
2617 realsize, ARCH_KMALLOC_MINALIGN, 2620 realsize, ARCH_KMALLOC_MINALIGN,
2618 SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) { 2621 SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
2619 kfree(s); 2622 kfree(s);
2620 kfree(text); 2623 kfree(text);
2621 goto unlock_out; 2624 goto unlock_out;
2622 } 2625 }
2623 2626
2624 list_add(&s->list, &slab_caches); 2627 list_add(&s->list, &slab_caches);
2625 kmalloc_caches_dma[index] = s; 2628 kmalloc_caches_dma[index] = s;
2626 2629
2627 schedule_work(&sysfs_add_work); 2630 schedule_work(&sysfs_add_work);
2628 2631
2629 unlock_out: 2632 unlock_out:
2630 up_write(&slub_lock); 2633 up_write(&slub_lock);
2631 out: 2634 out:
2632 return kmalloc_caches_dma[index]; 2635 return kmalloc_caches_dma[index];
2633 } 2636 }
2634 #endif 2637 #endif
2635 2638
2636 /* 2639 /*
2637 * Conversion table for small slabs sizes / 8 to the index in the 2640 * Conversion table for small slabs sizes / 8 to the index in the
2638 * kmalloc array. This is necessary for slabs < 192 since we have non power 2641 * kmalloc array. This is necessary for slabs < 192 since we have non power
2639 * of two cache sizes there. The size of larger slabs can be determined using 2642 * of two cache sizes there. The size of larger slabs can be determined using
2640 * fls. 2643 * fls.
2641 */ 2644 */
2642 static s8 size_index[24] = { 2645 static s8 size_index[24] = {
2643 3, /* 8 */ 2646 3, /* 8 */
2644 4, /* 16 */ 2647 4, /* 16 */
2645 5, /* 24 */ 2648 5, /* 24 */
2646 5, /* 32 */ 2649 5, /* 32 */
2647 6, /* 40 */ 2650 6, /* 40 */
2648 6, /* 48 */ 2651 6, /* 48 */
2649 6, /* 56 */ 2652 6, /* 56 */
2650 6, /* 64 */ 2653 6, /* 64 */
2651 1, /* 72 */ 2654 1, /* 72 */
2652 1, /* 80 */ 2655 1, /* 80 */
2653 1, /* 88 */ 2656 1, /* 88 */
2654 1, /* 96 */ 2657 1, /* 96 */
2655 7, /* 104 */ 2658 7, /* 104 */
2656 7, /* 112 */ 2659 7, /* 112 */
2657 7, /* 120 */ 2660 7, /* 120 */
2658 7, /* 128 */ 2661 7, /* 128 */
2659 2, /* 136 */ 2662 2, /* 136 */
2660 2, /* 144 */ 2663 2, /* 144 */
2661 2, /* 152 */ 2664 2, /* 152 */
2662 2, /* 160 */ 2665 2, /* 160 */
2663 2, /* 168 */ 2666 2, /* 168 */
2664 2, /* 176 */ 2667 2, /* 176 */
2665 2, /* 184 */ 2668 2, /* 184 */
2666 2 /* 192 */ 2669 2 /* 192 */
2667 }; 2670 };
2668 2671
2669 static struct kmem_cache *get_slab(size_t size, gfp_t flags) 2672 static struct kmem_cache *get_slab(size_t size, gfp_t flags)
2670 { 2673 {
2671 int index; 2674 int index;
2672 2675
2673 if (size <= 192) { 2676 if (size <= 192) {
2674 if (!size) 2677 if (!size)
2675 return ZERO_SIZE_PTR; 2678 return ZERO_SIZE_PTR;
2676 2679
2677 index = size_index[(size - 1) / 8]; 2680 index = size_index[(size - 1) / 8];
2678 } else 2681 } else
2679 index = fls(size - 1); 2682 index = fls(size - 1);
2680 2683
2681 #ifdef CONFIG_ZONE_DMA 2684 #ifdef CONFIG_ZONE_DMA
2682 if (unlikely((flags & SLUB_DMA))) 2685 if (unlikely((flags & SLUB_DMA)))
2683 return dma_kmalloc_cache(index, flags); 2686 return dma_kmalloc_cache(index, flags);
2684 2687
2685 #endif 2688 #endif
2686 return &kmalloc_caches[index]; 2689 return &kmalloc_caches[index];
2687 } 2690 }
2688 2691
2689 void *__kmalloc(size_t size, gfp_t flags) 2692 void *__kmalloc(size_t size, gfp_t flags)
2690 { 2693 {
2691 struct kmem_cache *s; 2694 struct kmem_cache *s;
2692 void *ret; 2695 void *ret;
2693 2696
2694 if (unlikely(size > SLUB_MAX_SIZE)) 2697 if (unlikely(size > SLUB_MAX_SIZE))
2695 return kmalloc_large(size, flags); 2698 return kmalloc_large(size, flags);
2696 2699
2697 s = get_slab(size, flags); 2700 s = get_slab(size, flags);
2698 2701
2699 if (unlikely(ZERO_OR_NULL_PTR(s))) 2702 if (unlikely(ZERO_OR_NULL_PTR(s)))
2700 return s; 2703 return s;
2701 2704
2702 ret = slab_alloc(s, flags, -1, _RET_IP_); 2705 ret = slab_alloc(s, flags, -1, _RET_IP_);
2703 2706
2704 trace_kmalloc(_RET_IP_, ret, size, s->size, flags); 2707 trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
2705 2708
2706 return ret; 2709 return ret;
2707 } 2710 }
2708 EXPORT_SYMBOL(__kmalloc); 2711 EXPORT_SYMBOL(__kmalloc);
2709 2712
2710 static void *kmalloc_large_node(size_t size, gfp_t flags, int node) 2713 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
2711 { 2714 {
2712 struct page *page = alloc_pages_node(node, flags | __GFP_COMP, 2715 struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
2713 get_order(size)); 2716 get_order(size));
2714 2717
2715 if (page) 2718 if (page)
2716 return page_address(page); 2719 return page_address(page);
2717 else 2720 else
2718 return NULL; 2721 return NULL;
2719 } 2722 }
2720 2723
2721 #ifdef CONFIG_NUMA 2724 #ifdef CONFIG_NUMA
2722 void *__kmalloc_node(size_t size, gfp_t flags, int node) 2725 void *__kmalloc_node(size_t size, gfp_t flags, int node)
2723 { 2726 {
2724 struct kmem_cache *s; 2727 struct kmem_cache *s;
2725 void *ret; 2728 void *ret;
2726 2729
2727 if (unlikely(size > SLUB_MAX_SIZE)) { 2730 if (unlikely(size > SLUB_MAX_SIZE)) {
2728 ret = kmalloc_large_node(size, flags, node); 2731 ret = kmalloc_large_node(size, flags, node);
2729 2732
2730 trace_kmalloc_node(_RET_IP_, ret, 2733 trace_kmalloc_node(_RET_IP_, ret,
2731 size, PAGE_SIZE << get_order(size), 2734 size, PAGE_SIZE << get_order(size),
2732 flags, node); 2735 flags, node);
2733 2736
2734 return ret; 2737 return ret;
2735 } 2738 }
2736 2739
2737 s = get_slab(size, flags); 2740 s = get_slab(size, flags);
2738 2741
2739 if (unlikely(ZERO_OR_NULL_PTR(s))) 2742 if (unlikely(ZERO_OR_NULL_PTR(s)))
2740 return s; 2743 return s;
2741 2744
2742 ret = slab_alloc(s, flags, node, _RET_IP_); 2745 ret = slab_alloc(s, flags, node, _RET_IP_);
2743 2746
2744 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); 2747 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
2745 2748
2746 return ret; 2749 return ret;
2747 } 2750 }
2748 EXPORT_SYMBOL(__kmalloc_node); 2751 EXPORT_SYMBOL(__kmalloc_node);
2749 #endif 2752 #endif
2750 2753
2751 size_t ksize(const void *object) 2754 size_t ksize(const void *object)
2752 { 2755 {
2753 struct page *page; 2756 struct page *page;
2754 struct kmem_cache *s; 2757 struct kmem_cache *s;
2755 2758
2756 if (unlikely(object == ZERO_SIZE_PTR)) 2759 if (unlikely(object == ZERO_SIZE_PTR))
2757 return 0; 2760 return 0;
2758 2761
2759 page = virt_to_head_page(object); 2762 page = virt_to_head_page(object);
2760 2763
2761 if (unlikely(!PageSlab(page))) { 2764 if (unlikely(!PageSlab(page))) {
2762 WARN_ON(!PageCompound(page)); 2765 WARN_ON(!PageCompound(page));
2763 return PAGE_SIZE << compound_order(page); 2766 return PAGE_SIZE << compound_order(page);
2764 } 2767 }
2765 s = page->slab; 2768 s = page->slab;
2766 2769
2767 #ifdef CONFIG_SLUB_DEBUG 2770 #ifdef CONFIG_SLUB_DEBUG
2768 /* 2771 /*
2769 * Debugging requires use of the padding between object 2772 * Debugging requires use of the padding between object
2770 * and whatever may come after it. 2773 * and whatever may come after it.
2771 */ 2774 */
2772 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) 2775 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
2773 return s->objsize; 2776 return s->objsize;
2774 2777
2775 #endif 2778 #endif
2776 /* 2779 /*
2777 * If we have the need to store the freelist pointer 2780 * If we have the need to store the freelist pointer
2778 * back there or track user information then we can 2781 * back there or track user information then we can
2779 * only use the space before that information. 2782 * only use the space before that information.
2780 */ 2783 */
2781 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER)) 2784 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER))
2782 return s->inuse; 2785 return s->inuse;
2783 /* 2786 /*
2784 * Else we can use all the padding etc for the allocation 2787 * Else we can use all the padding etc for the allocation
2785 */ 2788 */
2786 return s->size; 2789 return s->size;
2787 } 2790 }
2788 EXPORT_SYMBOL(ksize); 2791 EXPORT_SYMBOL(ksize);
2789 2792
2790 void kfree(const void *x) 2793 void kfree(const void *x)
2791 { 2794 {
2792 struct page *page; 2795 struct page *page;
2793 void *object = (void *)x; 2796 void *object = (void *)x;
2794 2797
2795 trace_kfree(_RET_IP_, x); 2798 trace_kfree(_RET_IP_, x);
2796 2799
2797 if (unlikely(ZERO_OR_NULL_PTR(x))) 2800 if (unlikely(ZERO_OR_NULL_PTR(x)))
2798 return; 2801 return;
2799 2802
2800 page = virt_to_head_page(x); 2803 page = virt_to_head_page(x);
2801 if (unlikely(!PageSlab(page))) { 2804 if (unlikely(!PageSlab(page))) {
2802 BUG_ON(!PageCompound(page)); 2805 BUG_ON(!PageCompound(page));
2803 put_page(page); 2806 put_page(page);
2804 return; 2807 return;
2805 } 2808 }
2806 slab_free(page->slab, page, object, _RET_IP_); 2809 slab_free(page->slab, page, object, _RET_IP_);
2807 } 2810 }
2808 EXPORT_SYMBOL(kfree); 2811 EXPORT_SYMBOL(kfree);
2809 2812
2810 /* 2813 /*
2811 * kmem_cache_shrink removes empty slabs from the partial lists and sorts 2814 * kmem_cache_shrink removes empty slabs from the partial lists and sorts
2812 * the remaining slabs by the number of items in use. The slabs with the 2815 * the remaining slabs by the number of items in use. The slabs with the
2813 * most items in use come first. New allocations will then fill those up 2816 * most items in use come first. New allocations will then fill those up
2814 * and thus they can be removed from the partial lists. 2817 * and thus they can be removed from the partial lists.
2815 * 2818 *
2816 * The slabs with the least items are placed last. This results in them 2819 * The slabs with the least items are placed last. This results in them
2817 * being allocated from last increasing the chance that the last objects 2820 * being allocated from last increasing the chance that the last objects
2818 * are freed in them. 2821 * are freed in them.
2819 */ 2822 */
2820 int kmem_cache_shrink(struct kmem_cache *s) 2823 int kmem_cache_shrink(struct kmem_cache *s)
2821 { 2824 {
2822 int node; 2825 int node;
2823 int i; 2826 int i;
2824 struct kmem_cache_node *n; 2827 struct kmem_cache_node *n;
2825 struct page *page; 2828 struct page *page;
2826 struct page *t; 2829 struct page *t;
2827 int objects = oo_objects(s->max); 2830 int objects = oo_objects(s->max);
2828 struct list_head *slabs_by_inuse = 2831 struct list_head *slabs_by_inuse =
2829 kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL); 2832 kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
2830 unsigned long flags; 2833 unsigned long flags;
2831 2834
2832 if (!slabs_by_inuse) 2835 if (!slabs_by_inuse)
2833 return -ENOMEM; 2836 return -ENOMEM;
2834 2837
2835 flush_all(s); 2838 flush_all(s);
2836 for_each_node_state(node, N_NORMAL_MEMORY) { 2839 for_each_node_state(node, N_NORMAL_MEMORY) {
2837 n = get_node(s, node); 2840 n = get_node(s, node);
2838 2841
2839 if (!n->nr_partial) 2842 if (!n->nr_partial)
2840 continue; 2843 continue;
2841 2844
2842 for (i = 0; i < objects; i++) 2845 for (i = 0; i < objects; i++)
2843 INIT_LIST_HEAD(slabs_by_inuse + i); 2846 INIT_LIST_HEAD(slabs_by_inuse + i);
2844 2847
2845 spin_lock_irqsave(&n->list_lock, flags); 2848 spin_lock_irqsave(&n->list_lock, flags);
2846 2849
2847 /* 2850 /*
2848 * Build lists indexed by the items in use in each slab. 2851 * Build lists indexed by the items in use in each slab.
2849 * 2852 *
2850 * Note that concurrent frees may occur while we hold the 2853 * Note that concurrent frees may occur while we hold the
2851 * list_lock. page->inuse here is the upper limit. 2854 * list_lock. page->inuse here is the upper limit.
2852 */ 2855 */
2853 list_for_each_entry_safe(page, t, &n->partial, lru) { 2856 list_for_each_entry_safe(page, t, &n->partial, lru) {
2854 if (!page->inuse && slab_trylock(page)) { 2857 if (!page->inuse && slab_trylock(page)) {
2855 /* 2858 /*
2856 * Must hold slab lock here because slab_free 2859 * Must hold slab lock here because slab_free
2857 * may have freed the last object and be 2860 * may have freed the last object and be
2858 * waiting to release the slab. 2861 * waiting to release the slab.
2859 */ 2862 */
2860 list_del(&page->lru); 2863 list_del(&page->lru);
2861 n->nr_partial--; 2864 n->nr_partial--;
2862 slab_unlock(page); 2865 slab_unlock(page);
2863 discard_slab(s, page); 2866 discard_slab(s, page);
2864 } else { 2867 } else {
2865 list_move(&page->lru, 2868 list_move(&page->lru,
2866 slabs_by_inuse + page->inuse); 2869 slabs_by_inuse + page->inuse);
2867 } 2870 }
2868 } 2871 }
2869 2872
2870 /* 2873 /*
2871 * Rebuild the partial list with the slabs filled up most 2874 * Rebuild the partial list with the slabs filled up most
2872 * first and the least used slabs at the end. 2875 * first and the least used slabs at the end.
2873 */ 2876 */
2874 for (i = objects - 1; i >= 0; i--) 2877 for (i = objects - 1; i >= 0; i--)
2875 list_splice(slabs_by_inuse + i, n->partial.prev); 2878 list_splice(slabs_by_inuse + i, n->partial.prev);
2876 2879
2877 spin_unlock_irqrestore(&n->list_lock, flags); 2880 spin_unlock_irqrestore(&n->list_lock, flags);
2878 } 2881 }
2879 2882
2880 kfree(slabs_by_inuse); 2883 kfree(slabs_by_inuse);
2881 return 0; 2884 return 0;
2882 } 2885 }
2883 EXPORT_SYMBOL(kmem_cache_shrink); 2886 EXPORT_SYMBOL(kmem_cache_shrink);
2884 2887
2885 #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG) 2888 #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
2886 static int slab_mem_going_offline_callback(void *arg) 2889 static int slab_mem_going_offline_callback(void *arg)
2887 { 2890 {
2888 struct kmem_cache *s; 2891 struct kmem_cache *s;
2889 2892
2890 down_read(&slub_lock); 2893 down_read(&slub_lock);
2891 list_for_each_entry(s, &slab_caches, list) 2894 list_for_each_entry(s, &slab_caches, list)
2892 kmem_cache_shrink(s); 2895 kmem_cache_shrink(s);
2893 up_read(&slub_lock); 2896 up_read(&slub_lock);
2894 2897
2895 return 0; 2898 return 0;
2896 } 2899 }
2897 2900
2898 static void slab_mem_offline_callback(void *arg) 2901 static void slab_mem_offline_callback(void *arg)
2899 { 2902 {
2900 struct kmem_cache_node *n; 2903 struct kmem_cache_node *n;
2901 struct kmem_cache *s; 2904 struct kmem_cache *s;
2902 struct memory_notify *marg = arg; 2905 struct memory_notify *marg = arg;
2903 int offline_node; 2906 int offline_node;
2904 2907
2905 offline_node = marg->status_change_nid; 2908 offline_node = marg->status_change_nid;
2906 2909
2907 /* 2910 /*
2908 * If the node still has available memory. we need kmem_cache_node 2911 * If the node still has available memory. we need kmem_cache_node
2909 * for it yet. 2912 * for it yet.
2910 */ 2913 */
2911 if (offline_node < 0) 2914 if (offline_node < 0)
2912 return; 2915 return;
2913 2916
2914 down_read(&slub_lock); 2917 down_read(&slub_lock);
2915 list_for_each_entry(s, &slab_caches, list) { 2918 list_for_each_entry(s, &slab_caches, list) {
2916 n = get_node(s, offline_node); 2919 n = get_node(s, offline_node);
2917 if (n) { 2920 if (n) {
2918 /* 2921 /*
2919 * if n->nr_slabs > 0, slabs still exist on the node 2922 * if n->nr_slabs > 0, slabs still exist on the node
2920 * that is going down. We were unable to free them, 2923 * that is going down. We were unable to free them,
2921 * and offline_pages() function shoudn't call this 2924 * and offline_pages() function shoudn't call this
2922 * callback. So, we must fail. 2925 * callback. So, we must fail.
2923 */ 2926 */
2924 BUG_ON(slabs_node(s, offline_node)); 2927 BUG_ON(slabs_node(s, offline_node));
2925 2928
2926 s->node[offline_node] = NULL; 2929 s->node[offline_node] = NULL;
2927 kmem_cache_free(kmalloc_caches, n); 2930 kmem_cache_free(kmalloc_caches, n);
2928 } 2931 }
2929 } 2932 }
2930 up_read(&slub_lock); 2933 up_read(&slub_lock);
2931 } 2934 }
2932 2935
2933 static int slab_mem_going_online_callback(void *arg) 2936 static int slab_mem_going_online_callback(void *arg)
2934 { 2937 {
2935 struct kmem_cache_node *n; 2938 struct kmem_cache_node *n;
2936 struct kmem_cache *s; 2939 struct kmem_cache *s;
2937 struct memory_notify *marg = arg; 2940 struct memory_notify *marg = arg;
2938 int nid = marg->status_change_nid; 2941 int nid = marg->status_change_nid;
2939 int ret = 0; 2942 int ret = 0;
2940 2943
2941 /* 2944 /*
2942 * If the node's memory is already available, then kmem_cache_node is 2945 * If the node's memory is already available, then kmem_cache_node is
2943 * already created. Nothing to do. 2946 * already created. Nothing to do.
2944 */ 2947 */
2945 if (nid < 0) 2948 if (nid < 0)
2946 return 0; 2949 return 0;
2947 2950
2948 /* 2951 /*
2949 * We are bringing a node online. No memory is available yet. We must 2952 * We are bringing a node online. No memory is available yet. We must
2950 * allocate a kmem_cache_node structure in order to bring the node 2953 * allocate a kmem_cache_node structure in order to bring the node
2951 * online. 2954 * online.
2952 */ 2955 */
2953 down_read(&slub_lock); 2956 down_read(&slub_lock);
2954 list_for_each_entry(s, &slab_caches, list) { 2957 list_for_each_entry(s, &slab_caches, list) {
2955 /* 2958 /*
2956 * XXX: kmem_cache_alloc_node will fallback to other nodes 2959 * XXX: kmem_cache_alloc_node will fallback to other nodes
2957 * since memory is not yet available from the node that 2960 * since memory is not yet available from the node that
2958 * is brought up. 2961 * is brought up.
2959 */ 2962 */
2960 n = kmem_cache_alloc(kmalloc_caches, GFP_KERNEL); 2963 n = kmem_cache_alloc(kmalloc_caches, GFP_KERNEL);
2961 if (!n) { 2964 if (!n) {
2962 ret = -ENOMEM; 2965 ret = -ENOMEM;
2963 goto out; 2966 goto out;
2964 } 2967 }
2965 init_kmem_cache_node(n, s); 2968 init_kmem_cache_node(n, s);
2966 s->node[nid] = n; 2969 s->node[nid] = n;
2967 } 2970 }
2968 out: 2971 out:
2969 up_read(&slub_lock); 2972 up_read(&slub_lock);
2970 return ret; 2973 return ret;
2971 } 2974 }
2972 2975
2973 static int slab_memory_callback(struct notifier_block *self, 2976 static int slab_memory_callback(struct notifier_block *self,
2974 unsigned long action, void *arg) 2977 unsigned long action, void *arg)
2975 { 2978 {
2976 int ret = 0; 2979 int ret = 0;
2977 2980
2978 switch (action) { 2981 switch (action) {
2979 case MEM_GOING_ONLINE: 2982 case MEM_GOING_ONLINE:
2980 ret = slab_mem_going_online_callback(arg); 2983 ret = slab_mem_going_online_callback(arg);
2981 break; 2984 break;
2982 case MEM_GOING_OFFLINE: 2985 case MEM_GOING_OFFLINE:
2983 ret = slab_mem_going_offline_callback(arg); 2986 ret = slab_mem_going_offline_callback(arg);
2984 break; 2987 break;
2985 case MEM_OFFLINE: 2988 case MEM_OFFLINE:
2986 case MEM_CANCEL_ONLINE: 2989 case MEM_CANCEL_ONLINE:
2987 slab_mem_offline_callback(arg); 2990 slab_mem_offline_callback(arg);
2988 break; 2991 break;
2989 case MEM_ONLINE: 2992 case MEM_ONLINE:
2990 case MEM_CANCEL_OFFLINE: 2993 case MEM_CANCEL_OFFLINE:
2991 break; 2994 break;
2992 } 2995 }
2993 if (ret) 2996 if (ret)
2994 ret = notifier_from_errno(ret); 2997 ret = notifier_from_errno(ret);
2995 else 2998 else
2996 ret = NOTIFY_OK; 2999 ret = NOTIFY_OK;
2997 return ret; 3000 return ret;
2998 } 3001 }
2999 3002
3000 #endif /* CONFIG_MEMORY_HOTPLUG */ 3003 #endif /* CONFIG_MEMORY_HOTPLUG */
3001 3004
3002 /******************************************************************** 3005 /********************************************************************
3003 * Basic setup of slabs 3006 * Basic setup of slabs
3004 *******************************************************************/ 3007 *******************************************************************/
3005 3008
3006 void __init kmem_cache_init(void) 3009 void __init kmem_cache_init(void)
3007 { 3010 {
3008 int i; 3011 int i;
3009 int caches = 0; 3012 int caches = 0;
3010 3013
3011 init_alloc_cpu(); 3014 init_alloc_cpu();
3012 3015
3013 #ifdef CONFIG_NUMA 3016 #ifdef CONFIG_NUMA
3014 /* 3017 /*
3015 * Must first have the slab cache available for the allocations of the 3018 * Must first have the slab cache available for the allocations of the
3016 * struct kmem_cache_node's. There is special bootstrap code in 3019 * struct kmem_cache_node's. There is special bootstrap code in
3017 * kmem_cache_open for slab_state == DOWN. 3020 * kmem_cache_open for slab_state == DOWN.
3018 */ 3021 */
3019 create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node", 3022 create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
3020 sizeof(struct kmem_cache_node), GFP_KERNEL); 3023 sizeof(struct kmem_cache_node), GFP_KERNEL);
3021 kmalloc_caches[0].refcount = -1; 3024 kmalloc_caches[0].refcount = -1;
3022 caches++; 3025 caches++;
3023 3026
3024 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 3027 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
3025 #endif 3028 #endif
3026 3029
3027 /* Able to allocate the per node structures */ 3030 /* Able to allocate the per node structures */
3028 slab_state = PARTIAL; 3031 slab_state = PARTIAL;
3029 3032
3030 /* Caches that are not of the two-to-the-power-of size */ 3033 /* Caches that are not of the two-to-the-power-of size */
3031 if (KMALLOC_MIN_SIZE <= 64) { 3034 if (KMALLOC_MIN_SIZE <= 64) {
3032 create_kmalloc_cache(&kmalloc_caches[1], 3035 create_kmalloc_cache(&kmalloc_caches[1],
3033 "kmalloc-96", 96, GFP_KERNEL); 3036 "kmalloc-96", 96, GFP_KERNEL);
3034 caches++; 3037 caches++;
3035 create_kmalloc_cache(&kmalloc_caches[2], 3038 create_kmalloc_cache(&kmalloc_caches[2],
3036 "kmalloc-192", 192, GFP_KERNEL); 3039 "kmalloc-192", 192, GFP_KERNEL);
3037 caches++; 3040 caches++;
3038 } 3041 }
3039 3042
3040 for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) { 3043 for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
3041 create_kmalloc_cache(&kmalloc_caches[i], 3044 create_kmalloc_cache(&kmalloc_caches[i],
3042 "kmalloc", 1 << i, GFP_KERNEL); 3045 "kmalloc", 1 << i, GFP_KERNEL);
3043 caches++; 3046 caches++;
3044 } 3047 }
3045 3048
3046 3049
3047 /* 3050 /*
3048 * Patch up the size_index table if we have strange large alignment 3051 * Patch up the size_index table if we have strange large alignment
3049 * requirements for the kmalloc array. This is only the case for 3052 * requirements for the kmalloc array. This is only the case for
3050 * MIPS it seems. The standard arches will not generate any code here. 3053 * MIPS it seems. The standard arches will not generate any code here.
3051 * 3054 *
3052 * Largest permitted alignment is 256 bytes due to the way we 3055 * Largest permitted alignment is 256 bytes due to the way we
3053 * handle the index determination for the smaller caches. 3056 * handle the index determination for the smaller caches.
3054 * 3057 *
3055 * Make sure that nothing crazy happens if someone starts tinkering 3058 * Make sure that nothing crazy happens if someone starts tinkering
3056 * around with ARCH_KMALLOC_MINALIGN 3059 * around with ARCH_KMALLOC_MINALIGN
3057 */ 3060 */
3058 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 || 3061 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
3059 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1))); 3062 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
3060 3063
3061 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) 3064 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8)
3062 size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW; 3065 size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;
3063 3066
3064 if (KMALLOC_MIN_SIZE == 128) { 3067 if (KMALLOC_MIN_SIZE == 128) {
3065 /* 3068 /*
3066 * The 192 byte sized cache is not used if the alignment 3069 * The 192 byte sized cache is not used if the alignment
3067 * is 128 byte. Redirect kmalloc to use the 256 byte cache 3070 * is 128 byte. Redirect kmalloc to use the 256 byte cache
3068 * instead. 3071 * instead.
3069 */ 3072 */
3070 for (i = 128 + 8; i <= 192; i += 8) 3073 for (i = 128 + 8; i <= 192; i += 8)
3071 size_index[(i - 1) / 8] = 8; 3074 size_index[(i - 1) / 8] = 8;
3072 } 3075 }
3073 3076
3074 slab_state = UP; 3077 slab_state = UP;
3075 3078
3076 /* Provide the correct kmalloc names now that the caches are up */ 3079 /* Provide the correct kmalloc names now that the caches are up */
3077 for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) 3080 for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
3078 kmalloc_caches[i]. name = 3081 kmalloc_caches[i]. name =
3079 kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i); 3082 kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
3080 3083
3081 #ifdef CONFIG_SMP 3084 #ifdef CONFIG_SMP
3082 register_cpu_notifier(&slab_notifier); 3085 register_cpu_notifier(&slab_notifier);
3083 kmem_size = offsetof(struct kmem_cache, cpu_slab) + 3086 kmem_size = offsetof(struct kmem_cache, cpu_slab) +
3084 nr_cpu_ids * sizeof(struct kmem_cache_cpu *); 3087 nr_cpu_ids * sizeof(struct kmem_cache_cpu *);
3085 #else 3088 #else
3086 kmem_size = sizeof(struct kmem_cache); 3089 kmem_size = sizeof(struct kmem_cache);
3087 #endif 3090 #endif
3088 3091
3089 printk(KERN_INFO 3092 printk(KERN_INFO
3090 "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d," 3093 "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
3091 " CPUs=%d, Nodes=%d\n", 3094 " CPUs=%d, Nodes=%d\n",
3092 caches, cache_line_size(), 3095 caches, cache_line_size(),
3093 slub_min_order, slub_max_order, slub_min_objects, 3096 slub_min_order, slub_max_order, slub_min_objects,
3094 nr_cpu_ids, nr_node_ids); 3097 nr_cpu_ids, nr_node_ids);
3095 } 3098 }
3096 3099
3097 /* 3100 /*
3098 * Find a mergeable slab cache 3101 * Find a mergeable slab cache
3099 */ 3102 */
3100 static int slab_unmergeable(struct kmem_cache *s) 3103 static int slab_unmergeable(struct kmem_cache *s)
3101 { 3104 {
3102 if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE)) 3105 if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
3103 return 1; 3106 return 1;
3104 3107
3105 if (s->ctor) 3108 if (s->ctor)
3106 return 1; 3109 return 1;
3107 3110
3108 /* 3111 /*
3109 * We may have set a slab to be unmergeable during bootstrap. 3112 * We may have set a slab to be unmergeable during bootstrap.
3110 */ 3113 */
3111 if (s->refcount < 0) 3114 if (s->refcount < 0)
3112 return 1; 3115 return 1;
3113 3116
3114 return 0; 3117 return 0;
3115 } 3118 }
3116 3119
3117 static struct kmem_cache *find_mergeable(size_t size, 3120 static struct kmem_cache *find_mergeable(size_t size,
3118 size_t align, unsigned long flags, const char *name, 3121 size_t align, unsigned long flags, const char *name,
3119 void (*ctor)(void *)) 3122 void (*ctor)(void *))
3120 { 3123 {
3121 struct kmem_cache *s; 3124 struct kmem_cache *s;
3122 3125
3123 if (slub_nomerge || (flags & SLUB_NEVER_MERGE)) 3126 if (slub_nomerge || (flags & SLUB_NEVER_MERGE))
3124 return NULL; 3127 return NULL;
3125 3128
3126 if (ctor) 3129 if (ctor)
3127 return NULL; 3130 return NULL;
3128 3131
3129 size = ALIGN(size, sizeof(void *)); 3132 size = ALIGN(size, sizeof(void *));
3130 align = calculate_alignment(flags, align, size); 3133 align = calculate_alignment(flags, align, size);
3131 size = ALIGN(size, align); 3134 size = ALIGN(size, align);
3132 flags = kmem_cache_flags(size, flags, name, NULL); 3135 flags = kmem_cache_flags(size, flags, name, NULL);
3133 3136
3134 list_for_each_entry(s, &slab_caches, list) { 3137 list_for_each_entry(s, &slab_caches, list) {
3135 if (slab_unmergeable(s)) 3138 if (slab_unmergeable(s))
3136 continue; 3139 continue;
3137 3140
3138 if (size > s->size) 3141 if (size > s->size)
3139 continue; 3142 continue;
3140 3143
3141 if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME)) 3144 if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
3142 continue; 3145 continue;
3143 /* 3146 /*
3144 * Check if alignment is compatible. 3147 * Check if alignment is compatible.
3145 * Courtesy of Adrian Drzewiecki 3148 * Courtesy of Adrian Drzewiecki
3146 */ 3149 */
3147 if ((s->size & ~(align - 1)) != s->size) 3150 if ((s->size & ~(align - 1)) != s->size)
3148 continue; 3151 continue;
3149 3152
3150 if (s->size - size >= sizeof(void *)) 3153 if (s->size - size >= sizeof(void *))
3151 continue; 3154 continue;
3152 3155
3153 return s; 3156 return s;
3154 } 3157 }
3155 return NULL; 3158 return NULL;
3156 } 3159 }
3157 3160
3158 struct kmem_cache *kmem_cache_create(const char *name, size_t size, 3161 struct kmem_cache *kmem_cache_create(const char *name, size_t size,
3159 size_t align, unsigned long flags, void (*ctor)(void *)) 3162 size_t align, unsigned long flags, void (*ctor)(void *))
3160 { 3163 {
3161 struct kmem_cache *s; 3164 struct kmem_cache *s;
3162 3165
3163 down_write(&slub_lock); 3166 down_write(&slub_lock);
3164 s = find_mergeable(size, align, flags, name, ctor); 3167 s = find_mergeable(size, align, flags, name, ctor);
3165 if (s) { 3168 if (s) {
3166 int cpu; 3169 int cpu;
3167 3170
3168 s->refcount++; 3171 s->refcount++;
3169 /* 3172 /*
3170 * Adjust the object sizes so that we clear 3173 * Adjust the object sizes so that we clear
3171 * the complete object on kzalloc. 3174 * the complete object on kzalloc.
3172 */ 3175 */
3173 s->objsize = max(s->objsize, (int)size); 3176 s->objsize = max(s->objsize, (int)size);
3174 3177
3175 /* 3178 /*
3176 * And then we need to update the object size in the 3179 * And then we need to update the object size in the
3177 * per cpu structures 3180 * per cpu structures
3178 */ 3181 */
3179 for_each_online_cpu(cpu) 3182 for_each_online_cpu(cpu)
3180 get_cpu_slab(s, cpu)->objsize = s->objsize; 3183 get_cpu_slab(s, cpu)->objsize = s->objsize;
3181 3184
3182 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); 3185 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
3183 up_write(&slub_lock); 3186 up_write(&slub_lock);
3184 3187
3185 if (sysfs_slab_alias(s, name)) { 3188 if (sysfs_slab_alias(s, name)) {
3186 down_write(&slub_lock); 3189 down_write(&slub_lock);
3187 s->refcount--; 3190 s->refcount--;
3188 up_write(&slub_lock); 3191 up_write(&slub_lock);
3189 goto err; 3192 goto err;
3190 } 3193 }
3191 return s; 3194 return s;
3192 } 3195 }
3193 3196
3194 s = kmalloc(kmem_size, GFP_KERNEL); 3197 s = kmalloc(kmem_size, GFP_KERNEL);
3195 if (s) { 3198 if (s) {
3196 if (kmem_cache_open(s, GFP_KERNEL, name, 3199 if (kmem_cache_open(s, GFP_KERNEL, name,
3197 size, align, flags, ctor)) { 3200 size, align, flags, ctor)) {
3198 list_add(&s->list, &slab_caches); 3201 list_add(&s->list, &slab_caches);
3199 up_write(&slub_lock); 3202 up_write(&slub_lock);
3200 if (sysfs_slab_add(s)) { 3203 if (sysfs_slab_add(s)) {
3201 down_write(&slub_lock); 3204 down_write(&slub_lock);
3202 list_del(&s->list); 3205 list_del(&s->list);
3203 up_write(&slub_lock); 3206 up_write(&slub_lock);
3204 kfree(s); 3207 kfree(s);
3205 goto err; 3208 goto err;
3206 } 3209 }
3207 return s; 3210 return s;
3208 } 3211 }
3209 kfree(s); 3212 kfree(s);
3210 } 3213 }
3211 up_write(&slub_lock); 3214 up_write(&slub_lock);
3212 3215
3213 err: 3216 err:
3214 if (flags & SLAB_PANIC) 3217 if (flags & SLAB_PANIC)
3215 panic("Cannot create slabcache %s\n", name); 3218 panic("Cannot create slabcache %s\n", name);
3216 else 3219 else
3217 s = NULL; 3220 s = NULL;
3218 return s; 3221 return s;
3219 } 3222 }
3220 EXPORT_SYMBOL(kmem_cache_create); 3223 EXPORT_SYMBOL(kmem_cache_create);
3221 3224
3222 #ifdef CONFIG_SMP 3225 #ifdef CONFIG_SMP
3223 /* 3226 /*
3224 * Use the cpu notifier to insure that the cpu slabs are flushed when 3227 * Use the cpu notifier to insure that the cpu slabs are flushed when
3225 * necessary. 3228 * necessary.
3226 */ 3229 */
3227 static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, 3230 static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb,
3228 unsigned long action, void *hcpu) 3231 unsigned long action, void *hcpu)
3229 { 3232 {
3230 long cpu = (long)hcpu; 3233 long cpu = (long)hcpu;
3231 struct kmem_cache *s; 3234 struct kmem_cache *s;
3232 unsigned long flags; 3235 unsigned long flags;
3233 3236
3234 switch (action) { 3237 switch (action) {
3235 case CPU_UP_PREPARE: 3238 case CPU_UP_PREPARE:
3236 case CPU_UP_PREPARE_FROZEN: 3239 case CPU_UP_PREPARE_FROZEN:
3237 init_alloc_cpu_cpu(cpu); 3240 init_alloc_cpu_cpu(cpu);
3238 down_read(&slub_lock); 3241 down_read(&slub_lock);
3239 list_for_each_entry(s, &slab_caches, list) 3242 list_for_each_entry(s, &slab_caches, list)
3240 s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu, 3243 s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu,
3241 GFP_KERNEL); 3244 GFP_KERNEL);
3242 up_read(&slub_lock); 3245 up_read(&slub_lock);
3243 break; 3246 break;
3244 3247
3245 case CPU_UP_CANCELED: 3248 case CPU_UP_CANCELED:
3246 case CPU_UP_CANCELED_FROZEN: 3249 case CPU_UP_CANCELED_FROZEN:
3247 case CPU_DEAD: 3250 case CPU_DEAD:
3248 case CPU_DEAD_FROZEN: 3251 case CPU_DEAD_FROZEN:
3249 down_read(&slub_lock); 3252 down_read(&slub_lock);
3250 list_for_each_entry(s, &slab_caches, list) { 3253 list_for_each_entry(s, &slab_caches, list) {
3251 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); 3254 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
3252 3255
3253 local_irq_save(flags); 3256 local_irq_save(flags);
3254 __flush_cpu_slab(s, cpu); 3257 __flush_cpu_slab(s, cpu);
3255 local_irq_restore(flags); 3258 local_irq_restore(flags);
3256 free_kmem_cache_cpu(c, cpu); 3259 free_kmem_cache_cpu(c, cpu);
3257 s->cpu_slab[cpu] = NULL; 3260 s->cpu_slab[cpu] = NULL;
3258 } 3261 }
3259 up_read(&slub_lock); 3262 up_read(&slub_lock);
3260 break; 3263 break;
3261 default: 3264 default:
3262 break; 3265 break;
3263 } 3266 }
3264 return NOTIFY_OK; 3267 return NOTIFY_OK;
3265 } 3268 }
3266 3269
3267 static struct notifier_block __cpuinitdata slab_notifier = { 3270 static struct notifier_block __cpuinitdata slab_notifier = {
3268 .notifier_call = slab_cpuup_callback 3271 .notifier_call = slab_cpuup_callback
3269 }; 3272 };
3270 3273
3271 #endif 3274 #endif
3272 3275
3273 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) 3276 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
3274 { 3277 {
3275 struct kmem_cache *s; 3278 struct kmem_cache *s;
3276 void *ret; 3279 void *ret;
3277 3280
3278 if (unlikely(size > SLUB_MAX_SIZE)) 3281 if (unlikely(size > SLUB_MAX_SIZE))
3279 return kmalloc_large(size, gfpflags); 3282 return kmalloc_large(size, gfpflags);
3280 3283
3281 s = get_slab(size, gfpflags); 3284 s = get_slab(size, gfpflags);
3282 3285
3283 if (unlikely(ZERO_OR_NULL_PTR(s))) 3286 if (unlikely(ZERO_OR_NULL_PTR(s)))
3284 return s; 3287 return s;
3285 3288
3286 ret = slab_alloc(s, gfpflags, -1, caller); 3289 ret = slab_alloc(s, gfpflags, -1, caller);
3287 3290
3288 /* Honor the call site pointer we recieved. */ 3291 /* Honor the call site pointer we recieved. */
3289 trace_kmalloc(caller, ret, size, s->size, gfpflags); 3292 trace_kmalloc(caller, ret, size, s->size, gfpflags);
3290 3293
3291 return ret; 3294 return ret;
3292 } 3295 }
3293 3296
3294 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, 3297 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
3295 int node, unsigned long caller) 3298 int node, unsigned long caller)
3296 { 3299 {
3297 struct kmem_cache *s; 3300 struct kmem_cache *s;
3298 void *ret; 3301 void *ret;
3299 3302
3300 if (unlikely(size > SLUB_MAX_SIZE)) 3303 if (unlikely(size > SLUB_MAX_SIZE))
3301 return kmalloc_large_node(size, gfpflags, node); 3304 return kmalloc_large_node(size, gfpflags, node);
3302 3305
3303 s = get_slab(size, gfpflags); 3306 s = get_slab(size, gfpflags);
3304 3307
3305 if (unlikely(ZERO_OR_NULL_PTR(s))) 3308 if (unlikely(ZERO_OR_NULL_PTR(s)))
3306 return s; 3309 return s;
3307 3310
3308 ret = slab_alloc(s, gfpflags, node, caller); 3311 ret = slab_alloc(s, gfpflags, node, caller);
3309 3312
3310 /* Honor the call site pointer we recieved. */ 3313 /* Honor the call site pointer we recieved. */
3311 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); 3314 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
3312 3315
3313 return ret; 3316 return ret;
3314 } 3317 }
3315 3318
3316 #ifdef CONFIG_SLUB_DEBUG 3319 #ifdef CONFIG_SLUB_DEBUG
3317 static unsigned long count_partial(struct kmem_cache_node *n, 3320 static unsigned long count_partial(struct kmem_cache_node *n,
3318 int (*get_count)(struct page *)) 3321 int (*get_count)(struct page *))
3319 { 3322 {
3320 unsigned long flags; 3323 unsigned long flags;
3321 unsigned long x = 0; 3324 unsigned long x = 0;
3322 struct page *page; 3325 struct page *page;
3323 3326
3324 spin_lock_irqsave(&n->list_lock, flags); 3327 spin_lock_irqsave(&n->list_lock, flags);
3325 list_for_each_entry(page, &n->partial, lru) 3328 list_for_each_entry(page, &n->partial, lru)
3326 x += get_count(page); 3329 x += get_count(page);
3327 spin_unlock_irqrestore(&n->list_lock, flags); 3330 spin_unlock_irqrestore(&n->list_lock, flags);
3328 return x; 3331 return x;
3329 } 3332 }
3330 3333
3331 static int count_inuse(struct page *page) 3334 static int count_inuse(struct page *page)
3332 { 3335 {
3333 return page->inuse; 3336 return page->inuse;
3334 } 3337 }
3335 3338
3336 static int count_total(struct page *page) 3339 static int count_total(struct page *page)
3337 { 3340 {
3338 return page->objects; 3341 return page->objects;
3339 } 3342 }
3340 3343
3341 static int count_free(struct page *page) 3344 static int count_free(struct page *page)
3342 { 3345 {
3343 return page->objects - page->inuse; 3346 return page->objects - page->inuse;
3344 } 3347 }
3345 3348
3346 static int validate_slab(struct kmem_cache *s, struct page *page, 3349 static int validate_slab(struct kmem_cache *s, struct page *page,
3347 unsigned long *map) 3350 unsigned long *map)
3348 { 3351 {
3349 void *p; 3352 void *p;
3350 void *addr = page_address(page); 3353 void *addr = page_address(page);
3351 3354
3352 if (!check_slab(s, page) || 3355 if (!check_slab(s, page) ||
3353 !on_freelist(s, page, NULL)) 3356 !on_freelist(s, page, NULL))
3354 return 0; 3357 return 0;
3355 3358
3356 /* Now we know that a valid freelist exists */ 3359 /* Now we know that a valid freelist exists */
3357 bitmap_zero(map, page->objects); 3360 bitmap_zero(map, page->objects);
3358 3361
3359 for_each_free_object(p, s, page->freelist) { 3362 for_each_free_object(p, s, page->freelist) {
3360 set_bit(slab_index(p, s, addr), map); 3363 set_bit(slab_index(p, s, addr), map);
3361 if (!check_object(s, page, p, 0)) 3364 if (!check_object(s, page, p, 0))
3362 return 0; 3365 return 0;
3363 } 3366 }
3364 3367
3365 for_each_object(p, s, addr, page->objects) 3368 for_each_object(p, s, addr, page->objects)
3366 if (!test_bit(slab_index(p, s, addr), map)) 3369 if (!test_bit(slab_index(p, s, addr), map))
3367 if (!check_object(s, page, p, 1)) 3370 if (!check_object(s, page, p, 1))
3368 return 0; 3371 return 0;
3369 return 1; 3372 return 1;
3370 } 3373 }
3371 3374
3372 static void validate_slab_slab(struct kmem_cache *s, struct page *page, 3375 static void validate_slab_slab(struct kmem_cache *s, struct page *page,
3373 unsigned long *map) 3376 unsigned long *map)
3374 { 3377 {
3375 if (slab_trylock(page)) { 3378 if (slab_trylock(page)) {
3376 validate_slab(s, page, map); 3379 validate_slab(s, page, map);
3377 slab_unlock(page); 3380 slab_unlock(page);
3378 } else 3381 } else
3379 printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n", 3382 printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n",
3380 s->name, page); 3383 s->name, page);
3381 3384
3382 if (s->flags & DEBUG_DEFAULT_FLAGS) { 3385 if (s->flags & DEBUG_DEFAULT_FLAGS) {
3383 if (!PageSlubDebug(page)) 3386 if (!PageSlubDebug(page))
3384 printk(KERN_ERR "SLUB %s: SlubDebug not set " 3387 printk(KERN_ERR "SLUB %s: SlubDebug not set "
3385 "on slab 0x%p\n", s->name, page); 3388 "on slab 0x%p\n", s->name, page);
3386 } else { 3389 } else {
3387 if (PageSlubDebug(page)) 3390 if (PageSlubDebug(page))
3388 printk(KERN_ERR "SLUB %s: SlubDebug set on " 3391 printk(KERN_ERR "SLUB %s: SlubDebug set on "
3389 "slab 0x%p\n", s->name, page); 3392 "slab 0x%p\n", s->name, page);
3390 } 3393 }
3391 } 3394 }
3392 3395
3393 static int validate_slab_node(struct kmem_cache *s, 3396 static int validate_slab_node(struct kmem_cache *s,
3394 struct kmem_cache_node *n, unsigned long *map) 3397 struct kmem_cache_node *n, unsigned long *map)
3395 { 3398 {
3396 unsigned long count = 0; 3399 unsigned long count = 0;
3397 struct page *page; 3400 struct page *page;
3398 unsigned long flags; 3401 unsigned long flags;
3399 3402
3400 spin_lock_irqsave(&n->list_lock, flags); 3403 spin_lock_irqsave(&n->list_lock, flags);
3401 3404
3402 list_for_each_entry(page, &n->partial, lru) { 3405 list_for_each_entry(page, &n->partial, lru) {
3403 validate_slab_slab(s, page, map); 3406 validate_slab_slab(s, page, map);
3404 count++; 3407 count++;
3405 } 3408 }
3406 if (count != n->nr_partial) 3409 if (count != n->nr_partial)
3407 printk(KERN_ERR "SLUB %s: %ld partial slabs counted but " 3410 printk(KERN_ERR "SLUB %s: %ld partial slabs counted but "
3408 "counter=%ld\n", s->name, count, n->nr_partial); 3411 "counter=%ld\n", s->name, count, n->nr_partial);
3409 3412
3410 if (!(s->flags & SLAB_STORE_USER)) 3413 if (!(s->flags & SLAB_STORE_USER))
3411 goto out; 3414 goto out;
3412 3415
3413 list_for_each_entry(page, &n->full, lru) { 3416 list_for_each_entry(page, &n->full, lru) {
3414 validate_slab_slab(s, page, map); 3417 validate_slab_slab(s, page, map);
3415 count++; 3418 count++;
3416 } 3419 }
3417 if (count != atomic_long_read(&n->nr_slabs)) 3420 if (count != atomic_long_read(&n->nr_slabs))
3418 printk(KERN_ERR "SLUB: %s %ld slabs counted but " 3421 printk(KERN_ERR "SLUB: %s %ld slabs counted but "
3419 "counter=%ld\n", s->name, count, 3422 "counter=%ld\n", s->name, count,
3420 atomic_long_read(&n->nr_slabs)); 3423 atomic_long_read(&n->nr_slabs));
3421 3424
3422 out: 3425 out:
3423 spin_unlock_irqrestore(&n->list_lock, flags); 3426 spin_unlock_irqrestore(&n->list_lock, flags);
3424 return count; 3427 return count;
3425 } 3428 }
3426 3429
3427 static long validate_slab_cache(struct kmem_cache *s) 3430 static long validate_slab_cache(struct kmem_cache *s)
3428 { 3431 {
3429 int node; 3432 int node;
3430 unsigned long count = 0; 3433 unsigned long count = 0;
3431 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * 3434 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
3432 sizeof(unsigned long), GFP_KERNEL); 3435 sizeof(unsigned long), GFP_KERNEL);
3433 3436
3434 if (!map) 3437 if (!map)
3435 return -ENOMEM; 3438 return -ENOMEM;
3436 3439
3437 flush_all(s); 3440 flush_all(s);
3438 for_each_node_state(node, N_NORMAL_MEMORY) { 3441 for_each_node_state(node, N_NORMAL_MEMORY) {
3439 struct kmem_cache_node *n = get_node(s, node); 3442 struct kmem_cache_node *n = get_node(s, node);
3440 3443
3441 count += validate_slab_node(s, n, map); 3444 count += validate_slab_node(s, n, map);
3442 } 3445 }
3443 kfree(map); 3446 kfree(map);
3444 return count; 3447 return count;
3445 } 3448 }
3446 3449
3447 #ifdef SLUB_RESILIENCY_TEST 3450 #ifdef SLUB_RESILIENCY_TEST
3448 static void resiliency_test(void) 3451 static void resiliency_test(void)
3449 { 3452 {
3450 u8 *p; 3453 u8 *p;
3451 3454
3452 printk(KERN_ERR "SLUB resiliency testing\n"); 3455 printk(KERN_ERR "SLUB resiliency testing\n");
3453 printk(KERN_ERR "-----------------------\n"); 3456 printk(KERN_ERR "-----------------------\n");
3454 printk(KERN_ERR "A. Corruption after allocation\n"); 3457 printk(KERN_ERR "A. Corruption after allocation\n");
3455 3458
3456 p = kzalloc(16, GFP_KERNEL); 3459 p = kzalloc(16, GFP_KERNEL);
3457 p[16] = 0x12; 3460 p[16] = 0x12;
3458 printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer" 3461 printk(KERN_ERR "\n1. kmalloc-16: Clobber Redzone/next pointer"
3459 " 0x12->0x%p\n\n", p + 16); 3462 " 0x12->0x%p\n\n", p + 16);
3460 3463
3461 validate_slab_cache(kmalloc_caches + 4); 3464 validate_slab_cache(kmalloc_caches + 4);
3462 3465
3463 /* Hmmm... The next two are dangerous */ 3466 /* Hmmm... The next two are dangerous */
3464 p = kzalloc(32, GFP_KERNEL); 3467 p = kzalloc(32, GFP_KERNEL);
3465 p[32 + sizeof(void *)] = 0x34; 3468 p[32 + sizeof(void *)] = 0x34;
3466 printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab" 3469 printk(KERN_ERR "\n2. kmalloc-32: Clobber next pointer/next slab"
3467 " 0x34 -> -0x%p\n", p); 3470 " 0x34 -> -0x%p\n", p);
3468 printk(KERN_ERR 3471 printk(KERN_ERR
3469 "If allocated object is overwritten then not detectable\n\n"); 3472 "If allocated object is overwritten then not detectable\n\n");
3470 3473
3471 validate_slab_cache(kmalloc_caches + 5); 3474 validate_slab_cache(kmalloc_caches + 5);
3472 p = kzalloc(64, GFP_KERNEL); 3475 p = kzalloc(64, GFP_KERNEL);
3473 p += 64 + (get_cycles() & 0xff) * sizeof(void *); 3476 p += 64 + (get_cycles() & 0xff) * sizeof(void *);
3474 *p = 0x56; 3477 *p = 0x56;
3475 printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", 3478 printk(KERN_ERR "\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
3476 p); 3479 p);
3477 printk(KERN_ERR 3480 printk(KERN_ERR
3478 "If allocated object is overwritten then not detectable\n\n"); 3481 "If allocated object is overwritten then not detectable\n\n");
3479 validate_slab_cache(kmalloc_caches + 6); 3482 validate_slab_cache(kmalloc_caches + 6);
3480 3483
3481 printk(KERN_ERR "\nB. Corruption after free\n"); 3484 printk(KERN_ERR "\nB. Corruption after free\n");
3482 p = kzalloc(128, GFP_KERNEL); 3485 p = kzalloc(128, GFP_KERNEL);
3483 kfree(p); 3486 kfree(p);
3484 *p = 0x78; 3487 *p = 0x78;
3485 printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); 3488 printk(KERN_ERR "1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
3486 validate_slab_cache(kmalloc_caches + 7); 3489 validate_slab_cache(kmalloc_caches + 7);
3487 3490
3488 p = kzalloc(256, GFP_KERNEL); 3491 p = kzalloc(256, GFP_KERNEL);
3489 kfree(p); 3492 kfree(p);
3490 p[50] = 0x9a; 3493 p[50] = 0x9a;
3491 printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", 3494 printk(KERN_ERR "\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n",
3492 p); 3495 p);
3493 validate_slab_cache(kmalloc_caches + 8); 3496 validate_slab_cache(kmalloc_caches + 8);
3494 3497
3495 p = kzalloc(512, GFP_KERNEL); 3498 p = kzalloc(512, GFP_KERNEL);
3496 kfree(p); 3499 kfree(p);
3497 p[512] = 0xab; 3500 p[512] = 0xab;
3498 printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); 3501 printk(KERN_ERR "\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
3499 validate_slab_cache(kmalloc_caches + 9); 3502 validate_slab_cache(kmalloc_caches + 9);
3500 } 3503 }
3501 #else 3504 #else
3502 static void resiliency_test(void) {}; 3505 static void resiliency_test(void) {};
3503 #endif 3506 #endif
3504 3507
3505 /* 3508 /*
3506 * Generate lists of code addresses where slabcache objects are allocated 3509 * Generate lists of code addresses where slabcache objects are allocated
3507 * and freed. 3510 * and freed.
3508 */ 3511 */
3509 3512
3510 struct location { 3513 struct location {
3511 unsigned long count; 3514 unsigned long count;
3512 unsigned long addr; 3515 unsigned long addr;
3513 long long sum_time; 3516 long long sum_time;
3514 long min_time; 3517 long min_time;
3515 long max_time; 3518 long max_time;
3516 long min_pid; 3519 long min_pid;
3517 long max_pid; 3520 long max_pid;
3518 DECLARE_BITMAP(cpus, NR_CPUS); 3521 DECLARE_BITMAP(cpus, NR_CPUS);
3519 nodemask_t nodes; 3522 nodemask_t nodes;
3520 }; 3523 };
3521 3524
3522 struct loc_track { 3525 struct loc_track {
3523 unsigned long max; 3526 unsigned long max;
3524 unsigned long count; 3527 unsigned long count;
3525 struct location *loc; 3528 struct location *loc;
3526 }; 3529 };
3527 3530
3528 static void free_loc_track(struct loc_track *t) 3531 static void free_loc_track(struct loc_track *t)
3529 { 3532 {
3530 if (t->max) 3533 if (t->max)
3531 free_pages((unsigned long)t->loc, 3534 free_pages((unsigned long)t->loc,
3532 get_order(sizeof(struct location) * t->max)); 3535 get_order(sizeof(struct location) * t->max));
3533 } 3536 }
3534 3537
3535 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) 3538 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
3536 { 3539 {
3537 struct location *l; 3540 struct location *l;
3538 int order; 3541 int order;
3539 3542
3540 order = get_order(sizeof(struct location) * max); 3543 order = get_order(sizeof(struct location) * max);
3541 3544
3542 l = (void *)__get_free_pages(flags, order); 3545 l = (void *)__get_free_pages(flags, order);
3543 if (!l) 3546 if (!l)
3544 return 0; 3547 return 0;
3545 3548
3546 if (t->count) { 3549 if (t->count) {
3547 memcpy(l, t->loc, sizeof(struct location) * t->count); 3550 memcpy(l, t->loc, sizeof(struct location) * t->count);
3548 free_loc_track(t); 3551 free_loc_track(t);
3549 } 3552 }
3550 t->max = max; 3553 t->max = max;
3551 t->loc = l; 3554 t->loc = l;
3552 return 1; 3555 return 1;
3553 } 3556 }
3554 3557
3555 static int add_location(struct loc_track *t, struct kmem_cache *s, 3558 static int add_location(struct loc_track *t, struct kmem_cache *s,
3556 const struct track *track) 3559 const struct track *track)
3557 { 3560 {
3558 long start, end, pos; 3561 long start, end, pos;
3559 struct location *l; 3562 struct location *l;
3560 unsigned long caddr; 3563 unsigned long caddr;
3561 unsigned long age = jiffies - track->when; 3564 unsigned long age = jiffies - track->when;
3562 3565
3563 start = -1; 3566 start = -1;
3564 end = t->count; 3567 end = t->count;
3565 3568
3566 for ( ; ; ) { 3569 for ( ; ; ) {
3567 pos = start + (end - start + 1) / 2; 3570 pos = start + (end - start + 1) / 2;
3568 3571
3569 /* 3572 /*
3570 * There is nothing at "end". If we end up there 3573 * There is nothing at "end". If we end up there
3571 * we need to add something to before end. 3574 * we need to add something to before end.
3572 */ 3575 */
3573 if (pos == end) 3576 if (pos == end)
3574 break; 3577 break;
3575 3578
3576 caddr = t->loc[pos].addr; 3579 caddr = t->loc[pos].addr;
3577 if (track->addr == caddr) { 3580 if (track->addr == caddr) {
3578 3581
3579 l = &t->loc[pos]; 3582 l = &t->loc[pos];
3580 l->count++; 3583 l->count++;
3581 if (track->when) { 3584 if (track->when) {
3582 l->sum_time += age; 3585 l->sum_time += age;
3583 if (age < l->min_time) 3586 if (age < l->min_time)
3584 l->min_time = age; 3587 l->min_time = age;
3585 if (age > l->max_time) 3588 if (age > l->max_time)
3586 l->max_time = age; 3589 l->max_time = age;
3587 3590
3588 if (track->pid < l->min_pid) 3591 if (track->pid < l->min_pid)
3589 l->min_pid = track->pid; 3592 l->min_pid = track->pid;
3590 if (track->pid > l->max_pid) 3593 if (track->pid > l->max_pid)
3591 l->max_pid = track->pid; 3594 l->max_pid = track->pid;
3592 3595
3593 cpumask_set_cpu(track->cpu, 3596 cpumask_set_cpu(track->cpu,
3594 to_cpumask(l->cpus)); 3597 to_cpumask(l->cpus));
3595 } 3598 }
3596 node_set(page_to_nid(virt_to_page(track)), l->nodes); 3599 node_set(page_to_nid(virt_to_page(track)), l->nodes);
3597 return 1; 3600 return 1;
3598 } 3601 }
3599 3602
3600 if (track->addr < caddr) 3603 if (track->addr < caddr)
3601 end = pos; 3604 end = pos;
3602 else 3605 else
3603 start = pos; 3606 start = pos;
3604 } 3607 }
3605 3608
3606 /* 3609 /*
3607 * Not found. Insert new tracking element. 3610 * Not found. Insert new tracking element.
3608 */ 3611 */
3609 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) 3612 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
3610 return 0; 3613 return 0;
3611 3614
3612 l = t->loc + pos; 3615 l = t->loc + pos;
3613 if (pos < t->count) 3616 if (pos < t->count)
3614 memmove(l + 1, l, 3617 memmove(l + 1, l,
3615 (t->count - pos) * sizeof(struct location)); 3618 (t->count - pos) * sizeof(struct location));
3616 t->count++; 3619 t->count++;
3617 l->count = 1; 3620 l->count = 1;
3618 l->addr = track->addr; 3621 l->addr = track->addr;
3619 l->sum_time = age; 3622 l->sum_time = age;
3620 l->min_time = age; 3623 l->min_time = age;
3621 l->max_time = age; 3624 l->max_time = age;
3622 l->min_pid = track->pid; 3625 l->min_pid = track->pid;
3623 l->max_pid = track->pid; 3626 l->max_pid = track->pid;
3624 cpumask_clear(to_cpumask(l->cpus)); 3627 cpumask_clear(to_cpumask(l->cpus));
3625 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); 3628 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
3626 nodes_clear(l->nodes); 3629 nodes_clear(l->nodes);
3627 node_set(page_to_nid(virt_to_page(track)), l->nodes); 3630 node_set(page_to_nid(virt_to_page(track)), l->nodes);
3628 return 1; 3631 return 1;
3629 } 3632 }
3630 3633
3631 static void process_slab(struct loc_track *t, struct kmem_cache *s, 3634 static void process_slab(struct loc_track *t, struct kmem_cache *s,
3632 struct page *page, enum track_item alloc) 3635 struct page *page, enum track_item alloc)
3633 { 3636 {
3634 void *addr = page_address(page); 3637 void *addr = page_address(page);
3635 DECLARE_BITMAP(map, page->objects); 3638 DECLARE_BITMAP(map, page->objects);
3636 void *p; 3639 void *p;
3637 3640
3638 bitmap_zero(map, page->objects); 3641 bitmap_zero(map, page->objects);
3639 for_each_free_object(p, s, page->freelist) 3642 for_each_free_object(p, s, page->freelist)
3640 set_bit(slab_index(p, s, addr), map); 3643 set_bit(slab_index(p, s, addr), map);
3641 3644
3642 for_each_object(p, s, addr, page->objects) 3645 for_each_object(p, s, addr, page->objects)
3643 if (!test_bit(slab_index(p, s, addr), map)) 3646 if (!test_bit(slab_index(p, s, addr), map))
3644 add_location(t, s, get_track(s, p, alloc)); 3647 add_location(t, s, get_track(s, p, alloc));
3645 } 3648 }
3646 3649
3647 static int list_locations(struct kmem_cache *s, char *buf, 3650 static int list_locations(struct kmem_cache *s, char *buf,
3648 enum track_item alloc) 3651 enum track_item alloc)
3649 { 3652 {
3650 int len = 0; 3653 int len = 0;
3651 unsigned long i; 3654 unsigned long i;
3652 struct loc_track t = { 0, 0, NULL }; 3655 struct loc_track t = { 0, 0, NULL };
3653 int node; 3656 int node;
3654 3657
3655 if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), 3658 if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
3656 GFP_TEMPORARY)) 3659 GFP_TEMPORARY))
3657 return sprintf(buf, "Out of memory\n"); 3660 return sprintf(buf, "Out of memory\n");
3658 3661
3659 /* Push back cpu slabs */ 3662 /* Push back cpu slabs */
3660 flush_all(s); 3663 flush_all(s);
3661 3664
3662 for_each_node_state(node, N_NORMAL_MEMORY) { 3665 for_each_node_state(node, N_NORMAL_MEMORY) {
3663 struct kmem_cache_node *n = get_node(s, node); 3666 struct kmem_cache_node *n = get_node(s, node);
3664 unsigned long flags; 3667 unsigned long flags;
3665 struct page *page; 3668 struct page *page;
3666 3669
3667 if (!atomic_long_read(&n->nr_slabs)) 3670 if (!atomic_long_read(&n->nr_slabs))
3668 continue; 3671 continue;
3669 3672
3670 spin_lock_irqsave(&n->list_lock, flags); 3673 spin_lock_irqsave(&n->list_lock, flags);
3671 list_for_each_entry(page, &n->partial, lru) 3674 list_for_each_entry(page, &n->partial, lru)
3672 process_slab(&t, s, page, alloc); 3675 process_slab(&t, s, page, alloc);
3673 list_for_each_entry(page, &n->full, lru) 3676 list_for_each_entry(page, &n->full, lru)
3674 process_slab(&t, s, page, alloc); 3677 process_slab(&t, s, page, alloc);
3675 spin_unlock_irqrestore(&n->list_lock, flags); 3678 spin_unlock_irqrestore(&n->list_lock, flags);
3676 } 3679 }
3677 3680
3678 for (i = 0; i < t.count; i++) { 3681 for (i = 0; i < t.count; i++) {
3679 struct location *l = &t.loc[i]; 3682 struct location *l = &t.loc[i];
3680 3683
3681 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100) 3684 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
3682 break; 3685 break;
3683 len += sprintf(buf + len, "%7ld ", l->count); 3686 len += sprintf(buf + len, "%7ld ", l->count);
3684 3687
3685 if (l->addr) 3688 if (l->addr)
3686 len += sprint_symbol(buf + len, (unsigned long)l->addr); 3689 len += sprint_symbol(buf + len, (unsigned long)l->addr);
3687 else 3690 else
3688 len += sprintf(buf + len, "<not-available>"); 3691 len += sprintf(buf + len, "<not-available>");
3689 3692
3690 if (l->sum_time != l->min_time) { 3693 if (l->sum_time != l->min_time) {
3691 len += sprintf(buf + len, " age=%ld/%ld/%ld", 3694 len += sprintf(buf + len, " age=%ld/%ld/%ld",
3692 l->min_time, 3695 l->min_time,
3693 (long)div_u64(l->sum_time, l->count), 3696 (long)div_u64(l->sum_time, l->count),
3694 l->max_time); 3697 l->max_time);
3695 } else 3698 } else
3696 len += sprintf(buf + len, " age=%ld", 3699 len += sprintf(buf + len, " age=%ld",
3697 l->min_time); 3700 l->min_time);
3698 3701
3699 if (l->min_pid != l->max_pid) 3702 if (l->min_pid != l->max_pid)
3700 len += sprintf(buf + len, " pid=%ld-%ld", 3703 len += sprintf(buf + len, " pid=%ld-%ld",
3701 l->min_pid, l->max_pid); 3704 l->min_pid, l->max_pid);
3702 else 3705 else
3703 len += sprintf(buf + len, " pid=%ld", 3706 len += sprintf(buf + len, " pid=%ld",
3704 l->min_pid); 3707 l->min_pid);
3705 3708
3706 if (num_online_cpus() > 1 && 3709 if (num_online_cpus() > 1 &&
3707 !cpumask_empty(to_cpumask(l->cpus)) && 3710 !cpumask_empty(to_cpumask(l->cpus)) &&
3708 len < PAGE_SIZE - 60) { 3711 len < PAGE_SIZE - 60) {
3709 len += sprintf(buf + len, " cpus="); 3712 len += sprintf(buf + len, " cpus=");
3710 len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50, 3713 len += cpulist_scnprintf(buf + len, PAGE_SIZE - len - 50,
3711 to_cpumask(l->cpus)); 3714 to_cpumask(l->cpus));
3712 } 3715 }
3713 3716
3714 if (num_online_nodes() > 1 && !nodes_empty(l->nodes) && 3717 if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
3715 len < PAGE_SIZE - 60) { 3718 len < PAGE_SIZE - 60) {
3716 len += sprintf(buf + len, " nodes="); 3719 len += sprintf(buf + len, " nodes=");
3717 len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50, 3720 len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
3718 l->nodes); 3721 l->nodes);
3719 } 3722 }
3720 3723
3721 len += sprintf(buf + len, "\n"); 3724 len += sprintf(buf + len, "\n");
3722 } 3725 }
3723 3726
3724 free_loc_track(&t); 3727 free_loc_track(&t);
3725 if (!t.count) 3728 if (!t.count)
3726 len += sprintf(buf, "No data\n"); 3729 len += sprintf(buf, "No data\n");
3727 return len; 3730 return len;
3728 } 3731 }
3729 3732
3730 enum slab_stat_type { 3733 enum slab_stat_type {
3731 SL_ALL, /* All slabs */ 3734 SL_ALL, /* All slabs */
3732 SL_PARTIAL, /* Only partially allocated slabs */ 3735 SL_PARTIAL, /* Only partially allocated slabs */
3733 SL_CPU, /* Only slabs used for cpu caches */ 3736 SL_CPU, /* Only slabs used for cpu caches */
3734 SL_OBJECTS, /* Determine allocated objects not slabs */ 3737 SL_OBJECTS, /* Determine allocated objects not slabs */
3735 SL_TOTAL /* Determine object capacity not slabs */ 3738 SL_TOTAL /* Determine object capacity not slabs */
3736 }; 3739 };
3737 3740
3738 #define SO_ALL (1 << SL_ALL) 3741 #define SO_ALL (1 << SL_ALL)
3739 #define SO_PARTIAL (1 << SL_PARTIAL) 3742 #define SO_PARTIAL (1 << SL_PARTIAL)
3740 #define SO_CPU (1 << SL_CPU) 3743 #define SO_CPU (1 << SL_CPU)
3741 #define SO_OBJECTS (1 << SL_OBJECTS) 3744 #define SO_OBJECTS (1 << SL_OBJECTS)
3742 #define SO_TOTAL (1 << SL_TOTAL) 3745 #define SO_TOTAL (1 << SL_TOTAL)
3743 3746
3744 static ssize_t show_slab_objects(struct kmem_cache *s, 3747 static ssize_t show_slab_objects(struct kmem_cache *s,
3745 char *buf, unsigned long flags) 3748 char *buf, unsigned long flags)
3746 { 3749 {
3747 unsigned long total = 0; 3750 unsigned long total = 0;
3748 int node; 3751 int node;
3749 int x; 3752 int x;
3750 unsigned long *nodes; 3753 unsigned long *nodes;
3751 unsigned long *per_cpu; 3754 unsigned long *per_cpu;
3752 3755
3753 nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); 3756 nodes = kzalloc(2 * sizeof(unsigned long) * nr_node_ids, GFP_KERNEL);
3754 if (!nodes) 3757 if (!nodes)
3755 return -ENOMEM; 3758 return -ENOMEM;
3756 per_cpu = nodes + nr_node_ids; 3759 per_cpu = nodes + nr_node_ids;
3757 3760
3758 if (flags & SO_CPU) { 3761 if (flags & SO_CPU) {
3759 int cpu; 3762 int cpu;
3760 3763
3761 for_each_possible_cpu(cpu) { 3764 for_each_possible_cpu(cpu) {
3762 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); 3765 struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
3763 3766
3764 if (!c || c->node < 0) 3767 if (!c || c->node < 0)
3765 continue; 3768 continue;
3766 3769
3767 if (c->page) { 3770 if (c->page) {
3768 if (flags & SO_TOTAL) 3771 if (flags & SO_TOTAL)
3769 x = c->page->objects; 3772 x = c->page->objects;
3770 else if (flags & SO_OBJECTS) 3773 else if (flags & SO_OBJECTS)
3771 x = c->page->inuse; 3774 x = c->page->inuse;
3772 else 3775 else
3773 x = 1; 3776 x = 1;
3774 3777
3775 total += x; 3778 total += x;
3776 nodes[c->node] += x; 3779 nodes[c->node] += x;
3777 } 3780 }
3778 per_cpu[c->node]++; 3781 per_cpu[c->node]++;
3779 } 3782 }
3780 } 3783 }
3781 3784
3782 if (flags & SO_ALL) { 3785 if (flags & SO_ALL) {
3783 for_each_node_state(node, N_NORMAL_MEMORY) { 3786 for_each_node_state(node, N_NORMAL_MEMORY) {
3784 struct kmem_cache_node *n = get_node(s, node); 3787 struct kmem_cache_node *n = get_node(s, node);
3785 3788
3786 if (flags & SO_TOTAL) 3789 if (flags & SO_TOTAL)
3787 x = atomic_long_read(&n->total_objects); 3790 x = atomic_long_read(&n->total_objects);
3788 else if (flags & SO_OBJECTS) 3791 else if (flags & SO_OBJECTS)
3789 x = atomic_long_read(&n->total_objects) - 3792 x = atomic_long_read(&n->total_objects) -
3790 count_partial(n, count_free); 3793 count_partial(n, count_free);
3791 3794
3792 else 3795 else
3793 x = atomic_long_read(&n->nr_slabs); 3796 x = atomic_long_read(&n->nr_slabs);
3794 total += x; 3797 total += x;
3795 nodes[node] += x; 3798 nodes[node] += x;
3796 } 3799 }
3797 3800
3798 } else if (flags & SO_PARTIAL) { 3801 } else if (flags & SO_PARTIAL) {
3799 for_each_node_state(node, N_NORMAL_MEMORY) { 3802 for_each_node_state(node, N_NORMAL_MEMORY) {
3800 struct kmem_cache_node *n = get_node(s, node); 3803 struct kmem_cache_node *n = get_node(s, node);
3801 3804
3802 if (flags & SO_TOTAL) 3805 if (flags & SO_TOTAL)
3803 x = count_partial(n, count_total); 3806 x = count_partial(n, count_total);
3804 else if (flags & SO_OBJECTS) 3807 else if (flags & SO_OBJECTS)
3805 x = count_partial(n, count_inuse); 3808 x = count_partial(n, count_inuse);
3806 else 3809 else
3807 x = n->nr_partial; 3810 x = n->nr_partial;
3808 total += x; 3811 total += x;
3809 nodes[node] += x; 3812 nodes[node] += x;
3810 } 3813 }
3811 } 3814 }
3812 x = sprintf(buf, "%lu", total); 3815 x = sprintf(buf, "%lu", total);
3813 #ifdef CONFIG_NUMA 3816 #ifdef CONFIG_NUMA
3814 for_each_node_state(node, N_NORMAL_MEMORY) 3817 for_each_node_state(node, N_NORMAL_MEMORY)
3815 if (nodes[node]) 3818 if (nodes[node])
3816 x += sprintf(buf + x, " N%d=%lu", 3819 x += sprintf(buf + x, " N%d=%lu",
3817 node, nodes[node]); 3820 node, nodes[node]);
3818 #endif 3821 #endif
3819 kfree(nodes); 3822 kfree(nodes);
3820 return x + sprintf(buf + x, "\n"); 3823 return x + sprintf(buf + x, "\n");
3821 } 3824 }
3822 3825
3823 static int any_slab_objects(struct kmem_cache *s) 3826 static int any_slab_objects(struct kmem_cache *s)
3824 { 3827 {
3825 int node; 3828 int node;
3826 3829
3827 for_each_online_node(node) { 3830 for_each_online_node(node) {
3828 struct kmem_cache_node *n = get_node(s, node); 3831 struct kmem_cache_node *n = get_node(s, node);
3829 3832
3830 if (!n) 3833 if (!n)
3831 continue; 3834 continue;
3832 3835
3833 if (atomic_long_read(&n->total_objects)) 3836 if (atomic_long_read(&n->total_objects))
3834 return 1; 3837 return 1;
3835 } 3838 }
3836 return 0; 3839 return 0;
3837 } 3840 }
3838 3841
3839 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) 3842 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
3840 #define to_slab(n) container_of(n, struct kmem_cache, kobj); 3843 #define to_slab(n) container_of(n, struct kmem_cache, kobj);
3841 3844
3842 struct slab_attribute { 3845 struct slab_attribute {
3843 struct attribute attr; 3846 struct attribute attr;
3844 ssize_t (*show)(struct kmem_cache *s, char *buf); 3847 ssize_t (*show)(struct kmem_cache *s, char *buf);
3845 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); 3848 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
3846 }; 3849 };
3847 3850
3848 #define SLAB_ATTR_RO(_name) \ 3851 #define SLAB_ATTR_RO(_name) \
3849 static struct slab_attribute _name##_attr = __ATTR_RO(_name) 3852 static struct slab_attribute _name##_attr = __ATTR_RO(_name)
3850 3853
3851 #define SLAB_ATTR(_name) \ 3854 #define SLAB_ATTR(_name) \
3852 static struct slab_attribute _name##_attr = \ 3855 static struct slab_attribute _name##_attr = \
3853 __ATTR(_name, 0644, _name##_show, _name##_store) 3856 __ATTR(_name, 0644, _name##_show, _name##_store)
3854 3857
3855 static ssize_t slab_size_show(struct kmem_cache *s, char *buf) 3858 static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
3856 { 3859 {
3857 return sprintf(buf, "%d\n", s->size); 3860 return sprintf(buf, "%d\n", s->size);
3858 } 3861 }
3859 SLAB_ATTR_RO(slab_size); 3862 SLAB_ATTR_RO(slab_size);
3860 3863
3861 static ssize_t align_show(struct kmem_cache *s, char *buf) 3864 static ssize_t align_show(struct kmem_cache *s, char *buf)
3862 { 3865 {
3863 return sprintf(buf, "%d\n", s->align); 3866 return sprintf(buf, "%d\n", s->align);
3864 } 3867 }
3865 SLAB_ATTR_RO(align); 3868 SLAB_ATTR_RO(align);
3866 3869
3867 static ssize_t object_size_show(struct kmem_cache *s, char *buf) 3870 static ssize_t object_size_show(struct kmem_cache *s, char *buf)
3868 { 3871 {
3869 return sprintf(buf, "%d\n", s->objsize); 3872 return sprintf(buf, "%d\n", s->objsize);
3870 } 3873 }
3871 SLAB_ATTR_RO(object_size); 3874 SLAB_ATTR_RO(object_size);
3872 3875
3873 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) 3876 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
3874 { 3877 {
3875 return sprintf(buf, "%d\n", oo_objects(s->oo)); 3878 return sprintf(buf, "%d\n", oo_objects(s->oo));
3876 } 3879 }
3877 SLAB_ATTR_RO(objs_per_slab); 3880 SLAB_ATTR_RO(objs_per_slab);
3878 3881
3879 static ssize_t order_store(struct kmem_cache *s, 3882 static ssize_t order_store(struct kmem_cache *s,
3880 const char *buf, size_t length) 3883 const char *buf, size_t length)
3881 { 3884 {
3882 unsigned long order; 3885 unsigned long order;
3883 int err; 3886 int err;
3884 3887
3885 err = strict_strtoul(buf, 10, &order); 3888 err = strict_strtoul(buf, 10, &order);
3886 if (err) 3889 if (err)
3887 return err; 3890 return err;
3888 3891
3889 if (order > slub_max_order || order < slub_min_order) 3892 if (order > slub_max_order || order < slub_min_order)
3890 return -EINVAL; 3893 return -EINVAL;
3891 3894
3892 calculate_sizes(s, order); 3895 calculate_sizes(s, order);
3893 return length; 3896 return length;
3894 } 3897 }
3895 3898
3896 static ssize_t order_show(struct kmem_cache *s, char *buf) 3899 static ssize_t order_show(struct kmem_cache *s, char *buf)
3897 { 3900 {
3898 return sprintf(buf, "%d\n", oo_order(s->oo)); 3901 return sprintf(buf, "%d\n", oo_order(s->oo));
3899 } 3902 }
3900 SLAB_ATTR(order); 3903 SLAB_ATTR(order);
3901 3904
3902 static ssize_t min_partial_show(struct kmem_cache *s, char *buf) 3905 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
3903 { 3906 {
3904 return sprintf(buf, "%lu\n", s->min_partial); 3907 return sprintf(buf, "%lu\n", s->min_partial);
3905 } 3908 }
3906 3909
3907 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, 3910 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
3908 size_t length) 3911 size_t length)
3909 { 3912 {
3910 unsigned long min; 3913 unsigned long min;
3911 int err; 3914 int err;
3912 3915
3913 err = strict_strtoul(buf, 10, &min); 3916 err = strict_strtoul(buf, 10, &min);
3914 if (err) 3917 if (err)
3915 return err; 3918 return err;
3916 3919
3917 set_min_partial(s, min); 3920 set_min_partial(s, min);
3918 return length; 3921 return length;
3919 } 3922 }
3920 SLAB_ATTR(min_partial); 3923 SLAB_ATTR(min_partial);
3921 3924
3922 static ssize_t ctor_show(struct kmem_cache *s, char *buf) 3925 static ssize_t ctor_show(struct kmem_cache *s, char *buf)
3923 { 3926 {
3924 if (s->ctor) { 3927 if (s->ctor) {
3925 int n = sprint_symbol(buf, (unsigned long)s->ctor); 3928 int n = sprint_symbol(buf, (unsigned long)s->ctor);
3926 3929
3927 return n + sprintf(buf + n, "\n"); 3930 return n + sprintf(buf + n, "\n");
3928 } 3931 }
3929 return 0; 3932 return 0;
3930 } 3933 }
3931 SLAB_ATTR_RO(ctor); 3934 SLAB_ATTR_RO(ctor);
3932 3935
3933 static ssize_t aliases_show(struct kmem_cache *s, char *buf) 3936 static ssize_t aliases_show(struct kmem_cache *s, char *buf)
3934 { 3937 {
3935 return sprintf(buf, "%d\n", s->refcount - 1); 3938 return sprintf(buf, "%d\n", s->refcount - 1);
3936 } 3939 }
3937 SLAB_ATTR_RO(aliases); 3940 SLAB_ATTR_RO(aliases);
3938 3941
3939 static ssize_t slabs_show(struct kmem_cache *s, char *buf) 3942 static ssize_t slabs_show(struct kmem_cache *s, char *buf)
3940 { 3943 {
3941 return show_slab_objects(s, buf, SO_ALL); 3944 return show_slab_objects(s, buf, SO_ALL);
3942 } 3945 }
3943 SLAB_ATTR_RO(slabs); 3946 SLAB_ATTR_RO(slabs);
3944 3947
3945 static ssize_t partial_show(struct kmem_cache *s, char *buf) 3948 static ssize_t partial_show(struct kmem_cache *s, char *buf)
3946 { 3949 {
3947 return show_slab_objects(s, buf, SO_PARTIAL); 3950 return show_slab_objects(s, buf, SO_PARTIAL);
3948 } 3951 }
3949 SLAB_ATTR_RO(partial); 3952 SLAB_ATTR_RO(partial);
3950 3953
3951 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) 3954 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
3952 { 3955 {
3953 return show_slab_objects(s, buf, SO_CPU); 3956 return show_slab_objects(s, buf, SO_CPU);
3954 } 3957 }
3955 SLAB_ATTR_RO(cpu_slabs); 3958 SLAB_ATTR_RO(cpu_slabs);
3956 3959
3957 static ssize_t objects_show(struct kmem_cache *s, char *buf) 3960 static ssize_t objects_show(struct kmem_cache *s, char *buf)
3958 { 3961 {
3959 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); 3962 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
3960 } 3963 }
3961 SLAB_ATTR_RO(objects); 3964 SLAB_ATTR_RO(objects);
3962 3965
3963 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) 3966 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
3964 { 3967 {
3965 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); 3968 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
3966 } 3969 }
3967 SLAB_ATTR_RO(objects_partial); 3970 SLAB_ATTR_RO(objects_partial);
3968 3971
3969 static ssize_t total_objects_show(struct kmem_cache *s, char *buf) 3972 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
3970 { 3973 {
3971 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); 3974 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
3972 } 3975 }
3973 SLAB_ATTR_RO(total_objects); 3976 SLAB_ATTR_RO(total_objects);
3974 3977
3975 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) 3978 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
3976 { 3979 {
3977 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); 3980 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
3978 } 3981 }
3979 3982
3980 static ssize_t sanity_checks_store(struct kmem_cache *s, 3983 static ssize_t sanity_checks_store(struct kmem_cache *s,
3981 const char *buf, size_t length) 3984 const char *buf, size_t length)
3982 { 3985 {
3983 s->flags &= ~SLAB_DEBUG_FREE; 3986 s->flags &= ~SLAB_DEBUG_FREE;
3984 if (buf[0] == '1') 3987 if (buf[0] == '1')
3985 s->flags |= SLAB_DEBUG_FREE; 3988 s->flags |= SLAB_DEBUG_FREE;
3986 return length; 3989 return length;
3987 } 3990 }
3988 SLAB_ATTR(sanity_checks); 3991 SLAB_ATTR(sanity_checks);
3989 3992
3990 static ssize_t trace_show(struct kmem_cache *s, char *buf) 3993 static ssize_t trace_show(struct kmem_cache *s, char *buf)
3991 { 3994 {
3992 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); 3995 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
3993 } 3996 }
3994 3997
3995 static ssize_t trace_store(struct kmem_cache *s, const char *buf, 3998 static ssize_t trace_store(struct kmem_cache *s, const char *buf,
3996 size_t length) 3999 size_t length)
3997 { 4000 {
3998 s->flags &= ~SLAB_TRACE; 4001 s->flags &= ~SLAB_TRACE;
3999 if (buf[0] == '1') 4002 if (buf[0] == '1')
4000 s->flags |= SLAB_TRACE; 4003 s->flags |= SLAB_TRACE;
4001 return length; 4004 return length;
4002 } 4005 }
4003 SLAB_ATTR(trace); 4006 SLAB_ATTR(trace);
4004 4007
4005 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) 4008 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
4006 { 4009 {
4007 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); 4010 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
4008 } 4011 }
4009 4012
4010 static ssize_t reclaim_account_store(struct kmem_cache *s, 4013 static ssize_t reclaim_account_store(struct kmem_cache *s,
4011 const char *buf, size_t length) 4014 const char *buf, size_t length)
4012 { 4015 {
4013 s->flags &= ~SLAB_RECLAIM_ACCOUNT; 4016 s->flags &= ~SLAB_RECLAIM_ACCOUNT;
4014 if (buf[0] == '1') 4017 if (buf[0] == '1')
4015 s->flags |= SLAB_RECLAIM_ACCOUNT; 4018 s->flags |= SLAB_RECLAIM_ACCOUNT;
4016 return length; 4019 return length;
4017 } 4020 }
4018 SLAB_ATTR(reclaim_account); 4021 SLAB_ATTR(reclaim_account);
4019 4022
4020 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) 4023 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
4021 { 4024 {
4022 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); 4025 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
4023 } 4026 }
4024 SLAB_ATTR_RO(hwcache_align); 4027 SLAB_ATTR_RO(hwcache_align);
4025 4028
4026 #ifdef CONFIG_ZONE_DMA 4029 #ifdef CONFIG_ZONE_DMA
4027 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) 4030 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
4028 { 4031 {
4029 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); 4032 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
4030 } 4033 }
4031 SLAB_ATTR_RO(cache_dma); 4034 SLAB_ATTR_RO(cache_dma);
4032 #endif 4035 #endif
4033 4036
4034 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) 4037 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
4035 { 4038 {
4036 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); 4039 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU));
4037 } 4040 }
4038 SLAB_ATTR_RO(destroy_by_rcu); 4041 SLAB_ATTR_RO(destroy_by_rcu);
4039 4042
4040 static ssize_t red_zone_show(struct kmem_cache *s, char *buf) 4043 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
4041 { 4044 {
4042 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); 4045 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
4043 } 4046 }
4044 4047
4045 static ssize_t red_zone_store(struct kmem_cache *s, 4048 static ssize_t red_zone_store(struct kmem_cache *s,
4046 const char *buf, size_t length) 4049 const char *buf, size_t length)
4047 { 4050 {
4048 if (any_slab_objects(s)) 4051 if (any_slab_objects(s))
4049 return -EBUSY; 4052 return -EBUSY;
4050 4053
4051 s->flags &= ~SLAB_RED_ZONE; 4054 s->flags &= ~SLAB_RED_ZONE;
4052 if (buf[0] == '1') 4055 if (buf[0] == '1')
4053 s->flags |= SLAB_RED_ZONE; 4056 s->flags |= SLAB_RED_ZONE;
4054 calculate_sizes(s, -1); 4057 calculate_sizes(s, -1);
4055 return length; 4058 return length;
4056 } 4059 }
4057 SLAB_ATTR(red_zone); 4060 SLAB_ATTR(red_zone);
4058 4061
4059 static ssize_t poison_show(struct kmem_cache *s, char *buf) 4062 static ssize_t poison_show(struct kmem_cache *s, char *buf)
4060 { 4063 {
4061 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); 4064 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
4062 } 4065 }
4063 4066
4064 static ssize_t poison_store(struct kmem_cache *s, 4067 static ssize_t poison_store(struct kmem_cache *s,
4065 const char *buf, size_t length) 4068 const char *buf, size_t length)
4066 { 4069 {
4067 if (any_slab_objects(s)) 4070 if (any_slab_objects(s))
4068 return -EBUSY; 4071 return -EBUSY;
4069 4072
4070 s->flags &= ~SLAB_POISON; 4073 s->flags &= ~SLAB_POISON;
4071 if (buf[0] == '1') 4074 if (buf[0] == '1')
4072 s->flags |= SLAB_POISON; 4075 s->flags |= SLAB_POISON;
4073 calculate_sizes(s, -1); 4076 calculate_sizes(s, -1);
4074 return length; 4077 return length;
4075 } 4078 }
4076 SLAB_ATTR(poison); 4079 SLAB_ATTR(poison);
4077 4080
4078 static ssize_t store_user_show(struct kmem_cache *s, char *buf) 4081 static ssize_t store_user_show(struct kmem_cache *s, char *buf)
4079 { 4082 {
4080 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); 4083 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
4081 } 4084 }
4082 4085
4083 static ssize_t store_user_store(struct kmem_cache *s, 4086 static ssize_t store_user_store(struct kmem_cache *s,
4084 const char *buf, size_t length) 4087 const char *buf, size_t length)
4085 { 4088 {
4086 if (any_slab_objects(s)) 4089 if (any_slab_objects(s))
4087 return -EBUSY; 4090 return -EBUSY;
4088 4091
4089 s->flags &= ~SLAB_STORE_USER; 4092 s->flags &= ~SLAB_STORE_USER;
4090 if (buf[0] == '1') 4093 if (buf[0] == '1')
4091 s->flags |= SLAB_STORE_USER; 4094 s->flags |= SLAB_STORE_USER;
4092 calculate_sizes(s, -1); 4095 calculate_sizes(s, -1);
4093 return length; 4096 return length;
4094 } 4097 }
4095 SLAB_ATTR(store_user); 4098 SLAB_ATTR(store_user);
4096 4099
4097 static ssize_t validate_show(struct kmem_cache *s, char *buf) 4100 static ssize_t validate_show(struct kmem_cache *s, char *buf)
4098 { 4101 {
4099 return 0; 4102 return 0;
4100 } 4103 }
4101 4104
4102 static ssize_t validate_store(struct kmem_cache *s, 4105 static ssize_t validate_store(struct kmem_cache *s,
4103 const char *buf, size_t length) 4106 const char *buf, size_t length)
4104 { 4107 {
4105 int ret = -EINVAL; 4108 int ret = -EINVAL;
4106 4109
4107 if (buf[0] == '1') { 4110 if (buf[0] == '1') {
4108 ret = validate_slab_cache(s); 4111 ret = validate_slab_cache(s);
4109 if (ret >= 0) 4112 if (ret >= 0)
4110 ret = length; 4113 ret = length;
4111 } 4114 }
4112 return ret; 4115 return ret;
4113 } 4116 }
4114 SLAB_ATTR(validate); 4117 SLAB_ATTR(validate);
4115 4118
4116 static ssize_t shrink_show(struct kmem_cache *s, char *buf) 4119 static ssize_t shrink_show(struct kmem_cache *s, char *buf)
4117 { 4120 {
4118 return 0; 4121 return 0;
4119 } 4122 }
4120 4123
4121 static ssize_t shrink_store(struct kmem_cache *s, 4124 static ssize_t shrink_store(struct kmem_cache *s,
4122 const char *buf, size_t length) 4125 const char *buf, size_t length)
4123 { 4126 {
4124 if (buf[0] == '1') { 4127 if (buf[0] == '1') {
4125 int rc = kmem_cache_shrink(s); 4128 int rc = kmem_cache_shrink(s);
4126 4129
4127 if (rc) 4130 if (rc)
4128 return rc; 4131 return rc;
4129 } else 4132 } else
4130 return -EINVAL; 4133 return -EINVAL;
4131 return length; 4134 return length;
4132 } 4135 }
4133 SLAB_ATTR(shrink); 4136 SLAB_ATTR(shrink);
4134 4137
4135 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) 4138 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf)
4136 { 4139 {
4137 if (!(s->flags & SLAB_STORE_USER)) 4140 if (!(s->flags & SLAB_STORE_USER))
4138 return -ENOSYS; 4141 return -ENOSYS;
4139 return list_locations(s, buf, TRACK_ALLOC); 4142 return list_locations(s, buf, TRACK_ALLOC);
4140 } 4143 }
4141 SLAB_ATTR_RO(alloc_calls); 4144 SLAB_ATTR_RO(alloc_calls);
4142 4145
4143 static ssize_t free_calls_show(struct kmem_cache *s, char *buf) 4146 static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
4144 { 4147 {
4145 if (!(s->flags & SLAB_STORE_USER)) 4148 if (!(s->flags & SLAB_STORE_USER))
4146 return -ENOSYS; 4149 return -ENOSYS;
4147 return list_locations(s, buf, TRACK_FREE); 4150 return list_locations(s, buf, TRACK_FREE);
4148 } 4151 }
4149 SLAB_ATTR_RO(free_calls); 4152 SLAB_ATTR_RO(free_calls);
4150 4153
4151 #ifdef CONFIG_NUMA 4154 #ifdef CONFIG_NUMA
4152 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) 4155 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
4153 { 4156 {
4154 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10); 4157 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10);
4155 } 4158 }
4156 4159
4157 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, 4160 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
4158 const char *buf, size_t length) 4161 const char *buf, size_t length)
4159 { 4162 {
4160 unsigned long ratio; 4163 unsigned long ratio;
4161 int err; 4164 int err;
4162 4165
4163 err = strict_strtoul(buf, 10, &ratio); 4166 err = strict_strtoul(buf, 10, &ratio);
4164 if (err) 4167 if (err)
4165 return err; 4168 return err;
4166 4169
4167 if (ratio <= 100) 4170 if (ratio <= 100)
4168 s->remote_node_defrag_ratio = ratio * 10; 4171 s->remote_node_defrag_ratio = ratio * 10;
4169 4172
4170 return length; 4173 return length;
4171 } 4174 }
4172 SLAB_ATTR(remote_node_defrag_ratio); 4175 SLAB_ATTR(remote_node_defrag_ratio);
4173 #endif 4176 #endif
4174 4177
4175 #ifdef CONFIG_SLUB_STATS 4178 #ifdef CONFIG_SLUB_STATS
4176 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) 4179 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
4177 { 4180 {
4178 unsigned long sum = 0; 4181 unsigned long sum = 0;
4179 int cpu; 4182 int cpu;
4180 int len; 4183 int len;
4181 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); 4184 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL);
4182 4185
4183 if (!data) 4186 if (!data)
4184 return -ENOMEM; 4187 return -ENOMEM;
4185 4188
4186 for_each_online_cpu(cpu) { 4189 for_each_online_cpu(cpu) {
4187 unsigned x = get_cpu_slab(s, cpu)->stat[si]; 4190 unsigned x = get_cpu_slab(s, cpu)->stat[si];
4188 4191
4189 data[cpu] = x; 4192 data[cpu] = x;
4190 sum += x; 4193 sum += x;
4191 } 4194 }
4192 4195
4193 len = sprintf(buf, "%lu", sum); 4196 len = sprintf(buf, "%lu", sum);
4194 4197
4195 #ifdef CONFIG_SMP 4198 #ifdef CONFIG_SMP
4196 for_each_online_cpu(cpu) { 4199 for_each_online_cpu(cpu) {
4197 if (data[cpu] && len < PAGE_SIZE - 20) 4200 if (data[cpu] && len < PAGE_SIZE - 20)
4198 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]); 4201 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
4199 } 4202 }
4200 #endif 4203 #endif
4201 kfree(data); 4204 kfree(data);
4202 return len + sprintf(buf + len, "\n"); 4205 return len + sprintf(buf + len, "\n");
4203 } 4206 }
4204 4207
4205 #define STAT_ATTR(si, text) \ 4208 #define STAT_ATTR(si, text) \
4206 static ssize_t text##_show(struct kmem_cache *s, char *buf) \ 4209 static ssize_t text##_show(struct kmem_cache *s, char *buf) \
4207 { \ 4210 { \
4208 return show_stat(s, buf, si); \ 4211 return show_stat(s, buf, si); \
4209 } \ 4212 } \
4210 SLAB_ATTR_RO(text); \ 4213 SLAB_ATTR_RO(text); \
4211 4214
4212 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 4215 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
4213 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 4216 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
4214 STAT_ATTR(FREE_FASTPATH, free_fastpath); 4217 STAT_ATTR(FREE_FASTPATH, free_fastpath);
4215 STAT_ATTR(FREE_SLOWPATH, free_slowpath); 4218 STAT_ATTR(FREE_SLOWPATH, free_slowpath);
4216 STAT_ATTR(FREE_FROZEN, free_frozen); 4219 STAT_ATTR(FREE_FROZEN, free_frozen);
4217 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 4220 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
4218 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 4221 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
4219 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 4222 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
4220 STAT_ATTR(ALLOC_SLAB, alloc_slab); 4223 STAT_ATTR(ALLOC_SLAB, alloc_slab);
4221 STAT_ATTR(ALLOC_REFILL, alloc_refill); 4224 STAT_ATTR(ALLOC_REFILL, alloc_refill);
4222 STAT_ATTR(FREE_SLAB, free_slab); 4225 STAT_ATTR(FREE_SLAB, free_slab);
4223 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 4226 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
4224 STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 4227 STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
4225 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 4228 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
4226 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); 4229 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
4227 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); 4230 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
4228 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 4231 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
4229 STAT_ATTR(ORDER_FALLBACK, order_fallback); 4232 STAT_ATTR(ORDER_FALLBACK, order_fallback);
4230 #endif 4233 #endif
4231 4234
4232 static struct attribute *slab_attrs[] = { 4235 static struct attribute *slab_attrs[] = {
4233 &slab_size_attr.attr, 4236 &slab_size_attr.attr,
4234 &object_size_attr.attr, 4237 &object_size_attr.attr,
4235 &objs_per_slab_attr.attr, 4238 &objs_per_slab_attr.attr,
4236 &order_attr.attr, 4239 &order_attr.attr,
4237 &min_partial_attr.attr, 4240 &min_partial_attr.attr,
4238 &objects_attr.attr, 4241 &objects_attr.attr,
4239 &objects_partial_attr.attr, 4242 &objects_partial_attr.attr,
4240 &total_objects_attr.attr, 4243 &total_objects_attr.attr,
4241 &slabs_attr.attr, 4244 &slabs_attr.attr,
4242 &partial_attr.attr, 4245 &partial_attr.attr,
4243 &cpu_slabs_attr.attr, 4246 &cpu_slabs_attr.attr,
4244 &ctor_attr.attr, 4247 &ctor_attr.attr,
4245 &aliases_attr.attr, 4248 &aliases_attr.attr,
4246 &align_attr.attr, 4249 &align_attr.attr,
4247 &sanity_checks_attr.attr, 4250 &sanity_checks_attr.attr,
4248 &trace_attr.attr, 4251 &trace_attr.attr,
4249 &hwcache_align_attr.attr, 4252 &hwcache_align_attr.attr,
4250 &reclaim_account_attr.attr, 4253 &reclaim_account_attr.attr,
4251 &destroy_by_rcu_attr.attr, 4254 &destroy_by_rcu_attr.attr,
4252 &red_zone_attr.attr, 4255 &red_zone_attr.attr,
4253 &poison_attr.attr, 4256 &poison_attr.attr,
4254 &store_user_attr.attr, 4257 &store_user_attr.attr,
4255 &validate_attr.attr, 4258 &validate_attr.attr,
4256 &shrink_attr.attr, 4259 &shrink_attr.attr,
4257 &alloc_calls_attr.attr, 4260 &alloc_calls_attr.attr,
4258 &free_calls_attr.attr, 4261 &free_calls_attr.attr,
4259 #ifdef CONFIG_ZONE_DMA 4262 #ifdef CONFIG_ZONE_DMA
4260 &cache_dma_attr.attr, 4263 &cache_dma_attr.attr,
4261 #endif 4264 #endif
4262 #ifdef CONFIG_NUMA 4265 #ifdef CONFIG_NUMA
4263 &remote_node_defrag_ratio_attr.attr, 4266 &remote_node_defrag_ratio_attr.attr,
4264 #endif 4267 #endif
4265 #ifdef CONFIG_SLUB_STATS 4268 #ifdef CONFIG_SLUB_STATS
4266 &alloc_fastpath_attr.attr, 4269 &alloc_fastpath_attr.attr,
4267 &alloc_slowpath_attr.attr, 4270 &alloc_slowpath_attr.attr,
4268 &free_fastpath_attr.attr, 4271 &free_fastpath_attr.attr,
4269 &free_slowpath_attr.attr, 4272 &free_slowpath_attr.attr,
4270 &free_frozen_attr.attr, 4273 &free_frozen_attr.attr,
4271 &free_add_partial_attr.attr, 4274 &free_add_partial_attr.attr,
4272 &free_remove_partial_attr.attr, 4275 &free_remove_partial_attr.attr,
4273 &alloc_from_partial_attr.attr, 4276 &alloc_from_partial_attr.attr,
4274 &alloc_slab_attr.attr, 4277 &alloc_slab_attr.attr,
4275 &alloc_refill_attr.attr, 4278 &alloc_refill_attr.attr,
4276 &free_slab_attr.attr, 4279 &free_slab_attr.attr,
4277 &cpuslab_flush_attr.attr, 4280 &cpuslab_flush_attr.attr,
4278 &deactivate_full_attr.attr, 4281 &deactivate_full_attr.attr,
4279 &deactivate_empty_attr.attr, 4282 &deactivate_empty_attr.attr,
4280 &deactivate_to_head_attr.attr, 4283 &deactivate_to_head_attr.attr,
4281 &deactivate_to_tail_attr.attr, 4284 &deactivate_to_tail_attr.attr,
4282 &deactivate_remote_frees_attr.attr, 4285 &deactivate_remote_frees_attr.attr,
4283 &order_fallback_attr.attr, 4286 &order_fallback_attr.attr,
4284 #endif 4287 #endif
4285 NULL 4288 NULL
4286 }; 4289 };
4287 4290
4288 static struct attribute_group slab_attr_group = { 4291 static struct attribute_group slab_attr_group = {
4289 .attrs = slab_attrs, 4292 .attrs = slab_attrs,
4290 }; 4293 };
4291 4294
4292 static ssize_t slab_attr_show(struct kobject *kobj, 4295 static ssize_t slab_attr_show(struct kobject *kobj,
4293 struct attribute *attr, 4296 struct attribute *attr,
4294 char *buf) 4297 char *buf)
4295 { 4298 {
4296 struct slab_attribute *attribute; 4299 struct slab_attribute *attribute;
4297 struct kmem_cache *s; 4300 struct kmem_cache *s;
4298 int err; 4301 int err;
4299 4302
4300 attribute = to_slab_attr(attr); 4303 attribute = to_slab_attr(attr);
4301 s = to_slab(kobj); 4304 s = to_slab(kobj);
4302 4305
4303 if (!attribute->show) 4306 if (!attribute->show)
4304 return -EIO; 4307 return -EIO;
4305 4308
4306 err = attribute->show(s, buf); 4309 err = attribute->show(s, buf);
4307 4310
4308 return err; 4311 return err;
4309 } 4312 }
4310 4313
4311 static ssize_t slab_attr_store(struct kobject *kobj, 4314 static ssize_t slab_attr_store(struct kobject *kobj,
4312 struct attribute *attr, 4315 struct attribute *attr,
4313 const char *buf, size_t len) 4316 const char *buf, size_t len)
4314 { 4317 {
4315 struct slab_attribute *attribute; 4318 struct slab_attribute *attribute;
4316 struct kmem_cache *s; 4319 struct kmem_cache *s;
4317 int err; 4320 int err;
4318 4321
4319 attribute = to_slab_attr(attr); 4322 attribute = to_slab_attr(attr);
4320 s = to_slab(kobj); 4323 s = to_slab(kobj);
4321 4324
4322 if (!attribute->store) 4325 if (!attribute->store)
4323 return -EIO; 4326 return -EIO;
4324 4327
4325 err = attribute->store(s, buf, len); 4328 err = attribute->store(s, buf, len);
4326 4329
4327 return err; 4330 return err;
4328 } 4331 }
4329 4332
4330 static void kmem_cache_release(struct kobject *kobj) 4333 static void kmem_cache_release(struct kobject *kobj)
4331 { 4334 {
4332 struct kmem_cache *s = to_slab(kobj); 4335 struct kmem_cache *s = to_slab(kobj);
4333 4336
4334 kfree(s); 4337 kfree(s);
4335 } 4338 }
4336 4339
4337 static struct sysfs_ops slab_sysfs_ops = { 4340 static struct sysfs_ops slab_sysfs_ops = {
4338 .show = slab_attr_show, 4341 .show = slab_attr_show,
4339 .store = slab_attr_store, 4342 .store = slab_attr_store,
4340 }; 4343 };
4341 4344
4342 static struct kobj_type slab_ktype = { 4345 static struct kobj_type slab_ktype = {
4343 .sysfs_ops = &slab_sysfs_ops, 4346 .sysfs_ops = &slab_sysfs_ops,
4344 .release = kmem_cache_release 4347 .release = kmem_cache_release
4345 }; 4348 };
4346 4349
4347 static int uevent_filter(struct kset *kset, struct kobject *kobj) 4350 static int uevent_filter(struct kset *kset, struct kobject *kobj)
4348 { 4351 {
4349 struct kobj_type *ktype = get_ktype(kobj); 4352 struct kobj_type *ktype = get_ktype(kobj);
4350 4353
4351 if (ktype == &slab_ktype) 4354 if (ktype == &slab_ktype)
4352 return 1; 4355 return 1;
4353 return 0; 4356 return 0;
4354 } 4357 }
4355 4358
4356 static struct kset_uevent_ops slab_uevent_ops = { 4359 static struct kset_uevent_ops slab_uevent_ops = {
4357 .filter = uevent_filter, 4360 .filter = uevent_filter,
4358 }; 4361 };
4359 4362
4360 static struct kset *slab_kset; 4363 static struct kset *slab_kset;
4361 4364
4362 #define ID_STR_LENGTH 64 4365 #define ID_STR_LENGTH 64
4363 4366
4364 /* Create a unique string id for a slab cache: 4367 /* Create a unique string id for a slab cache:
4365 * 4368 *
4366 * Format :[flags-]size 4369 * Format :[flags-]size
4367 */ 4370 */
4368 static char *create_unique_id(struct kmem_cache *s) 4371 static char *create_unique_id(struct kmem_cache *s)
4369 { 4372 {
4370 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); 4373 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
4371 char *p = name; 4374 char *p = name;
4372 4375
4373 BUG_ON(!name); 4376 BUG_ON(!name);
4374 4377
4375 *p++ = ':'; 4378 *p++ = ':';
4376 /* 4379 /*
4377 * First flags affecting slabcache operations. We will only 4380 * First flags affecting slabcache operations. We will only
4378 * get here for aliasable slabs so we do not need to support 4381 * get here for aliasable slabs so we do not need to support
4379 * too many flags. The flags here must cover all flags that 4382 * too many flags. The flags here must cover all flags that
4380 * are matched during merging to guarantee that the id is 4383 * are matched during merging to guarantee that the id is
4381 * unique. 4384 * unique.
4382 */ 4385 */
4383 if (s->flags & SLAB_CACHE_DMA) 4386 if (s->flags & SLAB_CACHE_DMA)
4384 *p++ = 'd'; 4387 *p++ = 'd';
4385 if (s->flags & SLAB_RECLAIM_ACCOUNT) 4388 if (s->flags & SLAB_RECLAIM_ACCOUNT)
4386 *p++ = 'a'; 4389 *p++ = 'a';
4387 if (s->flags & SLAB_DEBUG_FREE) 4390 if (s->flags & SLAB_DEBUG_FREE)
4388 *p++ = 'F'; 4391 *p++ = 'F';
4389 if (p != name + 1) 4392 if (p != name + 1)
4390 *p++ = '-'; 4393 *p++ = '-';
4391 p += sprintf(p, "%07d", s->size); 4394 p += sprintf(p, "%07d", s->size);
4392 BUG_ON(p > name + ID_STR_LENGTH - 1); 4395 BUG_ON(p > name + ID_STR_LENGTH - 1);
4393 return name; 4396 return name;
4394 } 4397 }
4395 4398
4396 static int sysfs_slab_add(struct kmem_cache *s) 4399 static int sysfs_slab_add(struct kmem_cache *s)
4397 { 4400 {
4398 int err; 4401 int err;
4399 const char *name; 4402 const char *name;
4400 int unmergeable; 4403 int unmergeable;
4401 4404
4402 if (slab_state < SYSFS) 4405 if (slab_state < SYSFS)
4403 /* Defer until later */ 4406 /* Defer until later */
4404 return 0; 4407 return 0;
4405 4408
4406 unmergeable = slab_unmergeable(s); 4409 unmergeable = slab_unmergeable(s);
4407 if (unmergeable) { 4410 if (unmergeable) {
4408 /* 4411 /*
4409 * Slabcache can never be merged so we can use the name proper. 4412 * Slabcache can never be merged so we can use the name proper.
4410 * This is typically the case for debug situations. In that 4413 * This is typically the case for debug situations. In that
4411 * case we can catch duplicate names easily. 4414 * case we can catch duplicate names easily.
4412 */ 4415 */
4413 sysfs_remove_link(&slab_kset->kobj, s->name); 4416 sysfs_remove_link(&slab_kset->kobj, s->name);
4414 name = s->name; 4417 name = s->name;
4415 } else { 4418 } else {
4416 /* 4419 /*
4417 * Create a unique name for the slab as a target 4420 * Create a unique name for the slab as a target
4418 * for the symlinks. 4421 * for the symlinks.
4419 */ 4422 */
4420 name = create_unique_id(s); 4423 name = create_unique_id(s);
4421 } 4424 }
4422 4425
4423 s->kobj.kset = slab_kset; 4426 s->kobj.kset = slab_kset;
4424 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name); 4427 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, name);
4425 if (err) { 4428 if (err) {
4426 kobject_put(&s->kobj); 4429 kobject_put(&s->kobj);
4427 return err; 4430 return err;
4428 } 4431 }
4429 4432
4430 err = sysfs_create_group(&s->kobj, &slab_attr_group); 4433 err = sysfs_create_group(&s->kobj, &slab_attr_group);
4431 if (err) 4434 if (err)
4432 return err; 4435 return err;
4433 kobject_uevent(&s->kobj, KOBJ_ADD); 4436 kobject_uevent(&s->kobj, KOBJ_ADD);
4434 if (!unmergeable) { 4437 if (!unmergeable) {
4435 /* Setup first alias */ 4438 /* Setup first alias */
4436 sysfs_slab_alias(s, s->name); 4439 sysfs_slab_alias(s, s->name);
4437 kfree(name); 4440 kfree(name);
4438 } 4441 }
4439 return 0; 4442 return 0;
4440 } 4443 }
4441 4444
4442 static void sysfs_slab_remove(struct kmem_cache *s) 4445 static void sysfs_slab_remove(struct kmem_cache *s)
4443 { 4446 {
4444 kobject_uevent(&s->kobj, KOBJ_REMOVE); 4447 kobject_uevent(&s->kobj, KOBJ_REMOVE);
4445 kobject_del(&s->kobj); 4448 kobject_del(&s->kobj);
4446 kobject_put(&s->kobj); 4449 kobject_put(&s->kobj);
4447 } 4450 }
4448 4451
4449 /* 4452 /*
4450 * Need to buffer aliases during bootup until sysfs becomes 4453 * Need to buffer aliases during bootup until sysfs becomes
4451 * available lest we lose that information. 4454 * available lest we lose that information.
4452 */ 4455 */
4453 struct saved_alias { 4456 struct saved_alias {
4454 struct kmem_cache *s; 4457 struct kmem_cache *s;
4455 const char *name; 4458 const char *name;
4456 struct saved_alias *next; 4459 struct saved_alias *next;
4457 }; 4460 };
4458 4461
4459 static struct saved_alias *alias_list; 4462 static struct saved_alias *alias_list;
4460 4463
4461 static int sysfs_slab_alias(struct kmem_cache *s, const char *name) 4464 static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
4462 { 4465 {
4463 struct saved_alias *al; 4466 struct saved_alias *al;
4464 4467
4465 if (slab_state == SYSFS) { 4468 if (slab_state == SYSFS) {
4466 /* 4469 /*
4467 * If we have a leftover link then remove it. 4470 * If we have a leftover link then remove it.
4468 */ 4471 */
4469 sysfs_remove_link(&slab_kset->kobj, name); 4472 sysfs_remove_link(&slab_kset->kobj, name);
4470 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 4473 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
4471 } 4474 }
4472 4475
4473 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); 4476 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
4474 if (!al) 4477 if (!al)
4475 return -ENOMEM; 4478 return -ENOMEM;
4476 4479
4477 al->s = s; 4480 al->s = s;
4478 al->name = name; 4481 al->name = name;
4479 al->next = alias_list; 4482 al->next = alias_list;
4480 alias_list = al; 4483 alias_list = al;
4481 return 0; 4484 return 0;
4482 } 4485 }
4483 4486
4484 static int __init slab_sysfs_init(void) 4487 static int __init slab_sysfs_init(void)
4485 { 4488 {
4486 struct kmem_cache *s; 4489 struct kmem_cache *s;
4487 int err; 4490 int err;
4488 4491
4489 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); 4492 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj);
4490 if (!slab_kset) { 4493 if (!slab_kset) {
4491 printk(KERN_ERR "Cannot register slab subsystem.\n"); 4494 printk(KERN_ERR "Cannot register slab subsystem.\n");
4492 return -ENOSYS; 4495 return -ENOSYS;
4493 } 4496 }
4494 4497
4495 slab_state = SYSFS; 4498 slab_state = SYSFS;
4496 4499
4497 list_for_each_entry(s, &slab_caches, list) { 4500 list_for_each_entry(s, &slab_caches, list) {
4498 err = sysfs_slab_add(s); 4501 err = sysfs_slab_add(s);
4499 if (err) 4502 if (err)
4500 printk(KERN_ERR "SLUB: Unable to add boot slab %s" 4503 printk(KERN_ERR "SLUB: Unable to add boot slab %s"
4501 " to sysfs\n", s->name); 4504 " to sysfs\n", s->name);
4502 } 4505 }
4503 4506
4504 while (alias_list) { 4507 while (alias_list) {
4505 struct saved_alias *al = alias_list; 4508 struct saved_alias *al = alias_list;
4506 4509
4507 alias_list = alias_list->next; 4510 alias_list = alias_list->next;
4508 err = sysfs_slab_alias(al->s, al->name); 4511 err = sysfs_slab_alias(al->s, al->name);
4509 if (err) 4512 if (err)
4510 printk(KERN_ERR "SLUB: Unable to add boot slab alias" 4513 printk(KERN_ERR "SLUB: Unable to add boot slab alias"
4511 " %s to sysfs\n", s->name); 4514 " %s to sysfs\n", s->name);
4512 kfree(al); 4515 kfree(al);
4513 } 4516 }
4514 4517
4515 resiliency_test(); 4518 resiliency_test();
4516 return 0; 4519 return 0;
4517 } 4520 }
4518 4521
4519 __initcall(slab_sysfs_init); 4522 __initcall(slab_sysfs_init);
4520 #endif 4523 #endif
4521 4524
4522 /* 4525 /*
4523 * The /proc/slabinfo ABI 4526 * The /proc/slabinfo ABI
4524 */ 4527 */
4525 #ifdef CONFIG_SLABINFO 4528 #ifdef CONFIG_SLABINFO
4526 static void print_slabinfo_header(struct seq_file *m) 4529 static void print_slabinfo_header(struct seq_file *m)
4527 { 4530 {
4528 seq_puts(m, "slabinfo - version: 2.1\n"); 4531 seq_puts(m, "slabinfo - version: 2.1\n");
4529 seq_puts(m, "# name <active_objs> <num_objs> <objsize> " 4532 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
4530 "<objperslab> <pagesperslab>"); 4533 "<objperslab> <pagesperslab>");
4531 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 4534 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
4532 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 4535 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
4533 seq_putc(m, '\n'); 4536 seq_putc(m, '\n');
4534 } 4537 }
4535 4538
4536 static void *s_start(struct seq_file *m, loff_t *pos) 4539 static void *s_start(struct seq_file *m, loff_t *pos)
4537 { 4540 {
4538 loff_t n = *pos; 4541 loff_t n = *pos;
4539 4542
4540 down_read(&slub_lock); 4543 down_read(&slub_lock);
4541 if (!n) 4544 if (!n)
4542 print_slabinfo_header(m); 4545 print_slabinfo_header(m);
4543 4546
4544 return seq_list_start(&slab_caches, *pos); 4547 return seq_list_start(&slab_caches, *pos);
4545 } 4548 }
4546 4549
4547 static void *s_next(struct seq_file *m, void *p, loff_t *pos) 4550 static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4548 { 4551 {
4549 return seq_list_next(p, &slab_caches, pos); 4552 return seq_list_next(p, &slab_caches, pos);
4550 } 4553 }
4551 4554
4552 static void s_stop(struct seq_file *m, void *p) 4555 static void s_stop(struct seq_file *m, void *p)
4553 { 4556 {
4554 up_read(&slub_lock); 4557 up_read(&slub_lock);
4555 } 4558 }
4556 4559
4557 static int s_show(struct seq_file *m, void *p) 4560 static int s_show(struct seq_file *m, void *p)
4558 { 4561 {
4559 unsigned long nr_partials = 0; 4562 unsigned long nr_partials = 0;
4560 unsigned long nr_slabs = 0; 4563 unsigned long nr_slabs = 0;
4561 unsigned long nr_inuse = 0; 4564 unsigned long nr_inuse = 0;
4562 unsigned long nr_objs = 0; 4565 unsigned long nr_objs = 0;
4563 unsigned long nr_free = 0; 4566 unsigned long nr_free = 0;
4564 struct kmem_cache *s; 4567 struct kmem_cache *s;
4565 int node; 4568 int node;
4566 4569
4567 s = list_entry(p, struct kmem_cache, list); 4570 s = list_entry(p, struct kmem_cache, list);
4568 4571
4569 for_each_online_node(node) { 4572 for_each_online_node(node) {
4570 struct kmem_cache_node *n = get_node(s, node); 4573 struct kmem_cache_node *n = get_node(s, node);
4571 4574
4572 if (!n) 4575 if (!n)
4573 continue; 4576 continue;
4574 4577
4575 nr_partials += n->nr_partial; 4578 nr_partials += n->nr_partial;
4576 nr_slabs += atomic_long_read(&n->nr_slabs); 4579 nr_slabs += atomic_long_read(&n->nr_slabs);
4577 nr_objs += atomic_long_read(&n->total_objects); 4580 nr_objs += atomic_long_read(&n->total_objects);
4578 nr_free += count_partial(n, count_free); 4581 nr_free += count_partial(n, count_free);
4579 } 4582 }
4580 4583
4581 nr_inuse = nr_objs - nr_free; 4584 nr_inuse = nr_objs - nr_free;
4582 4585
4583 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse, 4586 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
4584 nr_objs, s->size, oo_objects(s->oo), 4587 nr_objs, s->size, oo_objects(s->oo),
4585 (1 << oo_order(s->oo))); 4588 (1 << oo_order(s->oo)));
4586 seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0); 4589 seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0);
4587 seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs, 4590 seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
4588 0UL); 4591 0UL);
4589 seq_putc(m, '\n'); 4592 seq_putc(m, '\n');
4590 return 0; 4593 return 0;
4591 } 4594 }
4592 4595
4593 static const struct seq_operations slabinfo_op = { 4596 static const struct seq_operations slabinfo_op = {
4594 .start = s_start, 4597 .start = s_start,
4595 .next = s_next, 4598 .next = s_next,
4596 .stop = s_stop, 4599 .stop = s_stop,
4597 .show = s_show, 4600 .show = s_show,
4598 }; 4601 };
4599 4602
4600 static int slabinfo_open(struct inode *inode, struct file *file) 4603 static int slabinfo_open(struct inode *inode, struct file *file)
4601 { 4604 {
4602 return seq_open(file, &slabinfo_op); 4605 return seq_open(file, &slabinfo_op);
4603 } 4606 }
4604 4607
4605 static const struct file_operations proc_slabinfo_operations = { 4608 static const struct file_operations proc_slabinfo_operations = {
4606 .open = slabinfo_open, 4609 .open = slabinfo_open,
4607 .read = seq_read, 4610 .read = seq_read,
4608 .llseek = seq_lseek, 4611 .llseek = seq_lseek,
4609 .release = seq_release, 4612 .release = seq_release,
4610 }; 4613 };
4611 4614
4612 static int __init slab_proc_init(void) 4615 static int __init slab_proc_init(void)
4613 { 4616 {
4614 proc_create("slabinfo",S_IWUSR|S_IRUGO,NULL,&proc_slabinfo_operations); 4617 proc_create("slabinfo",S_IWUSR|S_IRUGO,NULL,&proc_slabinfo_operations);
4615 return 0; 4618 return 0;
4616 } 4619 }
4617 module_init(slab_proc_init); 4620 module_init(slab_proc_init);
4618 #endif /* CONFIG_SLABINFO */ 4621 #endif /* CONFIG_SLABINFO */
4619 4622