Doug / smarc-fsl-linux-kernel

1

/*

1

/*

2

* mm/kmemleak.c

2

* mm/kmemleak.c

3

*

3

*

4

5

* Written by Catalin Marinas <catalin.marinas@arm.com>

5

* Written by Catalin Marinas <catalin.marinas@arm.com>

6

*

6

*

7

* This program is free software; you can redistribute it and/or modify

7

* This program is free software; you can redistribute it and/or modify

8

* it under the terms of the GNU General Public License version 2 as

8

* it under the terms of the GNU General Public License version 2 as

9

* published by the Free Software Foundation.

9

* published by the Free Software Foundation.

10

*

10

*

11

* This program is distributed in the hope that it will be useful,

11

* This program is distributed in the hope that it will be useful,

12

* but WITHOUT ANY WARRANTY; without even the implied warranty of

12

* but WITHOUT ANY WARRANTY; without even the implied warranty of

13

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

13

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

14

* GNU General Public License for more details.

14

* GNU General Public License for more details.

15

*

15

*

16

* You should have received a copy of the GNU General Public License

16

* You should have received a copy of the GNU General Public License

17

* along with this program; if not, write to the Free Software

17

* along with this program; if not, write to the Free Software

18

* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

18

* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

19

*

19

*

20

*

20

*

21

* For more information on the algorithm and kmemleak usage, please see

21

* For more information on the algorithm and kmemleak usage, please see

22

* Documentation/kmemleak.txt.

22

* Documentation/kmemleak.txt.

23

*

23

*

24

* Notes on locking

24

* Notes on locking

25

* ----------------

25

* ----------------

26

*

26

*

27

* The following locks and mutexes are used by kmemleak:

27

* The following locks and mutexes are used by kmemleak:

28

*

28

*

29

* - kmemleak_lock (rwlock): protects the object_list modifications and

29

* - kmemleak_lock (rwlock): protects the object_list modifications and

30

* accesses to the object_tree_root. The object_list is the main list

30

* accesses to the object_tree_root. The object_list is the main list

31

* holding the metadata (struct kmemleak_object) for the allocated memory

31

* holding the metadata (struct kmemleak_object) for the allocated memory

32

* blocks. The object_tree_root is a priority search tree used to look-up

32

* blocks. The object_tree_root is a priority search tree used to look-up

33

* metadata based on a pointer to the corresponding memory block. The

33

* metadata based on a pointer to the corresponding memory block. The

34

* kmemleak_object structures are added to the object_list and

34

* kmemleak_object structures are added to the object_list and

35

* object_tree_root in the create_object() function called from the

35

* object_tree_root in the create_object() function called from the

36

* kmemleak_alloc() callback and removed in delete_object() called from the

36

* kmemleak_alloc() callback and removed in delete_object() called from the

37

* kmemleak_free() callback

37

* kmemleak_free() callback

38

* - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to

38

* - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to

39

* the metadata (e.g. count) are protected by this lock. Note that some

39

* the metadata (e.g. count) are protected by this lock. Note that some

40

* members of this structure may be protected by other means (atomic or

40

* members of this structure may be protected by other means (atomic or

41

* kmemleak_lock). This lock is also held when scanning the corresponding

41

* kmemleak_lock). This lock is also held when scanning the corresponding

42

* memory block to avoid the kernel freeing it via the kmemleak_free()

42

* memory block to avoid the kernel freeing it via the kmemleak_free()

43

* callback. This is less heavyweight than holding a global lock like

43

* callback. This is less heavyweight than holding a global lock like

44

* kmemleak_lock during scanning

44

* kmemleak_lock during scanning

45

* - scan_mutex (mutex): ensures that only one thread may scan the memory for

45

* - scan_mutex (mutex): ensures that only one thread may scan the memory for

46

* unreferenced objects at a time. The gray_list contains the objects which

46

* unreferenced objects at a time. The gray_list contains the objects which

47

* are already referenced or marked as false positives and need to be

47

* are already referenced or marked as false positives and need to be

48

* scanned. This list is only modified during a scanning episode when the

48

* scanned. This list is only modified during a scanning episode when the

49

* scan_mutex is held. At the end of a scan, the gray_list is always empty.

49

* scan_mutex is held. At the end of a scan, the gray_list is always empty.

50

* Note that the kmemleak_object.use_count is incremented when an object is

50

* Note that the kmemleak_object.use_count is incremented when an object is

51

* added to the gray_list and therefore cannot be freed. This mutex also

51

* added to the gray_list and therefore cannot be freed. This mutex also

52

* prevents multiple users of the "kmemleak" debugfs file together with

52

* prevents multiple users of the "kmemleak" debugfs file together with

53

* modifications to the memory scanning parameters including the scan_thread

53

* modifications to the memory scanning parameters including the scan_thread

54

* pointer

54

* pointer

55

*

55

*

56

* The kmemleak_object structures have a use_count incremented or decremented

56

* The kmemleak_object structures have a use_count incremented or decremented

57

* using the get_object()/put_object() functions. When the use_count becomes

57

* using the get_object()/put_object() functions. When the use_count becomes

58

* 0, this count can no longer be incremented and put_object() schedules the

58

* 0, this count can no longer be incremented and put_object() schedules the

59

* kmemleak_object freeing via an RCU callback. All calls to the get_object()

59

* kmemleak_object freeing via an RCU callback. All calls to the get_object()

60

* function must be protected by rcu_read_lock() to avoid accessing a freed

60

* function must be protected by rcu_read_lock() to avoid accessing a freed

61

* structure.

61

* structure.

62

*/

62

*/

63

64

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

64

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

65

66

#include <linux/init.h>

66

#include <linux/init.h>

67

#include <linux/kernel.h>

67

#include <linux/kernel.h>

68

#include <linux/list.h>

68

#include <linux/list.h>

69

#include <linux/sched.h>

69

#include <linux/sched.h>

70

#include <linux/jiffies.h>

70

#include <linux/jiffies.h>

71

#include <linux/delay.h>

71

#include <linux/delay.h>

72

#include <linux/module.h>

72

#include <linux/module.h>

73

#include <linux/kthread.h>

73

#include <linux/kthread.h>

74

#include <linux/prio_tree.h>

74

#include <linux/prio_tree.h>

75

#include <linux/fs.h>

75

#include <linux/fs.h>

76

#include <linux/debugfs.h>

76

#include <linux/debugfs.h>

77

#include <linux/seq_file.h>

77

#include <linux/seq_file.h>

78

#include <linux/cpumask.h>

78

#include <linux/cpumask.h>

79

#include <linux/spinlock.h>

79

#include <linux/spinlock.h>

80

#include <linux/mutex.h>

80

#include <linux/mutex.h>

81

#include <linux/rcupdate.h>

81

#include <linux/rcupdate.h>

82

#include <linux/stacktrace.h>

82

#include <linux/stacktrace.h>

83

#include <linux/cache.h>

83

#include <linux/cache.h>

84

#include <linux/percpu.h>

84

#include <linux/percpu.h>

85

#include <linux/hardirq.h>

85

#include <linux/hardirq.h>

86

#include <linux/mmzone.h>

86

#include <linux/mmzone.h>

87

#include <linux/slab.h>

87

#include <linux/slab.h>

88

#include <linux/thread_info.h>

88

#include <linux/thread_info.h>

89

#include <linux/err.h>

89

#include <linux/err.h>

90

#include <linux/uaccess.h>

90

#include <linux/uaccess.h>

91

#include <linux/string.h>

91

#include <linux/string.h>

92

#include <linux/nodemask.h>

92

#include <linux/nodemask.h>

93

#include <linux/mm.h>

93

#include <linux/mm.h>

94

#include <linux/workqueue.h>

94

#include <linux/workqueue.h>

95

#include <linux/crc32.h>

95

#include <linux/crc32.h>

96

97

#include <asm/sections.h>

97

#include <asm/sections.h>

98

#include <asm/processor.h>

98

#include <asm/processor.h>

99

#include <asm/atomic.h>

99

#include <asm/atomic.h>

100

101

#include <linux/kmemcheck.h>

101

#include <linux/kmemcheck.h>

102

#include <linux/kmemleak.h>

102

#include <linux/kmemleak.h>

103

104

/*

104

/*

105

* Kmemleak configuration and common defines.

105

* Kmemleak configuration and common defines.

106

*/

106

*/

107

#define MAX_TRACE 16 /* stack trace length */

107

#define MAX_TRACE 16 /* stack trace length */

108

#define MSECS_MIN_AGE 5000 /* minimum object age for reporting */

108

#define MSECS_MIN_AGE 5000 /* minimum object age for reporting */

109

#define SECS_FIRST_SCAN 60 /* delay before the first scan */

109

#define SECS_FIRST_SCAN 60 /* delay before the first scan */

110

#define SECS_SCAN_WAIT 600 /* subsequent auto scanning delay */

110

#define SECS_SCAN_WAIT 600 /* subsequent auto scanning delay */

111

#define MAX_SCAN_SIZE 4096 /* maximum size of a scanned block */

111

#define MAX_SCAN_SIZE 4096 /* maximum size of a scanned block */

112

113

#define BYTES_PER_POINTER sizeof(void *)

113

#define BYTES_PER_POINTER sizeof(void *)

114

115

/* GFP bitmask for kmemleak internal allocations */

115

/* GFP bitmask for kmemleak internal allocations */

116

#define GFP_KMEMLEAK_MASK (GFP_KERNEL | GFP_ATOMIC)

116

#define GFP_KMEMLEAK_MASK (GFP_KERNEL | GFP_ATOMIC)

117

118

/* scanning area inside a memory block */

118

/* scanning area inside a memory block */

119

struct kmemleak_scan_area {

119

struct kmemleak_scan_area {

120

struct hlist_node node;

120

struct hlist_node node;

121

unsigned long start;

121

unsigned long start;

122

size_t size;

122

size_t size;

123

};

123

};

124

125

#define KMEMLEAK_GREY 0

125

#define KMEMLEAK_GREY 0

126

#define KMEMLEAK_BLACK -1

126

#define KMEMLEAK_BLACK -1

127

128

/*

128

/*

129

* Structure holding the metadata for each allocated memory block.

129

* Structure holding the metadata for each allocated memory block.

130

* Modifications to such objects should be made while holding the

130

* Modifications to such objects should be made while holding the

131

* object->lock. Insertions or deletions from object_list, gray_list or

131

* object->lock. Insertions or deletions from object_list, gray_list or

132

* tree_node are already protected by the corresponding locks or mutex (see

132

* tree_node are already protected by the corresponding locks or mutex (see

133

* the notes on locking above). These objects are reference-counted

133

* the notes on locking above). These objects are reference-counted

134

* (use_count) and freed using the RCU mechanism.

134

* (use_count) and freed using the RCU mechanism.

135

*/

135

*/

136

struct kmemleak_object {

136

struct kmemleak_object {

137

spinlock_t lock;

137

spinlock_t lock;

138

unsigned long flags; /* object status flags */

138

unsigned long flags; /* object status flags */

139

struct list_head object_list;

139

struct list_head object_list;

140

struct list_head gray_list;

140

struct list_head gray_list;

141

struct prio_tree_node tree_node;

141

struct prio_tree_node tree_node;

142

struct rcu_head rcu; /* object_list lockless traversal */

142

struct rcu_head rcu; /* object_list lockless traversal */

143

/* object usage count; object freed when use_count == 0 */

143

/* object usage count; object freed when use_count == 0 */

144

atomic_t use_count;

144

atomic_t use_count;

145

unsigned long pointer;

145

unsigned long pointer;

146

size_t size;

146

size_t size;

147

/* minimum number of a pointers found before it is considered leak */

147

/* minimum number of a pointers found before it is considered leak */

148

int min_count;

148

int min_count;

149

/* the total number of pointers found pointing to this object */

149

/* the total number of pointers found pointing to this object */

150

int count;

150

int count;

151

/* checksum for detecting modified objects */

151

/* checksum for detecting modified objects */

152

u32 checksum;

152

u32 checksum;

153

/* memory ranges to be scanned inside an object (empty for all) */

153

/* memory ranges to be scanned inside an object (empty for all) */

154

struct hlist_head area_list;

154

struct hlist_head area_list;

155

unsigned long trace[MAX_TRACE];

155

unsigned long trace[MAX_TRACE];

156

unsigned int trace_len;

156

unsigned int trace_len;

157

unsigned long jiffies; /* creation timestamp */

157

unsigned long jiffies; /* creation timestamp */

158

pid_t pid; /* pid of the current task */

158

pid_t pid; /* pid of the current task */

159

char comm[TASK_COMM_LEN]; /* executable name */

159

char comm[TASK_COMM_LEN]; /* executable name */

160

};

160

};

161

162

/* flag representing the memory block allocation status */

162

/* flag representing the memory block allocation status */

163

#define OBJECT_ALLOCATED (1 << 0)

163

#define OBJECT_ALLOCATED (1 << 0)

164

/* flag set after the first reporting of an unreference object */

164

/* flag set after the first reporting of an unreference object */

165

#define OBJECT_REPORTED (1 << 1)

165

#define OBJECT_REPORTED (1 << 1)

166

/* flag set to not scan the object */

166

/* flag set to not scan the object */

167

#define OBJECT_NO_SCAN (1 << 2)

167

#define OBJECT_NO_SCAN (1 << 2)

168

169

/* number of bytes to print per line; must be 16 or 32 */

169

/* number of bytes to print per line; must be 16 or 32 */

170

#define HEX_ROW_SIZE 16

170

#define HEX_ROW_SIZE 16

171

/* number of bytes to print at a time (1, 2, 4, 8) */

171

/* number of bytes to print at a time (1, 2, 4, 8) */

172

#define HEX_GROUP_SIZE 1

172

#define HEX_GROUP_SIZE 1

173

/* include ASCII after the hex output */

173

/* include ASCII after the hex output */

174

#define HEX_ASCII 1

174

#define HEX_ASCII 1

175

/* max number of lines to be printed */

175

/* max number of lines to be printed */

176

#define HEX_MAX_LINES 2

176

#define HEX_MAX_LINES 2

177

178

/* the list of all allocated objects */

178

/* the list of all allocated objects */

179

static LIST_HEAD(object_list);

179

static LIST_HEAD(object_list);

180

/* the list of gray-colored objects (see color_gray comment below) */

180

/* the list of gray-colored objects (see color_gray comment below) */

181

static LIST_HEAD(gray_list);

181

static LIST_HEAD(gray_list);

182

/* prio search tree for object boundaries */

182

/* prio search tree for object boundaries */

183

static struct prio_tree_root object_tree_root;

183

static struct prio_tree_root object_tree_root;

184

/* rw_lock protecting the access to object_list and prio_tree_root */

184

/* rw_lock protecting the access to object_list and prio_tree_root */

185

static DEFINE_RWLOCK(kmemleak_lock);

185

static DEFINE_RWLOCK(kmemleak_lock);

186

187

/* allocation caches for kmemleak internal data */

187

/* allocation caches for kmemleak internal data */

188

static struct kmem_cache *object_cache;

188

static struct kmem_cache *object_cache;

189

static struct kmem_cache *scan_area_cache;

189

static struct kmem_cache *scan_area_cache;

190

191

/* set if tracing memory operations is enabled */

191

/* set if tracing memory operations is enabled */

192

static atomic_t kmemleak_enabled = ATOMIC_INIT(0);

192

static atomic_t kmemleak_enabled = ATOMIC_INIT(0);

193

/* set in the late_initcall if there were no errors */

193

/* set in the late_initcall if there were no errors */

194

static atomic_t kmemleak_initialized = ATOMIC_INIT(0);

194

static atomic_t kmemleak_initialized = ATOMIC_INIT(0);

195

/* enables or disables early logging of the memory operations */

195

/* enables or disables early logging of the memory operations */

196

static atomic_t kmemleak_early_log = ATOMIC_INIT(1);

196

static atomic_t kmemleak_early_log = ATOMIC_INIT(1);

197

/* set if a fata kmemleak error has occurred */

197

/* set if a fata kmemleak error has occurred */

198

static atomic_t kmemleak_error = ATOMIC_INIT(0);

198

static atomic_t kmemleak_error = ATOMIC_INIT(0);

199

200

/* minimum and maximum address that may be valid pointers */

200

/* minimum and maximum address that may be valid pointers */

201

static unsigned long min_addr = ULONG_MAX;

201

static unsigned long min_addr = ULONG_MAX;

202

static unsigned long max_addr;

202

static unsigned long max_addr;

203

204

static struct task_struct *scan_thread;

204

static struct task_struct *scan_thread;

205

/* used to avoid reporting of recently allocated objects */

205

/* used to avoid reporting of recently allocated objects */

206

static unsigned long jiffies_min_age;

206

static unsigned long jiffies_min_age;

207

static unsigned long jiffies_last_scan;

207

static unsigned long jiffies_last_scan;

208

/* delay between automatic memory scannings */

208

/* delay between automatic memory scannings */

209

static signed long jiffies_scan_wait;

209

static signed long jiffies_scan_wait;

210

/* enables or disables the task stacks scanning */

210

/* enables or disables the task stacks scanning */

211

static int kmemleak_stack_scan = 1;

211

static int kmemleak_stack_scan = 1;

212

/* protects the memory scanning, parameters and debug/kmemleak file access */

212

/* protects the memory scanning, parameters and debug/kmemleak file access */

213

static DEFINE_MUTEX(scan_mutex);

213

static DEFINE_MUTEX(scan_mutex);

214

/* setting kmemleak=on, will set this var, skipping the disable */

215

static int kmemleak_skip_disable;

214

216

217

215

/*

218

/*

216

* Early object allocation/freeing logging. Kmemleak is initialized after the

219

* Early object allocation/freeing logging. Kmemleak is initialized after the

217

* kernel allocator. However, both the kernel allocator and kmemleak may

220

* kernel allocator. However, both the kernel allocator and kmemleak may

218

* allocate memory blocks which need to be tracked. Kmemleak defines an

221

* allocate memory blocks which need to be tracked. Kmemleak defines an

219

* arbitrary buffer to hold the allocation/freeing information before it is

222

* arbitrary buffer to hold the allocation/freeing information before it is

220

* fully initialized.

223

* fully initialized.

221

*/

224

*/

222

225

223

/* kmemleak operation type for early logging */

226

/* kmemleak operation type for early logging */

224

enum {

227

enum {

225

KMEMLEAK_ALLOC,

228

KMEMLEAK_ALLOC,

226

KMEMLEAK_FREE,

229

KMEMLEAK_FREE,

227

KMEMLEAK_FREE_PART,

230

KMEMLEAK_FREE_PART,

228

KMEMLEAK_NOT_LEAK,

231

KMEMLEAK_NOT_LEAK,

229

KMEMLEAK_IGNORE,

232

KMEMLEAK_IGNORE,

230

KMEMLEAK_SCAN_AREA,

233

KMEMLEAK_SCAN_AREA,

231

KMEMLEAK_NO_SCAN

234

KMEMLEAK_NO_SCAN

232

};

235

};

233

236

234

/*

237

/*

235

* Structure holding the information passed to kmemleak callbacks during the

238

* Structure holding the information passed to kmemleak callbacks during the

236

* early logging.

239

* early logging.

237

*/

240

*/

238

struct early_log {

241

struct early_log {

239

int op_type; /* kmemleak operation type */

242

int op_type; /* kmemleak operation type */

240

const void *ptr; /* allocated/freed memory block */

243

const void *ptr; /* allocated/freed memory block */

241

size_t size; /* memory block size */

244

size_t size; /* memory block size */

242

int min_count; /* minimum reference count */

245

int min_count; /* minimum reference count */

243

unsigned long trace[MAX_TRACE]; /* stack trace */

246

unsigned long trace[MAX_TRACE]; /* stack trace */

244

unsigned int trace_len; /* stack trace length */

247

unsigned int trace_len; /* stack trace length */

245

};

248

};

246

249

247

/* early logging buffer and current position */

250

/* early logging buffer and current position */

248

static struct early_log

251

static struct early_log

249

early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;

252

early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;

250

static int crt_early_log __initdata;

253

static int crt_early_log __initdata;

251

254

252

static void kmemleak_disable(void);

255

static void kmemleak_disable(void);

253

256

254

/*

257

/*

255

* Print a warning and dump the stack trace.

258

* Print a warning and dump the stack trace.

256

*/

259

*/

257

#define kmemleak_warn(x...) do { \

260

#define kmemleak_warn(x...) do { \

258

pr_warning(x); \

261

pr_warning(x); \

259

dump_stack(); \

262

dump_stack(); \

260

} while (0)

263

} while (0)

261

264

262

/*

265

/*

263

* Macro invoked when a serious kmemleak condition occured and cannot be

266

* Macro invoked when a serious kmemleak condition occured and cannot be

264

* recovered from. Kmemleak will be disabled and further allocation/freeing

267

* recovered from. Kmemleak will be disabled and further allocation/freeing

265

* tracing no longer available.

268

* tracing no longer available.

266

*/

269

*/

267

#define kmemleak_stop(x...) do { \

270

#define kmemleak_stop(x...) do { \

268

kmemleak_warn(x); \

271

kmemleak_warn(x); \

269

kmemleak_disable(); \

272

kmemleak_disable(); \

270

} while (0)

273

} while (0)

271

274

272

/*

275

/*

273

* Printing of the objects hex dump to the seq file. The number of lines to be

276

* Printing of the objects hex dump to the seq file. The number of lines to be

274

* printed is limited to HEX_MAX_LINES to prevent seq file spamming. The

277

* printed is limited to HEX_MAX_LINES to prevent seq file spamming. The

275

* actual number of printed bytes depends on HEX_ROW_SIZE. It must be called

278

* actual number of printed bytes depends on HEX_ROW_SIZE. It must be called

276

* with the object->lock held.

279

* with the object->lock held.

277

*/

280

*/

278

static void hex_dump_object(struct seq_file *seq,

281

static void hex_dump_object(struct seq_file *seq,

279

struct kmemleak_object *object)

282

struct kmemleak_object *object)

280

{

283

{

281

const u8 *ptr = (const u8 *)object->pointer;

284

const u8 *ptr = (const u8 *)object->pointer;

282

int i, len, remaining;

285

int i, len, remaining;

283

unsigned char linebuf[HEX_ROW_SIZE * 5];

286

unsigned char linebuf[HEX_ROW_SIZE * 5];

284

287

285

/* limit the number of lines to HEX_MAX_LINES */

288

/* limit the number of lines to HEX_MAX_LINES */

286

remaining = len =

289

remaining = len =

287

min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));

290

min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));

288

291

289

seq_printf(seq, " hex dump (first %d bytes):\n", len);

292

seq_printf(seq, " hex dump (first %d bytes):\n", len);

290

for (i = 0; i < len; i += HEX_ROW_SIZE) {

293

for (i = 0; i < len; i += HEX_ROW_SIZE) {

291

int linelen = min(remaining, HEX_ROW_SIZE);

294

int linelen = min(remaining, HEX_ROW_SIZE);

292

295

293

remaining -= HEX_ROW_SIZE;

296

remaining -= HEX_ROW_SIZE;

294

hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE,

297

hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE,

295

HEX_GROUP_SIZE, linebuf, sizeof(linebuf),

298

HEX_GROUP_SIZE, linebuf, sizeof(linebuf),

296

HEX_ASCII);

299

HEX_ASCII);

297

seq_printf(seq, " %s\n", linebuf);

300

seq_printf(seq, " %s\n", linebuf);

298

}

301

}

299

}

302

}

300

303

301

/*

304

/*

302

* Object colors, encoded with count and min_count:

305

* Object colors, encoded with count and min_count:

303

* - white - orphan object, not enough references to it (count < min_count)

306

* - white - orphan object, not enough references to it (count < min_count)

304

* - gray - not orphan, not marked as false positive (min_count == 0) or

307

* - gray - not orphan, not marked as false positive (min_count == 0) or

305

* sufficient references to it (count >= min_count)

308

* sufficient references to it (count >= min_count)

306

* - black - ignore, it doesn't contain references (e.g. text section)

309

* - black - ignore, it doesn't contain references (e.g. text section)

307

* (min_count == -1). No function defined for this color.

310

* (min_count == -1). No function defined for this color.

308

* Newly created objects don't have any color assigned (object->count == -1)

311

* Newly created objects don't have any color assigned (object->count == -1)

309

* before the next memory scan when they become white.

312

* before the next memory scan when they become white.

310

*/

313

*/

311

static bool color_white(const struct kmemleak_object *object)

314

static bool color_white(const struct kmemleak_object *object)

312

{

315

{

313

return object->count != KMEMLEAK_BLACK &&

316

return object->count != KMEMLEAK_BLACK &&

314

object->count < object->min_count;

317

object->count < object->min_count;

315

}

318

}

316

319

317

static bool color_gray(const struct kmemleak_object *object)

320

static bool color_gray(const struct kmemleak_object *object)

318

{

321

{

319

return object->min_count != KMEMLEAK_BLACK &&

322

return object->min_count != KMEMLEAK_BLACK &&

320

object->count >= object->min_count;

323

object->count >= object->min_count;

321

}

324

}

322

325

323

/*

326

/*

324

* Objects are considered unreferenced only if their color is white, they have

327

* Objects are considered unreferenced only if their color is white, they have

325

* not be deleted and have a minimum age to avoid false positives caused by

328

* not be deleted and have a minimum age to avoid false positives caused by

326

* pointers temporarily stored in CPU registers.

329

* pointers temporarily stored in CPU registers.

327

*/

330

*/

328

static bool unreferenced_object(struct kmemleak_object *object)

331

static bool unreferenced_object(struct kmemleak_object *object)

329

{

332

{

330

return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&

333

return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&

331

time_before_eq(object->jiffies + jiffies_min_age,

334

time_before_eq(object->jiffies + jiffies_min_age,

332

jiffies_last_scan);

335

jiffies_last_scan);

333

}

336

}

334

337

335

/*

338

/*

336

* Printing of the unreferenced objects information to the seq file. The

339

* Printing of the unreferenced objects information to the seq file. The

337

* print_unreferenced function must be called with the object->lock held.

340

* print_unreferenced function must be called with the object->lock held.

338

*/

341

*/

339

static void print_unreferenced(struct seq_file *seq,

342

static void print_unreferenced(struct seq_file *seq,

340

struct kmemleak_object *object)

343

struct kmemleak_object *object)

341

{

344

{

342

int i;

345

int i;

343

unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);

346

unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);

344

347

345

seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",

348

seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",

346

object->pointer, object->size);

349

object->pointer, object->size);

347

seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",

350

seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",

348

object->comm, object->pid, object->jiffies,

351

object->comm, object->pid, object->jiffies,

349

msecs_age / 1000, msecs_age % 1000);

352

msecs_age / 1000, msecs_age % 1000);

350

hex_dump_object(seq, object);

353

hex_dump_object(seq, object);

351

seq_printf(seq, " backtrace:\n");

354

seq_printf(seq, " backtrace:\n");

352

355

353

for (i = 0; i < object->trace_len; i++) {

356

for (i = 0; i < object->trace_len; i++) {

354

void *ptr = (void *)object->trace[i];

357

void *ptr = (void *)object->trace[i];

355

seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);

358

seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);

356

}

359

}

357

}

360

}

358

361

359

/*

362

/*

360

* Print the kmemleak_object information. This function is used mainly for

363

* Print the kmemleak_object information. This function is used mainly for

361

* debugging special cases when kmemleak operations. It must be called with

364

* debugging special cases when kmemleak operations. It must be called with

362

* the object->lock held.

365

* the object->lock held.

363

*/

366

*/

364

static void dump_object_info(struct kmemleak_object *object)

367

static void dump_object_info(struct kmemleak_object *object)

365

{

368

{

366

struct stack_trace trace;

369

struct stack_trace trace;

367

370

368

trace.nr_entries = object->trace_len;

371

trace.nr_entries = object->trace_len;

369

trace.entries = object->trace;

372

trace.entries = object->trace;

370

373

371

pr_notice("Object 0x%08lx (size %zu):\n",

374

pr_notice("Object 0x%08lx (size %zu):\n",

372

object->tree_node.start, object->size);

375

object->tree_node.start, object->size);

373

pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",

376

pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",

374

object->comm, object->pid, object->jiffies);

377

object->comm, object->pid, object->jiffies);

375

pr_notice(" min_count = %d\n", object->min_count);

378

pr_notice(" min_count = %d\n", object->min_count);

376

pr_notice(" count = %d\n", object->count);

379

pr_notice(" count = %d\n", object->count);

377

pr_notice(" flags = 0x%lx\n", object->flags);

380

pr_notice(" flags = 0x%lx\n", object->flags);

378

pr_notice(" checksum = %d\n", object->checksum);

381

pr_notice(" checksum = %d\n", object->checksum);

379

pr_notice(" backtrace:\n");

382

pr_notice(" backtrace:\n");

380

print_stack_trace(&trace, 4);

383

print_stack_trace(&trace, 4);

381

}

384

}

382

385

383

/*

386

/*

384

* Look-up a memory block metadata (kmemleak_object) in the priority search

387

* Look-up a memory block metadata (kmemleak_object) in the priority search

385

* tree based on a pointer value. If alias is 0, only values pointing to the

388

* tree based on a pointer value. If alias is 0, only values pointing to the

386

* beginning of the memory block are allowed. The kmemleak_lock must be held

389

* beginning of the memory block are allowed. The kmemleak_lock must be held

387

* when calling this function.

390

* when calling this function.

388

*/

391

*/

389

static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)

392

static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)

390

{

393

{

391

struct prio_tree_node *node;

394

struct prio_tree_node *node;

392

struct prio_tree_iter iter;

395

struct prio_tree_iter iter;

393

struct kmemleak_object *object;

396

struct kmemleak_object *object;

394

397

395

prio_tree_iter_init(&iter, &object_tree_root, ptr, ptr);

398

prio_tree_iter_init(&iter, &object_tree_root, ptr, ptr);

396

node = prio_tree_next(&iter);

399

node = prio_tree_next(&iter);

397

if (node) {

400

if (node) {

398

object = prio_tree_entry(node, struct kmemleak_object,

401

object = prio_tree_entry(node, struct kmemleak_object,

399

tree_node);

402

tree_node);

400

if (!alias && object->pointer != ptr) {

403

if (!alias && object->pointer != ptr) {

401

pr_warning("Found object by alias at 0x%08lx\n", ptr);

404

pr_warning("Found object by alias at 0x%08lx\n", ptr);

402

dump_stack();

405

dump_stack();

403

dump_object_info(object);

406

dump_object_info(object);

404

object = NULL;

407

object = NULL;

405

}

408

}

406

} else

409

} else

407

object = NULL;

410

object = NULL;

408

411

409

return object;

412

return object;

410

}

413

}

411

414

412

/*

415

/*

413

* Increment the object use_count. Return 1 if successful or 0 otherwise. Note

416

* Increment the object use_count. Return 1 if successful or 0 otherwise. Note

414

* that once an object's use_count reached 0, the RCU freeing was already

417

* that once an object's use_count reached 0, the RCU freeing was already

415

* registered and the object should no longer be used. This function must be

418

* registered and the object should no longer be used. This function must be

416

* called under the protection of rcu_read_lock().

419

* called under the protection of rcu_read_lock().

417

*/

420

*/

418

static int get_object(struct kmemleak_object *object)

421

static int get_object(struct kmemleak_object *object)

419

{

422

{

420

return atomic_inc_not_zero(&object->use_count);

423

return atomic_inc_not_zero(&object->use_count);

421

}

424

}

422

425

423

/*

426

/*

424

* RCU callback to free a kmemleak_object.

427

* RCU callback to free a kmemleak_object.

425

*/

428

*/

426

static void free_object_rcu(struct rcu_head *rcu)

429

static void free_object_rcu(struct rcu_head *rcu)

427

{

430

{

428

struct hlist_node *elem, *tmp;

431

struct hlist_node *elem, *tmp;

429

struct kmemleak_scan_area *area;

432

struct kmemleak_scan_area *area;

430

struct kmemleak_object *object =

433

struct kmemleak_object *object =

431

container_of(rcu, struct kmemleak_object, rcu);

434

container_of(rcu, struct kmemleak_object, rcu);

432

435

433

/*

436

/*

434

* Once use_count is 0 (guaranteed by put_object), there is no other

437

* Once use_count is 0 (guaranteed by put_object), there is no other

435

* code accessing this object, hence no need for locking.

438

* code accessing this object, hence no need for locking.

436

*/

439

*/

437

hlist_for_each_entry_safe(area, elem, tmp, &object->area_list, node) {

440

hlist_for_each_entry_safe(area, elem, tmp, &object->area_list, node) {

438

hlist_del(elem);

441

hlist_del(elem);

439

kmem_cache_free(scan_area_cache, area);

442

kmem_cache_free(scan_area_cache, area);

440

}

443

}

441

kmem_cache_free(object_cache, object);

444

kmem_cache_free(object_cache, object);

442

}

445

}

443

446

444

/*

447

/*

445

* Decrement the object use_count. Once the count is 0, free the object using

448

* Decrement the object use_count. Once the count is 0, free the object using

446

* an RCU callback. Since put_object() may be called via the kmemleak_free() ->

449

* an RCU callback. Since put_object() may be called via the kmemleak_free() ->

447

* delete_object() path, the delayed RCU freeing ensures that there is no

450

* delete_object() path, the delayed RCU freeing ensures that there is no

448

* recursive call to the kernel allocator. Lock-less RCU object_list traversal

451

* recursive call to the kernel allocator. Lock-less RCU object_list traversal

449

* is also possible.

452

* is also possible.

450

*/

453

*/

451

static void put_object(struct kmemleak_object *object)

454

static void put_object(struct kmemleak_object *object)

452

{

455

{

453

if (!atomic_dec_and_test(&object->use_count))

456

if (!atomic_dec_and_test(&object->use_count))

454

return;

457

return;

455

458

456

/* should only get here after delete_object was called */

459

/* should only get here after delete_object was called */

457

WARN_ON(object->flags & OBJECT_ALLOCATED);

460

WARN_ON(object->flags & OBJECT_ALLOCATED);

458

461

459

call_rcu(&object->rcu, free_object_rcu);

462

call_rcu(&object->rcu, free_object_rcu);

460

}

463

}

461

464

462

/*

465

/*

463

* Look up an object in the prio search tree and increase its use_count.

466

* Look up an object in the prio search tree and increase its use_count.

464

*/

467

*/

465

static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)

468

static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)

466

{

469

{

467

unsigned long flags;

470

unsigned long flags;

468

struct kmemleak_object *object = NULL;

471

struct kmemleak_object *object = NULL;

469

472

470

rcu_read_lock();

473

rcu_read_lock();

471

read_lock_irqsave(&kmemleak_lock, flags);

474

read_lock_irqsave(&kmemleak_lock, flags);

472

if (ptr >= min_addr && ptr < max_addr)

475

if (ptr >= min_addr && ptr < max_addr)

473

object = lookup_object(ptr, alias);

476

object = lookup_object(ptr, alias);

474

read_unlock_irqrestore(&kmemleak_lock, flags);

477

read_unlock_irqrestore(&kmemleak_lock, flags);

475

478

476

/* check whether the object is still available */

479

/* check whether the object is still available */

477

if (object && !get_object(object))

480

if (object && !get_object(object))

478

object = NULL;

481

object = NULL;

479

rcu_read_unlock();

482

rcu_read_unlock();

480

483

481

return object;

484

return object;

482

}

485

}

483

486

484

/*

487

/*

485

* Save stack trace to the given array of MAX_TRACE size.

488

* Save stack trace to the given array of MAX_TRACE size.

486

*/

489

*/

487

static int __save_stack_trace(unsigned long *trace)

490

static int __save_stack_trace(unsigned long *trace)

488

{

491

{

489

struct stack_trace stack_trace;

492

struct stack_trace stack_trace;

490

493

491

stack_trace.max_entries = MAX_TRACE;

494

stack_trace.max_entries = MAX_TRACE;

492

stack_trace.nr_entries = 0;

495

stack_trace.nr_entries = 0;

493

stack_trace.entries = trace;

496

stack_trace.entries = trace;

494

stack_trace.skip = 2;

497

stack_trace.skip = 2;

495

save_stack_trace(&stack_trace);

498

save_stack_trace(&stack_trace);

496

499

497

return stack_trace.nr_entries;

500

return stack_trace.nr_entries;

498

}

501

}

499

502

500

/*

503

/*

501

* Create the metadata (struct kmemleak_object) corresponding to an allocated

504

* Create the metadata (struct kmemleak_object) corresponding to an allocated

502

* memory block and add it to the object_list and object_tree_root.

505

* memory block and add it to the object_list and object_tree_root.

503

*/

506

*/

504

static struct kmemleak_object *create_object(unsigned long ptr, size_t size,

507

static struct kmemleak_object *create_object(unsigned long ptr, size_t size,

505

int min_count, gfp_t gfp)

508

int min_count, gfp_t gfp)

506

{

509

{

507

unsigned long flags;

510

unsigned long flags;

508

struct kmemleak_object *object;

511

struct kmemleak_object *object;

509

struct prio_tree_node *node;

512

struct prio_tree_node *node;

510

513

511

object = kmem_cache_alloc(object_cache, gfp & GFP_KMEMLEAK_MASK);

514

object = kmem_cache_alloc(object_cache, gfp & GFP_KMEMLEAK_MASK);

512

if (!object) {

515

if (!object) {

513

kmemleak_stop("Cannot allocate a kmemleak_object structure\n");

516

kmemleak_stop("Cannot allocate a kmemleak_object structure\n");

514

return NULL;

517

return NULL;

515

}

518

}

516

519

517

INIT_LIST_HEAD(&object->object_list);

520

INIT_LIST_HEAD(&object->object_list);

518

INIT_LIST_HEAD(&object->gray_list);

521

INIT_LIST_HEAD(&object->gray_list);

519

INIT_HLIST_HEAD(&object->area_list);

522

INIT_HLIST_HEAD(&object->area_list);

520

spin_lock_init(&object->lock);

523

spin_lock_init(&object->lock);

521

atomic_set(&object->use_count, 1);

524

atomic_set(&object->use_count, 1);

522

object->flags = OBJECT_ALLOCATED;

525

object->flags = OBJECT_ALLOCATED;

523

object->pointer = ptr;

526

object->pointer = ptr;

524

object->size = size;

527

object->size = size;

525

object->min_count = min_count;

528

object->min_count = min_count;

526

object->count = 0; /* white color initially */

529

object->count = 0; /* white color initially */

527

object->jiffies = jiffies;

530

object->jiffies = jiffies;

528

object->checksum = 0;

531

object->checksum = 0;

529

532

530

/* task information */

533

/* task information */

531

if (in_irq()) {

534

if (in_irq()) {

532

object->pid = 0;

535

object->pid = 0;

533

strncpy(object->comm, "hardirq", sizeof(object->comm));

536

strncpy(object->comm, "hardirq", sizeof(object->comm));

534

} else if (in_softirq()) {

537

} else if (in_softirq()) {

535

object->pid = 0;

538

object->pid = 0;

536

strncpy(object->comm, "softirq", sizeof(object->comm));

539

strncpy(object->comm, "softirq", sizeof(object->comm));

537

} else {

540

} else {

538

object->pid = current->pid;

541

object->pid = current->pid;

539

/*

542

/*

540

* There is a small chance of a race with set_task_comm(),

543

* There is a small chance of a race with set_task_comm(),

541

* however using get_task_comm() here may cause locking

544

* however using get_task_comm() here may cause locking

542

* dependency issues with current->alloc_lock. In the worst

545

* dependency issues with current->alloc_lock. In the worst

543

* case, the command line is not correct.

546

* case, the command line is not correct.

544

*/

547

*/

545

strncpy(object->comm, current->comm, sizeof(object->comm));

548

strncpy(object->comm, current->comm, sizeof(object->comm));

546

}

549

}

547

550

548

/* kernel backtrace */

551

/* kernel backtrace */

549

object->trace_len = __save_stack_trace(object->trace);

552

object->trace_len = __save_stack_trace(object->trace);

550

553

551

INIT_PRIO_TREE_NODE(&object->tree_node);

554

INIT_PRIO_TREE_NODE(&object->tree_node);

552

object->tree_node.start = ptr;

555

object->tree_node.start = ptr;

553

object->tree_node.last = ptr + size - 1;

556

object->tree_node.last = ptr + size - 1;

554

557

555

write_lock_irqsave(&kmemleak_lock, flags);

558

write_lock_irqsave(&kmemleak_lock, flags);

556

559

557

min_addr = min(min_addr, ptr);

560

min_addr = min(min_addr, ptr);

558

max_addr = max(max_addr, ptr + size);

561

max_addr = max(max_addr, ptr + size);

559

node = prio_tree_insert(&object_tree_root, &object->tree_node);

562

node = prio_tree_insert(&object_tree_root, &object->tree_node);

560

/*

563

/*

561

* The code calling the kernel does not yet have the pointer to the

564

* The code calling the kernel does not yet have the pointer to the

562

* memory block to be able to free it. However, we still hold the

565

* memory block to be able to free it. However, we still hold the

563

* kmemleak_lock here in case parts of the kernel started freeing

566

* kmemleak_lock here in case parts of the kernel started freeing

564

* random memory blocks.

567

* random memory blocks.

565

*/

568

*/

566

if (node != &object->tree_node) {

569

if (node != &object->tree_node) {

567

kmemleak_stop("Cannot insert 0x%lx into the object search tree "

570

kmemleak_stop("Cannot insert 0x%lx into the object search tree "

568

"(already existing)\n", ptr);

571

"(already existing)\n", ptr);

569

object = lookup_object(ptr, 1);

572

object = lookup_object(ptr, 1);

570

spin_lock(&object->lock);

573

spin_lock(&object->lock);

571

dump_object_info(object);

574

dump_object_info(object);

572

spin_unlock(&object->lock);

575

spin_unlock(&object->lock);

573

576

574

goto out;

577

goto out;

575

}

578

}

576

list_add_tail_rcu(&object->object_list, &object_list);

579

list_add_tail_rcu(&object->object_list, &object_list);

577

out:

580

out:

578

write_unlock_irqrestore(&kmemleak_lock, flags);

581

write_unlock_irqrestore(&kmemleak_lock, flags);

579

return object;

582

return object;

580

}

583

}

581

584

582

/*

585

/*

583

* Remove the metadata (struct kmemleak_object) for a memory block from the

586

* Remove the metadata (struct kmemleak_object) for a memory block from the

584

* object_list and object_tree_root and decrement its use_count.

587

* object_list and object_tree_root and decrement its use_count.

585

*/

588

*/

586

static void __delete_object(struct kmemleak_object *object)

589

static void __delete_object(struct kmemleak_object *object)

587

{

590

{

588

unsigned long flags;

591

unsigned long flags;

589

592

590

write_lock_irqsave(&kmemleak_lock, flags);

593

write_lock_irqsave(&kmemleak_lock, flags);

591

prio_tree_remove(&object_tree_root, &object->tree_node);

594

prio_tree_remove(&object_tree_root, &object->tree_node);

592

list_del_rcu(&object->object_list);

595

list_del_rcu(&object->object_list);

593

write_unlock_irqrestore(&kmemleak_lock, flags);

596

write_unlock_irqrestore(&kmemleak_lock, flags);

594

597

595

WARN_ON(!(object->flags & OBJECT_ALLOCATED));

598

WARN_ON(!(object->flags & OBJECT_ALLOCATED));

596

WARN_ON(atomic_read(&object->use_count) < 2);

599

WARN_ON(atomic_read(&object->use_count) < 2);

597

600

598

/*

601

/*

599

* Locking here also ensures that the corresponding memory block

602

* Locking here also ensures that the corresponding memory block

600

* cannot be freed when it is being scanned.

603

* cannot be freed when it is being scanned.

601

*/

604

*/

602

spin_lock_irqsave(&object->lock, flags);

605

spin_lock_irqsave(&object->lock, flags);

603

object->flags &= ~OBJECT_ALLOCATED;

606

object->flags &= ~OBJECT_ALLOCATED;

604

spin_unlock_irqrestore(&object->lock, flags);

607

spin_unlock_irqrestore(&object->lock, flags);

605

put_object(object);

608

put_object(object);

606

}

609

}

607

610

608

/*

611

/*

609

* Look up the metadata (struct kmemleak_object) corresponding to ptr and

612

* Look up the metadata (struct kmemleak_object) corresponding to ptr and

610

* delete it.

613

* delete it.

611

*/

614

*/

612

static void delete_object_full(unsigned long ptr)

615

static void delete_object_full(unsigned long ptr)

613

{

616

{

614

struct kmemleak_object *object;

617

struct kmemleak_object *object;

615

618

616

object = find_and_get_object(ptr, 0);

619

object = find_and_get_object(ptr, 0);

617

if (!object) {

620

if (!object) {

618

#ifdef DEBUG

621

#ifdef DEBUG

619

kmemleak_warn("Freeing unknown object at 0x%08lx\n",

622

kmemleak_warn("Freeing unknown object at 0x%08lx\n",

620

ptr);

623

ptr);

621

#endif

624

#endif

622

return;

625

return;

623

}

626

}

624

__delete_object(object);

627

__delete_object(object);

625

put_object(object);

628

put_object(object);

626

}

629

}

627

630

628

/*

631

/*

629

* Look up the metadata (struct kmemleak_object) corresponding to ptr and

632

* Look up the metadata (struct kmemleak_object) corresponding to ptr and

630

* delete it. If the memory block is partially freed, the function may create

633

* delete it. If the memory block is partially freed, the function may create

631

* additional metadata for the remaining parts of the block.

634

* additional metadata for the remaining parts of the block.

632

*/

635

*/

633

static void delete_object_part(unsigned long ptr, size_t size)

636

static void delete_object_part(unsigned long ptr, size_t size)

634

{

637

{

635

struct kmemleak_object *object;

638

struct kmemleak_object *object;

636

unsigned long start, end;

639

unsigned long start, end;

637

640

638

object = find_and_get_object(ptr, 1);

641

object = find_and_get_object(ptr, 1);

639

if (!object) {

642

if (!object) {

640

#ifdef DEBUG

643

#ifdef DEBUG

641

kmemleak_warn("Partially freeing unknown object at 0x%08lx "

644

kmemleak_warn("Partially freeing unknown object at 0x%08lx "

642

"(size %zu)\n", ptr, size);

645

"(size %zu)\n", ptr, size);

643

#endif

646

#endif

644

return;

647

return;

645

}

648

}

646

__delete_object(object);

649

__delete_object(object);

647

650

648

/*

651

/*

649

* Create one or two objects that may result from the memory block

652

* Create one or two objects that may result from the memory block

650

* split. Note that partial freeing is only done by free_bootmem() and

653

* split. Note that partial freeing is only done by free_bootmem() and

651

* this happens before kmemleak_init() is called. The path below is

654

* this happens before kmemleak_init() is called. The path below is

652

* only executed during early log recording in kmemleak_init(), so

655

* only executed during early log recording in kmemleak_init(), so

653

* GFP_KERNEL is enough.

656

* GFP_KERNEL is enough.

654

*/

657

*/

655

start = object->pointer;

658

start = object->pointer;

656

end = object->pointer + object->size;

659

end = object->pointer + object->size;

657

if (ptr > start)

660

if (ptr > start)

658

create_object(start, ptr - start, object->min_count,

661

create_object(start, ptr - start, object->min_count,

659

GFP_KERNEL);

662

GFP_KERNEL);

660

if (ptr + size < end)

663

if (ptr + size < end)

661

create_object(ptr + size, end - ptr - size, object->min_count,

664

create_object(ptr + size, end - ptr - size, object->min_count,

662

GFP_KERNEL);

665

GFP_KERNEL);

663

666

664

put_object(object);

667

put_object(object);

665

}

668

}

666

669

667

static void __paint_it(struct kmemleak_object *object, int color)

670

static void __paint_it(struct kmemleak_object *object, int color)

668

{

671

{

669

object->min_count = color;

672

object->min_count = color;

670

if (color == KMEMLEAK_BLACK)

673

if (color == KMEMLEAK_BLACK)

671

object->flags |= OBJECT_NO_SCAN;

674

object->flags |= OBJECT_NO_SCAN;

672

}

675

}

673

676

674

static void paint_it(struct kmemleak_object *object, int color)

677

static void paint_it(struct kmemleak_object *object, int color)

675

{

678

{

676

unsigned long flags;

679

unsigned long flags;

677

680

678

spin_lock_irqsave(&object->lock, flags);

681

spin_lock_irqsave(&object->lock, flags);

679

__paint_it(object, color);

682

__paint_it(object, color);

680

spin_unlock_irqrestore(&object->lock, flags);

683

spin_unlock_irqrestore(&object->lock, flags);

681

}

684

}

682

685

683

static void paint_ptr(unsigned long ptr, int color)

686

static void paint_ptr(unsigned long ptr, int color)

684

{

687

{

685

struct kmemleak_object *object;

688

struct kmemleak_object *object;

686

689

687

object = find_and_get_object(ptr, 0);

690

object = find_and_get_object(ptr, 0);

688

if (!object) {

691

if (!object) {

689

kmemleak_warn("Trying to color unknown object "

692

kmemleak_warn("Trying to color unknown object "

690

"at 0x%08lx as %s\n", ptr,

693

"at 0x%08lx as %s\n", ptr,

691

(color == KMEMLEAK_GREY) ? "Grey" :

694

(color == KMEMLEAK_GREY) ? "Grey" :

692

(color == KMEMLEAK_BLACK) ? "Black" : "Unknown");

695

(color == KMEMLEAK_BLACK) ? "Black" : "Unknown");

693

return;

696

return;

694

}

697

}

695

paint_it(object, color);

698

paint_it(object, color);

696

put_object(object);

699

put_object(object);

697

}

700

}

698

701

699

/*

702

/*

700

* Make a object permanently as gray-colored so that it can no longer be

703

* Make a object permanently as gray-colored so that it can no longer be

701

* reported as a leak. This is used in general to mark a false positive.

704

* reported as a leak. This is used in general to mark a false positive.

702

*/

705

*/

703

static void make_gray_object(unsigned long ptr)

706

static void make_gray_object(unsigned long ptr)

704

{

707

{

705

paint_ptr(ptr, KMEMLEAK_GREY);

708

paint_ptr(ptr, KMEMLEAK_GREY);

706

}

709

}

707

710

708

/*

711

/*

709

* Mark the object as black-colored so that it is ignored from scans and

712

* Mark the object as black-colored so that it is ignored from scans and

710

* reporting.

713

* reporting.

711

*/

714

*/

712

static void make_black_object(unsigned long ptr)

715

static void make_black_object(unsigned long ptr)

713

{

716

{

714

paint_ptr(ptr, KMEMLEAK_BLACK);

717

paint_ptr(ptr, KMEMLEAK_BLACK);

715

}

718

}

716

719

717

/*

720

/*

718

* Add a scanning area to the object. If at least one such area is added,

721

* Add a scanning area to the object. If at least one such area is added,

719

* kmemleak will only scan these ranges rather than the whole memory block.

722

* kmemleak will only scan these ranges rather than the whole memory block.

720

*/

723

*/

721

static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)

724

static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)

722

{

725

{

723

unsigned long flags;

726

unsigned long flags;

724

struct kmemleak_object *object;

727

struct kmemleak_object *object;

725

struct kmemleak_scan_area *area;

728

struct kmemleak_scan_area *area;

726

729

727

object = find_and_get_object(ptr, 1);

730

object = find_and_get_object(ptr, 1);

728

if (!object) {

731

if (!object) {

729

kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",

732

kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",

730

ptr);

733

ptr);

731

return;

734

return;

732

}

735

}

733

736

734

area = kmem_cache_alloc(scan_area_cache, gfp & GFP_KMEMLEAK_MASK);

737

area = kmem_cache_alloc(scan_area_cache, gfp & GFP_KMEMLEAK_MASK);

735

if (!area) {

738

if (!area) {

736

kmemleak_warn("Cannot allocate a scan area\n");

739

kmemleak_warn("Cannot allocate a scan area\n");

737

goto out;

740

goto out;

738

}

741

}

739

742

740

spin_lock_irqsave(&object->lock, flags);

743

spin_lock_irqsave(&object->lock, flags);

741

if (ptr + size > object->pointer + object->size) {

744

if (ptr + size > object->pointer + object->size) {

742

kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);

745

kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);

743

dump_object_info(object);

746

dump_object_info(object);

744

kmem_cache_free(scan_area_cache, area);

747

kmem_cache_free(scan_area_cache, area);

745

goto out_unlock;

748

goto out_unlock;

746

}

749

}

747

750

748

INIT_HLIST_NODE(&area->node);

751

INIT_HLIST_NODE(&area->node);

749

area->start = ptr;

752

area->start = ptr;

750

area->size = size;

753

area->size = size;

751

754

752

hlist_add_head(&area->node, &object->area_list);

755

hlist_add_head(&area->node, &object->area_list);

753

out_unlock:

756

out_unlock:

754

spin_unlock_irqrestore(&object->lock, flags);

757

spin_unlock_irqrestore(&object->lock, flags);

755

out:

758

out:

756

put_object(object);

759

put_object(object);

757

}

760

}

758

761

759

/*

762

/*

760

* Set the OBJECT_NO_SCAN flag for the object corresponding to the give

763

* Set the OBJECT_NO_SCAN flag for the object corresponding to the give

761

* pointer. Such object will not be scanned by kmemleak but references to it

764

* pointer. Such object will not be scanned by kmemleak but references to it

762

* are searched.

765

* are searched.

763

*/

766

*/

764

static void object_no_scan(unsigned long ptr)

767

static void object_no_scan(unsigned long ptr)

765

{

768

{

766

unsigned long flags;

769

unsigned long flags;

767

struct kmemleak_object *object;

770

struct kmemleak_object *object;

768

771

769

object = find_and_get_object(ptr, 0);

772

object = find_and_get_object(ptr, 0);

770

if (!object) {

773

if (!object) {

771

kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);

774

kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);

772

return;

775

return;

773

}

776

}

774

777

775

spin_lock_irqsave(&object->lock, flags);

778

spin_lock_irqsave(&object->lock, flags);

776

object->flags |= OBJECT_NO_SCAN;

779

object->flags |= OBJECT_NO_SCAN;

777

spin_unlock_irqrestore(&object->lock, flags);

780

spin_unlock_irqrestore(&object->lock, flags);

778

put_object(object);

781

put_object(object);

779

}

782

}

780

783

781

/*

784

/*

782

* Log an early kmemleak_* call to the early_log buffer. These calls will be

785

* Log an early kmemleak_* call to the early_log buffer. These calls will be

783

* processed later once kmemleak is fully initialized.

786

* processed later once kmemleak is fully initialized.

784

*/

787

*/

785

static void __init log_early(int op_type, const void *ptr, size_t size,

788

static void __init log_early(int op_type, const void *ptr, size_t size,

786

int min_count)

789

int min_count)

787

{

790

{

788

unsigned long flags;

791

unsigned long flags;

789

struct early_log *log;

792

struct early_log *log;

790

793

791

if (crt_early_log >= ARRAY_SIZE(early_log)) {

794

if (crt_early_log >= ARRAY_SIZE(early_log)) {

792

pr_warning("Early log buffer exceeded, "

795

pr_warning("Early log buffer exceeded, "

793

"please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n");

796

"please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n");

794

kmemleak_disable();

797

kmemleak_disable();

795

return;

798

return;

796

}

799

}

797

800

798

/*

801

/*

799

* There is no need for locking since the kernel is still in UP mode

802

* There is no need for locking since the kernel is still in UP mode

800

* at this stage. Disabling the IRQs is enough.

803

* at this stage. Disabling the IRQs is enough.

801

*/

804

*/

802

local_irq_save(flags);

805

local_irq_save(flags);

803

log = &early_log[crt_early_log];

806

log = &early_log[crt_early_log];

804

log->op_type = op_type;

807

log->op_type = op_type;

805

log->ptr = ptr;

808

log->ptr = ptr;

806

log->size = size;

809

log->size = size;

807

log->min_count = min_count;

810

log->min_count = min_count;

808

if (op_type == KMEMLEAK_ALLOC)

811

if (op_type == KMEMLEAK_ALLOC)

809

log->trace_len = __save_stack_trace(log->trace);

812

log->trace_len = __save_stack_trace(log->trace);

810

crt_early_log++;

813

crt_early_log++;

811

local_irq_restore(flags);

814

local_irq_restore(flags);

812

}

815

}

813

816

814

/*

817

/*

815

* Log an early allocated block and populate the stack trace.

818

* Log an early allocated block and populate the stack trace.

816

*/

819

*/

817

static void early_alloc(struct early_log *log)

820

static void early_alloc(struct early_log *log)

818

{

821

{

819

struct kmemleak_object *object;

822

struct kmemleak_object *object;

820

unsigned long flags;

823

unsigned long flags;

821

int i;

824

int i;

822

825

823

if (!atomic_read(&kmemleak_enabled) || !log->ptr || IS_ERR(log->ptr))

826

if (!atomic_read(&kmemleak_enabled) || !log->ptr || IS_ERR(log->ptr))

824

return;

827

return;

825

828

826

/*

829

/*

827

* RCU locking needed to ensure object is not freed via put_object().

830

* RCU locking needed to ensure object is not freed via put_object().

828

*/

831

*/

829

rcu_read_lock();

832

rcu_read_lock();

830

object = create_object((unsigned long)log->ptr, log->size,

833

object = create_object((unsigned long)log->ptr, log->size,

831

log->min_count, GFP_ATOMIC);

834

log->min_count, GFP_ATOMIC);

832

if (!object)

835

if (!object)

833

goto out;

836

goto out;

834

spin_lock_irqsave(&object->lock, flags);

837

spin_lock_irqsave(&object->lock, flags);

835

for (i = 0; i < log->trace_len; i++)

838

for (i = 0; i < log->trace_len; i++)

836

object->trace[i] = log->trace[i];

839

object->trace[i] = log->trace[i];

837

object->trace_len = log->trace_len;

840

object->trace_len = log->trace_len;

838

spin_unlock_irqrestore(&object->lock, flags);

841

spin_unlock_irqrestore(&object->lock, flags);

839

out:

842

out:

840

rcu_read_unlock();

843

rcu_read_unlock();

841

}

844

}

842

845

843

/*

846

/*

844

* Memory allocation function callback. This function is called from the

847

* Memory allocation function callback. This function is called from the

845

* kernel allocators when a new block is allocated (kmem_cache_alloc, kmalloc,

848

* kernel allocators when a new block is allocated (kmem_cache_alloc, kmalloc,

846

* vmalloc etc.).

849

* vmalloc etc.).

847

*/

850

*/

848

void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,

851

void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,

849

gfp_t gfp)

852

gfp_t gfp)

850

{

853

{

851

pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);

854

pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);

852

855

853

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

856

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

854

create_object((unsigned long)ptr, size, min_count, gfp);

857

create_object((unsigned long)ptr, size, min_count, gfp);

855

else if (atomic_read(&kmemleak_early_log))

858

else if (atomic_read(&kmemleak_early_log))

856

log_early(KMEMLEAK_ALLOC, ptr, size, min_count);

859

log_early(KMEMLEAK_ALLOC, ptr, size, min_count);

857

}

860

}

858

EXPORT_SYMBOL_GPL(kmemleak_alloc);

861

EXPORT_SYMBOL_GPL(kmemleak_alloc);

859

862

860

/*

863

/*

861

* Memory freeing function callback. This function is called from the kernel

864

* Memory freeing function callback. This function is called from the kernel

862

* allocators when a block is freed (kmem_cache_free, kfree, vfree etc.).

865

* allocators when a block is freed (kmem_cache_free, kfree, vfree etc.).

863

*/

866

*/

864

void __ref kmemleak_free(const void *ptr)

867

void __ref kmemleak_free(const void *ptr)

865

{

868

{

866

pr_debug("%s(0x%p)\n", __func__, ptr);

869

pr_debug("%s(0x%p)\n", __func__, ptr);

867

870

868

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

871

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

869

delete_object_full((unsigned long)ptr);

872

delete_object_full((unsigned long)ptr);

870

else if (atomic_read(&kmemleak_early_log))

873

else if (atomic_read(&kmemleak_early_log))

871

log_early(KMEMLEAK_FREE, ptr, 0, 0);

874

log_early(KMEMLEAK_FREE, ptr, 0, 0);

872

}

875

}

873

EXPORT_SYMBOL_GPL(kmemleak_free);

876

EXPORT_SYMBOL_GPL(kmemleak_free);

874

877

875

/*

878

/*

876

* Partial memory freeing function callback. This function is usually called

879

* Partial memory freeing function callback. This function is usually called

877

* from bootmem allocator when (part of) a memory block is freed.

880

* from bootmem allocator when (part of) a memory block is freed.

878

*/

881

*/

879

void __ref kmemleak_free_part(const void *ptr, size_t size)

882

void __ref kmemleak_free_part(const void *ptr, size_t size)

880

{

883

{

881

pr_debug("%s(0x%p)\n", __func__, ptr);

884

pr_debug("%s(0x%p)\n", __func__, ptr);

882

885

883

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

886

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

884

delete_object_part((unsigned long)ptr, size);

887

delete_object_part((unsigned long)ptr, size);

885

else if (atomic_read(&kmemleak_early_log))

888

else if (atomic_read(&kmemleak_early_log))

886

log_early(KMEMLEAK_FREE_PART, ptr, size, 0);

889

log_early(KMEMLEAK_FREE_PART, ptr, size, 0);

887

}

890

}

888

EXPORT_SYMBOL_GPL(kmemleak_free_part);

891

EXPORT_SYMBOL_GPL(kmemleak_free_part);

889

892

890

/*

893

/*

891

* Mark an already allocated memory block as a false positive. This will cause

894

* Mark an already allocated memory block as a false positive. This will cause

892

* the block to no longer be reported as leak and always be scanned.

895

* the block to no longer be reported as leak and always be scanned.

893

*/

896

*/

894

void __ref kmemleak_not_leak(const void *ptr)

897

void __ref kmemleak_not_leak(const void *ptr)

895

{

898

{

896

pr_debug("%s(0x%p)\n", __func__, ptr);

899

pr_debug("%s(0x%p)\n", __func__, ptr);

897

900

898

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

901

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

899

make_gray_object((unsigned long)ptr);

902

make_gray_object((unsigned long)ptr);

900

else if (atomic_read(&kmemleak_early_log))

903

else if (atomic_read(&kmemleak_early_log))

901

log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);

904

log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);

902

}

905

}

903

EXPORT_SYMBOL(kmemleak_not_leak);

906

EXPORT_SYMBOL(kmemleak_not_leak);

904

907

905

/*

908

/*

906

* Ignore a memory block. This is usually done when it is known that the

909

* Ignore a memory block. This is usually done when it is known that the

907

* corresponding block is not a leak and does not contain any references to

910

* corresponding block is not a leak and does not contain any references to

908

* other allocated memory blocks.

911

* other allocated memory blocks.

909

*/

912

*/

910

void __ref kmemleak_ignore(const void *ptr)

913

void __ref kmemleak_ignore(const void *ptr)

911

{

914

{

912

pr_debug("%s(0x%p)\n", __func__, ptr);

915

pr_debug("%s(0x%p)\n", __func__, ptr);

913

916

914

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

917

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

915

make_black_object((unsigned long)ptr);

918

make_black_object((unsigned long)ptr);

916

else if (atomic_read(&kmemleak_early_log))

919

else if (atomic_read(&kmemleak_early_log))

917

log_early(KMEMLEAK_IGNORE, ptr, 0, 0);

920

log_early(KMEMLEAK_IGNORE, ptr, 0, 0);

918

}

921

}

919

EXPORT_SYMBOL(kmemleak_ignore);

922

EXPORT_SYMBOL(kmemleak_ignore);

920

923

921

/*

924

/*

922

* Limit the range to be scanned in an allocated memory block.

925

* Limit the range to be scanned in an allocated memory block.

923

*/

926

*/

924

void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)

927

void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)

925

{

928

{

926

pr_debug("%s(0x%p)\n", __func__, ptr);

929

pr_debug("%s(0x%p)\n", __func__, ptr);

927

930

928

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

931

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

929

add_scan_area((unsigned long)ptr, size, gfp);

932

add_scan_area((unsigned long)ptr, size, gfp);

930

else if (atomic_read(&kmemleak_early_log))

933

else if (atomic_read(&kmemleak_early_log))

931

log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);

934

log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);

932

}

935

}

933

EXPORT_SYMBOL(kmemleak_scan_area);

936

EXPORT_SYMBOL(kmemleak_scan_area);

934

937

935

/*

938

/*

936

* Inform kmemleak not to scan the given memory block.

939

* Inform kmemleak not to scan the given memory block.

937

*/

940

*/

938

void __ref kmemleak_no_scan(const void *ptr)

941

void __ref kmemleak_no_scan(const void *ptr)

939

{

942

{

940

pr_debug("%s(0x%p)\n", __func__, ptr);

943

pr_debug("%s(0x%p)\n", __func__, ptr);

941

944

942

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

945

if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))

943

object_no_scan((unsigned long)ptr);

946

object_no_scan((unsigned long)ptr);

944

else if (atomic_read(&kmemleak_early_log))

947

else if (atomic_read(&kmemleak_early_log))

945

log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);

948

log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);

946

}

949

}

947

EXPORT_SYMBOL(kmemleak_no_scan);

950

EXPORT_SYMBOL(kmemleak_no_scan);

948

951

949

/*

952

/*

950

* Update an object's checksum and return true if it was modified.

953

* Update an object's checksum and return true if it was modified.

951

*/

954

*/

952

static bool update_checksum(struct kmemleak_object *object)

955

static bool update_checksum(struct kmemleak_object *object)

953

{

956

{

954

u32 old_csum = object->checksum;

957

u32 old_csum = object->checksum;

955

958

956

if (!kmemcheck_is_obj_initialized(object->pointer, object->size))

959

if (!kmemcheck_is_obj_initialized(object->pointer, object->size))

957

return false;

960

return false;

958

961

959

object->checksum = crc32(0, (void *)object->pointer, object->size);

962

object->checksum = crc32(0, (void *)object->pointer, object->size);

960

return object->checksum != old_csum;

963

return object->checksum != old_csum;

961

}

964

}

962

965

963

/*

966

/*

964

* Memory scanning is a long process and it needs to be interruptable. This

967

* Memory scanning is a long process and it needs to be interruptable. This

965

* function checks whether such interrupt condition occured.

968

* function checks whether such interrupt condition occured.

966

*/

969

*/

967

static int scan_should_stop(void)

970

static int scan_should_stop(void)

968

{

971

{

969

if (!atomic_read(&kmemleak_enabled))

972

if (!atomic_read(&kmemleak_enabled))

970

return 1;

973

return 1;

971

974

972

/*

975

/*

973

* This function may be called from either process or kthread context,

976

* This function may be called from either process or kthread context,

974

* hence the need to check for both stop conditions.

977

* hence the need to check for both stop conditions.

975

*/

978

*/

976

if (current->mm)

979

if (current->mm)

977

return signal_pending(current);

980

return signal_pending(current);

978

else

981

else

979

return kthread_should_stop();

982

return kthread_should_stop();

980

983

981

return 0;

984

return 0;

982

}

985

}

983

986

984

/*

987

/*

985

* Scan a memory block (exclusive range) for valid pointers and add those

988

* Scan a memory block (exclusive range) for valid pointers and add those

986

* found to the gray list.

989

* found to the gray list.

987

*/

990

*/

988

static void scan_block(void *_start, void *_end,

991

static void scan_block(void *_start, void *_end,

989

struct kmemleak_object *scanned, int allow_resched)

992

struct kmemleak_object *scanned, int allow_resched)

990

{

993

{

991

unsigned long *ptr;

994

unsigned long *ptr;

992

unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);

995

unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);

993

unsigned long *end = _end - (BYTES_PER_POINTER - 1);

996

unsigned long *end = _end - (BYTES_PER_POINTER - 1);

994

997

995

for (ptr = start; ptr < end; ptr++) {

998

for (ptr = start; ptr < end; ptr++) {

996

struct kmemleak_object *object;

999

struct kmemleak_object *object;

997

unsigned long flags;

1000

unsigned long flags;

998

unsigned long pointer;

1001

unsigned long pointer;

999

1002

1000

if (allow_resched)

1003

if (allow_resched)

1001

cond_resched();

1004

cond_resched();

1002

if (scan_should_stop())

1005

if (scan_should_stop())

1003

break;

1006

break;

1004

1007

1005

/* don't scan uninitialized memory */

1008

/* don't scan uninitialized memory */

1006

if (!kmemcheck_is_obj_initialized((unsigned long)ptr,

1009

if (!kmemcheck_is_obj_initialized((unsigned long)ptr,

1007

BYTES_PER_POINTER))

1010

BYTES_PER_POINTER))

1008

continue;

1011

continue;

1009

1012

1010

pointer = *ptr;

1013

pointer = *ptr;

1011

1014

1012

object = find_and_get_object(pointer, 1);

1015

object = find_and_get_object(pointer, 1);

1013

if (!object)

1016

if (!object)

1014

continue;

1017

continue;

1015

if (object == scanned) {

1018

if (object == scanned) {

1016

/* self referenced, ignore */

1019

/* self referenced, ignore */

1017

put_object(object);

1020

put_object(object);

1018

continue;

1021

continue;

1019

}

1022

}

1020

1023

1021

/*

1024

/*

1022

* Avoid the lockdep recursive warning on object->lock being

1025

* Avoid the lockdep recursive warning on object->lock being

1023

* previously acquired in scan_object(). These locks are

1026

* previously acquired in scan_object(). These locks are

1024

* enclosed by scan_mutex.

1027

* enclosed by scan_mutex.

1025

*/

1028

*/

1026

spin_lock_irqsave_nested(&object->lock, flags,

1029

spin_lock_irqsave_nested(&object->lock, flags,

1027

SINGLE_DEPTH_NESTING);

1030

SINGLE_DEPTH_NESTING);

1028

if (!color_white(object)) {

1031

if (!color_white(object)) {

1029

/* non-orphan, ignored or new */

1032

/* non-orphan, ignored or new */

1030

spin_unlock_irqrestore(&object->lock, flags);

1033

spin_unlock_irqrestore(&object->lock, flags);

1031

put_object(object);

1034

put_object(object);

1032

continue;

1035

continue;

1033

}

1036

}

1034

1037

1035

/*

1038

/*

1036

* Increase the object's reference count (number of pointers

1039

* Increase the object's reference count (number of pointers

1037

* to the memory block). If this count reaches the required

1040

* to the memory block). If this count reaches the required

1038

* minimum, the object's color will become gray and it will be

1041

* minimum, the object's color will become gray and it will be

1039

* added to the gray_list.

1042

* added to the gray_list.

1040

*/

1043

*/

1041

object->count++;

1044

object->count++;

1042

if (color_gray(object)) {

1045

if (color_gray(object)) {

1043

list_add_tail(&object->gray_list, &gray_list);

1046

list_add_tail(&object->gray_list, &gray_list);

1044

spin_unlock_irqrestore(&object->lock, flags);

1047

spin_unlock_irqrestore(&object->lock, flags);

1045

continue;

1048

continue;

1046

}

1049

}

1047

1050

1048

spin_unlock_irqrestore(&object->lock, flags);

1051

spin_unlock_irqrestore(&object->lock, flags);

1049

put_object(object);

1052

put_object(object);

1050

}

1053

}

1051

}

1054

}

1052

1055

1053

/*

1056

/*

1054

* Scan a memory block corresponding to a kmemleak_object. A condition is

1057

* Scan a memory block corresponding to a kmemleak_object. A condition is

1055

* that object->use_count >= 1.

1058

* that object->use_count >= 1.

1056

*/

1059

*/

1057

static void scan_object(struct kmemleak_object *object)

1060

static void scan_object(struct kmemleak_object *object)

1058

{

1061

{

1059

struct kmemleak_scan_area *area;

1062

struct kmemleak_scan_area *area;

1060

struct hlist_node *elem;

1063

struct hlist_node *elem;

1061

unsigned long flags;

1064

unsigned long flags;

1062

1065

1063

/*

1066

/*

1064

* Once the object->lock is acquired, the corresponding memory block

1067

* Once the object->lock is acquired, the corresponding memory block

1065

* cannot be freed (the same lock is acquired in delete_object).

1068

* cannot be freed (the same lock is acquired in delete_object).

1066

*/

1069

*/

1067

spin_lock_irqsave(&object->lock, flags);

1070

spin_lock_irqsave(&object->lock, flags);

1068

if (object->flags & OBJECT_NO_SCAN)

1071

if (object->flags & OBJECT_NO_SCAN)

1069

goto out;

1072

goto out;

1070

if (!(object->flags & OBJECT_ALLOCATED))

1073

if (!(object->flags & OBJECT_ALLOCATED))

1071

/* already freed object */

1074

/* already freed object */

1072

goto out;

1075

goto out;

1073

if (hlist_empty(&object->area_list)) {

1076

if (hlist_empty(&object->area_list)) {

1074

void *start = (void *)object->pointer;

1077

void *start = (void *)object->pointer;

1075

void *end = (void *)(object->pointer + object->size);

1078

void *end = (void *)(object->pointer + object->size);

1076

1079

1077

while (start < end && (object->flags & OBJECT_ALLOCATED) &&

1080

while (start < end && (object->flags & OBJECT_ALLOCATED) &&

1078

!(object->flags & OBJECT_NO_SCAN)) {

1081

!(object->flags & OBJECT_NO_SCAN)) {

1079

scan_block(start, min(start + MAX_SCAN_SIZE, end),

1082

scan_block(start, min(start + MAX_SCAN_SIZE, end),

1080

object, 0);

1083

object, 0);

1081

start += MAX_SCAN_SIZE;

1084

start += MAX_SCAN_SIZE;

1082

1085

1083

spin_unlock_irqrestore(&object->lock, flags);

1086

spin_unlock_irqrestore(&object->lock, flags);

1084

cond_resched();

1087

cond_resched();

1085

spin_lock_irqsave(&object->lock, flags);

1088

spin_lock_irqsave(&object->lock, flags);

1086

}

1089

}

1087

} else

1090

} else

1088

hlist_for_each_entry(area, elem, &object->area_list, node)

1091

hlist_for_each_entry(area, elem, &object->area_list, node)

1089

scan_block((void *)area->start,

1092

scan_block((void *)area->start,

1090

(void *)(area->start + area->size),

1093

(void *)(area->start + area->size),

1091

object, 0);

1094

object, 0);

1092

out:

1095

out:

1093

spin_unlock_irqrestore(&object->lock, flags);

1096

spin_unlock_irqrestore(&object->lock, flags);

1094

}

1097

}

1095

1098

1096

/*

1099

/*

1097

* Scan the objects already referenced (gray objects). More objects will be

1100

* Scan the objects already referenced (gray objects). More objects will be

1098

* referenced and, if there are no memory leaks, all the objects are scanned.

1101

* referenced and, if there are no memory leaks, all the objects are scanned.

1099

*/

1102

*/

1100

static void scan_gray_list(void)

1103

static void scan_gray_list(void)

1101

{

1104

{

1102

struct kmemleak_object *object, *tmp;

1105

struct kmemleak_object *object, *tmp;

1103

1106

1104

/*

1107

/*

1105

* The list traversal is safe for both tail additions and removals

1108

* The list traversal is safe for both tail additions and removals

1106

* from inside the loop. The kmemleak objects cannot be freed from

1109

* from inside the loop. The kmemleak objects cannot be freed from

1107

* outside the loop because their use_count was incremented.

1110

* outside the loop because their use_count was incremented.

1108

*/

1111

*/

1109

object = list_entry(gray_list.next, typeof(*object), gray_list);

1112

object = list_entry(gray_list.next, typeof(*object), gray_list);

1110

while (&object->gray_list != &gray_list) {

1113

while (&object->gray_list != &gray_list) {

1111

cond_resched();

1114

cond_resched();

1112

1115

1113

/* may add new objects to the list */

1116

/* may add new objects to the list */

1114

if (!scan_should_stop())

1117

if (!scan_should_stop())

1115

scan_object(object);

1118

scan_object(object);

1116

1119

1117

tmp = list_entry(object->gray_list.next, typeof(*object),

1120

tmp = list_entry(object->gray_list.next, typeof(*object),

1118

gray_list);

1121

gray_list);

1119

1122

1120

/* remove the object from the list and release it */

1123

/* remove the object from the list and release it */

1121

list_del(&object->gray_list);

1124

list_del(&object->gray_list);

1122

put_object(object);

1125

put_object(object);

1123

1126

1124

object = tmp;

1127

object = tmp;

1125

}

1128

}

1126

WARN_ON(!list_empty(&gray_list));

1129

WARN_ON(!list_empty(&gray_list));

1127

}

1130

}

1128

1131

1129

/*

1132

/*

1130

* Scan data sections and all the referenced memory blocks allocated via the

1133

* Scan data sections and all the referenced memory blocks allocated via the

1131

* kernel's standard allocators. This function must be called with the

1134

* kernel's standard allocators. This function must be called with the

1132

* scan_mutex held.

1135

* scan_mutex held.

1133

*/

1136

*/

1134

static void kmemleak_scan(void)

1137

static void kmemleak_scan(void)

1135

{

1138

{

1136

unsigned long flags;

1139

unsigned long flags;

1137

struct kmemleak_object *object;

1140

struct kmemleak_object *object;

1138

int i;

1141

int i;

1139

int new_leaks = 0;

1142

int new_leaks = 0;

1140

1143

1141

jiffies_last_scan = jiffies;

1144

jiffies_last_scan = jiffies;

1142

1145

1143

/* prepare the kmemleak_object's */

1146

/* prepare the kmemleak_object's */

1144

rcu_read_lock();

1147

rcu_read_lock();

1145

list_for_each_entry_rcu(object, &object_list, object_list) {

1148

list_for_each_entry_rcu(object, &object_list, object_list) {

1146

spin_lock_irqsave(&object->lock, flags);

1149

spin_lock_irqsave(&object->lock, flags);

1147

#ifdef DEBUG

1150

#ifdef DEBUG

1148

/*

1151

/*

1149

* With a few exceptions there should be a maximum of

1152

* With a few exceptions there should be a maximum of

1150

* 1 reference to any object at this point.

1153

* 1 reference to any object at this point.

1151

*/

1154

*/

1152

if (atomic_read(&object->use_count) > 1) {

1155

if (atomic_read(&object->use_count) > 1) {

1153

pr_debug("object->use_count = %d\n",

1156

pr_debug("object->use_count = %d\n",

1154

atomic_read(&object->use_count));

1157

atomic_read(&object->use_count));

1155

dump_object_info(object);

1158

dump_object_info(object);

1156

}

1159

}

1157

#endif

1160

#endif

1158

/* reset the reference count (whiten the object) */

1161

/* reset the reference count (whiten the object) */

1159

object->count = 0;

1162

object->count = 0;

1160

if (color_gray(object) && get_object(object))

1163

if (color_gray(object) && get_object(object))

1161

list_add_tail(&object->gray_list, &gray_list);

1164

list_add_tail(&object->gray_list, &gray_list);

1162

1165

1163

spin_unlock_irqrestore(&object->lock, flags);

1166

spin_unlock_irqrestore(&object->lock, flags);

1164

}

1167

}

1165

rcu_read_unlock();

1168

rcu_read_unlock();

1166

1169

1167

/* data/bss scanning */

1170

/* data/bss scanning */

1168

scan_block(_sdata, _edata, NULL, 1);

1171

scan_block(_sdata, _edata, NULL, 1);

1169

scan_block(__bss_start, __bss_stop, NULL, 1);

1172

scan_block(__bss_start, __bss_stop, NULL, 1);

1170

1173

1171

#ifdef CONFIG_SMP

1174

#ifdef CONFIG_SMP

1172

/* per-cpu sections scanning */

1175

/* per-cpu sections scanning */

1173

for_each_possible_cpu(i)

1176

for_each_possible_cpu(i)

1174

scan_block(__per_cpu_start + per_cpu_offset(i),

1177

scan_block(__per_cpu_start + per_cpu_offset(i),

1175

__per_cpu_end + per_cpu_offset(i), NULL, 1);

1178

__per_cpu_end + per_cpu_offset(i), NULL, 1);

1176

#endif

1179

#endif

1177

1180

1178

/*

1181

/*

1179

* Struct page scanning for each node. The code below is not yet safe

1182

* Struct page scanning for each node. The code below is not yet safe

1180

* with MEMORY_HOTPLUG.

1183

* with MEMORY_HOTPLUG.

1181

*/

1184

*/

1182

for_each_online_node(i) {

1185

for_each_online_node(i) {

1183

pg_data_t *pgdat = NODE_DATA(i);

1186

pg_data_t *pgdat = NODE_DATA(i);

1184

unsigned long start_pfn = pgdat->node_start_pfn;

1187

unsigned long start_pfn = pgdat->node_start_pfn;

1185

unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;

1188

unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;

1186

unsigned long pfn;

1189

unsigned long pfn;

1187

1190

1188

for (pfn = start_pfn; pfn < end_pfn; pfn++) {

1191

for (pfn = start_pfn; pfn < end_pfn; pfn++) {

1189

struct page *page;

1192

struct page *page;

1190

1193

1191

if (!pfn_valid(pfn))

1194

if (!pfn_valid(pfn))

1192

continue;

1195

continue;

1193

page = pfn_to_page(pfn);

1196

page = pfn_to_page(pfn);

1194

/* only scan if page is in use */

1197

/* only scan if page is in use */

1195

if (page_count(page) == 0)

1198

if (page_count(page) == 0)

1196

continue;

1199

continue;

1197

scan_block(page, page + 1, NULL, 1);

1200

scan_block(page, page + 1, NULL, 1);

1198

}

1201

}

1199

}

1202

}

1200

1203

1201

/*

1204

/*

1202

* Scanning the task stacks (may introduce false negatives).

1205

* Scanning the task stacks (may introduce false negatives).

1203

*/

1206

*/

1204

if (kmemleak_stack_scan) {

1207

if (kmemleak_stack_scan) {

1205

struct task_struct *p, *g;

1208

struct task_struct *p, *g;

1206

1209

1207

read_lock(&tasklist_lock);

1210

read_lock(&tasklist_lock);

1208

do_each_thread(g, p) {

1211

do_each_thread(g, p) {

1209

scan_block(task_stack_page(p), task_stack_page(p) +

1212

scan_block(task_stack_page(p), task_stack_page(p) +

1210

THREAD_SIZE, NULL, 0);

1213

THREAD_SIZE, NULL, 0);

1211

} while_each_thread(g, p);

1214

} while_each_thread(g, p);

1212

read_unlock(&tasklist_lock);

1215

read_unlock(&tasklist_lock);

1213

}

1216

}

1214

1217

1215

/*

1218

/*

1216

* Scan the objects already referenced from the sections scanned

1219

* Scan the objects already referenced from the sections scanned

1217

* above.

1220

* above.

1218

*/

1221

*/

1219

scan_gray_list();

1222

scan_gray_list();

1220

1223

1221

/*

1224

/*

1222

* Check for new or unreferenced objects modified since the previous

1225

* Check for new or unreferenced objects modified since the previous

1223

* scan and color them gray until the next scan.

1226

* scan and color them gray until the next scan.

1224

*/

1227

*/

1225

rcu_read_lock();

1228

rcu_read_lock();

1226

list_for_each_entry_rcu(object, &object_list, object_list) {

1229

list_for_each_entry_rcu(object, &object_list, object_list) {

1227

spin_lock_irqsave(&object->lock, flags);

1230

spin_lock_irqsave(&object->lock, flags);

1228

if (color_white(object) && (object->flags & OBJECT_ALLOCATED)

1231

if (color_white(object) && (object->flags & OBJECT_ALLOCATED)

1229

&& update_checksum(object) && get_object(object)) {

1232

&& update_checksum(object) && get_object(object)) {

1230

/* color it gray temporarily */

1233

/* color it gray temporarily */

1231

object->count = object->min_count;

1234

object->count = object->min_count;

1232

list_add_tail(&object->gray_list, &gray_list);

1235

list_add_tail(&object->gray_list, &gray_list);

1233

}

1236

}

1234

spin_unlock_irqrestore(&object->lock, flags);

1237

spin_unlock_irqrestore(&object->lock, flags);

1235

}

1238

}

1236

rcu_read_unlock();

1239

rcu_read_unlock();

1237

1240

1238

/*

1241

/*

1239

* Re-scan the gray list for modified unreferenced objects.

1242

* Re-scan the gray list for modified unreferenced objects.

1240

*/

1243

*/

1241

scan_gray_list();

1244

scan_gray_list();

1242

1245

1243

/*

1246

/*

1244

* If scanning was stopped do not report any new unreferenced objects.

1247

* If scanning was stopped do not report any new unreferenced objects.

1245

*/

1248

*/

1246

if (scan_should_stop())

1249

if (scan_should_stop())

1247

return;

1250

return;

1248

1251

1249

/*

1252

/*

1250

* Scanning result reporting.

1253

* Scanning result reporting.

1251

*/

1254

*/

1252

rcu_read_lock();

1255

rcu_read_lock();

1253

list_for_each_entry_rcu(object, &object_list, object_list) {

1256

list_for_each_entry_rcu(object, &object_list, object_list) {

1254

spin_lock_irqsave(&object->lock, flags);

1257

spin_lock_irqsave(&object->lock, flags);

1255

if (unreferenced_object(object) &&

1258

if (unreferenced_object(object) &&

1256

!(object->flags & OBJECT_REPORTED)) {

1259

!(object->flags & OBJECT_REPORTED)) {

1257

object->flags |= OBJECT_REPORTED;

1260

object->flags |= OBJECT_REPORTED;

1258

new_leaks++;

1261

new_leaks++;

1259

}

1262

}

1260

spin_unlock_irqrestore(&object->lock, flags);

1263

spin_unlock_irqrestore(&object->lock, flags);

1261

}

1264

}

1262

rcu_read_unlock();

1265

rcu_read_unlock();

1263

1266

1264

if (new_leaks)

1267

if (new_leaks)

1265

pr_info("%d new suspected memory leaks (see "

1268

pr_info("%d new suspected memory leaks (see "

1266

"/sys/kernel/debug/kmemleak)\n", new_leaks);

1269

"/sys/kernel/debug/kmemleak)\n", new_leaks);

1267

1270

1268

}

1271

}

1269

1272

1270

/*

1273

/*

1271

* Thread function performing automatic memory scanning. Unreferenced objects

1274

* Thread function performing automatic memory scanning. Unreferenced objects

1272

* at the end of a memory scan are reported but only the first time.

1275

* at the end of a memory scan are reported but only the first time.

1273

*/

1276

*/

1274

static int kmemleak_scan_thread(void *arg)

1277

static int kmemleak_scan_thread(void *arg)

1275

{

1278

{

1276

static int first_run = 1;

1279

static int first_run = 1;

1277

1280

1278

pr_info("Automatic memory scanning thread started\n");

1281

pr_info("Automatic memory scanning thread started\n");

1279

set_user_nice(current, 10);

1282

set_user_nice(current, 10);

1280

1283

1281

/*

1284

/*

1282

* Wait before the first scan to allow the system to fully initialize.

1285

* Wait before the first scan to allow the system to fully initialize.

1283

*/

1286

*/

1284

if (first_run) {

1287

if (first_run) {

1285

first_run = 0;

1288

first_run = 0;

1286

ssleep(SECS_FIRST_SCAN);

1289

ssleep(SECS_FIRST_SCAN);

1287

}

1290

}

1288

1291

1289

while (!kthread_should_stop()) {

1292

while (!kthread_should_stop()) {

1290

signed long timeout = jiffies_scan_wait;

1293

signed long timeout = jiffies_scan_wait;

1291

1294

1292

mutex_lock(&scan_mutex);

1295

mutex_lock(&scan_mutex);

1293

kmemleak_scan();

1296

kmemleak_scan();

1294

mutex_unlock(&scan_mutex);

1297

mutex_unlock(&scan_mutex);

1295

1298

1296

/* wait before the next scan */

1299

/* wait before the next scan */

1297

while (timeout && !kthread_should_stop())

1300

while (timeout && !kthread_should_stop())

1298

timeout = schedule_timeout_interruptible(timeout);

1301

timeout = schedule_timeout_interruptible(timeout);

1299

}

1302

}

1300

1303

1301

pr_info("Automatic memory scanning thread ended\n");

1304

pr_info("Automatic memory scanning thread ended\n");

1302

1305

1303

return 0;

1306

return 0;

1304

}

1307

}

1305

1308

1306

/*

1309

/*

1307

* Start the automatic memory scanning thread. This function must be called

1310

* Start the automatic memory scanning thread. This function must be called

1308

* with the scan_mutex held.

1311

* with the scan_mutex held.

1309

*/

1312

*/

1310

static void start_scan_thread(void)

1313

static void start_scan_thread(void)

1311

{

1314

{

1312

if (scan_thread)

1315

if (scan_thread)

1313

return;

1316

return;

1314

scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");

1317

scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");

1315

if (IS_ERR(scan_thread)) {

1318

if (IS_ERR(scan_thread)) {

1316

pr_warning("Failed to create the scan thread\n");

1319

pr_warning("Failed to create the scan thread\n");

1317

scan_thread = NULL;

1320

scan_thread = NULL;

1318

}

1321

}

1319

}

1322

}

1320

1323

1321

/*

1324

/*

1322

* Stop the automatic memory scanning thread. This function must be called

1325

* Stop the automatic memory scanning thread. This function must be called

1323

* with the scan_mutex held.

1326

* with the scan_mutex held.

1324

*/

1327

*/

1325

static void stop_scan_thread(void)

1328

static void stop_scan_thread(void)

1326

{

1329

{

1327

if (scan_thread) {

1330

if (scan_thread) {

1328

kthread_stop(scan_thread);

1331

kthread_stop(scan_thread);

1329

scan_thread = NULL;

1332

scan_thread = NULL;

1330

}

1333

}

1331

}

1334

}

1332

1335

1333

/*

1336

/*

1334

* Iterate over the object_list and return the first valid object at or after

1337

* Iterate over the object_list and return the first valid object at or after

1335

* the required position with its use_count incremented. The function triggers

1338

* the required position with its use_count incremented. The function triggers

1336

* a memory scanning when the pos argument points to the first position.

1339

* a memory scanning when the pos argument points to the first position.

1337

*/

1340

*/

1338

static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)

1341

static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)

1339

{

1342

{

1340

struct kmemleak_object *object;

1343

struct kmemleak_object *object;

1341

loff_t n = *pos;

1344

loff_t n = *pos;

1342

int err;

1345

int err;

1343

1346

1344

err = mutex_lock_interruptible(&scan_mutex);

1347

err = mutex_lock_interruptible(&scan_mutex);

1345

if (err < 0)

1348

if (err < 0)

1346

return ERR_PTR(err);

1349

return ERR_PTR(err);

1347

1350

1348

rcu_read_lock();

1351

rcu_read_lock();

1349

list_for_each_entry_rcu(object, &object_list, object_list) {

1352

list_for_each_entry_rcu(object, &object_list, object_list) {

1350

if (n-- > 0)

1353

if (n-- > 0)

1351

continue;

1354

continue;

1352

if (get_object(object))

1355

if (get_object(object))

1353

goto out;

1356

goto out;

1354

}

1357

}

1355

object = NULL;

1358

object = NULL;

1356

out:

1359

out:

1357

return object;

1360

return object;

1358

}

1361

}

1359

1362

1360

/*

1363

/*

1361

* Return the next object in the object_list. The function decrements the

1364

* Return the next object in the object_list. The function decrements the

1362

* use_count of the previous object and increases that of the next one.

1365

* use_count of the previous object and increases that of the next one.

1363

*/

1366

*/

1364

static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)

1367

static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)

1365

{

1368

{

1366

struct kmemleak_object *prev_obj = v;

1369

struct kmemleak_object *prev_obj = v;

1367

struct kmemleak_object *next_obj = NULL;

1370

struct kmemleak_object *next_obj = NULL;

1368

struct list_head *n = &prev_obj->object_list;

1371

struct list_head *n = &prev_obj->object_list;

1369

1372

1370

++(*pos);

1373

++(*pos);

1371

1374

1372

list_for_each_continue_rcu(n, &object_list) {

1375

list_for_each_continue_rcu(n, &object_list) {

1373

next_obj = list_entry(n, struct kmemleak_object, object_list);

1376

next_obj = list_entry(n, struct kmemleak_object, object_list);

1374

if (get_object(next_obj))

1377

if (get_object(next_obj))

1375

break;

1378

break;

1376

}

1379

}

1377

1380

1378

put_object(prev_obj);

1381

put_object(prev_obj);

1379

return next_obj;

1382

return next_obj;

1380

}

1383

}

1381

1384

1382

/*

1385

/*

1383

* Decrement the use_count of the last object required, if any.

1386

* Decrement the use_count of the last object required, if any.

1384

*/

1387

*/

1385

static void kmemleak_seq_stop(struct seq_file *seq, void *v)

1388

static void kmemleak_seq_stop(struct seq_file *seq, void *v)

1386

{

1389

{

1387

if (!IS_ERR(v)) {

1390

if (!IS_ERR(v)) {

1388

/*

1391

/*

1389

* kmemleak_seq_start may return ERR_PTR if the scan_mutex

1392

* kmemleak_seq_start may return ERR_PTR if the scan_mutex

1390

* waiting was interrupted, so only release it if !IS_ERR.

1393

* waiting was interrupted, so only release it if !IS_ERR.

1391

*/

1394

*/

1392

rcu_read_unlock();

1395

rcu_read_unlock();

1393

mutex_unlock(&scan_mutex);

1396

mutex_unlock(&scan_mutex);

1394

if (v)

1397

if (v)

1395

put_object(v);

1398

put_object(v);

1396

}

1399

}

1397

}

1400

}

1398

1401

1399

/*

1402

/*

1400

* Print the information for an unreferenced object to the seq file.

1403

* Print the information for an unreferenced object to the seq file.

1401

*/

1404

*/

1402

static int kmemleak_seq_show(struct seq_file *seq, void *v)

1405

static int kmemleak_seq_show(struct seq_file *seq, void *v)

1403

{

1406

{

1404

struct kmemleak_object *object = v;

1407

struct kmemleak_object *object = v;

1405

unsigned long flags;

1408

unsigned long flags;

1406

1409

1407

spin_lock_irqsave(&object->lock, flags);

1410

spin_lock_irqsave(&object->lock, flags);

1408

if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))

1411

if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))

1409

print_unreferenced(seq, object);

1412

print_unreferenced(seq, object);

1410

spin_unlock_irqrestore(&object->lock, flags);

1413

spin_unlock_irqrestore(&object->lock, flags);

1411

return 0;

1414

return 0;

1412

}

1415

}

1413

1416

1414

static const struct seq_operations kmemleak_seq_ops = {

1417

static const struct seq_operations kmemleak_seq_ops = {

1415

.start = kmemleak_seq_start,

1418

.start = kmemleak_seq_start,

1416

.next = kmemleak_seq_next,

1419

.next = kmemleak_seq_next,

1417

.stop = kmemleak_seq_stop,

1420

.stop = kmemleak_seq_stop,

1418

.show = kmemleak_seq_show,

1421

.show = kmemleak_seq_show,

1419

};

1422

};

1420

1423

1421

static int kmemleak_open(struct inode *inode, struct file *file)

1424

static int kmemleak_open(struct inode *inode, struct file *file)

1422

{

1425

{

1423

if (!atomic_read(&kmemleak_enabled))

1426

if (!atomic_read(&kmemleak_enabled))

1424

return -EBUSY;

1427

return -EBUSY;

1425

1428

1426

return seq_open(file, &kmemleak_seq_ops);

1429

return seq_open(file, &kmemleak_seq_ops);

1427

}

1430

}

1428

1431

1429

static int kmemleak_release(struct inode *inode, struct file *file)

1432

static int kmemleak_release(struct inode *inode, struct file *file)

1430

{

1433

{

1431

return seq_release(inode, file);

1434

return seq_release(inode, file);

1432

}

1435

}

1433

1436

1434

static int dump_str_object_info(const char *str)

1437

static int dump_str_object_info(const char *str)

1435

{

1438

{

1436

unsigned long flags;

1439

unsigned long flags;

1437

struct kmemleak_object *object;

1440

struct kmemleak_object *object;

1438

unsigned long addr;

1441

unsigned long addr;

1439

1442

1440

addr= simple_strtoul(str, NULL, 0);

1443

addr= simple_strtoul(str, NULL, 0);

1441

object = find_and_get_object(addr, 0);

1444

object = find_and_get_object(addr, 0);

1442

if (!object) {

1445

if (!object) {

1443

pr_info("Unknown object at 0x%08lx\n", addr);

1446

pr_info("Unknown object at 0x%08lx\n", addr);

1444

return -EINVAL;

1447

return -EINVAL;

1445

}

1448

}

1446

1449

1447

spin_lock_irqsave(&object->lock, flags);

1450

spin_lock_irqsave(&object->lock, flags);

1448

dump_object_info(object);

1451

dump_object_info(object);

1449

spin_unlock_irqrestore(&object->lock, flags);

1452

spin_unlock_irqrestore(&object->lock, flags);

1450

1453

1451

put_object(object);

1454

put_object(object);

1452

return 0;

1455

return 0;

1453

}

1456

}

1454

1457

1455

/*

1458

/*

1456

* We use grey instead of black to ensure we can do future scans on the same

1459

* We use grey instead of black to ensure we can do future scans on the same

1457

* objects. If we did not do future scans these black objects could

1460

* objects. If we did not do future scans these black objects could

1458

* potentially contain references to newly allocated objects in the future and

1461

* potentially contain references to newly allocated objects in the future and

1459

* we'd end up with false positives.

1462

* we'd end up with false positives.

1460

*/

1463

*/

1461

static void kmemleak_clear(void)

1464

static void kmemleak_clear(void)

1462

{

1465

{

1463

struct kmemleak_object *object;

1466

struct kmemleak_object *object;

1464

unsigned long flags;

1467

unsigned long flags;

1465

1468

1466

rcu_read_lock();

1469

rcu_read_lock();

1467

list_for_each_entry_rcu(object, &object_list, object_list) {

1470

list_for_each_entry_rcu(object, &object_list, object_list) {

1468

spin_lock_irqsave(&object->lock, flags);

1471

spin_lock_irqsave(&object->lock, flags);

1469

if ((object->flags & OBJECT_REPORTED) &&

1472

if ((object->flags & OBJECT_REPORTED) &&

1470

unreferenced_object(object))

1473

unreferenced_object(object))

1471

__paint_it(object, KMEMLEAK_GREY);

1474

__paint_it(object, KMEMLEAK_GREY);

1472

spin_unlock_irqrestore(&object->lock, flags);

1475

spin_unlock_irqrestore(&object->lock, flags);

1473

}

1476

}

1474

rcu_read_unlock();

1477

rcu_read_unlock();

1475

}

1478

}

1476

1479

1477

/*

1480

/*

1478

* File write operation to configure kmemleak at run-time. The following

1481

* File write operation to configure kmemleak at run-time. The following

1479

* commands can be written to the /sys/kernel/debug/kmemleak file:

1482

* commands can be written to the /sys/kernel/debug/kmemleak file:

1480

* off - disable kmemleak (irreversible)

1483

* off - disable kmemleak (irreversible)

1481

* stack=on - enable the task stacks scanning

1484

* stack=on - enable the task stacks scanning

1482

* stack=off - disable the tasks stacks scanning

1485

* stack=off - disable the tasks stacks scanning

1483

* scan=on - start the automatic memory scanning thread

1486

* scan=on - start the automatic memory scanning thread

1484

* scan=off - stop the automatic memory scanning thread

1487

* scan=off - stop the automatic memory scanning thread

1485

* scan=... - set the automatic memory scanning period in seconds (0 to

1488

* scan=... - set the automatic memory scanning period in seconds (0 to

1486

* disable it)

1489

* disable it)

1487

* scan - trigger a memory scan

1490

* scan - trigger a memory scan

1488

* clear - mark all current reported unreferenced kmemleak objects as

1491

* clear - mark all current reported unreferenced kmemleak objects as

1489

* grey to ignore printing them

1492

* grey to ignore printing them

1490

* dump=... - dump information about the object found at the given address

1493

* dump=... - dump information about the object found at the given address

1491

*/

1494

*/

1492

static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,

1495

static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,

1493

size_t size, loff_t *ppos)

1496

size_t size, loff_t *ppos)

1494

{

1497

{

1495

char buf[64];

1498

char buf[64];

1496

int buf_size;

1499

int buf_size;

1497

int ret;

1500

int ret;

1498

1501

1499

buf_size = min(size, (sizeof(buf) - 1));

1502

buf_size = min(size, (sizeof(buf) - 1));

1500

if (strncpy_from_user(buf, user_buf, buf_size) < 0)

1503

if (strncpy_from_user(buf, user_buf, buf_size) < 0)

1501

return -EFAULT;

1504

return -EFAULT;

1502

buf[buf_size] = 0;

1505

buf[buf_size] = 0;

1503

1506

1504

ret = mutex_lock_interruptible(&scan_mutex);

1507

ret = mutex_lock_interruptible(&scan_mutex);

1505

if (ret < 0)

1508

if (ret < 0)

1506

return ret;

1509

return ret;

1507

1510

1508

if (strncmp(buf, "off", 3) == 0)

1511

if (strncmp(buf, "off", 3) == 0)

1509

kmemleak_disable();

1512

kmemleak_disable();

1510

else if (strncmp(buf, "stack=on", 8) == 0)

1513

else if (strncmp(buf, "stack=on", 8) == 0)

1511

kmemleak_stack_scan = 1;

1514

kmemleak_stack_scan = 1;

1512

else if (strncmp(buf, "stack=off", 9) == 0)

1515

else if (strncmp(buf, "stack=off", 9) == 0)

1513

kmemleak_stack_scan = 0;

1516

kmemleak_stack_scan = 0;

1514

else if (strncmp(buf, "scan=on", 7) == 0)

1517

else if (strncmp(buf, "scan=on", 7) == 0)

1515

start_scan_thread();

1518

start_scan_thread();

1516

else if (strncmp(buf, "scan=off", 8) == 0)

1519

else if (strncmp(buf, "scan=off", 8) == 0)

1517

stop_scan_thread();

1520

stop_scan_thread();

1518

else if (strncmp(buf, "scan=", 5) == 0) {

1521

else if (strncmp(buf, "scan=", 5) == 0) {

1519

unsigned long secs;

1522

unsigned long secs;

1520

1523

1521

ret = strict_strtoul(buf + 5, 0, &secs);

1524

ret = strict_strtoul(buf + 5, 0, &secs);

1522

if (ret < 0)

1525

if (ret < 0)

1523

goto out;

1526

goto out;

1524

stop_scan_thread();

1527

stop_scan_thread();

1525

if (secs) {

1528

if (secs) {

1526

jiffies_scan_wait = msecs_to_jiffies(secs * 1000);

1529

jiffies_scan_wait = msecs_to_jiffies(secs * 1000);

1527

start_scan_thread();

1530

start_scan_thread();

1528

}

1531

}

1529

} else if (strncmp(buf, "scan", 4) == 0)

1532

} else if (strncmp(buf, "scan", 4) == 0)

1530

kmemleak_scan();

1533

kmemleak_scan();

1531

else if (strncmp(buf, "clear", 5) == 0)

1534

else if (strncmp(buf, "clear", 5) == 0)

1532

kmemleak_clear();

1535

kmemleak_clear();

1533

else if (strncmp(buf, "dump=", 5) == 0)

1536

else if (strncmp(buf, "dump=", 5) == 0)

1534

ret = dump_str_object_info(buf + 5);

1537

ret = dump_str_object_info(buf + 5);

1535

else

1538

else

1536

ret = -EINVAL;

1539

ret = -EINVAL;

1537

1540

1538

out:

1541

out:

1539

mutex_unlock(&scan_mutex);

1542

mutex_unlock(&scan_mutex);

1540

if (ret < 0)

1543

if (ret < 0)

1541

return ret;

1544

return ret;

1542

1545

1543

/* ignore the rest of the buffer, only one command at a time */

1546

/* ignore the rest of the buffer, only one command at a time */

1544

*ppos += size;

1547

*ppos += size;

1545

return size;

1548

return size;

1546

}

1549

}

1547

1550

1548

static const struct file_operations kmemleak_fops = {

1551

static const struct file_operations kmemleak_fops = {

1549

.owner = THIS_MODULE,

1552

.owner = THIS_MODULE,

1550

.open = kmemleak_open,

1553

.open = kmemleak_open,

1551

.read = seq_read,

1554

.read = seq_read,

1552

.write = kmemleak_write,

1555

.write = kmemleak_write,

1553

.llseek = seq_lseek,

1556

.llseek = seq_lseek,

1554

.release = kmemleak_release,

1557

.release = kmemleak_release,

1555

};

1558

};

1556

1559

1557

/*

1560

/*

1558

* Perform the freeing of the kmemleak internal objects after waiting for any

1561

* Perform the freeing of the kmemleak internal objects after waiting for any

1559

* current memory scan to complete.

1562

* current memory scan to complete.

1560

*/

1563

*/

1561

static void kmemleak_do_cleanup(struct work_struct *work)

1564

static void kmemleak_do_cleanup(struct work_struct *work)

1562

{

1565

{

1563

struct kmemleak_object *object;

1566

struct kmemleak_object *object;

1564

1567

1565

mutex_lock(&scan_mutex);

1568

mutex_lock(&scan_mutex);

1566

stop_scan_thread();

1569

stop_scan_thread();

1567

1570

1568

rcu_read_lock();

1571

rcu_read_lock();

1569

list_for_each_entry_rcu(object, &object_list, object_list)

1572

list_for_each_entry_rcu(object, &object_list, object_list)

1570

delete_object_full(object->pointer);

1573

delete_object_full(object->pointer);

1571

rcu_read_unlock();

1574

rcu_read_unlock();

1572

mutex_unlock(&scan_mutex);

1575

mutex_unlock(&scan_mutex);

1573

}

1576

}

1574

1577

1575

static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);

1578

static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);

1576

1579

1577

/*

1580

/*

1578

* Disable kmemleak. No memory allocation/freeing will be traced once this

1581

* Disable kmemleak. No memory allocation/freeing will be traced once this

1579

* function is called. Disabling kmemleak is an irreversible operation.

1582

* function is called. Disabling kmemleak is an irreversible operation.

1580

*/

1583

*/

1581

static void kmemleak_disable(void)

1584

static void kmemleak_disable(void)

1582

{

1585

{

1583

/* atomically check whether it was already invoked */

1586

/* atomically check whether it was already invoked */

1584

if (atomic_cmpxchg(&kmemleak_error, 0, 1))

1587

if (atomic_cmpxchg(&kmemleak_error, 0, 1))

1585

return;

1588

return;

1586

1589

1587

/* stop any memory operation tracing */

1590

/* stop any memory operation tracing */

1588

atomic_set(&kmemleak_early_log, 0);

1591

atomic_set(&kmemleak_early_log, 0);

1589

atomic_set(&kmemleak_enabled, 0);

1592

atomic_set(&kmemleak_enabled, 0);

1590

1593

1591

/* check whether it is too early for a kernel thread */

1594

/* check whether it is too early for a kernel thread */

1592

if (atomic_read(&kmemleak_initialized))

1595

if (atomic_read(&kmemleak_initialized))

1593

schedule_work(&cleanup_work);

1596

schedule_work(&cleanup_work);

1594

1597

1595

pr_info("Kernel memory leak detector disabled\n");

1598

pr_info("Kernel memory leak detector disabled\n");

1596

}

1599

}

1597

1600

1598

/*

1601

/*

1599

* Allow boot-time kmemleak disabling (enabled by default).

1602

* Allow boot-time kmemleak disabling (enabled by default).

1600

*/

1603

*/

1601

static int kmemleak_boot_config(char *str)

1604

static int kmemleak_boot_config(char *str)

1602

{

1605

{

1603

if (!str)

1606

if (!str)

1604

return -EINVAL;

1607

return -EINVAL;

1605

if (strcmp(str, "off") == 0)

1608

if (strcmp(str, "off") == 0)

1606

kmemleak_disable();

1609

kmemleak_disable();

1607

else if (strcmp(str, "on") != 0)

1610

else if (strcmp(str, "on") == 0)

1611

kmemleak_skip_disable = 1;

1612

else

1608

return -EINVAL;

1613

return -EINVAL;

1609

return 0;

1614

return 0;

1610

}

1615

}

1611

early_param("kmemleak", kmemleak_boot_config);

1616

early_param("kmemleak", kmemleak_boot_config);

1612

1617

1613

/*

1618

/*

1614

* Kmemleak initialization.

1619

* Kmemleak initialization.

1615

*/

1620

*/

1616

void __init kmemleak_init(void)

1621

void __init kmemleak_init(void)

1617

{

1622

{

1618

int i;

1623

int i;

1619

unsigned long flags;

1624

unsigned long flags;

1625

1626

#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF

1627

if (!kmemleak_skip_disable) {

1628

kmemleak_disable();

1629

return;

1630

}

1631

#endif

1620

1632

1621

jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);

1633

jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);

1622

jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);

1634

jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);

1623

1635

1624

object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);

1636

object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);

1625

scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);

1637

scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);

1626

INIT_PRIO_TREE_ROOT(&object_tree_root);

1638

INIT_PRIO_TREE_ROOT(&object_tree_root);

1627

1639

1628

/* the kernel is still in UP mode, so disabling the IRQs is enough */

1640

/* the kernel is still in UP mode, so disabling the IRQs is enough */

1629

local_irq_save(flags);

1641

local_irq_save(flags);

1630

if (!atomic_read(&kmemleak_error)) {

1642

if (!atomic_read(&kmemleak_error)) {

1631

atomic_set(&kmemleak_enabled, 1);

1643

atomic_set(&kmemleak_enabled, 1);

1632

atomic_set(&kmemleak_early_log, 0);

1644

atomic_set(&kmemleak_early_log, 0);

1633

}

1645

}

1634

local_irq_restore(flags);

1646

local_irq_restore(flags);

1635

1647

1636

/*

1648

/*

1637

* This is the point where tracking allocations is safe. Automatic

1649

* This is the point where tracking allocations is safe. Automatic

1638

* scanning is started during the late initcall. Add the early logged

1650

* scanning is started during the late initcall. Add the early logged

1639

* callbacks to the kmemleak infrastructure.

1651

* callbacks to the kmemleak infrastructure.

1640

*/

1652

*/

1641

for (i = 0; i < crt_early_log; i++) {

1653

for (i = 0; i < crt_early_log; i++) {

1642

struct early_log *log = &early_log[i];

1654

struct early_log *log = &early_log[i];

1643

1655

1644

switch (log->op_type) {

1656

switch (log->op_type) {

1645

case KMEMLEAK_ALLOC:

1657

case KMEMLEAK_ALLOC:

1646

early_alloc(log);

1658

early_alloc(log);

1647

break;

1659

break;

1648

case KMEMLEAK_FREE:

1660

case KMEMLEAK_FREE:

1649

kmemleak_free(log->ptr);

1661

kmemleak_free(log->ptr);

1650

break;

1662

break;

1651

case KMEMLEAK_FREE_PART:

1663

case KMEMLEAK_FREE_PART:

1652

kmemleak_free_part(log->ptr, log->size);

1664

kmemleak_free_part(log->ptr, log->size);

1653

break;

1665

break;

1654

case KMEMLEAK_NOT_LEAK:

1666

case KMEMLEAK_NOT_LEAK:

1655

kmemleak_not_leak(log->ptr);

1667

kmemleak_not_leak(log->ptr);

1656

break;

1668

break;

1657

case KMEMLEAK_IGNORE:

1669

case KMEMLEAK_IGNORE:

1658

kmemleak_ignore(log->ptr);

1670

kmemleak_ignore(log->ptr);

1659

break;

1671

break;

1660

case KMEMLEAK_SCAN_AREA:

1672

case KMEMLEAK_SCAN_AREA:

1661

kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);

1673

kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);

1662

break;

1674

break;

1663

case KMEMLEAK_NO_SCAN:

1675

case KMEMLEAK_NO_SCAN:

1664

kmemleak_no_scan(log->ptr);

1676

kmemleak_no_scan(log->ptr);

1665

break;

1677

break;

1666

default:

1678

default:

1667

WARN_ON(1);

1679

WARN_ON(1);

1668

}

1680

}

1669

}

1681

}

1670

}

1682

}

1671

1683

1672

/*

1684

/*

1673

* Late initialization function.

1685

* Late initialization function.

1674

*/

1686

*/

1675

static int __init kmemleak_late_init(void)

1687

static int __init kmemleak_late_init(void)

1676

{

1688

{

1677

struct dentry *dentry;

1689

struct dentry *dentry;

1678

1690

1679

atomic_set(&kmemleak_initialized, 1);

1691

atomic_set(&kmemleak_initialized, 1);

1680

1692

1681

if (atomic_read(&kmemleak_error)) {

1693

if (atomic_read(&kmemleak_error)) {

1682

/*

1694

/*

1683

* Some error occured and kmemleak was disabled. There is a

1695

* Some error occured and kmemleak was disabled. There is a

1684

* small chance that kmemleak_disable() was called immediately

1696

* small chance that kmemleak_disable() was called immediately

1685

* after setting kmemleak_initialized and we may end up with

1697

* after setting kmemleak_initialized and we may end up with

1686

* two clean-up threads but serialized by scan_mutex.

1698

* two clean-up threads but serialized by scan_mutex.

1687

*/

1699

*/

1688

schedule_work(&cleanup_work);

1700

schedule_work(&cleanup_work);

1689

return -ENOMEM;

1701

return -ENOMEM;

1690

}

1702

}

1691

1703

1692

dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,

1704

dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,

1693

&kmemleak_fops);

1705

&kmemleak_fops);

1694

if (!dentry)

1706

if (!dentry)

1695

pr_warning("Failed to create the debugfs kmemleak file\n");

1707

pr_warning("Failed to create the debugfs kmemleak file\n");

1696

mutex_lock(&scan_mutex);

1708

mutex_lock(&scan_mutex);

1697

start_scan_thread();

1709

start_scan_thread();

1698

mutex_unlock(&scan_mutex);

1710

mutex_unlock(&scan_mutex);

1699

1711

1700

pr_info("Kernel memory leak detector initialized\n");

1712

pr_info("Kernel memory leak detector initialized\n");

1701

1713

1702

return 0;

1714

return 0;

1703

}

1715

}

1704

late_initcall(kmemleak_late_init);

1716

late_initcall(kmemleak_late_init);

1705

1717

GITLAB

kmemleak: Introduce a default off mode for kmemleak

 config PRINTK_TIME
 	bool "Show timing information on printks"
 	depends on PRINTK
 	help
 	  Selecting this option causes timing information to be
 	  included in printk output.  This allows you to measure
 	  the interval between kernel operations, including bootup
 	  operations.  This is useful for identifying long delays
 	  in kernel startup.
 config ENABLE_WARN_DEPRECATED
 	bool "Enable __deprecated logic"
 	default y
 	help
 	  Enable the __deprecated logic in the kernel build.
 	  Disable this to suppress the "warning: 'foo' is deprecated
 	  (declared at kernel/power/somefile.c:1234)" messages.
 config ENABLE_MUST_CHECK
 	bool "Enable __must_check logic"
 	default y
 	help
 	  Enable the __must_check logic in the kernel build.  Disable this to
 	  suppress the "warning: ignoring return value of 'foo', declared with
 	  attribute warn_unused_result" messages.
 config FRAME_WARN
 	int "Warn for stack frames larger than (needs gcc 4.4)"
 	range 0 8192
 	default 1024 if !64BIT
 	default 2048 if 64BIT
 	help
 	  Tell gcc to warn at build time for stack frames larger than this.
 	  Setting this too low will cause a lot of warnings.
 	  Setting it to 0 disables the warning.
 	  Requires gcc 4.4
 config MAGIC_SYSRQ
 	bool "Magic SysRq key"
 	depends on !UML
 	help
 	  If you say Y here, you will have some control over the system even
 	  if the system crashes for example during kernel debugging (e.g., you
 	  will be able to flush the buffer cache to disk, reboot the system
 	  immediately or dump some status information). This is accomplished
 	  by pressing various keys while holding SysRq (Alt+PrintScreen). It
 	  also works on a serial console (on PC hardware at least), if you
 	  send a BREAK and then within 5 seconds a command keypress. The
 	  keys are documented in <file:Documentation/sysrq.txt>. Don't say Y
 	  unless you really know what this hack does.
 config STRIP_ASM_SYMS
 	bool "Strip assembler-generated symbols during link"
 	default n
 	help
 	  Strip internal assembler-generated symbols during a link (symbols
 	  that look like '.Lxxx') so they don't pollute the output of
 	  get_wchan() and suchlike.
 config UNUSED_SYMBOLS
 	bool "Enable unused/obsolete exported symbols"
 	default y if X86
 	help
 	  Unused but exported symbols make the kernel needlessly bigger.  For
 	  that reason most of these unused exports will soon be removed.  This
 	  option is provided temporarily to provide a transition period in case
 	  some external kernel module needs one of these symbols anyway. If you
 	  encounter such a case in your module, consider if you are actually
 	  using the right API.  (rationale: since nobody in the kernel is using
 	  this in a module, there is a pretty good chance it's actually the
 	  wrong interface to use).  If you really need the symbol, please send a
 	  mail to the linux kernel mailing list mentioning the symbol and why
 	  you really need it, and what the merge plan to the mainline kernel for
 	  your module is.
 config DEBUG_FS
 	bool "Debug Filesystem"
 	depends on SYSFS
 	help
 	  debugfs is a virtual file system that kernel developers use to put
 	  debugging files into.  Enable this option to be able to read and
 	  write to these files.
 	  For detailed documentation on the debugfs API, see
 	  Documentation/DocBook/filesystems.
 	  If unsure, say N.
 config HEADERS_CHECK
 	bool "Run 'make headers_check' when building vmlinux"
 	depends on !UML
 	help
 	  This option will extract the user-visible kernel headers whenever
 	  building the kernel, and will run basic sanity checks on them to
 	  ensure that exported files do not attempt to include files which
 	  were not exported, etc.
 	  If you're making modifications to header files which are
 	  relevant for userspace, say 'Y', and check the headers
 	  exported to $(INSTALL_HDR_PATH) (usually 'usr/include' in
 	  your build tree), to make sure they're suitable.
 config DEBUG_SECTION_MISMATCH
 	bool "Enable full Section mismatch analysis"
 	depends on UNDEFINED || (BLACKFIN)
 	default y
 	# This option is on purpose disabled for now.
 	# It will be enabled when we are down to a reasonable number
 	# of section mismatch warnings (< 10 for an allyesconfig build)
 	help
 	  The section mismatch analysis checks if there are illegal
 	  references from one section to another section.
 	  Linux will during link or during runtime drop some sections
 	  and any use of code/data previously in these sections will
 	  most likely result in an oops.
 	  In the code functions and variables are annotated with
 	  __init, __devinit etc. (see full list in include/linux/init.h)
 	  which results in the code/data being placed in specific sections.
 	  The section mismatch analysis is always done after a full
 	  kernel build but enabling this option will in addition
 	  do the following:
 	  - Add the option -fno-inline-functions-called-once to gcc
 	    When inlining a function annotated __init in a non-init
 	    function we would lose the section information and thus
 	    the analysis would not catch the illegal reference.
 	    This option tells gcc to inline less but will also
 	    result in a larger kernel.
 	  - Run the section mismatch analysis for each module/built-in.o
 	    When we run the section mismatch analysis on vmlinux.o we
 	    lose valueble information about where the mismatch was
 	    introduced.
 	    Running the analysis for each module/built-in.o file
 	    will tell where the mismatch happens much closer to the
 	    source. The drawback is that we will report the same
 	    mismatch at least twice.
 	  - Enable verbose reporting from modpost to help solving
 	    the section mismatches reported.
 config DEBUG_KERNEL
 	bool "Kernel debugging"
 	help
 	  Say Y here if you are developing drivers or trying to debug and
 	  identify kernel problems.
 config DEBUG_SHIRQ
 	bool "Debug shared IRQ handlers"
 	depends on DEBUG_KERNEL && GENERIC_HARDIRQS
 	help
 	  Enable this to generate a spurious interrupt as soon as a shared
 	  interrupt handler is registered, and just before one is deregistered.
 	  Drivers ought to be able to handle interrupts coming in at those
 	  points; some don't and need to be caught.
 config DETECT_SOFTLOCKUP
 	bool "Detect Soft Lockups"
 	depends on DEBUG_KERNEL && !S390
 	default y
 	help
 	  Say Y here to enable the kernel to detect "soft lockups",
 	  which are bugs that cause the kernel to loop in kernel
 	  mode for more than 60 seconds, without giving other tasks a
 	  chance to run.
 	  When a soft-lockup is detected, the kernel will print the
 	  current stack trace (which you should report), but the
 	  system will stay locked up. This feature has negligible
 	  overhead.
 	  (Note that "hard lockups" are separate type of bugs that
 	   can be detected via the NMI-watchdog, on platforms that
 	   support it.)
 config BOOTPARAM_SOFTLOCKUP_PANIC
 	bool "Panic (Reboot) On Soft Lockups"
 	depends on DETECT_SOFTLOCKUP
 	help
 	  Say Y here to enable the kernel to panic on "soft lockups",
 	  which are bugs that cause the kernel to loop in kernel
 	  mode for more than 60 seconds, without giving other tasks a
 	  chance to run.
 	  The panic can be used in combination with panic_timeout,
 	  to cause the system to reboot automatically after a
 	  lockup has been detected. This feature is useful for
 	  high-availability systems that have uptime guarantees and
 	  where a lockup must be resolved ASAP.
 	  Say N if unsure.
 config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE
 	int
 	depends on DETECT_SOFTLOCKUP
 	range 0 1
 	default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC
 	default 1 if BOOTPARAM_SOFTLOCKUP_PANIC
 config DETECT_HUNG_TASK
 	bool "Detect Hung Tasks"
 	depends on DEBUG_KERNEL
 	default DETECT_SOFTLOCKUP
 	help
 	  Say Y here to enable the kernel to detect "hung tasks",
 	  which are bugs that cause the task to be stuck in
 	  uninterruptible "D" state indefinitiley.
 	  When a hung task is detected, the kernel will print the
 	  current stack trace (which you should report), but the
 	  task will stay in uninterruptible state. If lockdep is
 	  enabled then all held locks will also be reported. This
 	  feature has negligible overhead.
 config BOOTPARAM_HUNG_TASK_PANIC
 	bool "Panic (Reboot) On Hung Tasks"
 	depends on DETECT_HUNG_TASK
 	help
 	  Say Y here to enable the kernel to panic on "hung tasks",
 	  which are bugs that cause the kernel to leave a task stuck
 	  in uninterruptible "D" state.
 	  The panic can be used in combination with panic_timeout,
 	  to cause the system to reboot automatically after a
 	  hung task has been detected. This feature is useful for
 	  high-availability systems that have uptime guarantees and
 	  where a hung tasks must be resolved ASAP.
 	  Say N if unsure.
 config BOOTPARAM_HUNG_TASK_PANIC_VALUE
 	int
 	depends on DETECT_HUNG_TASK
 	range 0 1
 	default 0 if !BOOTPARAM_HUNG_TASK_PANIC
 	default 1 if BOOTPARAM_HUNG_TASK_PANIC
 config SCHED_DEBUG
 	bool "Collect scheduler debugging info"
 	depends on DEBUG_KERNEL && PROC_FS
 	default y
 	help
 	  If you say Y here, the /proc/sched_debug file will be provided
 	  that can help debug the scheduler. The runtime overhead of this
 	  option is minimal.
 config SCHEDSTATS
 	bool "Collect scheduler statistics"
 	depends on DEBUG_KERNEL && PROC_FS
 	help
 	  If you say Y here, additional code will be inserted into the
 	  scheduler and related routines to collect statistics about
 	  scheduler behavior and provide them in /proc/schedstat.  These
 	  stats may be useful for both tuning and debugging the scheduler
 	  If you aren't debugging the scheduler or trying to tune a specific
 	  application, you can say N to avoid the very slight overhead
 	  this adds.
 config TIMER_STATS
 	bool "Collect kernel timers statistics"
 	depends on DEBUG_KERNEL && PROC_FS
 	help
 	  If you say Y here, additional code will be inserted into the
 	  timer routines to collect statistics about kernel timers being
 	  reprogrammed. The statistics can be read from /proc/timer_stats.
 	  The statistics collection is started by writing 1 to /proc/timer_stats,
 	  writing 0 stops it. This feature is useful to collect information
 	  about timer usage patterns in kernel and userspace. This feature
 	  is lightweight if enabled in the kernel config but not activated
 	  (it defaults to deactivated on bootup and will only be activated
 	  if some application like powertop activates it explicitly).
 config DEBUG_OBJECTS
 	bool "Debug object operations"
 	depends on DEBUG_KERNEL
 	help
 	  If you say Y here, additional code will be inserted into the
 	  kernel to track the life time of various objects and validate
 	  the operations on those objects.
 config DEBUG_OBJECTS_SELFTEST
 	bool "Debug objects selftest"
 	depends on DEBUG_OBJECTS
 	help
 	  This enables the selftest of the object debug code.
 config DEBUG_OBJECTS_FREE
 	bool "Debug objects in freed memory"
 	depends on DEBUG_OBJECTS
 	help
 	  This enables checks whether a k/v free operation frees an area
 	  which contains an object which has not been deactivated
 	  properly. This can make kmalloc/kfree-intensive workloads
 	  much slower.
 config DEBUG_OBJECTS_TIMERS
 	bool "Debug timer objects"
 	depends on DEBUG_OBJECTS
 	help
 	  If you say Y here, additional code will be inserted into the
 	  timer routines to track the life time of timer objects and
 	  validate the timer operations.
 config DEBUG_OBJECTS_WORK
 	bool "Debug work objects"
 	depends on DEBUG_OBJECTS
 	help
 	  If you say Y here, additional code will be inserted into the
 	  work queue routines to track the life time of work objects and
 	  validate the work operations.
 config DEBUG_OBJECTS_ENABLE_DEFAULT
 	int "debug_objects bootup default value (0-1)"
         range 0 1
         default "1"
         depends on DEBUG_OBJECTS
         help
           Debug objects boot parameter default value
 config DEBUG_SLAB
 	bool "Debug slab memory allocations"
 	depends on DEBUG_KERNEL && SLAB && !KMEMCHECK
 	help
 	  Say Y here to have the kernel do limited verification on memory
 	  allocation as well as poisoning memory on free to catch use of freed
 	  memory. This can make kmalloc/kfree-intensive workloads much slower.
 config DEBUG_SLAB_LEAK
 	bool "Memory leak debugging"
 	depends on DEBUG_SLAB
 config SLUB_DEBUG_ON
 	bool "SLUB debugging on by default"
 	depends on SLUB && SLUB_DEBUG && !KMEMCHECK
 	default n
 	help
 	  Boot with debugging on by default. SLUB boots by default with
 	  the runtime debug capabilities switched off. Enabling this is
 	  equivalent to specifying the "slub_debug" parameter on boot.
 	  There is no support for more fine grained debug control like
 	  possible with slub_debug=xxx. SLUB debugging may be switched
 	  off in a kernel built with CONFIG_SLUB_DEBUG_ON by specifying
 	  "slub_debug=-".
 config SLUB_STATS
 	default n
 	bool "Enable SLUB performance statistics"
 	depends on SLUB && SLUB_DEBUG && SYSFS
 	help
 	  SLUB statistics are useful to debug SLUBs allocation behavior in
 	  order find ways to optimize the allocator. This should never be
 	  enabled for production use since keeping statistics slows down
 	  the allocator by a few percentage points. The slabinfo command
 	  supports the determination of the most active slabs to figure
 	  out which slabs are relevant to a particular load.
 	  Try running: slabinfo -DA
 config DEBUG_KMEMLEAK
 	bool "Kernel memory leak detector"
 	depends on DEBUG_KERNEL && EXPERIMENTAL && !MEMORY_HOTPLUG && \
 		(X86 || ARM || PPC || S390 || SPARC64 || SUPERH || MICROBLAZE)
 	select DEBUG_FS if SYSFS
 	select STACKTRACE if STACKTRACE_SUPPORT
 	select KALLSYMS
 	select CRC32
 	help
 	  Say Y here if you want to enable the memory leak
 	  detector. The memory allocation/freeing is traced in a way
 	  similar to the Boehm's conservative garbage collector, the
 	  difference being that the orphan objects are not freed but
 	  only shown in /sys/kernel/debug/kmemleak. Enabling this
 	  feature will introduce an overhead to memory
 	  allocations. See Documentation/kmemleak.txt for more
 	  details.
 	  Enabling DEBUG_SLAB or SLUB_DEBUG may increase the chances
 	  of finding leaks due to the slab objects poisoning.
 	  In order to access the kmemleak file, debugfs needs to be
 	  mounted (usually at /sys/kernel/debug).
 config DEBUG_KMEMLEAK_EARLY_LOG_SIZE
 	int "Maximum kmemleak early log entries"
 	depends on DEBUG_KMEMLEAK
 	range 200 40000
 	default 400
 	help
 	  Kmemleak must track all the memory allocations to avoid
 	  reporting false positives. Since memory may be allocated or
 	  freed before kmemleak is initialised, an early log buffer is
 	  used to store these actions. If kmemleak reports "early log
 	  buffer exceeded", please increase this value.
 config DEBUG_KMEMLEAK_TEST
 	tristate "Simple test for the kernel memory leak detector"
 	depends on DEBUG_KMEMLEAK
 	help
 	  Say Y or M here to build a test for the kernel memory leak
 	  detector. This option enables a module that explicitly leaks
 	  memory.
 	  If unsure, say N.
+config DEBUG_KMEMLEAK_DEFAULT_OFF
+	bool "Default kmemleak to off"
+	depends on DEBUG_KMEMLEAK
+	help
+	  Say Y here to disable kmemleak by default. It can then be enabled
+	  on the command line via kmemleak=on.
 config DEBUG_PREEMPT
 	bool "Debug preemptible kernel"
 	depends on DEBUG_KERNEL && PREEMPT && TRACE_IRQFLAGS_SUPPORT
 	default y
 	help
 	  If you say Y here then the kernel will use a debug variant of the
 	  commonly used smp_processor_id() function and will print warnings
 	  if kernel code uses it in a preemption-unsafe way. Also, the kernel
 	  will detect preemption count underflows.
 config DEBUG_RT_MUTEXES
 	bool "RT Mutex debugging, deadlock detection"
 	depends on DEBUG_KERNEL && RT_MUTEXES
 	help
 	 This allows rt mutex semantics violations and rt mutex related
 	 deadlocks (lockups) to be detected and reported automatically.
 config DEBUG_PI_LIST
 	bool
 	default y
 	depends on DEBUG_RT_MUTEXES
 config RT_MUTEX_TESTER
 	bool "Built-in scriptable tester for rt-mutexes"
 	depends on DEBUG_KERNEL && RT_MUTEXES
 	help
 	  This option enables a rt-mutex tester.
 config DEBUG_SPINLOCK
 	bool "Spinlock and rw-lock debugging: basic checks"
 	depends on DEBUG_KERNEL
 	help
 	  Say Y here and build SMP to catch missing spinlock initialization
 	  and certain other kinds of spinlock errors commonly made.  This is
 	  best used in conjunction with the NMI watchdog so that spinlock
 	  deadlocks are also debuggable.
 config DEBUG_MUTEXES
 	bool "Mutex debugging: basic checks"
 	depends on DEBUG_KERNEL
 	help
 	 This feature allows mutex semantics violations to be detected and
 	 reported.
 config DEBUG_LOCK_ALLOC
 	bool "Lock debugging: detect incorrect freeing of live locks"
 	depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
 	select DEBUG_SPINLOCK
 	select DEBUG_MUTEXES
 	select LOCKDEP
 	help
 	 This feature will check whether any held lock (spinlock, rwlock,
 	 mutex or rwsem) is incorrectly freed by the kernel, via any of the
 	 memory-freeing routines (kfree(), kmem_cache_free(), free_pages(),
 	 vfree(), etc.), whether a live lock is incorrectly reinitialized via
 	 spin_lock_init()/mutex_init()/etc., or whether there is any lock
 	 held during task exit.
 config PROVE_LOCKING
 	bool "Lock debugging: prove locking correctness"
 	depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
 	select LOCKDEP
 	select DEBUG_SPINLOCK
 	select DEBUG_MUTEXES
 	select DEBUG_LOCK_ALLOC
 	default n
 	help
 	 This feature enables the kernel to prove that all locking
 	 that occurs in the kernel runtime is mathematically
 	 correct: that under no circumstance could an arbitrary (and
 	 not yet triggered) combination of observed locking
 	 sequences (on an arbitrary number of CPUs, running an
 	 arbitrary number of tasks and interrupt contexts) cause a
 	 deadlock.
 	 In short, this feature enables the kernel to report locking
 	 related deadlocks before they actually occur.
 	 The proof does not depend on how hard and complex a
 	 deadlock scenario would be to trigger: how many
 	 participant CPUs, tasks and irq-contexts would be needed
 	 for it to trigger. The proof also does not depend on
 	 timing: if a race and a resulting deadlock is possible
 	 theoretically (no matter how unlikely the race scenario
 	 is), it will be proven so and will immediately be
 	 reported by the kernel (once the event is observed that
 	 makes the deadlock theoretically possible).
 	 If a deadlock is impossible (i.e. the locking rules, as
 	 observed by the kernel, are mathematically correct), the
 	 kernel reports nothing.
 	 NOTE: this feature can also be enabled for rwlocks, mutexes
 	 and rwsems - in which case all dependencies between these
 	 different locking variants are observed and mapped too, and
 	 the proof of observed correctness is also maintained for an
 	 arbitrary combination of these separate locking variants.
 	 For more details, see Documentation/lockdep-design.txt.
 config PROVE_RCU
 	bool "RCU debugging: prove RCU correctness"
 	depends on PROVE_LOCKING
 	default n
 	help
 	 This feature enables lockdep extensions that check for correct
 	 use of RCU APIs.  This is currently under development.  Say Y
 	 if you want to debug RCU usage or help work on the PROVE_RCU
 	 feature.
 	 Say N if you are unsure.
 config PROVE_RCU_REPEATEDLY
 	bool "RCU debugging: don't disable PROVE_RCU on first splat"
 	depends on PROVE_RCU
 	default n
 	help
 	 By itself, PROVE_RCU will disable checking upon issuing the
 	 first warning (or "splat").  This feature prevents such
 	 disabling, allowing multiple RCU-lockdep warnings to be printed
 	 on a single reboot.
 	 Say N if you are unsure.
 config LOCKDEP
 	bool
 	depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
 	select STACKTRACE
 	select FRAME_POINTER if !MIPS && !PPC && !ARM_UNWIND && !S390
 	select KALLSYMS
 	select KALLSYMS_ALL
 config LOCK_STAT
 	bool "Lock usage statistics"
 	depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
 	select LOCKDEP
 	select DEBUG_SPINLOCK
 	select DEBUG_MUTEXES
 	select DEBUG_LOCK_ALLOC
 	default n
 	help
 	 This feature enables tracking lock contention points
 	 For more details, see Documentation/lockstat.txt
 	 This also enables lock events required by "perf lock",
 	 subcommand of perf.
 	 If you want to use "perf lock", you also need to turn on
 	 CONFIG_EVENT_TRACING.
 	 CONFIG_LOCK_STAT defines "contended" and "acquired" lock events.
 	 (CONFIG_LOCKDEP defines "acquire" and "release" events.)
 config DEBUG_LOCKDEP
 	bool "Lock dependency engine debugging"
 	depends on DEBUG_KERNEL && LOCKDEP
 	help
 	  If you say Y here, the lock dependency engine will do
 	  additional runtime checks to debug itself, at the price
 	  of more runtime overhead.
 config TRACE_IRQFLAGS
 	depends on DEBUG_KERNEL
 	bool
 	default y
 	depends on TRACE_IRQFLAGS_SUPPORT
 	depends on PROVE_LOCKING
 config DEBUG_SPINLOCK_SLEEP
 	bool "Spinlock debugging: sleep-inside-spinlock checking"
 	depends on DEBUG_KERNEL
 	help
 	  If you say Y here, various routines which may sleep will become very
 	  noisy if they are called with a spinlock held.
 config DEBUG_LOCKING_API_SELFTESTS
 	bool "Locking API boot-time self-tests"
 	depends on DEBUG_KERNEL
 	help
 	  Say Y here if you want the kernel to run a short self-test during
 	  bootup. The self-test checks whether common types of locking bugs
 	  are detected by debugging mechanisms or not. (if you disable
 	  lock debugging then those bugs wont be detected of course.)
 	  The following locking APIs are covered: spinlocks, rwlocks,
 	  mutexes and rwsems.
 config STACKTRACE
 	bool
 	depends on STACKTRACE_SUPPORT
 config DEBUG_KOBJECT
 	bool "kobject debugging"
 	depends on DEBUG_KERNEL
 	help
 	  If you say Y here, some extra kobject debugging messages will be sent
 	  to the syslog.
 config DEBUG_HIGHMEM
 	bool "Highmem debugging"
 	depends on DEBUG_KERNEL && HIGHMEM
 	help
 	  This options enables addition error checking for high memory systems.
 	  Disable for production systems.
 config DEBUG_BUGVERBOSE
 	bool "Verbose BUG() reporting (adds 70K)" if DEBUG_KERNEL && EMBEDDED
 	depends on BUG
 	depends on ARM || AVR32 || M32R || M68K || SPARC32 || SPARC64 || \
 		   FRV || SUPERH || GENERIC_BUG || BLACKFIN || MN10300
 	default y
 	help
 	  Say Y here to make BUG() panics output the file name and line number
 	  of the BUG call as well as the EIP and oops trace.  This aids
 	  debugging but costs about 70-100K of memory.
 config DEBUG_INFO
 	bool "Compile the kernel with debug info"
 	depends on DEBUG_KERNEL
 	help
           If you say Y here the resulting kernel image will include
 	  debugging info resulting in a larger kernel image.
 	  This adds debug symbols to the kernel and modules (gcc -g), and
 	  is needed if you intend to use kernel crashdump or binary object
 	  tools like crash, kgdb, LKCD, gdb, etc on the kernel.
 	  Say Y here only if you plan to debug the kernel.
 	  If unsure, say N.
 config DEBUG_VM
 	bool "Debug VM"
 	depends on DEBUG_KERNEL
 	help
 	  Enable this to turn on extended checks in the virtual-memory system
           that may impact performance.
 	  If unsure, say N.
 config DEBUG_VIRTUAL
 	bool "Debug VM translations"
 	depends on DEBUG_KERNEL && X86
 	help
 	  Enable some costly sanity checks in virtual to page code. This can
 	  catch mistakes with virt_to_page() and friends.
 	  If unsure, say N.
 config DEBUG_NOMMU_REGIONS
 	bool "Debug the global anon/private NOMMU mapping region tree"
 	depends on DEBUG_KERNEL && !MMU
 	help
 	  This option causes the global tree of anonymous and private mapping
 	  regions to be regularly checked for invalid topology.
 config DEBUG_WRITECOUNT
 	bool "Debug filesystem writers count"
 	depends on DEBUG_KERNEL
 	help
 	  Enable this to catch wrong use of the writers count in struct
 	  vfsmount.  This will increase the size of each file struct by
 	  32 bits.
 	  If unsure, say N.
 config DEBUG_MEMORY_INIT
 	bool "Debug memory initialisation" if EMBEDDED
 	default !EMBEDDED
 	help
 	  Enable this for additional checks during memory initialisation.
 	  The sanity checks verify aspects of the VM such as the memory model
 	  and other information provided by the architecture. Verbose
 	  information will be printed at KERN_DEBUG loglevel depending
 	  on the mminit_loglevel= command-line option.
 	  If unsure, say Y
 config DEBUG_LIST
 	bool "Debug linked list manipulation"
 	depends on DEBUG_KERNEL
 	help
 	  Enable this to turn on extended checks in the linked-list
 	  walking routines.
 	  If unsure, say N.
 config DEBUG_SG
 	bool "Debug SG table operations"
 	depends on DEBUG_KERNEL
 	help
 	  Enable this to turn on checks on scatter-gather tables. This can
 	  help find problems with drivers that do not properly initialize
 	  their sg tables.
 	  If unsure, say N.
 config DEBUG_NOTIFIERS
 	bool "Debug notifier call chains"
 	depends on DEBUG_KERNEL
 	help
 	  Enable this to turn on sanity checking for notifier call chains.
 	  This is most useful for kernel developers to make sure that
 	  modules properly unregister themselves from notifier chains.
 	  This is a relatively cheap check but if you care about maximum
 	  performance, say N.
 config DEBUG_CREDENTIALS
 	bool "Debug credential management"
 	depends on DEBUG_KERNEL
 	help
 	  Enable this to turn on some debug checking for credential
 	  management.  The additional code keeps track of the number of
 	  pointers from task_structs to any given cred struct, and checks to
 	  see that this number never exceeds the usage count of the cred
 	  struct.
 	  Furthermore, if SELinux is enabled, this also checks that the
 	  security pointer in the cred struct is never seen to be invalid.
 	  If unsure, say N.
 #
 # Select this config option from the architecture Kconfig, if it
 # it is preferred to always offer frame pointers as a config
 # option on the architecture (regardless of KERNEL_DEBUG):
 #
 config ARCH_WANT_FRAME_POINTERS
 	bool
 	help
 config FRAME_POINTER
 	bool "Compile the kernel with frame pointers"
 	depends on DEBUG_KERNEL && \
 		(CRIS || M68K || M68KNOMMU || FRV || UML || \
 		 AVR32 || SUPERH || BLACKFIN || MN10300) || \
 		ARCH_WANT_FRAME_POINTERS
 	default y if (DEBUG_INFO && UML) || ARCH_WANT_FRAME_POINTERS
 	help
 	  If you say Y here the resulting kernel image will be slightly
 	  larger and slower, but it gives very useful debugging information
 	  in case of kernel bugs. (precise oopses/stacktraces/warnings)
 config BOOT_PRINTK_DELAY
 	bool "Delay each boot printk message by N milliseconds"
 	depends on DEBUG_KERNEL && PRINTK && GENERIC_CALIBRATE_DELAY
 	help
 	  This build option allows you to read kernel boot messages
 	  by inserting a short delay after each one.  The delay is
 	  specified in milliseconds on the kernel command line,
 	  using "boot_delay=N".
 	  It is likely that you would also need to use "lpj=M" to preset
 	  the "loops per jiffie" value.
 	  See a previous boot log for the "lpj" value to use for your
 	  system, and then set "lpj=M" before setting "boot_delay=N".
 	  NOTE:  Using this option may adversely affect SMP systems.
 	  I.e., processors other than the first one may not boot up.
 	  BOOT_PRINTK_DELAY also may cause DETECT_SOFTLOCKUP to detect
 	  what it believes to be lockup conditions.
 config RCU_TORTURE_TEST
 	tristate "torture tests for RCU"
 	depends on DEBUG_KERNEL
 	default n
 	help
 	  This option provides a kernel module that runs torture tests
 	  on the RCU infrastructure.  The kernel module may be built
 	  after the fact on the running kernel to be tested, if desired.
 	  Say Y here if you want RCU torture tests to be built into
 	  the kernel.
 	  Say M if you want the RCU torture tests to build as a module.
 	  Say N if you are unsure.
 config RCU_TORTURE_TEST_RUNNABLE
 	bool "torture tests for RCU runnable by default"
 	depends on RCU_TORTURE_TEST = y
 	default n
 	help
 	  This option provides a way to build the RCU torture tests
 	  directly into the kernel without them starting up at boot
 	  time.  You can use /proc/sys/kernel/rcutorture_runnable
 	  to manually override this setting.  This /proc file is
 	  available only when the RCU torture tests have been built
 	  into the kernel.
 	  Say Y here if you want the RCU torture tests to start during
 	  boot (you probably don't).
 	  Say N here if you want the RCU torture tests to start only
 	  after being manually enabled via /proc.
 config RCU_CPU_STALL_DETECTOR
 	bool "Check for stalled CPUs delaying RCU grace periods"
 	depends on TREE_RCU || TREE_PREEMPT_RCU
 	default y
 	help
 	  This option causes RCU to printk information on which
 	  CPUs are delaying the current grace period, but only when
 	  the grace period extends for excessive time periods.
 	  Say N if you want to disable such checks.
 	  Say Y if you are unsure.
 config RCU_CPU_STALL_VERBOSE
 	bool "Print additional per-task information for RCU_CPU_STALL_DETECTOR"
 	depends on RCU_CPU_STALL_DETECTOR && TREE_PREEMPT_RCU
 	default y
 	help
 	  This option causes RCU to printk detailed per-task information
 	  for any tasks that are stalling the current RCU grace period.
 	  Say N if you are unsure.
 	  Say Y if you want to enable such checks.
 config KPROBES_SANITY_TEST
 	bool "Kprobes sanity tests"
 	depends on DEBUG_KERNEL
 	depends on KPROBES
 	default n
 	help
 	  This option provides for testing basic kprobes functionality on
 	  boot. A sample kprobe, jprobe and kretprobe are inserted and
 	  verified for functionality.
 	  Say N if you are unsure.
 config BACKTRACE_SELF_TEST
 	tristate "Self test for the backtrace code"
 	depends on DEBUG_KERNEL
 	default n
 	help
 	  This option provides a kernel module that can be used to test
 	  the kernel stack backtrace code. This option is not useful
 	  for distributions or general kernels, but only for kernel
 	  developers working on architecture code.
 	  Note that if you want to also test saved backtraces, you will
 	  have to enable STACKTRACE as well.
 	  Say N if you are unsure.
 config DEBUG_BLOCK_EXT_DEVT
         bool "Force extended block device numbers and spread them"
 	depends on DEBUG_KERNEL
 	depends on BLOCK
 	default n
 	help
 	  BIG FAT WARNING: ENABLING THIS OPTION MIGHT BREAK BOOTING ON
 	  SOME DISTRIBUTIONS.  DO NOT ENABLE THIS UNLESS YOU KNOW WHAT
 	  YOU ARE DOING.  Distros, please enable this and fix whatever
 	  is broken.
 	  Conventionally, block device numbers are allocated from
 	  predetermined contiguous area.  However, extended block area
 	  may introduce non-contiguous block device numbers.  This
 	  option forces most block device numbers to be allocated from
 	  the extended space and spreads them to discover kernel or
 	  userland code paths which assume predetermined contiguous
 	  device number allocation.
 	  Note that turning on this debug option shuffles all the
 	  device numbers for all IDE and SCSI devices including libata
 	  ones, so root partition specified using device number
 	  directly (via rdev or root=MAJ:MIN) won't work anymore.
 	  Textual device names (root=/dev/sdXn) will continue to work.
 	  Say N if you are unsure.
 config DEBUG_FORCE_WEAK_PER_CPU
 	bool "Force weak per-cpu definitions"
 	depends on DEBUG_KERNEL
 	help
 	  s390 and alpha require percpu variables in modules to be
 	  defined weak to work around addressing range issue which
 	  puts the following two restrictions on percpu variable
 	  definitions.
 	  1. percpu symbols must be unique whether static or not
 	  2. percpu variables can't be defined inside a function
 	  To ensure that generic code follows the above rules, this
 	  option forces all percpu variables to be defined as weak.
 config LKDTM
 	tristate "Linux Kernel Dump Test Tool Module"
 	depends on DEBUG_FS
 	depends on BLOCK
 	default n
 	help
 	This module enables testing of the different dumping mechanisms by
 	inducing system failures at predefined crash points.
 	If you don't need it: say N
 	Choose M here to compile this code as a module. The module will be
 	called lkdtm.
 	Documentation on how to use the module can be found in
 	Documentation/fault-injection/provoke-crashes.txt
 config CPU_NOTIFIER_ERROR_INJECT
 	tristate "CPU notifier error injection module"
 	depends on HOTPLUG_CPU && DEBUG_KERNEL
 	help
 	  This option provides a kernel module that can be used to test
 	  the error handling of the cpu notifiers
 	  To compile this code as a module, choose M here: the module will
 	  be called cpu-notifier-error-inject.
 	  If unsure, say N.
 config FAULT_INJECTION
 	bool "Fault-injection framework"
 	depends on DEBUG_KERNEL
 	help
 	  Provide fault-injection framework.
 	  For more details, see Documentation/fault-injection/.
 config FAILSLAB
 	bool "Fault-injection capability for kmalloc"
 	depends on FAULT_INJECTION
 	depends on SLAB || SLUB
 	help
 	  Provide fault-injection capability for kmalloc.
 config FAIL_PAGE_ALLOC
 	bool "Fault-injection capabilitiy for alloc_pages()"
 	depends on FAULT_INJECTION
 	help
 	  Provide fault-injection capability for alloc_pages().
 config FAIL_MAKE_REQUEST
 	bool "Fault-injection capability for disk IO"
 	depends on FAULT_INJECTION && BLOCK
 	help
 	  Provide fault-injection capability for disk IO.
 config FAIL_IO_TIMEOUT
 	bool "Faul-injection capability for faking disk interrupts"
 	depends on FAULT_INJECTION && BLOCK
 	help
 	  Provide fault-injection capability on end IO handling. This
 	  will make the block layer "forget" an interrupt as configured,
 	  thus exercising the error handling.
 	  Only works with drivers that use the generic timeout handling,
 	  for others it wont do anything.
 config FAULT_INJECTION_DEBUG_FS
 	bool "Debugfs entries for fault-injection capabilities"
 	depends on FAULT_INJECTION && SYSFS && DEBUG_FS
 	help
 	  Enable configuration of fault-injection capabilities via debugfs.
 config FAULT_INJECTION_STACKTRACE_FILTER
 	bool "stacktrace filter for fault-injection capabilities"
 	depends on FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT
 	depends on !X86_64
 	select STACKTRACE
 	select FRAME_POINTER if !PPC && !S390
 	help
 	  Provide stacktrace filter for fault-injection capabilities
 config LATENCYTOP
 	bool "Latency measuring infrastructure"
 	select FRAME_POINTER if !MIPS && !PPC && !S390
 	select KALLSYMS
 	select KALLSYMS_ALL
 	select STACKTRACE
 	select SCHEDSTATS
 	select SCHED_DEBUG
 	depends on HAVE_LATENCYTOP_SUPPORT
 	help
 	  Enable this option if you want to use the LatencyTOP tool
 	  to find out which userspace is blocking on what kernel operations.
 config SYSCTL_SYSCALL_CHECK
 	bool "Sysctl checks"
 	depends on SYSCTL
 	---help---
 	  sys_sysctl uses binary paths that have been found challenging
 	  to properly maintain and use. This enables checks that help
 	  you to keep things correct.
 source mm/Kconfig.debug
 source kernel/trace/Kconfig
 config PROVIDE_OHCI1394_DMA_INIT
 	bool "Remote debugging over FireWire early on boot"
 	depends on PCI && X86
 	help
 	  If you want to debug problems which hang or crash the kernel early
 	  on boot and the crashing machine has a FireWire port, you can use
 	  this feature to remotely access the memory of the crashed machine
 	  over FireWire. This employs remote DMA as part of the OHCI1394
 	  specification which is now the standard for FireWire controllers.
 	  With remote DMA, you can monitor the printk buffer remotely using
 	  firescope and access all memory below 4GB using fireproxy from gdb.
 	  Even controlling a kernel debugger is possible using remote DMA.
 	  Usage:
 	  If ohci1394_dma=early is used as boot parameter, it will initialize
 	  all OHCI1394 controllers which are found in the PCI config space.
 	  As all changes to the FireWire bus such as enabling and disabling
 	  devices cause a bus reset and thereby disable remote DMA for all
 	  devices, be sure to have the cable plugged and FireWire enabled on
 	  the debugging host before booting the debug target for debugging.
 	  This code (~1k) is freed after boot. By then, the firewire stack
 	  in charge of the OHCI-1394 controllers should be used instead.
 	  See Documentation/debugging-via-ohci1394.txt for more information.
 config FIREWIRE_OHCI_REMOTE_DMA
 	bool "Remote debugging over FireWire with firewire-ohci"
 	depends on FIREWIRE_OHCI
 	help
 	  This option lets you use the FireWire bus for remote debugging
 	  with help of the firewire-ohci driver. It enables unfiltered
 	  remote DMA in firewire-ohci.
 	  See Documentation/debugging-via-ohci1394.txt for more information.
 	  If unsure, say N.
 config BUILD_DOCSRC
 	bool "Build targets in Documentation/ tree"
 	depends on HEADERS_CHECK
 	help
 	  This option attempts to build objects from the source files in the
 	  kernel Documentation/ tree.
 	  Say N if you are unsure.
 config DYNAMIC_DEBUG
 	bool "Enable dynamic printk() support"
 	default n
 	depends on PRINTK
 	depends on DEBUG_FS
 	help
 	  Compiles debug level messages into the kernel, which would not
 	  otherwise be available at runtime. These messages can then be
 	  enabled/disabled based on various levels of scope - per source file,
 	  function, module, format string, and line number. This mechanism
 	  implicitly enables all pr_debug() and dev_dbg() calls. The impact of
 	  this compile option is a larger kernel text size of about 2%.
 	  Usage:
 	  Dynamic debugging is controlled via the 'dynamic_debug/control' file,
 	  which is contained in the 'debugfs' filesystem. Thus, the debugfs
 	  filesystem must first be mounted before making use of this feature.
 	  We refer the control file as: <debugfs>/dynamic_debug/control. This
 	  file contains a list of the debug statements that can be enabled. The
 	  format for each line of the file is:
 		filename:lineno [module]function flags format
 	  filename : source file of the debug statement
 	  lineno : line number of the debug statement
 	  module : module that contains the debug statement
 	  function : function that contains the debug statement
           flags : 'p' means the line is turned 'on' for printing
           format : the format used for the debug statement
 	  From a live system:
 		nullarbor:~ # cat <debugfs>/dynamic_debug/control
 		# filename:lineno [module]function flags format
 		fs/aio.c:222 [aio]__put_ioctx - "__put_ioctx:\040freeing\040%p\012"
 		fs/aio.c:248 [aio]ioctx_alloc - "ENOMEM:\040nr_events\040too\040high\012"
 		fs/aio.c:1770 [aio]sys_io_cancel - "calling\040cancel\012"
 	  Example usage:
 		// enable the message at line 1603 of file svcsock.c
 		nullarbor:~ # echo -n 'file svcsock.c line 1603 +p' >
 						<debugfs>/dynamic_debug/control
 		// enable all the messages in file svcsock.c
 		nullarbor:~ # echo -n 'file svcsock.c +p' >
 						<debugfs>/dynamic_debug/control
 		// enable all the messages in the NFS server module
 		nullarbor:~ # echo -n 'module nfsd +p' >
 						<debugfs>/dynamic_debug/control
 		// enable all 12 messages in the function svc_process()
 		nullarbor:~ # echo -n 'func svc_process +p' >
 						<debugfs>/dynamic_debug/control
 		// disable all 12 messages in the function svc_process()
 		nullarbor:~ # echo -n 'func svc_process -p' >
 						<debugfs>/dynamic_debug/control
 	  See Documentation/dynamic-debug-howto.txt for additional information.
 config DMA_API_DEBUG
 	bool "Enable debugging of DMA-API usage"
 	depends on HAVE_DMA_API_DEBUG
 	help
 	  Enable this option to debug the use of the DMA API by device drivers.
 	  With this option you will be able to detect common bugs in device
 	  drivers like double-freeing of DMA mappings or freeing mappings that
 	  were never allocated.
 	  This option causes a performance degredation.  Use only if you want
 	  to debug device drivers. If unsure, say N.
 config ATOMIC64_SELFTEST
 	bool "Perform an atomic64_t self-test at boot"
 	help
 	  Enable this option to test the atomic64_t functions at boot.
 	  If unsure, say N.
 source "samples/Kconfig"
 source "lib/Kconfig.kgdb"
 source "lib/Kconfig.kmemcheck"

 /*
  * mm/kmemleak.c
  *
  * Copyright (C) 2008 ARM Limited
  * Written by Catalin Marinas <catalin.marinas@arm.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  *
  *
  * For more information on the algorithm and kmemleak usage, please see
  * Documentation/kmemleak.txt.
  *
  * Notes on locking
  * ----------------
  *
  * The following locks and mutexes are used by kmemleak:
  *
  * - kmemleak_lock (rwlock): protects the object_list modifications and
  *   accesses to the object_tree_root. The object_list is the main list
  *   holding the metadata (struct kmemleak_object) for the allocated memory
  *   blocks. The object_tree_root is a priority search tree used to look-up
  *   metadata based on a pointer to the corresponding memory block.  The
  *   kmemleak_object structures are added to the object_list and
  *   object_tree_root in the create_object() function called from the
  *   kmemleak_alloc() callback and removed in delete_object() called from the
  *   kmemleak_free() callback
  * - kmemleak_object.lock (spinlock): protects a kmemleak_object. Accesses to
  *   the metadata (e.g. count) are protected by this lock. Note that some
  *   members of this structure may be protected by other means (atomic or
  *   kmemleak_lock). This lock is also held when scanning the corresponding
  *   memory block to avoid the kernel freeing it via the kmemleak_free()
  *   callback. This is less heavyweight than holding a global lock like
  *   kmemleak_lock during scanning
  * - scan_mutex (mutex): ensures that only one thread may scan the memory for
  *   unreferenced objects at a time. The gray_list contains the objects which
  *   are already referenced or marked as false positives and need to be
  *   scanned. This list is only modified during a scanning episode when the
  *   scan_mutex is held. At the end of a scan, the gray_list is always empty.
  *   Note that the kmemleak_object.use_count is incremented when an object is
  *   added to the gray_list and therefore cannot be freed. This mutex also
  *   prevents multiple users of the "kmemleak" debugfs file together with
  *   modifications to the memory scanning parameters including the scan_thread
  *   pointer
  *
  * The kmemleak_object structures have a use_count incremented or decremented
  * using the get_object()/put_object() functions. When the use_count becomes
  * 0, this count can no longer be incremented and put_object() schedules the
  * kmemleak_object freeing via an RCU callback. All calls to the get_object()
  * function must be protected by rcu_read_lock() to avoid accessing a freed
  * structure.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/sched.h>
 #include <linux/jiffies.h>
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/kthread.h>
 #include <linux/prio_tree.h>
 #include <linux/fs.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/cpumask.h>
 #include <linux/spinlock.h>
 #include <linux/mutex.h>
 #include <linux/rcupdate.h>
 #include <linux/stacktrace.h>
 #include <linux/cache.h>
 #include <linux/percpu.h>
 #include <linux/hardirq.h>
 #include <linux/mmzone.h>
 #include <linux/slab.h>
 #include <linux/thread_info.h>
 #include <linux/err.h>
 #include <linux/uaccess.h>
 #include <linux/string.h>
 #include <linux/nodemask.h>
 #include <linux/mm.h>
 #include <linux/workqueue.h>
 #include <linux/crc32.h>
 #include <asm/sections.h>
 #include <asm/processor.h>
 #include <asm/atomic.h>
 #include <linux/kmemcheck.h>
 #include <linux/kmemleak.h>
 /*
  * Kmemleak configuration and common defines.
  */
 #define MAX_TRACE		16	/* stack trace length */
 #define MSECS_MIN_AGE		5000	/* minimum object age for reporting */
 #define SECS_FIRST_SCAN		60	/* delay before the first scan */
 #define SECS_SCAN_WAIT		600	/* subsequent auto scanning delay */
 #define MAX_SCAN_SIZE		4096	/* maximum size of a scanned block */
 #define BYTES_PER_POINTER	sizeof(void *)
 /* GFP bitmask for kmemleak internal allocations */
 #define GFP_KMEMLEAK_MASK	(GFP_KERNEL | GFP_ATOMIC)
 /* scanning area inside a memory block */
 struct kmemleak_scan_area {
 	struct hlist_node node;
 	unsigned long start;
 	size_t size;
 };
 #define KMEMLEAK_GREY	0
 #define KMEMLEAK_BLACK	-1
 /*
  * Structure holding the metadata for each allocated memory block.
  * Modifications to such objects should be made while holding the
  * object->lock. Insertions or deletions from object_list, gray_list or
  * tree_node are already protected by the corresponding locks or mutex (see
  * the notes on locking above). These objects are reference-counted
  * (use_count) and freed using the RCU mechanism.
  */
 struct kmemleak_object {
 	spinlock_t lock;
 	unsigned long flags;		/* object status flags */
 	struct list_head object_list;
 	struct list_head gray_list;
 	struct prio_tree_node tree_node;
 	struct rcu_head rcu;		/* object_list lockless traversal */
 	/* object usage count; object freed when use_count == 0 */
 	atomic_t use_count;
 	unsigned long pointer;
 	size_t size;
 	/* minimum number of a pointers found before it is considered leak */
 	int min_count;
 	/* the total number of pointers found pointing to this object */
 	int count;
 	/* checksum for detecting modified objects */
 	u32 checksum;
 	/* memory ranges to be scanned inside an object (empty for all) */
 	struct hlist_head area_list;
 	unsigned long trace[MAX_TRACE];
 	unsigned int trace_len;
 	unsigned long jiffies;		/* creation timestamp */
 	pid_t pid;			/* pid of the current task */
 	char comm[TASK_COMM_LEN];	/* executable name */
 };
 /* flag representing the memory block allocation status */
 #define OBJECT_ALLOCATED	(1 << 0)
 /* flag set after the first reporting of an unreference object */
 #define OBJECT_REPORTED		(1 << 1)
 /* flag set to not scan the object */
 #define OBJECT_NO_SCAN		(1 << 2)
 /* number of bytes to print per line; must be 16 or 32 */
 #define HEX_ROW_SIZE		16
 /* number of bytes to print at a time (1, 2, 4, 8) */
 #define HEX_GROUP_SIZE		1
 /* include ASCII after the hex output */
 #define HEX_ASCII		1
 /* max number of lines to be printed */
 #define HEX_MAX_LINES		2
 /* the list of all allocated objects */
 static LIST_HEAD(object_list);
 /* the list of gray-colored objects (see color_gray comment below) */
 static LIST_HEAD(gray_list);
 /* prio search tree for object boundaries */
 static struct prio_tree_root object_tree_root;
 /* rw_lock protecting the access to object_list and prio_tree_root */
 static DEFINE_RWLOCK(kmemleak_lock);
 /* allocation caches for kmemleak internal data */
 static struct kmem_cache *object_cache;
 static struct kmem_cache *scan_area_cache;
 /* set if tracing memory operations is enabled */
 static atomic_t kmemleak_enabled = ATOMIC_INIT(0);
 /* set in the late_initcall if there were no errors */
 static atomic_t kmemleak_initialized = ATOMIC_INIT(0);
 /* enables or disables early logging of the memory operations */
 static atomic_t kmemleak_early_log = ATOMIC_INIT(1);
 /* set if a fata kmemleak error has occurred */
 static atomic_t kmemleak_error = ATOMIC_INIT(0);
 /* minimum and maximum address that may be valid pointers */
 static unsigned long min_addr = ULONG_MAX;
 static unsigned long max_addr;
 static struct task_struct *scan_thread;
 /* used to avoid reporting of recently allocated objects */
 static unsigned long jiffies_min_age;
 static unsigned long jiffies_last_scan;
 /* delay between automatic memory scannings */
 static signed long jiffies_scan_wait;
 /* enables or disables the task stacks scanning */
 static int kmemleak_stack_scan = 1;
 /* protects the memory scanning, parameters and debug/kmemleak file access */
 static DEFINE_MUTEX(scan_mutex);
+/* setting kmemleak=on, will set this var, skipping the disable */
+static int kmemleak_skip_disable;
 /*
  * Early object allocation/freeing logging. Kmemleak is initialized after the
  * kernel allocator. However, both the kernel allocator and kmemleak may
  * allocate memory blocks which need to be tracked. Kmemleak defines an
  * arbitrary buffer to hold the allocation/freeing information before it is
  * fully initialized.
  */
 /* kmemleak operation type for early logging */
 enum {
 	KMEMLEAK_ALLOC,
 	KMEMLEAK_FREE,
 	KMEMLEAK_FREE_PART,
 	KMEMLEAK_NOT_LEAK,
 	KMEMLEAK_IGNORE,
 	KMEMLEAK_SCAN_AREA,
 	KMEMLEAK_NO_SCAN
 };
 /*
  * Structure holding the information passed to kmemleak callbacks during the
  * early logging.
  */
 struct early_log {
 	int op_type;			/* kmemleak operation type */
 	const void *ptr;		/* allocated/freed memory block */
 	size_t size;			/* memory block size */
 	int min_count;			/* minimum reference count */
 	unsigned long trace[MAX_TRACE];	/* stack trace */
 	unsigned int trace_len;		/* stack trace length */
 };
 /* early logging buffer and current position */
 static struct early_log
 	early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
 static int crt_early_log __initdata;
 static void kmemleak_disable(void);
 /*
  * Print a warning and dump the stack trace.
  */
 #define kmemleak_warn(x...)	do {	\
 	pr_warning(x);			\
 	dump_stack();			\
 } while (0)
 /*
  * Macro invoked when a serious kmemleak condition occured and cannot be
  * recovered from. Kmemleak will be disabled and further allocation/freeing
  * tracing no longer available.
  */
 #define kmemleak_stop(x...)	do {	\
 	kmemleak_warn(x);		\
 	kmemleak_disable();		\
 } while (0)
 /*
  * Printing of the objects hex dump to the seq file. The number of lines to be
  * printed is limited to HEX_MAX_LINES to prevent seq file spamming. The
  * actual number of printed bytes depends on HEX_ROW_SIZE. It must be called
  * with the object->lock held.
  */
 static void hex_dump_object(struct seq_file *seq,
 			    struct kmemleak_object *object)
 {
 	const u8 *ptr = (const u8 *)object->pointer;
 	int i, len, remaining;
 	unsigned char linebuf[HEX_ROW_SIZE * 5];
 	/* limit the number of lines to HEX_MAX_LINES */
 	remaining = len =
 		min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));
 	seq_printf(seq, "  hex dump (first %d bytes):\n", len);
 	for (i = 0; i < len; i += HEX_ROW_SIZE) {
 		int linelen = min(remaining, HEX_ROW_SIZE);
 		remaining -= HEX_ROW_SIZE;
 		hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE,
 				   HEX_GROUP_SIZE, linebuf, sizeof(linebuf),
 				   HEX_ASCII);
 		seq_printf(seq, "    %s\n", linebuf);
 	}
 }
 /*
  * Object colors, encoded with count and min_count:
  * - white - orphan object, not enough references to it (count < min_count)
  * - gray  - not orphan, not marked as false positive (min_count == 0) or
  *		sufficient references to it (count >= min_count)
  * - black - ignore, it doesn't contain references (e.g. text section)
  *		(min_count == -1). No function defined for this color.
  * Newly created objects don't have any color assigned (object->count == -1)
  * before the next memory scan when they become white.
  */
 static bool color_white(const struct kmemleak_object *object)
 {
 	return object->count != KMEMLEAK_BLACK &&
 		object->count < object->min_count;
 }
 static bool color_gray(const struct kmemleak_object *object)
 {
 	return object->min_count != KMEMLEAK_BLACK &&
 		object->count >= object->min_count;
 }
 /*
  * Objects are considered unreferenced only if their color is white, they have
  * not be deleted and have a minimum age to avoid false positives caused by
  * pointers temporarily stored in CPU registers.
  */
 static bool unreferenced_object(struct kmemleak_object *object)
 {
 	return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
 		time_before_eq(object->jiffies + jiffies_min_age,
 			       jiffies_last_scan);
 }
 /*
  * Printing of the unreferenced objects information to the seq file. The
  * print_unreferenced function must be called with the object->lock held.
  */
 static void print_unreferenced(struct seq_file *seq,
 			       struct kmemleak_object *object)
 {
 	int i;
 	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
 	seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
 		   object->pointer, object->size);
 	seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
 		   object->comm, object->pid, object->jiffies,
 		   msecs_age / 1000, msecs_age % 1000);
 	hex_dump_object(seq, object);
 	seq_printf(seq, "  backtrace:\n");
 	for (i = 0; i < object->trace_len; i++) {
 		void *ptr = (void *)object->trace[i];
 		seq_printf(seq, "    [<%p>] %pS\n", ptr, ptr);
 	}
 }
 /*
  * Print the kmemleak_object information. This function is used mainly for
  * debugging special cases when kmemleak operations. It must be called with
  * the object->lock held.
  */
 static void dump_object_info(struct kmemleak_object *object)
 {
 	struct stack_trace trace;
 	trace.nr_entries = object->trace_len;
 	trace.entries = object->trace;
 	pr_notice("Object 0x%08lx (size %zu):\n",
 		  object->tree_node.start, object->size);
 	pr_notice("  comm \"%s\", pid %d, jiffies %lu\n",
 		  object->comm, object->pid, object->jiffies);
 	pr_notice("  min_count = %d\n", object->min_count);
 	pr_notice("  count = %d\n", object->count);
 	pr_notice("  flags = 0x%lx\n", object->flags);
 	pr_notice("  checksum = %d\n", object->checksum);
 	pr_notice("  backtrace:\n");
 	print_stack_trace(&trace, 4);
 }
 /*
  * Look-up a memory block metadata (kmemleak_object) in the priority search
  * tree based on a pointer value. If alias is 0, only values pointing to the
  * beginning of the memory block are allowed. The kmemleak_lock must be held
  * when calling this function.
  */
 static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
 {
 	struct prio_tree_node *node;
 	struct prio_tree_iter iter;
 	struct kmemleak_object *object;
 	prio_tree_iter_init(&iter, &object_tree_root, ptr, ptr);
 	node = prio_tree_next(&iter);
 	if (node) {
 		object = prio_tree_entry(node, struct kmemleak_object,
 					 tree_node);
 		if (!alias && object->pointer != ptr) {
 			pr_warning("Found object by alias at 0x%08lx\n", ptr);
 			dump_stack();
 			dump_object_info(object);
 			object = NULL;
 		}
 	} else
 		object = NULL;
 	return object;
 }
 /*
  * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
  * that once an object's use_count reached 0, the RCU freeing was already
  * registered and the object should no longer be used. This function must be
  * called under the protection of rcu_read_lock().
  */
 static int get_object(struct kmemleak_object *object)
 {
 	return atomic_inc_not_zero(&object->use_count);
 }
 /*
  * RCU callback to free a kmemleak_object.
  */
 static void free_object_rcu(struct rcu_head *rcu)
 {
 	struct hlist_node *elem, *tmp;
 	struct kmemleak_scan_area *area;
 	struct kmemleak_object *object =
 		container_of(rcu, struct kmemleak_object, rcu);
 	/*
 	 * Once use_count is 0 (guaranteed by put_object), there is no other
 	 * code accessing this object, hence no need for locking.
 	 */
 	hlist_for_each_entry_safe(area, elem, tmp, &object->area_list, node) {
 		hlist_del(elem);
 		kmem_cache_free(scan_area_cache, area);
 	}
 	kmem_cache_free(object_cache, object);
 }
 /*
  * Decrement the object use_count. Once the count is 0, free the object using
  * an RCU callback. Since put_object() may be called via the kmemleak_free() ->
  * delete_object() path, the delayed RCU freeing ensures that there is no
  * recursive call to the kernel allocator. Lock-less RCU object_list traversal
  * is also possible.
  */
 static void put_object(struct kmemleak_object *object)
 {
 	if (!atomic_dec_and_test(&object->use_count))
 		return;
 	/* should only get here after delete_object was called */
 	WARN_ON(object->flags & OBJECT_ALLOCATED);
 	call_rcu(&object->rcu, free_object_rcu);
 }
 /*
  * Look up an object in the prio search tree and increase its use_count.
  */
 static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
 {
 	unsigned long flags;
 	struct kmemleak_object *object = NULL;
 	rcu_read_lock();
 	read_lock_irqsave(&kmemleak_lock, flags);
 	if (ptr >= min_addr && ptr < max_addr)
 		object = lookup_object(ptr, alias);
 	read_unlock_irqrestore(&kmemleak_lock, flags);
 	/* check whether the object is still available */
 	if (object && !get_object(object))
 		object = NULL;
 	rcu_read_unlock();
 	return object;
 }
 /*
  * Save stack trace to the given array of MAX_TRACE size.
  */
 static int __save_stack_trace(unsigned long *trace)
 {
 	struct stack_trace stack_trace;
 	stack_trace.max_entries = MAX_TRACE;
 	stack_trace.nr_entries = 0;
 	stack_trace.entries = trace;
 	stack_trace.skip = 2;
 	save_stack_trace(&stack_trace);
 	return stack_trace.nr_entries;
 }
 /*
  * Create the metadata (struct kmemleak_object) corresponding to an allocated
  * memory block and add it to the object_list and object_tree_root.
  */
 static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
 					     int min_count, gfp_t gfp)
 {
 	unsigned long flags;
 	struct kmemleak_object *object;
 	struct prio_tree_node *node;
 	object = kmem_cache_alloc(object_cache, gfp & GFP_KMEMLEAK_MASK);
 	if (!object) {
 		kmemleak_stop("Cannot allocate a kmemleak_object structure\n");
 		return NULL;
 	}
 	INIT_LIST_HEAD(&object->object_list);
 	INIT_LIST_HEAD(&object->gray_list);
 	INIT_HLIST_HEAD(&object->area_list);
 	spin_lock_init(&object->lock);
 	atomic_set(&object->use_count, 1);
 	object->flags = OBJECT_ALLOCATED;
 	object->pointer = ptr;
 	object->size = size;
 	object->min_count = min_count;
 	object->count = 0;			/* white color initially */
 	object->jiffies = jiffies;
 	object->checksum = 0;
 	/* task information */
 	if (in_irq()) {
 		object->pid = 0;
 		strncpy(object->comm, "hardirq", sizeof(object->comm));
 	} else if (in_softirq()) {
 		object->pid = 0;
 		strncpy(object->comm, "softirq", sizeof(object->comm));
 	} else {
 		object->pid = current->pid;
 		/*
 		 * There is a small chance of a race with set_task_comm(),
 		 * however using get_task_comm() here may cause locking
 		 * dependency issues with current->alloc_lock. In the worst
 		 * case, the command line is not correct.
 		 */
 		strncpy(object->comm, current->comm, sizeof(object->comm));
 	}
 	/* kernel backtrace */
 	object->trace_len = __save_stack_trace(object->trace);
 	INIT_PRIO_TREE_NODE(&object->tree_node);
 	object->tree_node.start = ptr;
 	object->tree_node.last = ptr + size - 1;
 	write_lock_irqsave(&kmemleak_lock, flags);
 	min_addr = min(min_addr, ptr);
 	max_addr = max(max_addr, ptr + size);
 	node = prio_tree_insert(&object_tree_root, &object->tree_node);
 	/*
 	 * The code calling the kernel does not yet have the pointer to the
 	 * memory block to be able to free it.  However, we still hold the
 	 * kmemleak_lock here in case parts of the kernel started freeing
 	 * random memory blocks.
 	 */
 	if (node != &object->tree_node) {
 		kmemleak_stop("Cannot insert 0x%lx into the object search tree "
 			      "(already existing)\n", ptr);
 		object = lookup_object(ptr, 1);
 		spin_lock(&object->lock);
 		dump_object_info(object);
 		spin_unlock(&object->lock);
 		goto out;
 	}
 	list_add_tail_rcu(&object->object_list, &object_list);
 out:
 	write_unlock_irqrestore(&kmemleak_lock, flags);
 	return object;
 }
 /*
  * Remove the metadata (struct kmemleak_object) for a memory block from the
  * object_list and object_tree_root and decrement its use_count.
  */
 static void __delete_object(struct kmemleak_object *object)
 {
 	unsigned long flags;
 	write_lock_irqsave(&kmemleak_lock, flags);
 	prio_tree_remove(&object_tree_root, &object->tree_node);
 	list_del_rcu(&object->object_list);
 	write_unlock_irqrestore(&kmemleak_lock, flags);
 	WARN_ON(!(object->flags & OBJECT_ALLOCATED));
 	WARN_ON(atomic_read(&object->use_count) < 2);
 	/*
 	 * Locking here also ensures that the corresponding memory block
 	 * cannot be freed when it is being scanned.
 	 */
 	spin_lock_irqsave(&object->lock, flags);
 	object->flags &= ~OBJECT_ALLOCATED;
 	spin_unlock_irqrestore(&object->lock, flags);
 	put_object(object);
 }
 /*
  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
  * delete it.
  */
 static void delete_object_full(unsigned long ptr)
 {
 	struct kmemleak_object *object;
 	object = find_and_get_object(ptr, 0);
 	if (!object) {
 #ifdef DEBUG
 		kmemleak_warn("Freeing unknown object at 0x%08lx\n",
 			      ptr);
 #endif
 		return;
 	}
 	__delete_object(object);
 	put_object(object);
 }
 /*
  * Look up the metadata (struct kmemleak_object) corresponding to ptr and
  * delete it. If the memory block is partially freed, the function may create
  * additional metadata for the remaining parts of the block.
  */
 static void delete_object_part(unsigned long ptr, size_t size)
 {
 	struct kmemleak_object *object;
 	unsigned long start, end;
 	object = find_and_get_object(ptr, 1);
 	if (!object) {
 #ifdef DEBUG
 		kmemleak_warn("Partially freeing unknown object at 0x%08lx "
 			      "(size %zu)\n", ptr, size);
 #endif
 		return;
 	}
 	__delete_object(object);
 	/*
 	 * Create one or two objects that may result from the memory block
 	 * split. Note that partial freeing is only done by free_bootmem() and
 	 * this happens before kmemleak_init() is called. The path below is
 	 * only executed during early log recording in kmemleak_init(), so
 	 * GFP_KERNEL is enough.
 	 */
 	start = object->pointer;
 	end = object->pointer + object->size;
 	if (ptr > start)
 		create_object(start, ptr - start, object->min_count,
 			      GFP_KERNEL);
 	if (ptr + size < end)
 		create_object(ptr + size, end - ptr - size, object->min_count,
 			      GFP_KERNEL);
 	put_object(object);
 }
 static void __paint_it(struct kmemleak_object *object, int color)
 {
 	object->min_count = color;
 	if (color == KMEMLEAK_BLACK)
 		object->flags |= OBJECT_NO_SCAN;
 }
 static void paint_it(struct kmemleak_object *object, int color)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&object->lock, flags);
 	__paint_it(object, color);
 	spin_unlock_irqrestore(&object->lock, flags);
 }
 static void paint_ptr(unsigned long ptr, int color)
 {
 	struct kmemleak_object *object;
 	object = find_and_get_object(ptr, 0);
 	if (!object) {
 		kmemleak_warn("Trying to color unknown object "
 			      "at 0x%08lx as %s\n", ptr,
 			      (color == KMEMLEAK_GREY) ? "Grey" :
 			      (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
 		return;
 	}
 	paint_it(object, color);
 	put_object(object);
 }
 /*
  * Make a object permanently as gray-colored so that it can no longer be
  * reported as a leak. This is used in general to mark a false positive.
  */
 static void make_gray_object(unsigned long ptr)
 {
 	paint_ptr(ptr, KMEMLEAK_GREY);
 }
 /*
  * Mark the object as black-colored so that it is ignored from scans and
  * reporting.
  */
 static void make_black_object(unsigned long ptr)
 {
 	paint_ptr(ptr, KMEMLEAK_BLACK);
 }
 /*
  * Add a scanning area to the object. If at least one such area is added,
  * kmemleak will only scan these ranges rather than the whole memory block.
  */
 static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
 {
 	unsigned long flags;
 	struct kmemleak_object *object;
 	struct kmemleak_scan_area *area;
 	object = find_and_get_object(ptr, 1);
 	if (!object) {
 		kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
 			      ptr);
 		return;
 	}
 	area = kmem_cache_alloc(scan_area_cache, gfp & GFP_KMEMLEAK_MASK);
 	if (!area) {
 		kmemleak_warn("Cannot allocate a scan area\n");
 		goto out;
 	}
 	spin_lock_irqsave(&object->lock, flags);
 	if (ptr + size > object->pointer + object->size) {
 		kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
 		dump_object_info(object);
 		kmem_cache_free(scan_area_cache, area);
 		goto out_unlock;
 	}
 	INIT_HLIST_NODE(&area->node);
 	area->start = ptr;
 	area->size = size;
 	hlist_add_head(&area->node, &object->area_list);
 out_unlock:
 	spin_unlock_irqrestore(&object->lock, flags);
 out:
 	put_object(object);
 }
 /*
  * Set the OBJECT_NO_SCAN flag for the object corresponding to the give
  * pointer. Such object will not be scanned by kmemleak but references to it
  * are searched.
  */
 static void object_no_scan(unsigned long ptr)
 {
 	unsigned long flags;
 	struct kmemleak_object *object;
 	object = find_and_get_object(ptr, 0);
 	if (!object) {
 		kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
 		return;
 	}
 	spin_lock_irqsave(&object->lock, flags);
 	object->flags |= OBJECT_NO_SCAN;
 	spin_unlock_irqrestore(&object->lock, flags);
 	put_object(object);
 }
 /*
  * Log an early kmemleak_* call to the early_log buffer. These calls will be
  * processed later once kmemleak is fully initialized.
  */
 static void __init log_early(int op_type, const void *ptr, size_t size,
 			     int min_count)
 {
 	unsigned long flags;
 	struct early_log *log;
 	if (crt_early_log >= ARRAY_SIZE(early_log)) {
 		pr_warning("Early log buffer exceeded, "
 			   "please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n");
 		kmemleak_disable();
 		return;
 	}
 	/*
 	 * There is no need for locking since the kernel is still in UP mode
 	 * at this stage. Disabling the IRQs is enough.
 	 */
 	local_irq_save(flags);
 	log = &early_log[crt_early_log];
 	log->op_type = op_type;
 	log->ptr = ptr;
 	log->size = size;
 	log->min_count = min_count;
 	if (op_type == KMEMLEAK_ALLOC)
 		log->trace_len = __save_stack_trace(log->trace);
 	crt_early_log++;
 	local_irq_restore(flags);
 }
 /*
  * Log an early allocated block and populate the stack trace.
  */
 static void early_alloc(struct early_log *log)
 {
 	struct kmemleak_object *object;
 	unsigned long flags;
 	int i;
 	if (!atomic_read(&kmemleak_enabled) || !log->ptr || IS_ERR(log->ptr))
 		return;
 	/*
 	 * RCU locking needed to ensure object is not freed via put_object().
 	 */
 	rcu_read_lock();
 	object = create_object((unsigned long)log->ptr, log->size,
 			       log->min_count, GFP_ATOMIC);
 	if (!object)
 		goto out;
 	spin_lock_irqsave(&object->lock, flags);
 	for (i = 0; i < log->trace_len; i++)
 		object->trace[i] = log->trace[i];
 	object->trace_len = log->trace_len;
 	spin_unlock_irqrestore(&object->lock, flags);
 out:
 	rcu_read_unlock();
 }
 /*
  * Memory allocation function callback. This function is called from the
  * kernel allocators when a new block is allocated (kmem_cache_alloc, kmalloc,
  * vmalloc etc.).
  */
 void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
 			  gfp_t gfp)
 {
 	pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		create_object((unsigned long)ptr, size, min_count, gfp);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
 }
 EXPORT_SYMBOL_GPL(kmemleak_alloc);
 /*
  * Memory freeing function callback. This function is called from the kernel
  * allocators when a block is freed (kmem_cache_free, kfree, vfree etc.).
  */
 void __ref kmemleak_free(const void *ptr)
 {
 	pr_debug("%s(0x%p)\n", __func__, ptr);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		delete_object_full((unsigned long)ptr);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_FREE, ptr, 0, 0);
 }
 EXPORT_SYMBOL_GPL(kmemleak_free);
 /*
  * Partial memory freeing function callback. This function is usually called
  * from bootmem allocator when (part of) a memory block is freed.
  */
 void __ref kmemleak_free_part(const void *ptr, size_t size)
 {
 	pr_debug("%s(0x%p)\n", __func__, ptr);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		delete_object_part((unsigned long)ptr, size);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
 }
 EXPORT_SYMBOL_GPL(kmemleak_free_part);
 /*
  * Mark an already allocated memory block as a false positive. This will cause
  * the block to no longer be reported as leak and always be scanned.
  */
 void __ref kmemleak_not_leak(const void *ptr)
 {
 	pr_debug("%s(0x%p)\n", __func__, ptr);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		make_gray_object((unsigned long)ptr);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
 }
 EXPORT_SYMBOL(kmemleak_not_leak);
 /*
  * Ignore a memory block. This is usually done when it is known that the
  * corresponding block is not a leak and does not contain any references to
  * other allocated memory blocks.
  */
 void __ref kmemleak_ignore(const void *ptr)
 {
 	pr_debug("%s(0x%p)\n", __func__, ptr);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		make_black_object((unsigned long)ptr);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
 }
 EXPORT_SYMBOL(kmemleak_ignore);
 /*
  * Limit the range to be scanned in an allocated memory block.
  */
 void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
 {
 	pr_debug("%s(0x%p)\n", __func__, ptr);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		add_scan_area((unsigned long)ptr, size, gfp);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
 }
 EXPORT_SYMBOL(kmemleak_scan_area);
 /*
  * Inform kmemleak not to scan the given memory block.
  */
 void __ref kmemleak_no_scan(const void *ptr)
 {
 	pr_debug("%s(0x%p)\n", __func__, ptr);
 	if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
 		object_no_scan((unsigned long)ptr);
 	else if (atomic_read(&kmemleak_early_log))
 		log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
 }
 EXPORT_SYMBOL(kmemleak_no_scan);
 /*
  * Update an object's checksum and return true if it was modified.
  */
 static bool update_checksum(struct kmemleak_object *object)
 {
 	u32 old_csum = object->checksum;
 	if (!kmemcheck_is_obj_initialized(object->pointer, object->size))
 		return false;
 	object->checksum = crc32(0, (void *)object->pointer, object->size);
 	return object->checksum != old_csum;
 }
 /*
  * Memory scanning is a long process and it needs to be interruptable. This
  * function checks whether such interrupt condition occured.
  */
 static int scan_should_stop(void)
 {
 	if (!atomic_read(&kmemleak_enabled))
 		return 1;
 	/*
 	 * This function may be called from either process or kthread context,
 	 * hence the need to check for both stop conditions.
 	 */
 	if (current->mm)
 		return signal_pending(current);
 	else
 		return kthread_should_stop();
 	return 0;
 }
 /*
  * Scan a memory block (exclusive range) for valid pointers and add those
  * found to the gray list.
  */
 static void scan_block(void *_start, void *_end,
 		       struct kmemleak_object *scanned, int allow_resched)
 {
 	unsigned long *ptr;
 	unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
 	unsigned long *end = _end - (BYTES_PER_POINTER - 1);
 	for (ptr = start; ptr < end; ptr++) {
 		struct kmemleak_object *object;
 		unsigned long flags;
 		unsigned long pointer;
 		if (allow_resched)
 			cond_resched();
 		if (scan_should_stop())
 			break;
 		/* don't scan uninitialized memory */
 		if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
 						  BYTES_PER_POINTER))
 			continue;
 		pointer = *ptr;
 		object = find_and_get_object(pointer, 1);
 		if (!object)
 			continue;
 		if (object == scanned) {
 			/* self referenced, ignore */
 			put_object(object);
 			continue;
 		}
 		/*
 		 * Avoid the lockdep recursive warning on object->lock being
 		 * previously acquired in scan_object(). These locks are
 		 * enclosed by scan_mutex.
 		 */
 		spin_lock_irqsave_nested(&object->lock, flags,
 					 SINGLE_DEPTH_NESTING);
 		if (!color_white(object)) {
 			/* non-orphan, ignored or new */
 			spin_unlock_irqrestore(&object->lock, flags);
 			put_object(object);
 			continue;
 		}
 		/*
 		 * Increase the object's reference count (number of pointers
 		 * to the memory block). If this count reaches the required
 		 * minimum, the object's color will become gray and it will be
 		 * added to the gray_list.
 		 */
 		object->count++;
 		if (color_gray(object)) {
 			list_add_tail(&object->gray_list, &gray_list);
 			spin_unlock_irqrestore(&object->lock, flags);
 			continue;
 		}
 		spin_unlock_irqrestore(&object->lock, flags);
 		put_object(object);
 	}
 }
 /*
  * Scan a memory block corresponding to a kmemleak_object. A condition is
  * that object->use_count >= 1.
  */
 static void scan_object(struct kmemleak_object *object)
 {
 	struct kmemleak_scan_area *area;
 	struct hlist_node *elem;
 	unsigned long flags;
 	/*
 	 * Once the object->lock is acquired, the corresponding memory block
 	 * cannot be freed (the same lock is acquired in delete_object).
 	 */
 	spin_lock_irqsave(&object->lock, flags);
 	if (object->flags & OBJECT_NO_SCAN)
 		goto out;
 	if (!(object->flags & OBJECT_ALLOCATED))
 		/* already freed object */
 		goto out;
 	if (hlist_empty(&object->area_list)) {
 		void *start = (void *)object->pointer;
 		void *end = (void *)(object->pointer + object->size);
 		while (start < end && (object->flags & OBJECT_ALLOCATED) &&
 		       !(object->flags & OBJECT_NO_SCAN)) {
 			scan_block(start, min(start + MAX_SCAN_SIZE, end),
 				   object, 0);
 			start += MAX_SCAN_SIZE;
 			spin_unlock_irqrestore(&object->lock, flags);
 			cond_resched();
 			spin_lock_irqsave(&object->lock, flags);
 		}
 	} else
 		hlist_for_each_entry(area, elem, &object->area_list, node)
 			scan_block((void *)area->start,
 				   (void *)(area->start + area->size),
 				   object, 0);
 out:
 	spin_unlock_irqrestore(&object->lock, flags);
 }
 /*
  * Scan the objects already referenced (gray objects). More objects will be
  * referenced and, if there are no memory leaks, all the objects are scanned.
  */
 static void scan_gray_list(void)
 {
 	struct kmemleak_object *object, *tmp;
 	/*
 	 * The list traversal is safe for both tail additions and removals
 	 * from inside the loop. The kmemleak objects cannot be freed from
 	 * outside the loop because their use_count was incremented.
 	 */
 	object = list_entry(gray_list.next, typeof(*object), gray_list);
 	while (&object->gray_list != &gray_list) {
 		cond_resched();
 		/* may add new objects to the list */
 		if (!scan_should_stop())
 			scan_object(object);
 		tmp = list_entry(object->gray_list.next, typeof(*object),
 				 gray_list);
 		/* remove the object from the list and release it */
 		list_del(&object->gray_list);
 		put_object(object);
 		object = tmp;
 	}
 	WARN_ON(!list_empty(&gray_list));
 }
 /*
  * Scan data sections and all the referenced memory blocks allocated via the
  * kernel's standard allocators. This function must be called with the
  * scan_mutex held.
  */
 static void kmemleak_scan(void)
 {
 	unsigned long flags;
 	struct kmemleak_object *object;
 	int i;
 	int new_leaks = 0;
 	jiffies_last_scan = jiffies;
 	/* prepare the kmemleak_object's */
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list) {
 		spin_lock_irqsave(&object->lock, flags);
 #ifdef DEBUG
 		/*
 		 * With a few exceptions there should be a maximum of
 		 * 1 reference to any object at this point.
 		 */
 		if (atomic_read(&object->use_count) > 1) {
 			pr_debug("object->use_count = %d\n",
 				 atomic_read(&object->use_count));
 			dump_object_info(object);
 		}
 #endif
 		/* reset the reference count (whiten the object) */
 		object->count = 0;
 		if (color_gray(object) && get_object(object))
 			list_add_tail(&object->gray_list, &gray_list);
 		spin_unlock_irqrestore(&object->lock, flags);
 	}
 	rcu_read_unlock();
 	/* data/bss scanning */
 	scan_block(_sdata, _edata, NULL, 1);
 	scan_block(__bss_start, __bss_stop, NULL, 1);
 #ifdef CONFIG_SMP
 	/* per-cpu sections scanning */
 	for_each_possible_cpu(i)
 		scan_block(__per_cpu_start + per_cpu_offset(i),
 			   __per_cpu_end + per_cpu_offset(i), NULL, 1);
 #endif
 	/*
 	 * Struct page scanning for each node. The code below is not yet safe
 	 * with MEMORY_HOTPLUG.
 	 */
 	for_each_online_node(i) {
 		pg_data_t *pgdat = NODE_DATA(i);
 		unsigned long start_pfn = pgdat->node_start_pfn;
 		unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
 		unsigned long pfn;
 		for (pfn = start_pfn; pfn < end_pfn; pfn++) {
 			struct page *page;
 			if (!pfn_valid(pfn))
 				continue;
 			page = pfn_to_page(pfn);
 			/* only scan if page is in use */
 			if (page_count(page) == 0)
 				continue;
 			scan_block(page, page + 1, NULL, 1);
 		}
 	}
 	/*
 	 * Scanning the task stacks (may introduce false negatives).
 	 */
 	if (kmemleak_stack_scan) {
 		struct task_struct *p, *g;
 		read_lock(&tasklist_lock);
 		do_each_thread(g, p) {
 			scan_block(task_stack_page(p), task_stack_page(p) +
 				   THREAD_SIZE, NULL, 0);
 		} while_each_thread(g, p);
 		read_unlock(&tasklist_lock);
 	}
 	/*
 	 * Scan the objects already referenced from the sections scanned
 	 * above.
 	 */
 	scan_gray_list();
 	/*
 	 * Check for new or unreferenced objects modified since the previous
 	 * scan and color them gray until the next scan.
 	 */
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list) {
 		spin_lock_irqsave(&object->lock, flags);
 		if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
 		    && update_checksum(object) && get_object(object)) {
 			/* color it gray temporarily */
 			object->count = object->min_count;
 			list_add_tail(&object->gray_list, &gray_list);
 		}
 		spin_unlock_irqrestore(&object->lock, flags);
 	}
 	rcu_read_unlock();
 	/*
 	 * Re-scan the gray list for modified unreferenced objects.
 	 */
 	scan_gray_list();
 	/*
 	 * If scanning was stopped do not report any new unreferenced objects.
 	 */
 	if (scan_should_stop())
 		return;
 	/*
 	 * Scanning result reporting.
 	 */
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list) {
 		spin_lock_irqsave(&object->lock, flags);
 		if (unreferenced_object(object) &&
 		    !(object->flags & OBJECT_REPORTED)) {
 			object->flags |= OBJECT_REPORTED;
 			new_leaks++;
 		}
 		spin_unlock_irqrestore(&object->lock, flags);
 	}
 	rcu_read_unlock();
 	if (new_leaks)
 		pr_info("%d new suspected memory leaks (see "
 			"/sys/kernel/debug/kmemleak)\n", new_leaks);
 }
 /*
  * Thread function performing automatic memory scanning. Unreferenced objects
  * at the end of a memory scan are reported but only the first time.
  */
 static int kmemleak_scan_thread(void *arg)
 {
 	static int first_run = 1;
 	pr_info("Automatic memory scanning thread started\n");
 	set_user_nice(current, 10);
 	/*
 	 * Wait before the first scan to allow the system to fully initialize.
 	 */
 	if (first_run) {
 		first_run = 0;
 		ssleep(SECS_FIRST_SCAN);
 	}
 	while (!kthread_should_stop()) {
 		signed long timeout = jiffies_scan_wait;
 		mutex_lock(&scan_mutex);
 		kmemleak_scan();
 		mutex_unlock(&scan_mutex);
 		/* wait before the next scan */
 		while (timeout && !kthread_should_stop())
 			timeout = schedule_timeout_interruptible(timeout);
 	}
 	pr_info("Automatic memory scanning thread ended\n");
 	return 0;
 }
 /*
  * Start the automatic memory scanning thread. This function must be called
  * with the scan_mutex held.
  */
 static void start_scan_thread(void)
 {
 	if (scan_thread)
 		return;
 	scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
 	if (IS_ERR(scan_thread)) {
 		pr_warning("Failed to create the scan thread\n");
 		scan_thread = NULL;
 	}
 }
 /*
  * Stop the automatic memory scanning thread. This function must be called
  * with the scan_mutex held.
  */
 static void stop_scan_thread(void)
 {
 	if (scan_thread) {
 		kthread_stop(scan_thread);
 		scan_thread = NULL;
 	}
 }
 /*
  * Iterate over the object_list and return the first valid object at or after
  * the required position with its use_count incremented. The function triggers
  * a memory scanning when the pos argument points to the first position.
  */
 static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
 {
 	struct kmemleak_object *object;
 	loff_t n = *pos;
 	int err;
 	err = mutex_lock_interruptible(&scan_mutex);
 	if (err < 0)
 		return ERR_PTR(err);
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list) {
 		if (n-- > 0)
 			continue;
 		if (get_object(object))
 			goto out;
 	}
 	object = NULL;
 out:
 	return object;
 }
 /*
  * Return the next object in the object_list. The function decrements the
  * use_count of the previous object and increases that of the next one.
  */
 static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 {
 	struct kmemleak_object *prev_obj = v;
 	struct kmemleak_object *next_obj = NULL;
 	struct list_head *n = &prev_obj->object_list;
 	++(*pos);
 	list_for_each_continue_rcu(n, &object_list) {
 		next_obj = list_entry(n, struct kmemleak_object, object_list);
 		if (get_object(next_obj))
 			break;
 	}
 	put_object(prev_obj);
 	return next_obj;
 }
 /*
  * Decrement the use_count of the last object required, if any.
  */
 static void kmemleak_seq_stop(struct seq_file *seq, void *v)
 {
 	if (!IS_ERR(v)) {
 		/*
 		 * kmemleak_seq_start may return ERR_PTR if the scan_mutex
 		 * waiting was interrupted, so only release it if !IS_ERR.
 		 */
 		rcu_read_unlock();
 		mutex_unlock(&scan_mutex);
 		if (v)
 			put_object(v);
 	}
 }
 /*
  * Print the information for an unreferenced object to the seq file.
  */
 static int kmemleak_seq_show(struct seq_file *seq, void *v)
 {
 	struct kmemleak_object *object = v;
 	unsigned long flags;
 	spin_lock_irqsave(&object->lock, flags);
 	if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
 		print_unreferenced(seq, object);
 	spin_unlock_irqrestore(&object->lock, flags);
 	return 0;
 }
 static const struct seq_operations kmemleak_seq_ops = {
 	.start = kmemleak_seq_start,
 	.next  = kmemleak_seq_next,
 	.stop  = kmemleak_seq_stop,
 	.show  = kmemleak_seq_show,
 };
 static int kmemleak_open(struct inode *inode, struct file *file)
 {
 	if (!atomic_read(&kmemleak_enabled))
 		return -EBUSY;
 	return seq_open(file, &kmemleak_seq_ops);
 }
 static int kmemleak_release(struct inode *inode, struct file *file)
 {
 	return seq_release(inode, file);
 }
 static int dump_str_object_info(const char *str)
 {
 	unsigned long flags;
 	struct kmemleak_object *object;
 	unsigned long addr;
 	addr= simple_strtoul(str, NULL, 0);
 	object = find_and_get_object(addr, 0);
 	if (!object) {
 		pr_info("Unknown object at 0x%08lx\n", addr);
 		return -EINVAL;
 	}
 	spin_lock_irqsave(&object->lock, flags);
 	dump_object_info(object);
 	spin_unlock_irqrestore(&object->lock, flags);
 	put_object(object);
 	return 0;
 }
 /*
  * We use grey instead of black to ensure we can do future scans on the same
  * objects. If we did not do future scans these black objects could
  * potentially contain references to newly allocated objects in the future and
  * we'd end up with false positives.
  */
 static void kmemleak_clear(void)
 {
 	struct kmemleak_object *object;
 	unsigned long flags;
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list) {
 		spin_lock_irqsave(&object->lock, flags);
 		if ((object->flags & OBJECT_REPORTED) &&
 		    unreferenced_object(object))
 			__paint_it(object, KMEMLEAK_GREY);
 		spin_unlock_irqrestore(&object->lock, flags);
 	}
 	rcu_read_unlock();
 }
 /*
  * File write operation to configure kmemleak at run-time. The following
  * commands can be written to the /sys/kernel/debug/kmemleak file:
  *   off	- disable kmemleak (irreversible)
  *   stack=on	- enable the task stacks scanning
  *   stack=off	- disable the tasks stacks scanning
  *   scan=on	- start the automatic memory scanning thread
  *   scan=off	- stop the automatic memory scanning thread
  *   scan=...	- set the automatic memory scanning period in seconds (0 to
  *		  disable it)
  *   scan	- trigger a memory scan
  *   clear	- mark all current reported unreferenced kmemleak objects as
  *		  grey to ignore printing them
  *   dump=...	- dump information about the object found at the given address
  */
 static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
 			      size_t size, loff_t *ppos)
 {
 	char buf[64];
 	int buf_size;
 	int ret;
 	buf_size = min(size, (sizeof(buf) - 1));
 	if (strncpy_from_user(buf, user_buf, buf_size) < 0)
 		return -EFAULT;
 	buf[buf_size] = 0;
 	ret = mutex_lock_interruptible(&scan_mutex);
 	if (ret < 0)
 		return ret;
 	if (strncmp(buf, "off", 3) == 0)
 		kmemleak_disable();
 	else if (strncmp(buf, "stack=on", 8) == 0)
 		kmemleak_stack_scan = 1;
 	else if (strncmp(buf, "stack=off", 9) == 0)
 		kmemleak_stack_scan = 0;
 	else if (strncmp(buf, "scan=on", 7) == 0)
 		start_scan_thread();
 	else if (strncmp(buf, "scan=off", 8) == 0)
 		stop_scan_thread();
 	else if (strncmp(buf, "scan=", 5) == 0) {
 		unsigned long secs;
 		ret = strict_strtoul(buf + 5, 0, &secs);
 		if (ret < 0)
 			goto out;
 		stop_scan_thread();
 		if (secs) {
 			jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
 			start_scan_thread();
 		}
 	} else if (strncmp(buf, "scan", 4) == 0)
 		kmemleak_scan();
 	else if (strncmp(buf, "clear", 5) == 0)
 		kmemleak_clear();
 	else if (strncmp(buf, "dump=", 5) == 0)
 		ret = dump_str_object_info(buf + 5);
 	else
 		ret = -EINVAL;
 out:
 	mutex_unlock(&scan_mutex);
 	if (ret < 0)
 		return ret;
 	/* ignore the rest of the buffer, only one command at a time */
 	*ppos += size;
 	return size;
 }
 static const struct file_operations kmemleak_fops = {
 	.owner		= THIS_MODULE,
 	.open		= kmemleak_open,
 	.read		= seq_read,
 	.write		= kmemleak_write,
 	.llseek		= seq_lseek,
 	.release	= kmemleak_release,
 };
 /*
  * Perform the freeing of the kmemleak internal objects after waiting for any
  * current memory scan to complete.
  */
 static void kmemleak_do_cleanup(struct work_struct *work)
 {
 	struct kmemleak_object *object;
 	mutex_lock(&scan_mutex);
 	stop_scan_thread();
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list)
 		delete_object_full(object->pointer);
 	rcu_read_unlock();
 	mutex_unlock(&scan_mutex);
 }
 static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
 /*
  * Disable kmemleak. No memory allocation/freeing will be traced once this
  * function is called. Disabling kmemleak is an irreversible operation.
  */
 static void kmemleak_disable(void)
 {
 	/* atomically check whether it was already invoked */
 	if (atomic_cmpxchg(&kmemleak_error, 0, 1))
 		return;
 	/* stop any memory operation tracing */
 	atomic_set(&kmemleak_early_log, 0);
 	atomic_set(&kmemleak_enabled, 0);
 	/* check whether it is too early for a kernel thread */
 	if (atomic_read(&kmemleak_initialized))
 		schedule_work(&cleanup_work);
 	pr_info("Kernel memory leak detector disabled\n");
 }
 /*
  * Allow boot-time kmemleak disabling (enabled by default).
  */
 static int kmemleak_boot_config(char *str)
 {
 	if (!str)
 		return -EINVAL;
 	if (strcmp(str, "off") == 0)
 		kmemleak_disable();
-	else if (strcmp(str, "on") != 0)
+	else if (strcmp(str, "on") == 0)
+		kmemleak_skip_disable = 1;
+	else
 		return -EINVAL;
 	return 0;
 }
 early_param("kmemleak", kmemleak_boot_config);
 /*
  * Kmemleak initialization.
  */
 void __init kmemleak_init(void)
 {
 	int i;
 	unsigned long flags;
+#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
+	if (!kmemleak_skip_disable) {
+		kmemleak_disable();
+		return;
+	}
+#endif
 	jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
 	jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
 	object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
 	scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
 	INIT_PRIO_TREE_ROOT(&object_tree_root);
 	/* the kernel is still in UP mode, so disabling the IRQs is enough */
 	local_irq_save(flags);
 	if (!atomic_read(&kmemleak_error)) {
 		atomic_set(&kmemleak_enabled, 1);
 		atomic_set(&kmemleak_early_log, 0);
 	}
 	local_irq_restore(flags);
 	/*
 	 * This is the point where tracking allocations is safe. Automatic
 	 * scanning is started during the late initcall. Add the early logged
 	 * callbacks to the kmemleak infrastructure.
 	 */
 	for (i = 0; i < crt_early_log; i++) {
 		struct early_log *log = &early_log[i];
 		switch (log->op_type) {
 		case KMEMLEAK_ALLOC:
 			early_alloc(log);
 			break;
 		case KMEMLEAK_FREE:
 			kmemleak_free(log->ptr);
 			break;
 		case KMEMLEAK_FREE_PART:
 			kmemleak_free_part(log->ptr, log->size);
 			break;
 		case KMEMLEAK_NOT_LEAK:
 			kmemleak_not_leak(log->ptr);
 			break;
 		case KMEMLEAK_IGNORE:
 			kmemleak_ignore(log->ptr);
 			break;
 		case KMEMLEAK_SCAN_AREA:
 			kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
 			break;
 		case KMEMLEAK_NO_SCAN:
 			kmemleak_no_scan(log->ptr);
 			break;
 		default:
 			WARN_ON(1);
 		}
 	}
 }
 /*
  * Late initialization function.
  */
 static int __init kmemleak_late_init(void)
 {
 	struct dentry *dentry;
 	atomic_set(&kmemleak_initialized, 1);
 	if (atomic_read(&kmemleak_error)) {
 		/*
 		 * Some error occured and kmemleak was disabled. There is a
 		 * small chance that kmemleak_disable() was called immediately
 		 * after setting kmemleak_initialized and we may end up with
 		 * two clean-up threads but serialized by scan_mutex.
 		 */
 		schedule_work(&cleanup_work);
 		return -ENOMEM;
 	}
 	dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
 				     &kmemleak_fops);
 	if (!dentry)
 		pr_warning("Failed to create the debugfs kmemleak file\n");
 	mutex_lock(&scan_mutex);
 	start_scan_thread();
 	mutex_unlock(&scan_mutex);
 	pr_info("Kernel memory leak detector initialized\n");
 	return 0;
 }
 late_initcall(kmemleak_late_init);