Doug / smarc-fsl-linux-kernel

1

/*

1

/*

2

* User-space Probes (UProbes)

2

* User-space Probes (UProbes)

3

*

3

*

4

* This program is free software; you can redistribute it and/or modify

4

* This program is free software; you can redistribute it and/or modify

5

* it under the terms of the GNU General Public License as published by

5

* it under the terms of the GNU General Public License as published by

6

* the Free Software Foundation; either version 2 of the License, or

6

* the Free Software Foundation; either version 2 of the License, or

7

* (at your option) any later version.

7

* (at your option) any later version.

8

*

8

*

9

* This program is distributed in the hope that it will be useful,

9

* This program is distributed in the hope that it will be useful,

10

* but WITHOUT ANY WARRANTY; without even the implied warranty of

10

* but WITHOUT ANY WARRANTY; without even the implied warranty of

11

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

11

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12

* GNU General Public License for more details.

12

* GNU General Public License for more details.

13

*

13

*

14

* You should have received a copy of the GNU General Public License

14

* You should have received a copy of the GNU General Public License

15

* along with this program; if not, write to the Free Software

15

* along with this program; if not, write to the Free Software

16

* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

16

* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

17

*

17

*

18

* Copyright (C) IBM Corporation, 2008-2012

18

* Copyright (C) IBM Corporation, 2008-2012

19

* Authors:

19

* Authors:

20

* Srikar Dronamraju

20

* Srikar Dronamraju

21

* Jim Keniston

21

* Jim Keniston

22

23

*/

23

*/

24

25

#include <linux/kernel.h>

25

#include <linux/kernel.h>

26

#include <linux/highmem.h>

26

#include <linux/highmem.h>

27

#include <linux/pagemap.h> /* read_mapping_page */

27

#include <linux/pagemap.h> /* read_mapping_page */

28

#include <linux/slab.h>

28

#include <linux/slab.h>

29

#include <linux/sched.h>

29

#include <linux/sched.h>

30

#include <linux/rmap.h> /* anon_vma_prepare */

30

#include <linux/rmap.h> /* anon_vma_prepare */

31

#include <linux/mmu_notifier.h> /* set_pte_at_notify */

31

#include <linux/mmu_notifier.h> /* set_pte_at_notify */

32

#include <linux/swap.h> /* try_to_free_swap */

32

#include <linux/swap.h> /* try_to_free_swap */

33

#include <linux/ptrace.h> /* user_enable_single_step */

33

#include <linux/ptrace.h> /* user_enable_single_step */

34

#include <linux/kdebug.h> /* notifier mechanism */

34

#include <linux/kdebug.h> /* notifier mechanism */

35

#include "../../mm/internal.h" /* munlock_vma_page */

35

#include "../../mm/internal.h" /* munlock_vma_page */

36

37

#include <linux/uprobes.h>

37

#include <linux/uprobes.h>

38

39

#define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)

39

#define UINSNS_PER_PAGE (PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)

40

#define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE

40

#define MAX_UPROBE_XOL_SLOTS UINSNS_PER_PAGE

41

42

static struct rb_root uprobes_tree = RB_ROOT;

42

static struct rb_root uprobes_tree = RB_ROOT;

43

44

static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */

44

static DEFINE_SPINLOCK(uprobes_treelock); /* serialize rbtree access */

45

46

#define UPROBES_HASH_SZ 13

46

#define UPROBES_HASH_SZ 13

47

48

/*

48

/*

49

* We need separate register/unregister and mmap/munmap lock hashes because

49

* We need separate register/unregister and mmap/munmap lock hashes because

50

* of mmap_sem nesting.

50

* of mmap_sem nesting.

51

*

51

*

52

* uprobe_register() needs to install probes on (potentially) all processes

52

* uprobe_register() needs to install probes on (potentially) all processes

53

* and thus needs to acquire multiple mmap_sems (consequtively, not

53

* and thus needs to acquire multiple mmap_sems (consequtively, not

54

* concurrently), whereas uprobe_mmap() is called while holding mmap_sem

54

* concurrently), whereas uprobe_mmap() is called while holding mmap_sem

55

* for the particular process doing the mmap.

55

* for the particular process doing the mmap.

56

*

56

*

57

* uprobe_register()->register_for_each_vma() needs to drop/acquire mmap_sem

57

* uprobe_register()->register_for_each_vma() needs to drop/acquire mmap_sem

58

* because of lock order against i_mmap_mutex. This means there's a hole in

58

* because of lock order against i_mmap_mutex. This means there's a hole in

59

* the register vma iteration where a mmap() can happen.

59

* the register vma iteration where a mmap() can happen.

60

*

60

*

61

* Thus uprobe_register() can race with uprobe_mmap() and we can try and

61

* Thus uprobe_register() can race with uprobe_mmap() and we can try and

62

* install a probe where one is already installed.

62

* install a probe where one is already installed.

63

*/

63

*/

64

65

/* serialize (un)register */

65

/* serialize (un)register */

66

static struct mutex uprobes_mutex[UPROBES_HASH_SZ];

66

static struct mutex uprobes_mutex[UPROBES_HASH_SZ];

67

68

#define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])

68

#define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])

69

70

/* serialize uprobe->pending_list */

70

/* serialize uprobe->pending_list */

71

static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];

71

static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];

72

#define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])

72

#define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])

73

74

/*

74

/*

75

* uprobe_events allows us to skip the uprobe_mmap if there are no uprobe

75

* uprobe_events allows us to skip the uprobe_mmap if there are no uprobe

76

* events active at this time. Probably a fine grained per inode count is

76

* events active at this time. Probably a fine grained per inode count is

77

* better?

77

* better?

78

*/

78

*/

79

static atomic_t uprobe_events = ATOMIC_INIT(0);

79

static atomic_t uprobe_events = ATOMIC_INIT(0);

80

81

struct uprobe {

81

struct uprobe {

82

struct rb_node rb_node; /* node in the rb tree */

82

struct rb_node rb_node; /* node in the rb tree */

83

atomic_t ref;

83

atomic_t ref;

84

struct rw_semaphore consumer_rwsem;

84

struct rw_semaphore consumer_rwsem;

85

struct list_head pending_list;

85

struct list_head pending_list;

86

struct uprobe_consumer *consumers;

86

struct uprobe_consumer *consumers;

87

struct inode *inode; /* Also hold a ref to inode */

87

struct inode *inode; /* Also hold a ref to inode */

88

loff_t offset;

88

loff_t offset;

89

int flags;

89

int flags;

90

struct arch_uprobe arch;

90

struct arch_uprobe arch;

91

};

91

};

92

93

/*

93

/*

94

* valid_vma: Verify if the specified vma is an executable vma

94

* valid_vma: Verify if the specified vma is an executable vma

95

* Relax restrictions while unregistering: vm_flags might have

95

* Relax restrictions while unregistering: vm_flags might have

96

* changed after breakpoint was inserted.

96

* changed after breakpoint was inserted.

97

* - is_register: indicates if we are in register context.

97

* - is_register: indicates if we are in register context.

98

* - Return 1 if the specified virtual address is in an

98

* - Return 1 if the specified virtual address is in an

99

* executable vma.

99

* executable vma.

100

*/

100

*/

101

static bool valid_vma(struct vm_area_struct *vma, bool is_register)

101

static bool valid_vma(struct vm_area_struct *vma, bool is_register)

102

{

102

{

103

if (!vma->vm_file)

103

if (!vma->vm_file)

104

return false;

104

return false;

105

106

if (!is_register)

106

if (!is_register)

107

return true;

107

return true;

108

109

if ((vma->vm_flags & (VM_HUGETLB|VM_READ|VM_WRITE|VM_EXEC|VM_SHARED))

109

if ((vma->vm_flags & (VM_HUGETLB|VM_READ|VM_WRITE|VM_EXEC|VM_SHARED))

110

== (VM_READ|VM_EXEC))

110

== (VM_READ|VM_EXEC))

111

return true;

111

return true;

112

113

return false;

113

return false;

114

}

114

}

115

116

static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)

116

static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)

117

{

117

{

118

return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);

118

return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);

119

}

119

}

120

121

static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)

121

static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)

122

{

122

{

123

return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);

123

return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);

124

}

124

}

125

126

/**

126

/**

127

* __replace_page - replace page in vma by new page.

127

* __replace_page - replace page in vma by new page.

128

* based on replace_page in mm/ksm.c

128

* based on replace_page in mm/ksm.c

129

*

129

*

130

* @vma: vma that holds the pte pointing to page

130

* @vma: vma that holds the pte pointing to page

131

* @addr: address the old @page is mapped at

131

* @addr: address the old @page is mapped at

132

* @page: the cowed page we are replacing by kpage

132

* @page: the cowed page we are replacing by kpage

133

* @kpage: the modified page we replace page by

133

* @kpage: the modified page we replace page by

134

*

134

*

135

* Returns 0 on success, -EFAULT on failure.

135

* Returns 0 on success, -EFAULT on failure.

136

*/

136

*/

137

static int __replace_page(struct vm_area_struct *vma, unsigned long addr,

137

static int __replace_page(struct vm_area_struct *vma, unsigned long addr,

138

struct page *page, struct page *kpage)

138

struct page *page, struct page *kpage)

139

{

139

{

140

struct mm_struct *mm = vma->vm_mm;

140

struct mm_struct *mm = vma->vm_mm;

141

spinlock_t *ptl;

141

spinlock_t *ptl;

142

pte_t *ptep;

142

pte_t *ptep;

143

int err;

143

int err;

144

145

/* For try_to_free_swap() and munlock_vma_page() below */

145

/* For try_to_free_swap() and munlock_vma_page() below */

146

lock_page(page);

146

lock_page(page);

147

148

err = -EAGAIN;

148

err = -EAGAIN;

149

ptep = page_check_address(page, mm, addr, &ptl, 0);

149

ptep = page_check_address(page, mm, addr, &ptl, 0);

150

if (!ptep)

150

if (!ptep)

151

goto unlock;

151

goto unlock;

152

153

get_page(kpage);

153

get_page(kpage);

154

page_add_new_anon_rmap(kpage, vma, addr);

154

page_add_new_anon_rmap(kpage, vma, addr);

155

156

if (!PageAnon(page)) {

156

if (!PageAnon(page)) {

157

dec_mm_counter(mm, MM_FILEPAGES);

157

dec_mm_counter(mm, MM_FILEPAGES);

158

inc_mm_counter(mm, MM_ANONPAGES);

158

inc_mm_counter(mm, MM_ANONPAGES);

159

}

159

}

160

161

flush_cache_page(vma, addr, pte_pfn(*ptep));

161

flush_cache_page(vma, addr, pte_pfn(*ptep));

162

ptep_clear_flush(vma, addr, ptep);

162

ptep_clear_flush(vma, addr, ptep);

163

set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));

163

set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));

164

165

page_remove_rmap(page);

165

page_remove_rmap(page);

166

if (!page_mapped(page))

166

if (!page_mapped(page))

167

try_to_free_swap(page);

167

try_to_free_swap(page);

168

pte_unmap_unlock(ptep, ptl);

168

pte_unmap_unlock(ptep, ptl);

169

170

if (vma->vm_flags & VM_LOCKED)

170

if (vma->vm_flags & VM_LOCKED)

171

munlock_vma_page(page);

171

munlock_vma_page(page);

172

put_page(page);

172

put_page(page);

173

174

err = 0;

174

err = 0;

175

unlock:

175

unlock:

176

unlock_page(page);

176

unlock_page(page);

177

return err;

177

return err;

178

}

178

}

179

180

/**

180

/**

181

* is_swbp_insn - check if instruction is breakpoint instruction.

181

* is_swbp_insn - check if instruction is breakpoint instruction.

182

* @insn: instruction to be checked.

182

* @insn: instruction to be checked.

183

* Default implementation of is_swbp_insn

183

* Default implementation of is_swbp_insn

184

* Returns true if @insn is a breakpoint instruction.

184

* Returns true if @insn is a breakpoint instruction.

185

*/

185

*/

186

bool __weak is_swbp_insn(uprobe_opcode_t *insn)

186

bool __weak is_swbp_insn(uprobe_opcode_t *insn)

187

{

187

{

188

return *insn == UPROBE_SWBP_INSN;

188

return *insn == UPROBE_SWBP_INSN;

189

}

189

}

190

191

/*

191

/*

192

* NOTE:

192

* NOTE:

193

* Expect the breakpoint instruction to be the smallest size instruction for

193

* Expect the breakpoint instruction to be the smallest size instruction for

194

* the architecture. If an arch has variable length instruction and the

194

* the architecture. If an arch has variable length instruction and the

195

* breakpoint instruction is not of the smallest length instruction

195

* breakpoint instruction is not of the smallest length instruction

196

* supported by that architecture then we need to modify read_opcode /

196

* supported by that architecture then we need to modify read_opcode /

197

* write_opcode accordingly. This would never be a problem for archs that

197

* write_opcode accordingly. This would never be a problem for archs that

198

* have fixed length instructions.

198

* have fixed length instructions.

199

*/

199

*/

200

201

/*

201

/*

202

* write_opcode - write the opcode at a given virtual address.

202

* write_opcode - write the opcode at a given virtual address.

203

* @auprobe: arch breakpointing information.

203

* @auprobe: arch breakpointing information.

204

* @mm: the probed process address space.

204

* @mm: the probed process address space.

205

* @vaddr: the virtual address to store the opcode.

205

* @vaddr: the virtual address to store the opcode.

206

* @opcode: opcode to be written at @vaddr.

206

* @opcode: opcode to be written at @vaddr.

207

*

207

*

208

* Called with mm->mmap_sem held (for read and with a reference to

208

* Called with mm->mmap_sem held (for read and with a reference to

209

* mm).

209

* mm).

210

*

210

*

211

* For mm @mm, write the opcode at @vaddr.

211

* For mm @mm, write the opcode at @vaddr.

212

* Return 0 (success) or a negative errno.

212

* Return 0 (success) or a negative errno.

213

*/

213

*/

214

static int write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,

214

static int write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,

215

unsigned long vaddr, uprobe_opcode_t opcode)

215

unsigned long vaddr, uprobe_opcode_t opcode)

216

{

216

{

217

struct page *old_page, *new_page;

217

struct page *old_page, *new_page;

218

void *vaddr_old, *vaddr_new;

218

void *vaddr_old, *vaddr_new;

219

struct vm_area_struct *vma;

219

struct vm_area_struct *vma;

220

int ret;

220

int ret;

221

222

retry:

222

retry:

223

/* Read the page with vaddr into memory */

223

/* Read the page with vaddr into memory */

224

ret = get_user_pages(NULL, mm, vaddr, 1, 0, 0, &old_page, &vma);

224

ret = get_user_pages(NULL, mm, vaddr, 1, 0, 0, &old_page, &vma);

225

if (ret <= 0)

225

if (ret <= 0)

226

return ret;

226

return ret;

227

228

ret = -ENOMEM;

228

ret = -ENOMEM;

229

new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);

229

new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);

230

if (!new_page)

230

if (!new_page)

231

goto put_old;

231

goto put_old;

232

233

__SetPageUptodate(new_page);

233

__SetPageUptodate(new_page);

234

235

/* copy the page now that we've got it stable */

235

/* copy the page now that we've got it stable */

236

vaddr_old = kmap_atomic(old_page);

236

vaddr_old = kmap_atomic(old_page);

237

vaddr_new = kmap_atomic(new_page);

237

vaddr_new = kmap_atomic(new_page);

238

239

memcpy(vaddr_new, vaddr_old, PAGE_SIZE);

239

memcpy(vaddr_new, vaddr_old, PAGE_SIZE);

240

memcpy(vaddr_new + (vaddr & ~PAGE_MASK), &opcode, UPROBE_SWBP_INSN_SIZE);

240

memcpy(vaddr_new + (vaddr & ~PAGE_MASK), &opcode, UPROBE_SWBP_INSN_SIZE);

241

242

kunmap_atomic(vaddr_new);

242

kunmap_atomic(vaddr_new);

243

kunmap_atomic(vaddr_old);

243

kunmap_atomic(vaddr_old);

244

245

ret = anon_vma_prepare(vma);

245

ret = anon_vma_prepare(vma);

246

if (ret)

246

if (ret)

247

goto put_new;

247

goto put_new;

248

249

ret = __replace_page(vma, vaddr, old_page, new_page);

249

ret = __replace_page(vma, vaddr, old_page, new_page);

250

251

put_new:

251

put_new:

252

page_cache_release(new_page);

252

page_cache_release(new_page);

253

put_old:

253

put_old:

254

put_page(old_page);

254

put_page(old_page);

255

256

if (unlikely(ret == -EAGAIN))

256

if (unlikely(ret == -EAGAIN))

257

goto retry;

257

goto retry;

258

return ret;

258

return ret;

259

}

259

}

260

261

/**

261

/**

262

* read_opcode - read the opcode at a given virtual address.

262

* read_opcode - read the opcode at a given virtual address.

263

* @mm: the probed process address space.

263

* @mm: the probed process address space.

264

* @vaddr: the virtual address to read the opcode.

264

* @vaddr: the virtual address to read the opcode.

265

* @opcode: location to store the read opcode.

265

* @opcode: location to store the read opcode.

266

*

266

*

267

* Called with mm->mmap_sem held (for read and with a reference to

267

* Called with mm->mmap_sem held (for read and with a reference to

268

* mm.

268

* mm.

269

*

269

*

270

* For mm @mm, read the opcode at @vaddr and store it in @opcode.

270

* For mm @mm, read the opcode at @vaddr and store it in @opcode.

271

* Return 0 (success) or a negative errno.

271

* Return 0 (success) or a negative errno.

272

*/

272

*/

273

static int read_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t *opcode)

273

static int read_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t *opcode)

274

{

274

{

275

struct page *page;

275

struct page *page;

276

void *vaddr_new;

276

void *vaddr_new;

277

int ret;

277

int ret;

278

279

ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL);

279

ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL);

280

if (ret <= 0)

280

if (ret <= 0)

281

return ret;

281

return ret;

282

283

vaddr_new = kmap_atomic(page);

283

vaddr_new = kmap_atomic(page);

284

vaddr &= ~PAGE_MASK;

284

vaddr &= ~PAGE_MASK;

285

memcpy(opcode, vaddr_new + vaddr, UPROBE_SWBP_INSN_SIZE);

285

memcpy(opcode, vaddr_new + vaddr, UPROBE_SWBP_INSN_SIZE);

286

kunmap_atomic(vaddr_new);

286

kunmap_atomic(vaddr_new);

287

288

put_page(page);

288

put_page(page);

289

290

return 0;

290

return 0;

291

}

291

}

292

293

static int is_swbp_at_addr(struct mm_struct *mm, unsigned long vaddr)

293

static int is_swbp_at_addr(struct mm_struct *mm, unsigned long vaddr)

294

{

294

{

295

uprobe_opcode_t opcode;

295

uprobe_opcode_t opcode;

296

int result;

296

int result;

297

298

if (current->mm == mm) {

298

if (current->mm == mm) {

299

pagefault_disable();

299

pagefault_disable();

300

result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,

300

result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,

301

sizeof(opcode));

301

sizeof(opcode));

302

pagefault_enable();

302

pagefault_enable();

303

304

if (likely(result == 0))

304

if (likely(result == 0))

305

goto out;

305

goto out;

306

}

306

}

307

308

result = read_opcode(mm, vaddr, &opcode);

308

result = read_opcode(mm, vaddr, &opcode);

309

if (result)

309

if (result)

310

return result;

310

return result;

311

out:

311

out:

312

if (is_swbp_insn(&opcode))

312

if (is_swbp_insn(&opcode))

313

return 1;

313

return 1;

314

315

return 0;

315

return 0;

316

}

316

}

317

318

/**

318

/**

319

* set_swbp - store breakpoint at a given address.

319

* set_swbp - store breakpoint at a given address.

320

* @auprobe: arch specific probepoint information.

320

* @auprobe: arch specific probepoint information.

321

* @mm: the probed process address space.

321

* @mm: the probed process address space.

322

* @vaddr: the virtual address to insert the opcode.

322

* @vaddr: the virtual address to insert the opcode.

323

*

323

*

324

* For mm @mm, store the breakpoint instruction at @vaddr.

324

* For mm @mm, store the breakpoint instruction at @vaddr.

325

* Return 0 (success) or a negative errno.

325

* Return 0 (success) or a negative errno.

326

*/

326

*/

327

int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)

327

int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)

328

{

328

{

329

int result;

329

int result;

330

/*

330

/*

331

* See the comment near uprobes_hash().

331

* See the comment near uprobes_hash().

332

*/

332

*/

333

result = is_swbp_at_addr(mm, vaddr);

333

result = is_swbp_at_addr(mm, vaddr);

334

if (result == 1)

334

if (result == 1)

335

return 0;

335

return 0;

336

337

if (result)

337

if (result)

338

return result;

338

return result;

339

340

return write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);

340

return write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);

341

}

341

}

342

343

/**

343

/**

344

* set_orig_insn - Restore the original instruction.

344

* set_orig_insn - Restore the original instruction.

345

* @mm: the probed process address space.

345

* @mm: the probed process address space.

346

* @auprobe: arch specific probepoint information.

346

* @auprobe: arch specific probepoint information.

347

* @vaddr: the virtual address to insert the opcode.

347

* @vaddr: the virtual address to insert the opcode.

348

*

348

*

349

* For mm @mm, restore the original opcode (opcode) at @vaddr.

349

* For mm @mm, restore the original opcode (opcode) at @vaddr.

350

* Return 0 (success) or a negative errno.

350

* Return 0 (success) or a negative errno.

351

*/

351

*/

352

int __weak

352

int __weak

353

set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)

353

set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)

354

{

354

{

355

int result;

355

int result;

356

357

result = is_swbp_at_addr(mm, vaddr);

357

result = is_swbp_at_addr(mm, vaddr);

358

if (!result)

358

if (!result)

359

return -EINVAL;

359

return -EINVAL;

360

361

if (result != 1)

361

if (result != 1)

362

return result;

362

return result;

363

364

return write_opcode(auprobe, mm, vaddr, *(uprobe_opcode_t *)auprobe->insn);

364

return write_opcode(auprobe, mm, vaddr, *(uprobe_opcode_t *)auprobe->insn);

365

}

365

}

366

367

static int match_uprobe(struct uprobe *l, struct uprobe *r)

367

static int match_uprobe(struct uprobe *l, struct uprobe *r)

368

{

368

{

369

if (l->inode < r->inode)

369

if (l->inode < r->inode)

370

return -1;

370

return -1;

371

372

if (l->inode > r->inode)

372

if (l->inode > r->inode)

373

return 1;

373

return 1;

374

375

if (l->offset < r->offset)

375

if (l->offset < r->offset)

376

return -1;

376

return -1;

377

378

if (l->offset > r->offset)

378

if (l->offset > r->offset)

379

return 1;

379

return 1;

380

381

return 0;

381

return 0;

382

}

382

}

383

384

static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)

384

static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)

385

{

385

{

386

struct uprobe u = { .inode = inode, .offset = offset };

386

struct uprobe u = { .inode = inode, .offset = offset };

387

struct rb_node *n = uprobes_tree.rb_node;

387

struct rb_node *n = uprobes_tree.rb_node;

388

struct uprobe *uprobe;

388

struct uprobe *uprobe;

389

int match;

389

int match;

390

391

while (n) {

391

while (n) {

392

uprobe = rb_entry(n, struct uprobe, rb_node);

392

uprobe = rb_entry(n, struct uprobe, rb_node);

393

match = match_uprobe(&u, uprobe);

393

match = match_uprobe(&u, uprobe);

394

if (!match) {

394

if (!match) {

395

atomic_inc(&uprobe->ref);

395

atomic_inc(&uprobe->ref);

396

return uprobe;

396

return uprobe;

397

}

397

}

398

399

if (match < 0)

399

if (match < 0)

400

n = n->rb_left;

400

n = n->rb_left;

401

else

401

else

402

n = n->rb_right;

402

n = n->rb_right;

403

}

403

}

404

return NULL;

404

return NULL;

405

}

405

}

406

407

/*

407

/*

408

* Find a uprobe corresponding to a given inode:offset

408

* Find a uprobe corresponding to a given inode:offset

409

* Acquires uprobes_treelock

409

* Acquires uprobes_treelock

410

*/

410

*/

411

static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)

411

static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)

412

{

412

{

413

struct uprobe *uprobe;

413

struct uprobe *uprobe;

414

415

spin_lock(&uprobes_treelock);

415

spin_lock(&uprobes_treelock);

416

uprobe = __find_uprobe(inode, offset);

416

uprobe = __find_uprobe(inode, offset);

417

spin_unlock(&uprobes_treelock);

417

spin_unlock(&uprobes_treelock);

418

419

return uprobe;

419

return uprobe;

420

}

420

}

421

422

static struct uprobe *__insert_uprobe(struct uprobe *uprobe)

422

static struct uprobe *__insert_uprobe(struct uprobe *uprobe)

423

{

423

{

424

struct rb_node **p = &uprobes_tree.rb_node;

424

struct rb_node **p = &uprobes_tree.rb_node;

425

struct rb_node *parent = NULL;

425

struct rb_node *parent = NULL;

426

struct uprobe *u;

426

struct uprobe *u;

427

int match;

427

int match;

428

429

while (*p) {

429

while (*p) {

430

parent = *p;

430

parent = *p;

431

u = rb_entry(parent, struct uprobe, rb_node);

431

u = rb_entry(parent, struct uprobe, rb_node);

432

match = match_uprobe(uprobe, u);

432

match = match_uprobe(uprobe, u);

433

if (!match) {

433

if (!match) {

434

atomic_inc(&u->ref);

434

atomic_inc(&u->ref);

435

return u;

435

return u;

436

}

436

}

437

438

if (match < 0)

438

if (match < 0)

439

p = &parent->rb_left;

439

p = &parent->rb_left;

440

else

440

else

441

p = &parent->rb_right;

441

p = &parent->rb_right;

442

443

}

443

}

444

445

u = NULL;

445

u = NULL;

446

rb_link_node(&uprobe->rb_node, parent, p);

446

rb_link_node(&uprobe->rb_node, parent, p);

447

rb_insert_color(&uprobe->rb_node, &uprobes_tree);

447

rb_insert_color(&uprobe->rb_node, &uprobes_tree);

448

/* get access + creation ref */

448

/* get access + creation ref */

449

atomic_set(&uprobe->ref, 2);

449

atomic_set(&uprobe->ref, 2);

450

451

return u;

451

return u;

452

}

452

}

453

454

/*

454

/*

455

* Acquire uprobes_treelock.

455

* Acquire uprobes_treelock.

456

* Matching uprobe already exists in rbtree;

456

* Matching uprobe already exists in rbtree;

457

* increment (access refcount) and return the matching uprobe.

457

* increment (access refcount) and return the matching uprobe.

458

*

458

*

459

* No matching uprobe; insert the uprobe in rb_tree;

459

* No matching uprobe; insert the uprobe in rb_tree;

460

* get a double refcount (access + creation) and return NULL.

460

* get a double refcount (access + creation) and return NULL.

461

*/

461

*/

462

static struct uprobe *insert_uprobe(struct uprobe *uprobe)

462

static struct uprobe *insert_uprobe(struct uprobe *uprobe)

463

{

463

{

464

struct uprobe *u;

464

struct uprobe *u;

465

466

spin_lock(&uprobes_treelock);

466

spin_lock(&uprobes_treelock);

467

u = __insert_uprobe(uprobe);

467

u = __insert_uprobe(uprobe);

468

spin_unlock(&uprobes_treelock);

468

spin_unlock(&uprobes_treelock);

469

470

/* For now assume that the instruction need not be single-stepped */

470

/* For now assume that the instruction need not be single-stepped */

471

uprobe->flags |= UPROBE_SKIP_SSTEP;

471

uprobe->flags |= UPROBE_SKIP_SSTEP;

472

473

return u;

473

return u;

474

}

474

}

475

476

static void put_uprobe(struct uprobe *uprobe)

476

static void put_uprobe(struct uprobe *uprobe)

477

{

477

{

478

if (atomic_dec_and_test(&uprobe->ref))

478

if (atomic_dec_and_test(&uprobe->ref))

479

kfree(uprobe);

479

kfree(uprobe);

480

}

480

}

481

482

static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)

482

static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)

483

{

483

{

484

struct uprobe *uprobe, *cur_uprobe;

484

struct uprobe *uprobe, *cur_uprobe;

485

486

uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);

486

uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);

487

if (!uprobe)

487

if (!uprobe)

488

return NULL;

488

return NULL;

489

490

uprobe->inode = igrab(inode);

490

uprobe->inode = igrab(inode);

491

uprobe->offset = offset;

491

uprobe->offset = offset;

492

init_rwsem(&uprobe->consumer_rwsem);

492

init_rwsem(&uprobe->consumer_rwsem);

493

494

/* add to uprobes_tree, sorted on inode:offset */

494

/* add to uprobes_tree, sorted on inode:offset */

495

cur_uprobe = insert_uprobe(uprobe);

495

cur_uprobe = insert_uprobe(uprobe);

496

497

/* a uprobe exists for this inode:offset combination */

497

/* a uprobe exists for this inode:offset combination */

498

if (cur_uprobe) {

498

if (cur_uprobe) {

499

kfree(uprobe);

499

kfree(uprobe);

500

uprobe = cur_uprobe;

500

uprobe = cur_uprobe;

501

iput(inode);

501

iput(inode);

502

} else {

502

} else {

503

atomic_inc(&uprobe_events);

503

atomic_inc(&uprobe_events);

504

}

504

}

505

506

return uprobe;

506

return uprobe;

507

}

507

}

508

509

static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)

509

static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)

510

{

510

{

511

struct uprobe_consumer *uc;

511

struct uprobe_consumer *uc;

512

513

if (!(uprobe->flags & UPROBE_RUN_HANDLER))

513

if (!(uprobe->flags & UPROBE_RUN_HANDLER))

514

return;

514

return;

515

516

down_read(&uprobe->consumer_rwsem);

516

down_read(&uprobe->consumer_rwsem);

517

for (uc = uprobe->consumers; uc; uc = uc->next) {

517

for (uc = uprobe->consumers; uc; uc = uc->next) {

518

if (!uc->filter || uc->filter(uc, current))

518

if (!uc->filter || uc->filter(uc, current))

519

uc->handler(uc, regs);

519

uc->handler(uc, regs);

520

}

520

}

521

up_read(&uprobe->consumer_rwsem);

521

up_read(&uprobe->consumer_rwsem);

522

}

522

}

523

524

/* Returns the previous consumer */

524

/* Returns the previous consumer */

525

static struct uprobe_consumer *

525

static struct uprobe_consumer *

526

consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)

526

consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)

527

{

527

{

528

down_write(&uprobe->consumer_rwsem);

528

down_write(&uprobe->consumer_rwsem);

529

uc->next = uprobe->consumers;

529

uc->next = uprobe->consumers;

530

uprobe->consumers = uc;

530

uprobe->consumers = uc;

531

up_write(&uprobe->consumer_rwsem);

531

up_write(&uprobe->consumer_rwsem);

532

533

return uc->next;

533

return uc->next;

534

}

534

}

535

536

/*

536

/*

537

* For uprobe @uprobe, delete the consumer @uc.

537

* For uprobe @uprobe, delete the consumer @uc.

538

* Return true if the @uc is deleted successfully

538

* Return true if the @uc is deleted successfully

539

* or return false.

539

* or return false.

540

*/

540

*/

541

static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)

541

static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)

542

{

542

{

543

struct uprobe_consumer **con;

543

struct uprobe_consumer **con;

544

bool ret = false;

544

bool ret = false;

545

546

down_write(&uprobe->consumer_rwsem);

546

down_write(&uprobe->consumer_rwsem);

547

for (con = &uprobe->consumers; *con; con = &(*con)->next) {

547

for (con = &uprobe->consumers; *con; con = &(*con)->next) {

548

if (*con == uc) {

548

if (*con == uc) {

549

*con = uc->next;

549

*con = uc->next;

550

ret = true;

550

ret = true;

551

break;

551

break;

552

}

552

}

553

}

553

}

554

up_write(&uprobe->consumer_rwsem);

554

up_write(&uprobe->consumer_rwsem);

555

556

return ret;

556

return ret;

557

}

557

}

558

559

static int

559

static int

560

__copy_insn(struct address_space *mapping, struct file *filp, char *insn,

560

__copy_insn(struct address_space *mapping, struct file *filp, char *insn,

561

unsigned long nbytes, loff_t offset)

561

unsigned long nbytes, loff_t offset)

562

{

562

{

563

struct page *page;

563

struct page *page;

564

void *vaddr;

564

void *vaddr;

565

unsigned long off;

565

unsigned long off;

566

pgoff_t idx;

566

pgoff_t idx;

567

568

if (!filp)

568

if (!filp)

569

return -EINVAL;

569

return -EINVAL;

570

571

if (!mapping->a_ops->readpage)

571

if (!mapping->a_ops->readpage)

572

return -EIO;

572

return -EIO;

573

574

idx = offset >> PAGE_CACHE_SHIFT;

574

idx = offset >> PAGE_CACHE_SHIFT;

575

off = offset & ~PAGE_MASK;

575

off = offset & ~PAGE_MASK;

576

577

/*

577

/*

578

* Ensure that the page that has the original instruction is

578

* Ensure that the page that has the original instruction is

579

* populated and in page-cache.

579

* populated and in page-cache.

580

*/

580

*/

581

page = read_mapping_page(mapping, idx, filp);

581

page = read_mapping_page(mapping, idx, filp);

582

if (IS_ERR(page))

582

if (IS_ERR(page))

583

return PTR_ERR(page);

583

return PTR_ERR(page);

584

585

vaddr = kmap_atomic(page);

585

vaddr = kmap_atomic(page);

586

memcpy(insn, vaddr + off, nbytes);

586

memcpy(insn, vaddr + off, nbytes);

587

kunmap_atomic(vaddr);

587

kunmap_atomic(vaddr);

588

page_cache_release(page);

588

page_cache_release(page);

589

590

return 0;

590

return 0;

591

}

591

}

592

593

static int copy_insn(struct uprobe *uprobe, struct file *filp)

593

static int copy_insn(struct uprobe *uprobe, struct file *filp)

594

{

594

{

595

struct address_space *mapping;

595

struct address_space *mapping;

596

unsigned long nbytes;

596

unsigned long nbytes;

597

int bytes;

597

int bytes;

598

599

nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK);

599

nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK);

600

mapping = uprobe->inode->i_mapping;

600

mapping = uprobe->inode->i_mapping;

601

602

/* Instruction at end of binary; copy only available bytes */

602

/* Instruction at end of binary; copy only available bytes */

603

if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size)

603

if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size)

604

bytes = uprobe->inode->i_size - uprobe->offset;

604

bytes = uprobe->inode->i_size - uprobe->offset;

605

else

605

else

606

bytes = MAX_UINSN_BYTES;

606

bytes = MAX_UINSN_BYTES;

607

608

/* Instruction at the page-boundary; copy bytes in second page */

608

/* Instruction at the page-boundary; copy bytes in second page */

609

if (nbytes < bytes) {

609

if (nbytes < bytes) {

610

int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes,

610

int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes,

611

bytes - nbytes, uprobe->offset + nbytes);

611

bytes - nbytes, uprobe->offset + nbytes);

612

if (err)

612

if (err)

613

return err;

613

return err;

614

bytes = nbytes;

614

bytes = nbytes;

615

}

615

}

616

return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset);

616

return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset);

617

}

617

}

618

619

/*

619

/*

620

* How mm->uprobes_state.count gets updated

620

* How mm->uprobes_state.count gets updated

621

* uprobe_mmap() increments the count if

621

* uprobe_mmap() increments the count if

622

* - it successfully adds a breakpoint.

622

* - it successfully adds a breakpoint.

623

* - it cannot add a breakpoint, but sees that there is a underlying

623

* - it cannot add a breakpoint, but sees that there is a underlying

624

* breakpoint (via a is_swbp_at_addr()).

624

* breakpoint (via a is_swbp_at_addr()).

625

*

625

*

626

* uprobe_munmap() decrements the count if

626

* uprobe_munmap() decrements the count if

627

* - it sees a underlying breakpoint, (via is_swbp_at_addr)

627

* - it sees a underlying breakpoint, (via is_swbp_at_addr)

628

* (Subsequent uprobe_unregister wouldnt find the breakpoint

628

* (Subsequent uprobe_unregister wouldnt find the breakpoint

629

* unless a uprobe_mmap kicks in, since the old vma would be

629

* unless a uprobe_mmap kicks in, since the old vma would be

630

* dropped just after uprobe_munmap.)

630

* dropped just after uprobe_munmap.)

631

*

631

*

632

* uprobe_register increments the count if:

632

* uprobe_register increments the count if:

633

* - it successfully adds a breakpoint.

633

* - it successfully adds a breakpoint.

634

*

634

*

635

* uprobe_unregister decrements the count if:

635

* uprobe_unregister decrements the count if:

636

* - it sees a underlying breakpoint and removes successfully.

636

* - it sees a underlying breakpoint and removes successfully.

637

* (via is_swbp_at_addr)

637

* (via is_swbp_at_addr)

638

* (Subsequent uprobe_munmap wouldnt find the breakpoint

638

* (Subsequent uprobe_munmap wouldnt find the breakpoint

639

* since there is no underlying breakpoint after the

639

* since there is no underlying breakpoint after the

640

* breakpoint removal.)

640

* breakpoint removal.)

641

*/

641

*/

642

static int

642

static int

643

install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,

643

install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,

644

struct vm_area_struct *vma, unsigned long vaddr)

644

struct vm_area_struct *vma, unsigned long vaddr)

645

{

645

{

646

bool first_uprobe;

646

bool first_uprobe;

647

int ret;

647

int ret;

648

649

/*

649

/*

650

* If probe is being deleted, unregister thread could be done with

650

* If probe is being deleted, unregister thread could be done with

651

* the vma-rmap-walk through. Adding a probe now can be fatal since

651

* the vma-rmap-walk through. Adding a probe now can be fatal since

652

* nobody will be able to cleanup. Also we could be from fork or

652

* nobody will be able to cleanup. Also we could be from fork or

653

* mremap path, where the probe might have already been inserted.

653

* mremap path, where the probe might have already been inserted.

654

* Hence behave as if probe already existed.

654

* Hence behave as if probe already existed.

655

*/

655

*/

656

if (!uprobe->consumers)

656

if (!uprobe->consumers)

657

return 0;

657

return 0;

658

659

if (!(uprobe->flags & UPROBE_COPY_INSN)) {

659

if (!(uprobe->flags & UPROBE_COPY_INSN)) {

660

ret = copy_insn(uprobe, vma->vm_file);

660

ret = copy_insn(uprobe, vma->vm_file);

661

if (ret)

661

if (ret)

662

return ret;

662

return ret;

663

664

if (is_swbp_insn((uprobe_opcode_t *)uprobe->arch.insn))

664

if (is_swbp_insn((uprobe_opcode_t *)uprobe->arch.insn))

665

return -ENOTSUPP;

665

return -ENOTSUPP;

666

667

ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);

667

ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);

668

if (ret)

668

if (ret)

669

return ret;

669

return ret;

670

671

/* write_opcode() assumes we don't cross page boundary */

671

/* write_opcode() assumes we don't cross page boundary */

672

BUG_ON((uprobe->offset & ~PAGE_MASK) +

672

BUG_ON((uprobe->offset & ~PAGE_MASK) +

673

UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);

673

UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);

674

675

uprobe->flags |= UPROBE_COPY_INSN;

675

uprobe->flags |= UPROBE_COPY_INSN;

676

}

676

}

677

678

/*

678

/*

679

* set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),

679

* set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),

680

* the task can hit this breakpoint right after __replace_page().

680

* the task can hit this breakpoint right after __replace_page().

681

*/

681

*/

682

first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);

682

first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);

683

if (first_uprobe)

683

if (first_uprobe)

684

set_bit(MMF_HAS_UPROBES, &mm->flags);

684

set_bit(MMF_HAS_UPROBES, &mm->flags);

685

686

ret = set_swbp(&uprobe->arch, mm, vaddr);

686

ret = set_swbp(&uprobe->arch, mm, vaddr);

687

if (!ret)

687

if (!ret)

688

clear_bit(MMF_RECALC_UPROBES, &mm->flags);

688

clear_bit(MMF_RECALC_UPROBES, &mm->flags);

689

else if (first_uprobe)

689

else if (first_uprobe)

690

clear_bit(MMF_HAS_UPROBES, &mm->flags);

690

clear_bit(MMF_HAS_UPROBES, &mm->flags);

691

692

return ret;

692

return ret;

693

}

693

}

694

695

static void

695

static void

696

remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)

696

remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)

697

{

697

{

698

/* can happen if uprobe_register() fails */

698

/* can happen if uprobe_register() fails */

699

if (!test_bit(MMF_HAS_UPROBES, &mm->flags))

699

if (!test_bit(MMF_HAS_UPROBES, &mm->flags))

700

return;

700

return;

701

702

set_bit(MMF_RECALC_UPROBES, &mm->flags);

702

set_bit(MMF_RECALC_UPROBES, &mm->flags);

703

set_orig_insn(&uprobe->arch, mm, vaddr);

703

set_orig_insn(&uprobe->arch, mm, vaddr);

704

}

704

}

705

706

/*

706

/*

707

* There could be threads that have already hit the breakpoint. They

707

* There could be threads that have already hit the breakpoint. They

708

* will recheck the current insn and restart if find_uprobe() fails.

708

* will recheck the current insn and restart if find_uprobe() fails.

709

* See find_active_uprobe().

709

* See find_active_uprobe().

710

*/

710

*/

711

static void delete_uprobe(struct uprobe *uprobe)

711

static void delete_uprobe(struct uprobe *uprobe)

712

{

712

{

713

spin_lock(&uprobes_treelock);

713

spin_lock(&uprobes_treelock);

714

rb_erase(&uprobe->rb_node, &uprobes_tree);

714

rb_erase(&uprobe->rb_node, &uprobes_tree);

715

spin_unlock(&uprobes_treelock);

715

spin_unlock(&uprobes_treelock);

716

iput(uprobe->inode);

716

iput(uprobe->inode);

717

put_uprobe(uprobe);

717

put_uprobe(uprobe);

718

atomic_dec(&uprobe_events);

718

atomic_dec(&uprobe_events);

719

}

719

}

720

721

struct map_info {

721

struct map_info {

722

struct map_info *next;

722

struct map_info *next;

723

struct mm_struct *mm;

723

struct mm_struct *mm;

724

unsigned long vaddr;

724

unsigned long vaddr;

725

};

725

};

726

727

static inline struct map_info *free_map_info(struct map_info *info)

727

static inline struct map_info *free_map_info(struct map_info *info)

728

{

728

{

729

struct map_info *next = info->next;

729

struct map_info *next = info->next;

730

kfree(info);

730

kfree(info);

731

return next;

731

return next;

732

}

732

}

733

734

static struct map_info *

734

static struct map_info *

735

build_map_info(struct address_space *mapping, loff_t offset, bool is_register)

735

build_map_info(struct address_space *mapping, loff_t offset, bool is_register)

736

{

736

{

737

unsigned long pgoff = offset >> PAGE_SHIFT;

737

unsigned long pgoff = offset >> PAGE_SHIFT;

738

struct prio_tree_iter iter;

738

struct prio_tree_iter iter;

739

struct vm_area_struct *vma;

739

struct vm_area_struct *vma;

740

struct map_info *curr = NULL;

740

struct map_info *curr = NULL;

741

struct map_info *prev = NULL;

741

struct map_info *prev = NULL;

742

struct map_info *info;

742

struct map_info *info;

743

int more = 0;

743

int more = 0;

744

745

again:

745

again:

746

mutex_lock(&mapping->i_mmap_mutex);

746

mutex_lock(&mapping->i_mmap_mutex);

747

vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {

747

vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {

748

if (!valid_vma(vma, is_register))

748

if (!valid_vma(vma, is_register))

749

continue;

749

continue;

750

751

if (!prev && !more) {

751

if (!prev && !more) {

752

/*

752

/*

753

* Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through

753

* Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through

754

* reclaim. This is optimistic, no harm done if it fails.

754

* reclaim. This is optimistic, no harm done if it fails.

755

*/

755

*/

756

prev = kmalloc(sizeof(struct map_info),

756

prev = kmalloc(sizeof(struct map_info),

757

GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);

757

GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);

758

if (prev)

758

if (prev)

759

prev->next = NULL;

759

prev->next = NULL;

760

}

760

}

761

if (!prev) {

761

if (!prev) {

762

more++;

762

more++;

763

continue;

763

continue;

764

}

764

}

765

766

if (!atomic_inc_not_zero(&vma->vm_mm->mm_users))

766

if (!atomic_inc_not_zero(&vma->vm_mm->mm_users))

767

continue;

767

continue;

768

769

info = prev;

769

info = prev;

770

prev = prev->next;

770

prev = prev->next;

771

info->next = curr;

771

info->next = curr;

772

curr = info;

772

curr = info;

773

774

info->mm = vma->vm_mm;

774

info->mm = vma->vm_mm;

775

info->vaddr = offset_to_vaddr(vma, offset);

775

info->vaddr = offset_to_vaddr(vma, offset);

776

}

776

}

777

mutex_unlock(&mapping->i_mmap_mutex);

777

mutex_unlock(&mapping->i_mmap_mutex);

778

779

if (!more)

779

if (!more)

780

goto out;

780

goto out;

781

782

prev = curr;

782

prev = curr;

783

while (curr) {

783

while (curr) {

784

mmput(curr->mm);

784

mmput(curr->mm);

785

curr = curr->next;

785

curr = curr->next;

786

}

786

}

787

788

do {

788

do {

789

info = kmalloc(sizeof(struct map_info), GFP_KERNEL);

789

info = kmalloc(sizeof(struct map_info), GFP_KERNEL);

790

if (!info) {

790

if (!info) {

791

curr = ERR_PTR(-ENOMEM);

791

curr = ERR_PTR(-ENOMEM);

792

goto out;

792

goto out;

793

}

793

}

794

info->next = prev;

794

info->next = prev;

795

prev = info;

795

prev = info;

796

} while (--more);

796

} while (--more);

797

798

goto again;

798

goto again;

799

out:

799

out:

800

while (prev)

800

while (prev)

801

prev = free_map_info(prev);

801

prev = free_map_info(prev);

802

return curr;

802

return curr;

803

}

803

}

804

805

static int register_for_each_vma(struct uprobe *uprobe, bool is_register)

805

static int register_for_each_vma(struct uprobe *uprobe, bool is_register)

806

{

806

{

807

struct map_info *info;

807

struct map_info *info;

808

int err = 0;

808

int err = 0;

809

810

info = build_map_info(uprobe->inode->i_mapping,

810

info = build_map_info(uprobe->inode->i_mapping,

811

uprobe->offset, is_register);

811

uprobe->offset, is_register);

812

if (IS_ERR(info))

812

if (IS_ERR(info))

813

return PTR_ERR(info);

813

return PTR_ERR(info);

814

815

while (info) {

815

while (info) {

816

struct mm_struct *mm = info->mm;

816

struct mm_struct *mm = info->mm;

817

struct vm_area_struct *vma;

817

struct vm_area_struct *vma;

818

819

if (err)

819

if (err)

820

goto free;

820

goto free;

821

822

down_write(&mm->mmap_sem);

822

down_write(&mm->mmap_sem);

823

vma = find_vma(mm, info->vaddr);

823

vma = find_vma(mm, info->vaddr);

824

if (!vma || !valid_vma(vma, is_register) ||

824

if (!vma || !valid_vma(vma, is_register) ||

825

vma->vm_file->f_mapping->host != uprobe->inode)

825

vma->vm_file->f_mapping->host != uprobe->inode)

826

goto unlock;

826

goto unlock;

827

828

if (vma->vm_start > info->vaddr ||

828

if (vma->vm_start > info->vaddr ||

829

vaddr_to_offset(vma, info->vaddr) != uprobe->offset)

829

vaddr_to_offset(vma, info->vaddr) != uprobe->offset)

830

goto unlock;

830

goto unlock;

831

832

if (is_register)

832

if (is_register)

833

err = install_breakpoint(uprobe, mm, vma, info->vaddr);

833

err = install_breakpoint(uprobe, mm, vma, info->vaddr);

834

else

834

else

835

remove_breakpoint(uprobe, mm, info->vaddr);

835

remove_breakpoint(uprobe, mm, info->vaddr);

836

837

unlock:

837

unlock:

838

up_write(&mm->mmap_sem);

838

up_write(&mm->mmap_sem);

839

free:

839

free:

840

mmput(mm);

840

mmput(mm);

841

info = free_map_info(info);

841

info = free_map_info(info);

842

}

842

}

843

844

return err;

844

return err;

845

}

845

}

846

847

static int __uprobe_register(struct uprobe *uprobe)

847

static int __uprobe_register(struct uprobe *uprobe)

848

{

848

{

849

return register_for_each_vma(uprobe, true);

849

return register_for_each_vma(uprobe, true);

850

}

850

}

851

852

static void __uprobe_unregister(struct uprobe *uprobe)

852

static void __uprobe_unregister(struct uprobe *uprobe)

853

{

853

{

854

if (!register_for_each_vma(uprobe, false))

854

if (!register_for_each_vma(uprobe, false))

855

delete_uprobe(uprobe);

855

delete_uprobe(uprobe);

856

857

/* TODO : cant unregister? schedule a worker thread */

857

/* TODO : cant unregister? schedule a worker thread */

858

}

858

}

859

860

/*

860

/*

861

* uprobe_register - register a probe

861

* uprobe_register - register a probe

862

* @inode: the file in which the probe has to be placed.

862

* @inode: the file in which the probe has to be placed.

863

* @offset: offset from the start of the file.

863

* @offset: offset from the start of the file.

864

* @uc: information on howto handle the probe..

864

* @uc: information on howto handle the probe..

865

*

865

*

866

* Apart from the access refcount, uprobe_register() takes a creation

866

* Apart from the access refcount, uprobe_register() takes a creation

867

* refcount (thro alloc_uprobe) if and only if this @uprobe is getting

867

* refcount (thro alloc_uprobe) if and only if this @uprobe is getting

868

* inserted into the rbtree (i.e first consumer for a @inode:@offset

868

* inserted into the rbtree (i.e first consumer for a @inode:@offset

869

* tuple). Creation refcount stops uprobe_unregister from freeing the

869

* tuple). Creation refcount stops uprobe_unregister from freeing the

870

* @uprobe even before the register operation is complete. Creation

870

* @uprobe even before the register operation is complete. Creation

871

* refcount is released when the last @uc for the @uprobe

871

* refcount is released when the last @uc for the @uprobe

872

* unregisters.

872

* unregisters.

873

*

873

*

874

* Return errno if it cannot successully install probes

874

* Return errno if it cannot successully install probes

875

* else return 0 (success)

875

* else return 0 (success)

876

*/

876

*/

877

int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)

877

int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)

878

{

878

{

879

struct uprobe *uprobe;

879

struct uprobe *uprobe;

880

int ret;

880

int ret;

881

882

if (!inode || !uc || uc->next)

882

if (!inode || !uc || uc->next)

883

return -EINVAL;

883

return -EINVAL;

884

885

if (offset > i_size_read(inode))

885

if (offset > i_size_read(inode))

886

return -EINVAL;

886

return -EINVAL;

887

888

ret = 0;

888

ret = 0;

889

mutex_lock(uprobes_hash(inode));

889

mutex_lock(uprobes_hash(inode));

890

uprobe = alloc_uprobe(inode, offset);

890

uprobe = alloc_uprobe(inode, offset);

891

892

if (uprobe && !consumer_add(uprobe, uc)) {

892

if (uprobe && !consumer_add(uprobe, uc)) {

893

ret = __uprobe_register(uprobe);

893

ret = __uprobe_register(uprobe);

894

if (ret) {

894

if (ret) {

895

uprobe->consumers = NULL;

895

uprobe->consumers = NULL;

896

__uprobe_unregister(uprobe);

896

__uprobe_unregister(uprobe);

897

} else {

897

} else {

898

uprobe->flags |= UPROBE_RUN_HANDLER;

898

uprobe->flags |= UPROBE_RUN_HANDLER;

899

}

899

}

900

}

900

}

901

902

mutex_unlock(uprobes_hash(inode));

902

mutex_unlock(uprobes_hash(inode));

903

if (uprobe)

903

if (uprobe)

904

put_uprobe(uprobe);

904

put_uprobe(uprobe);

905

906

return ret;

906

return ret;

907

}

907

}

908

909

/*

909

/*

910

* uprobe_unregister - unregister a already registered probe.

910

* uprobe_unregister - unregister a already registered probe.

911

* @inode: the file in which the probe has to be removed.

911

* @inode: the file in which the probe has to be removed.

912

* @offset: offset from the start of the file.

912

* @offset: offset from the start of the file.

913

* @uc: identify which probe if multiple probes are colocated.

913

* @uc: identify which probe if multiple probes are colocated.

914

*/

914

*/

915

void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)

915

void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)

916

{

916

{

917

struct uprobe *uprobe;

917

struct uprobe *uprobe;

918

919

if (!inode || !uc)

919

if (!inode || !uc)

920

return;

920

return;

921

922

uprobe = find_uprobe(inode, offset);

922

uprobe = find_uprobe(inode, offset);

923

if (!uprobe)

923

if (!uprobe)

924

return;

924

return;

925

926

mutex_lock(uprobes_hash(inode));

926

mutex_lock(uprobes_hash(inode));

927

928

if (consumer_del(uprobe, uc)) {

928

if (consumer_del(uprobe, uc)) {

929

if (!uprobe->consumers) {

929

if (!uprobe->consumers) {

930

__uprobe_unregister(uprobe);

930

__uprobe_unregister(uprobe);

931

uprobe->flags &= ~UPROBE_RUN_HANDLER;

931

uprobe->flags &= ~UPROBE_RUN_HANDLER;

932

}

932

}

933

}

933

}

934

935

mutex_unlock(uprobes_hash(inode));

935

mutex_unlock(uprobes_hash(inode));

936

if (uprobe)

936

if (uprobe)

937

put_uprobe(uprobe);

937

put_uprobe(uprobe);

938

}

938

}

939

940

static struct rb_node *

940

static struct rb_node *

941

find_node_in_range(struct inode *inode, loff_t min, loff_t max)

941

find_node_in_range(struct inode *inode, loff_t min, loff_t max)

942

{

942

{

943

struct rb_node *n = uprobes_tree.rb_node;

943

struct rb_node *n = uprobes_tree.rb_node;

944

945

while (n) {

945

while (n) {

946

struct uprobe *u = rb_entry(n, struct uprobe, rb_node);

946

struct uprobe *u = rb_entry(n, struct uprobe, rb_node);

947

948

if (inode < u->inode) {

948

if (inode < u->inode) {

949

n = n->rb_left;

949

n = n->rb_left;

950

} else if (inode > u->inode) {

950

} else if (inode > u->inode) {

951

n = n->rb_right;

951

n = n->rb_right;

952

} else {

952

} else {

953

if (max < u->offset)

953

if (max < u->offset)

954

n = n->rb_left;

954

n = n->rb_left;

955

else if (min > u->offset)

955

else if (min > u->offset)

956

n = n->rb_right;

956

n = n->rb_right;

957

else

957

else

958

break;

958

break;

959

}

959

}

960

}

960

}

961

962

return n;

962

return n;

963

}

963

}

964

965

/*

965

/*

966

* For a given range in vma, build a list of probes that need to be inserted.

966

* For a given range in vma, build a list of probes that need to be inserted.

967

*/

967

*/

968

static void build_probe_list(struct inode *inode,

968

static void build_probe_list(struct inode *inode,

969

struct vm_area_struct *vma,

969

struct vm_area_struct *vma,

970

unsigned long start, unsigned long end,

970

unsigned long start, unsigned long end,

971

struct list_head *head)

971

struct list_head *head)

972

{

972

{

973

loff_t min, max;

973

loff_t min, max;

974

struct rb_node *n, *t;

974

struct rb_node *n, *t;

975

struct uprobe *u;

975

struct uprobe *u;

976

977

INIT_LIST_HEAD(head);

977

INIT_LIST_HEAD(head);

978

min = vaddr_to_offset(vma, start);

978

min = vaddr_to_offset(vma, start);

979

max = min + (end - start) - 1;

979

max = min + (end - start) - 1;

980

981

spin_lock(&uprobes_treelock);

981

spin_lock(&uprobes_treelock);

982

n = find_node_in_range(inode, min, max);

982

n = find_node_in_range(inode, min, max);

983

if (n) {

983

if (n) {

984

for (t = n; t; t = rb_prev(t)) {

984

for (t = n; t; t = rb_prev(t)) {

985

u = rb_entry(t, struct uprobe, rb_node);

985

u = rb_entry(t, struct uprobe, rb_node);

986

if (u->inode != inode || u->offset < min)

986

if (u->inode != inode || u->offset < min)

987

break;

987

break;

988

list_add(&u->pending_list, head);

988

list_add(&u->pending_list, head);

989

atomic_inc(&u->ref);

989

atomic_inc(&u->ref);

990

}

990

}

991

for (t = n; (t = rb_next(t)); ) {

991

for (t = n; (t = rb_next(t)); ) {

992

u = rb_entry(t, struct uprobe, rb_node);

992

u = rb_entry(t, struct uprobe, rb_node);

993

if (u->inode != inode || u->offset > max)

993

if (u->inode != inode || u->offset > max)

994

break;

994

break;

995

list_add(&u->pending_list, head);

995

list_add(&u->pending_list, head);

996

atomic_inc(&u->ref);

996

atomic_inc(&u->ref);

997

}

997

}

998

}

998

}

999

spin_unlock(&uprobes_treelock);

999

spin_unlock(&uprobes_treelock);

1000

}

1000

}

1001

1002

/*

1002

/*

1003

* Called from mmap_region/vma_adjust with mm->mmap_sem acquired.

1003

* Called from mmap_region/vma_adjust with mm->mmap_sem acquired.

1004

*

1004

*

1005

* Currently we ignore all errors and always return 0, the callers

1005

* Currently we ignore all errors and always return 0, the callers

1006

* can't handle the failure anyway.

1006

* can't handle the failure anyway.

1007

*/

1007

*/

1008

int uprobe_mmap(struct vm_area_struct *vma)

1008

int uprobe_mmap(struct vm_area_struct *vma)

1009

{

1009

{

1010

struct list_head tmp_list;

1010

struct list_head tmp_list;

1011

struct uprobe *uprobe, *u;

1011

struct uprobe *uprobe, *u;

1012

struct inode *inode;

1012

struct inode *inode;

1013

1014

if (!atomic_read(&uprobe_events) || !valid_vma(vma, true))

1014

if (!atomic_read(&uprobe_events) || !valid_vma(vma, true))

1015

return 0;

1015

return 0;

1016

1017

inode = vma->vm_file->f_mapping->host;

1017

inode = vma->vm_file->f_mapping->host;

1018

if (!inode)

1018

if (!inode)

1019

return 0;

1019

return 0;

1020

1021

mutex_lock(uprobes_mmap_hash(inode));

1021

mutex_lock(uprobes_mmap_hash(inode));

1022

build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);

1022

build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);

1023

1024

list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {

1024

list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {

1025

if (!fatal_signal_pending(current)) {

1025

if (!fatal_signal_pending(current)) {

1026

unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);

1026

unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);

1027

install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);

1027

install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);

1028

}

1028

}

1029

put_uprobe(uprobe);

1029

put_uprobe(uprobe);

1030

}

1030

}

1031

mutex_unlock(uprobes_mmap_hash(inode));

1031

mutex_unlock(uprobes_mmap_hash(inode));

1032

1033

return 0;

1033

return 0;

1034

}

1034

}

1035

1036

static bool

1036

static bool

1037

vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)

1037

vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)

1038

{

1038

{

1039

loff_t min, max;

1039

loff_t min, max;

1040

struct inode *inode;

1040

struct inode *inode;

1041

struct rb_node *n;

1041

struct rb_node *n;

1042

1043

inode = vma->vm_file->f_mapping->host;

1043

inode = vma->vm_file->f_mapping->host;

1044

1045

min = vaddr_to_offset(vma, start);

1045

min = vaddr_to_offset(vma, start);

1046

max = min + (end - start) - 1;

1046

max = min + (end - start) - 1;

1047

1048

spin_lock(&uprobes_treelock);

1048

spin_lock(&uprobes_treelock);

1049

n = find_node_in_range(inode, min, max);

1049

n = find_node_in_range(inode, min, max);

1050

spin_unlock(&uprobes_treelock);

1050

spin_unlock(&uprobes_treelock);

1051

1052

return !!n;

1052

return !!n;

1053

}

1053

}

1054

1055

/*

1055

/*

1056

* Called in context of a munmap of a vma.

1056

* Called in context of a munmap of a vma.

1057

*/

1057

*/

1058

void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)

1058

void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)

1059

{

1059

{

1060

if (!atomic_read(&uprobe_events) || !valid_vma(vma, false))

1060

if (!atomic_read(&uprobe_events) || !valid_vma(vma, false))

1061

return;

1061

return;

1062

1063

if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */

1063

if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */

1064

return;

1064

return;

1065

1066

if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||

1066

if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||

1067

test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))

1067

test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))

1068

return;

1068

return;

1069

1070

if (vma_has_uprobes(vma, start, end))

1070

if (vma_has_uprobes(vma, start, end))

1071

set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);

1071

set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);

1072

}

1072

}

1073

1074

/* Slot allocation for XOL */

1074

/* Slot allocation for XOL */

1075

static int xol_add_vma(struct xol_area *area)

1075

static int xol_add_vma(struct xol_area *area)

1076

{

1076

{

1077

struct mm_struct *mm;

1077

struct mm_struct *mm;

1078

int ret;

1078

int ret;

1079

1080

area->page = alloc_page(GFP_HIGHUSER);

1080

area->page = alloc_page(GFP_HIGHUSER);

1081

if (!area->page)

1081

if (!area->page)

1082

return -ENOMEM;

1082

return -ENOMEM;

1083

1084

ret = -EALREADY;

1084

ret = -EALREADY;

1085

mm = current->mm;

1085

mm = current->mm;

1086

1087

down_write(&mm->mmap_sem);

1087

down_write(&mm->mmap_sem);

1088

if (mm->uprobes_state.xol_area)

1088

if (mm->uprobes_state.xol_area)

1089

goto fail;

1089

goto fail;

1090

1091

ret = -ENOMEM;

1091

ret = -ENOMEM;

1092

1093

/* Try to map as high as possible, this is only a hint. */

1093

/* Try to map as high as possible, this is only a hint. */

1094

area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0);

1094

area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0);

1095

if (area->vaddr & ~PAGE_MASK) {

1095

if (area->vaddr & ~PAGE_MASK) {

1096

ret = area->vaddr;

1096

ret = area->vaddr;

1097

goto fail;

1097

goto fail;

1098

}

1098

}

1099

1100

ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE,

1100

ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE,

1101

VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page);

1101

VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page);

1102

if (ret)

1102

if (ret)

1103

goto fail;

1103

goto fail;

1104

1105

smp_wmb(); /* pairs with get_xol_area() */

1105

smp_wmb(); /* pairs with get_xol_area() */

1106

mm->uprobes_state.xol_area = area;

1106

mm->uprobes_state.xol_area = area;

1107

ret = 0;

1107

ret = 0;

1108

1109

fail:

1109

fail:

1110

up_write(&mm->mmap_sem);

1110

up_write(&mm->mmap_sem);

1111

if (ret)

1111

if (ret)

1112

__free_page(area->page);

1112

__free_page(area->page);

1113

1114

return ret;

1114

return ret;

1115

}

1115

}

1116

1117

static struct xol_area *get_xol_area(struct mm_struct *mm)

1117

static struct xol_area *get_xol_area(struct mm_struct *mm)

1118

{

1118

{

1119

struct xol_area *area;

1119

struct xol_area *area;

1120

1121

area = mm->uprobes_state.xol_area;

1121

area = mm->uprobes_state.xol_area;

1122

smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */

1122

smp_read_barrier_depends(); /* pairs with wmb in xol_add_vma() */

1123

1124

return area;

1124

return area;

1125

}

1125

}

1126

1127

/*

1127

/*

1128

* xol_alloc_area - Allocate process's xol_area.

1128

* xol_alloc_area - Allocate process's xol_area.

1129

* This area will be used for storing instructions for execution out of

1129

* This area will be used for storing instructions for execution out of

1130

* line.

1130

* line.

1131

*

1131

*

1132

* Returns the allocated area or NULL.

1132

* Returns the allocated area or NULL.

1133

*/

1133

*/

1134

static struct xol_area *xol_alloc_area(void)

1134

static struct xol_area *xol_alloc_area(void)

1135

{

1135

{

1136

struct xol_area *area;

1136

struct xol_area *area;

1137

1138

area = kzalloc(sizeof(*area), GFP_KERNEL);

1138

area = kzalloc(sizeof(*area), GFP_KERNEL);

1139

if (unlikely(!area))

1139

if (unlikely(!area))

1140

return NULL;

1140

return NULL;

1141

1142

area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL);

1142

area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL);

1143

1144

if (!area->bitmap)

1144

if (!area->bitmap)

1145

goto fail;

1145

goto fail;

1146

1147

init_waitqueue_head(&area->wq);

1147

init_waitqueue_head(&area->wq);

1148

if (!xol_add_vma(area))

1148

if (!xol_add_vma(area))

1149

return area;

1149

return area;

1150

1151

fail:

1151

fail:

1152

kfree(area->bitmap);

1152

kfree(area->bitmap);

1153

kfree(area);

1153

kfree(area);

1154

1155

return get_xol_area(current->mm);

1155

return get_xol_area(current->mm);

1156

}

1156

}

1157

1158

/*

1158

/*

1159

* uprobe_clear_state - Free the area allocated for slots.

1159

* uprobe_clear_state - Free the area allocated for slots.

1160

*/

1160

*/

1161

void uprobe_clear_state(struct mm_struct *mm)

1161

void uprobe_clear_state(struct mm_struct *mm)

1162

{

1162

{

1163

struct xol_area *area = mm->uprobes_state.xol_area;

1163

struct xol_area *area = mm->uprobes_state.xol_area;

1164

1165

if (!area)

1165

if (!area)

1166

return;

1166

return;

1167

1168

put_page(area->page);

1168

put_page(area->page);

1169

kfree(area->bitmap);

1169

kfree(area->bitmap);

1170

kfree(area);

1170

kfree(area);

1171

}

1171

}

1172

1173

void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)

1173

void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)

1174

{

1174

{

1175

newmm->uprobes_state.xol_area = NULL;

1175

newmm->uprobes_state.xol_area = NULL;

1176

1177

if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {

1177

if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {

1178

set_bit(MMF_HAS_UPROBES, &newmm->flags);

1178

set_bit(MMF_HAS_UPROBES, &newmm->flags);

1179

/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */

1179

/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */

1180

set_bit(MMF_RECALC_UPROBES, &newmm->flags);

1180

set_bit(MMF_RECALC_UPROBES, &newmm->flags);

1181

}

1181

}

1182

}

1182

}

1183

1184

/*

1184

/*

1185

* - search for a free slot.

1185

* - search for a free slot.

1186

*/

1186

*/

1187

static unsigned long xol_take_insn_slot(struct xol_area *area)

1187

static unsigned long xol_take_insn_slot(struct xol_area *area)

1188

{

1188

{

1189

unsigned long slot_addr;

1189

unsigned long slot_addr;

1190

int slot_nr;

1190

int slot_nr;

1191

1192

do {

1192

do {

1193

slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);

1193

slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);

1194

if (slot_nr < UINSNS_PER_PAGE) {

1194

if (slot_nr < UINSNS_PER_PAGE) {

1195

if (!test_and_set_bit(slot_nr, area->bitmap))

1195

if (!test_and_set_bit(slot_nr, area->bitmap))

1196

break;

1196

break;

1197

1198

slot_nr = UINSNS_PER_PAGE;

1198

slot_nr = UINSNS_PER_PAGE;

1199

continue;

1199

continue;

1200

}

1200

}

1201

wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));

1201

wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));

1202

} while (slot_nr >= UINSNS_PER_PAGE);

1202

} while (slot_nr >= UINSNS_PER_PAGE);

1203

1204

slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);

1204

slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);

1205

atomic_inc(&area->slot_count);

1205

atomic_inc(&area->slot_count);

1206

1207

return slot_addr;

1207

return slot_addr;

1208

}

1208

}

1209

1210

/*

1210

/*

1211

* xol_get_insn_slot - If was not allocated a slot, then

1211

* xol_get_insn_slot - If was not allocated a slot, then

1212

* allocate a slot.

1212

* allocate a slot.

1213

* Returns the allocated slot address or 0.

1213

* Returns the allocated slot address or 0.

1214

*/

1214

*/

1215

static unsigned long xol_get_insn_slot(struct uprobe *uprobe, unsigned long slot_addr)

1215

static unsigned long xol_get_insn_slot(struct uprobe *uprobe, unsigned long slot_addr)

1216

{

1216

{

1217

struct xol_area *area;

1217

struct xol_area *area;

1218

unsigned long offset;

1218

unsigned long offset;

1219

void *vaddr;

1219

void *vaddr;

1220

1221

area = get_xol_area(current->mm);

1221

area = get_xol_area(current->mm);

1222

if (!area) {

1222

if (!area) {

1223

area = xol_alloc_area();

1223

area = xol_alloc_area();

1224

if (!area)

1224

if (!area)

1225

return 0;

1225

return 0;

1226

}

1226

}

1227

current->utask->xol_vaddr = xol_take_insn_slot(area);

1227

current->utask->xol_vaddr = xol_take_insn_slot(area);

1228

1229

/*

1229

/*

1230

* Initialize the slot if xol_vaddr points to valid

1230

* Initialize the slot if xol_vaddr points to valid

1231

* instruction slot.

1231

* instruction slot.

1232

*/

1232

*/

1233

if (unlikely(!current->utask->xol_vaddr))

1233

if (unlikely(!current->utask->xol_vaddr))

1234

return 0;

1234

return 0;

1235

1236

current->utask->vaddr = slot_addr;

1236

current->utask->vaddr = slot_addr;

1237

offset = current->utask->xol_vaddr & ~PAGE_MASK;

1237

offset = current->utask->xol_vaddr & ~PAGE_MASK;

1238

vaddr = kmap_atomic(area->page);

1238

vaddr = kmap_atomic(area->page);

1239

memcpy(vaddr + offset, uprobe->arch.insn, MAX_UINSN_BYTES);

1239

memcpy(vaddr + offset, uprobe->arch.insn, MAX_UINSN_BYTES);

1240

kunmap_atomic(vaddr);

1240

kunmap_atomic(vaddr);

1241

1242

return current->utask->xol_vaddr;

1242

return current->utask->xol_vaddr;

1243

}

1243

}

1244

1245

/*

1245

/*

1246

* xol_free_insn_slot - If slot was earlier allocated by

1246

* xol_free_insn_slot - If slot was earlier allocated by

1247

* @xol_get_insn_slot(), make the slot available for

1247

* @xol_get_insn_slot(), make the slot available for

1248

* subsequent requests.

1248

* subsequent requests.

1249

*/

1249

*/

1250

static void xol_free_insn_slot(struct task_struct *tsk)

1250

static void xol_free_insn_slot(struct task_struct *tsk)

1251

{

1251

{

1252

struct xol_area *area;

1252

struct xol_area *area;

1253

unsigned long vma_end;

1253

unsigned long vma_end;

1254

unsigned long slot_addr;

1254

unsigned long slot_addr;

1255

1256

if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)

1256

if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)

1257

return;

1257

return;

1258

1259

slot_addr = tsk->utask->xol_vaddr;

1259

slot_addr = tsk->utask->xol_vaddr;

1260

1261

if (unlikely(!slot_addr || IS_ERR_VALUE(slot_addr)))

1261

if (unlikely(!slot_addr || IS_ERR_VALUE(slot_addr)))

1262

return;

1262

return;

1263

1264

area = tsk->mm->uprobes_state.xol_area;

1264

area = tsk->mm->uprobes_state.xol_area;

1265

vma_end = area->vaddr + PAGE_SIZE;

1265

vma_end = area->vaddr + PAGE_SIZE;

1266

if (area->vaddr <= slot_addr && slot_addr < vma_end) {

1266

if (area->vaddr <= slot_addr && slot_addr < vma_end) {

1267

unsigned long offset;

1267

unsigned long offset;

1268

int slot_nr;

1268

int slot_nr;

1269

1270

offset = slot_addr - area->vaddr;

1270

offset = slot_addr - area->vaddr;

1271

slot_nr = offset / UPROBE_XOL_SLOT_BYTES;

1271

slot_nr = offset / UPROBE_XOL_SLOT_BYTES;

1272

if (slot_nr >= UINSNS_PER_PAGE)

1272

if (slot_nr >= UINSNS_PER_PAGE)

1273

return;

1273

return;

1274

1275

clear_bit(slot_nr, area->bitmap);

1275

clear_bit(slot_nr, area->bitmap);

1276

atomic_dec(&area->slot_count);

1276

atomic_dec(&area->slot_count);

1277

if (waitqueue_active(&area->wq))

1277

if (waitqueue_active(&area->wq))

1278

wake_up(&area->wq);

1278

wake_up(&area->wq);

1279

1280

tsk->utask->xol_vaddr = 0;

1280

tsk->utask->xol_vaddr = 0;

1281

}

1281

}

1282

}

1282

}

1283

1284

/**

1284

/**

1285

* uprobe_get_swbp_addr - compute address of swbp given post-swbp regs

1285

* uprobe_get_swbp_addr - compute address of swbp given post-swbp regs

1286

* @regs: Reflects the saved state of the task after it has hit a breakpoint

1286

* @regs: Reflects the saved state of the task after it has hit a breakpoint

1287

* instruction.

1287

* instruction.

1288

* Return the address of the breakpoint instruction.

1288

* Return the address of the breakpoint instruction.

1289

*/

1289

*/

1290

unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)

1290

unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)

1291

{

1291

{

1292

return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;

1292

return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;

1293

}

1293

}

1294

1295

/*

1295

/*

1296

* Called with no locks held.

1296

* Called with no locks held.

1297

* Called in context of a exiting or a exec-ing thread.

1297

* Called in context of a exiting or a exec-ing thread.

1298

*/

1298

*/

1299

void uprobe_free_utask(struct task_struct *t)

1299

void uprobe_free_utask(struct task_struct *t)

1300

{

1300

{

1301

struct uprobe_task *utask = t->utask;

1301

struct uprobe_task *utask = t->utask;

1302

1303

if (!utask)

1303

if (!utask)

1304

return;

1304

return;

1305

1306

if (utask->active_uprobe)

1306

if (utask->active_uprobe)

1307

put_uprobe(utask->active_uprobe);

1307

put_uprobe(utask->active_uprobe);

1308

1309

xol_free_insn_slot(t);

1309

xol_free_insn_slot(t);

1310

kfree(utask);

1310

kfree(utask);

1311

t->utask = NULL;

1311

t->utask = NULL;

1312

}

1312

}

1313

1314

/*

1314

/*

1315

* Called in context of a new clone/fork from copy_process.

1315

* Called in context of a new clone/fork from copy_process.

1316

*/

1316

*/

1317

void uprobe_copy_process(struct task_struct *t)

1317

void uprobe_copy_process(struct task_struct *t)

1318

{

1318

{

1319

t->utask = NULL;

1319

t->utask = NULL;

1320

}

1320

}

1321

1322

/*

1322

/*

1323

* Allocate a uprobe_task object for the task.

1323

* Allocate a uprobe_task object for the task.

1324

* Called when the thread hits a breakpoint for the first time.

1324

* Called when the thread hits a breakpoint for the first time.

1325

*

1325

*

1326

* Returns:

1326

* Returns:

1327

* - pointer to new uprobe_task on success

1327

* - pointer to new uprobe_task on success

1328

* - NULL otherwise

1328

* - NULL otherwise

1329

*/

1329

*/

1330

static struct uprobe_task *add_utask(void)

1330

static struct uprobe_task *add_utask(void)

1331

{

1331

{

1332

struct uprobe_task *utask;

1332

struct uprobe_task *utask;

1333

1334

utask = kzalloc(sizeof *utask, GFP_KERNEL);

1334

utask = kzalloc(sizeof *utask, GFP_KERNEL);

1335

if (unlikely(!utask))

1335

if (unlikely(!utask))

1336

return NULL;

1336

return NULL;

1337

1338

current->utask = utask;

1338

current->utask = utask;

1339

return utask;

1339

return utask;

1340

}

1340

}

1341

1342

/* Prepare to single-step probed instruction out of line. */

1342

/* Prepare to single-step probed instruction out of line. */

1343

static int

1343

static int

1344

pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long vaddr)

1344

pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long vaddr)

1345

{

1345

{

1346

if (xol_get_insn_slot(uprobe, vaddr) && !arch_uprobe_pre_xol(&uprobe->arch, regs))

1346

if (xol_get_insn_slot(uprobe, vaddr) && !arch_uprobe_pre_xol(&uprobe->arch, regs))

1347

return 0;

1347

return 0;

1348

1349

return -EFAULT;

1349

return -EFAULT;

1350

}

1350

}

1351

1352

/*

1352

/*

1353

* If we are singlestepping, then ensure this thread is not connected to

1353

* If we are singlestepping, then ensure this thread is not connected to

1354

* non-fatal signals until completion of singlestep. When xol insn itself

1354

* non-fatal signals until completion of singlestep. When xol insn itself

1355

* triggers the signal, restart the original insn even if the task is

1355

* triggers the signal, restart the original insn even if the task is

1356

* already SIGKILL'ed (since coredump should report the correct ip). This

1356

* already SIGKILL'ed (since coredump should report the correct ip). This

1357

* is even more important if the task has a handler for SIGSEGV/etc, The

1357

* is even more important if the task has a handler for SIGSEGV/etc, The

1358

* _same_ instruction should be repeated again after return from the signal

1358

* _same_ instruction should be repeated again after return from the signal

1359

* handler, and SSTEP can never finish in this case.

1359

* handler, and SSTEP can never finish in this case.

1360

*/

1360

*/

1361

bool uprobe_deny_signal(void)

1361

bool uprobe_deny_signal(void)

1362

{

1362

{

1363

struct task_struct *t = current;

1363

struct task_struct *t = current;

1364

struct uprobe_task *utask = t->utask;

1364

struct uprobe_task *utask = t->utask;

1365

1366

if (likely(!utask || !utask->active_uprobe))

1366

if (likely(!utask || !utask->active_uprobe))

1367

return false;

1367

return false;

1368

1369

WARN_ON_ONCE(utask->state != UTASK_SSTEP);

1369

WARN_ON_ONCE(utask->state != UTASK_SSTEP);

1370

1371

if (signal_pending(t)) {

1371

if (signal_pending(t)) {

1372

spin_lock_irq(&t->sighand->siglock);

1372

spin_lock_irq(&t->sighand->siglock);

1373

clear_tsk_thread_flag(t, TIF_SIGPENDING);

1373

clear_tsk_thread_flag(t, TIF_SIGPENDING);

1374

spin_unlock_irq(&t->sighand->siglock);

1374

spin_unlock_irq(&t->sighand->siglock);

1375

1376

if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {

1376

if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {

1377

utask->state = UTASK_SSTEP_TRAPPED;

1377

utask->state = UTASK_SSTEP_TRAPPED;

1378

set_tsk_thread_flag(t, TIF_UPROBE);

1378

set_tsk_thread_flag(t, TIF_UPROBE);

1379

set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);

1379

set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);

1380

}

1380

}

1381

}

1381

}

1382

1383

return true;

1383

return true;

1384

}

1384

}

1385

1386

/*

1386

/*

1387

* Avoid singlestepping the original instruction if the original instruction

1387

* Avoid singlestepping the original instruction if the original instruction

1388

* is a NOP or can be emulated.

1388

* is a NOP or can be emulated.

1389

*/

1389

*/

1390

static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs)

1390

static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs)

1391

{

1391

{

1392

if (uprobe->flags & UPROBE_SKIP_SSTEP) {

1392

if (uprobe->flags & UPROBE_SKIP_SSTEP) {

1393

if (arch_uprobe_skip_sstep(&uprobe->arch, regs))

1393

if (arch_uprobe_skip_sstep(&uprobe->arch, regs))

1394

return true;

1394

return true;

1395

uprobe->flags &= ~UPROBE_SKIP_SSTEP;

1395

uprobe->flags &= ~UPROBE_SKIP_SSTEP;

1396

}

1396

}

1397

return false;

1397

return false;

1398

}

1398

}

1399

1400

static void mmf_recalc_uprobes(struct mm_struct *mm)

1400

static void mmf_recalc_uprobes(struct mm_struct *mm)

1401

{

1401

{

1402

struct vm_area_struct *vma;

1402

struct vm_area_struct *vma;

1403

1404

for (vma = mm->mmap; vma; vma = vma->vm_next) {

1404

for (vma = mm->mmap; vma; vma = vma->vm_next) {

1405

if (!valid_vma(vma, false))

1405

if (!valid_vma(vma, false))

1406

continue;

1406

continue;

1407

/*

1407

/*

1408

* This is not strictly accurate, we can race with

1408

* This is not strictly accurate, we can race with

1409

* uprobe_unregister() and see the already removed

1409

* uprobe_unregister() and see the already removed

1410

* uprobe if delete_uprobe() was not yet called.

1410

* uprobe if delete_uprobe() was not yet called.

1411

*/

1411

*/

1412

if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))

1412

if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))

1413

return;

1413

return;

1414

}

1414

}

1415

1416

clear_bit(MMF_HAS_UPROBES, &mm->flags);

1416

clear_bit(MMF_HAS_UPROBES, &mm->flags);

1417

}

1417

}

1418

1419

static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)

1419

static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)

1420

{

1420

{

1421

struct mm_struct *mm = current->mm;

1421

struct mm_struct *mm = current->mm;

1422

struct uprobe *uprobe = NULL;

1422

struct uprobe *uprobe = NULL;

1423

struct vm_area_struct *vma;

1423

struct vm_area_struct *vma;

1424

1425

down_read(&mm->mmap_sem);

1425

down_read(&mm->mmap_sem);

1426

vma = find_vma(mm, bp_vaddr);

1426

vma = find_vma(mm, bp_vaddr);

1427

if (vma && vma->vm_start <= bp_vaddr) {

1427

if (vma && vma->vm_start <= bp_vaddr) {

1428

if (valid_vma(vma, false)) {

1428

if (valid_vma(vma, false)) {

1429

struct inode *inode = vma->vm_file->f_mapping->host;

1429

struct inode *inode = vma->vm_file->f_mapping->host;

1430

loff_t offset = vaddr_to_offset(vma, bp_vaddr);

1430

loff_t offset = vaddr_to_offset(vma, bp_vaddr);

1431

1432

uprobe = find_uprobe(inode, offset);

1432

uprobe = find_uprobe(inode, offset);

1433

}

1433

}

1434

1435

if (!uprobe)

1435

if (!uprobe)

1436

*is_swbp = is_swbp_at_addr(mm, bp_vaddr);

1436

*is_swbp = is_swbp_at_addr(mm, bp_vaddr);

1437

} else {

1437

} else {

1438

*is_swbp = -EFAULT;

1438

*is_swbp = -EFAULT;

1439

}

1439

}

1440

1441

if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))

1441

if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))

1442

mmf_recalc_uprobes(mm);

1442

mmf_recalc_uprobes(mm);

1443

up_read(&mm->mmap_sem);

1443

up_read(&mm->mmap_sem);

1444

1445

return uprobe;

1445

return uprobe;

1446

}

1446

}

1447

1448

void __weak arch_uprobe_enable_step(struct arch_uprobe *arch)

1448

void __weak arch_uprobe_enable_step(struct arch_uprobe *arch)

1449

{

1449

{

1450

user_enable_single_step(current);

1450

user_enable_single_step(current);

1451

}

1451

}

1452

1453

void __weak arch_uprobe_disable_step(struct arch_uprobe *arch)

1453

void __weak arch_uprobe_disable_step(struct arch_uprobe *arch)

1454

{

1454

{

1455

user_disable_single_step(current);

1455

user_disable_single_step(current);

1456

}

1456

}

1457

1458

/*

1458

/*

1459

* Run handler and ask thread to singlestep.

1459

* Run handler and ask thread to singlestep.

1460

* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.

1460

* Ensure all non-fatal signals cannot interrupt thread while it singlesteps.

1461

*/

1461

*/

1462

static void handle_swbp(struct pt_regs *regs)

1462

static void handle_swbp(struct pt_regs *regs)

1463

{

1463

{

1464

struct uprobe_task *utask;

1464

struct uprobe_task *utask;

1465

struct uprobe *uprobe;

1465

struct uprobe *uprobe;

1466

unsigned long bp_vaddr;

1466

unsigned long bp_vaddr;

1467

int uninitialized_var(is_swbp);

1467

int uninitialized_var(is_swbp);

1468

1469

bp_vaddr = uprobe_get_swbp_addr(regs);

1469

bp_vaddr = uprobe_get_swbp_addr(regs);

1470

uprobe = find_active_uprobe(bp_vaddr, &is_swbp);

1470

uprobe = find_active_uprobe(bp_vaddr, &is_swbp);

1471

1472

if (!uprobe) {

1472

if (!uprobe) {

1473

if (is_swbp > 0) {

1473

if (is_swbp > 0) {

1474

/* No matching uprobe; signal SIGTRAP. */

1474

/* No matching uprobe; signal SIGTRAP. */

1475

send_sig(SIGTRAP, current, 0);

1475

send_sig(SIGTRAP, current, 0);

1476

} else {

1476

} else {

1477

/*

1477

/*

1478

* Either we raced with uprobe_unregister() or we can't

1478

* Either we raced with uprobe_unregister() or we can't

1479

* access this memory. The latter is only possible if

1479

* access this memory. The latter is only possible if

1480

* another thread plays with our ->mm. In both cases

1480

* another thread plays with our ->mm. In both cases

1481

* we can simply restart. If this vma was unmapped we

1481

* we can simply restart. If this vma was unmapped we

1482

* can pretend this insn was not executed yet and get

1482

* can pretend this insn was not executed yet and get

1483

* the (correct) SIGSEGV after restart.

1483

* the (correct) SIGSEGV after restart.

1484

*/

1484

*/

1485

instruction_pointer_set(regs, bp_vaddr);

1485

instruction_pointer_set(regs, bp_vaddr);

1486

}

1486

}

1487

return;

1487

return;

1488

}

1488

}

1489

1490

utask = current->utask;

1490

utask = current->utask;

1491

if (!utask) {

1491

if (!utask) {

1492

utask = add_utask();

1492

utask = add_utask();

1493

/* Cannot allocate; re-execute the instruction. */

1493

/* Cannot allocate; re-execute the instruction. */

1494

if (!utask)

1494

if (!utask)

1495

goto restart;

1495

goto restart;

1496

}

1496

}

1497

1498

handler_chain(uprobe, regs);

1498

handler_chain(uprobe, regs);

1499

if (can_skip_sstep(uprobe, regs))

1499

if (can_skip_sstep(uprobe, regs))

1500

goto out;

1500

goto out;

1501

1502

if (!pre_ssout(uprobe, regs, bp_vaddr)) {

1502

if (!pre_ssout(uprobe, regs, bp_vaddr)) {

1503

arch_uprobe_enable_step(&uprobe->arch);

1503

arch_uprobe_enable_step(&uprobe->arch);

1504

utask->active_uprobe = uprobe;

1504

utask->active_uprobe = uprobe;

1505

utask->state = UTASK_SSTEP;

1505

utask->state = UTASK_SSTEP;

1506

return;

1506

return;

1507

}

1507

}

1508

1509

restart:

1509

restart:

1510

/*

1510

/*

1511

* cannot singlestep; cannot skip instruction;

1511

* cannot singlestep; cannot skip instruction;

1512

* re-execute the instruction.

1512

* re-execute the instruction.

1513

*/

1513

*/

1514

instruction_pointer_set(regs, bp_vaddr);

1514

instruction_pointer_set(regs, bp_vaddr);

1515

out:

1515

out:

1516

put_uprobe(uprobe);

1516

put_uprobe(uprobe);

1517

}

1517

}

1518

1519

/*

1519

/*

1520

* Perform required fix-ups and disable singlestep.

1520

* Perform required fix-ups and disable singlestep.

1521

* Allow pending signals to take effect.

1521

* Allow pending signals to take effect.

1522

*/

1522

*/

1523

static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)

1523

static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)

1524

{

1524

{

1525

struct uprobe *uprobe;

1525

struct uprobe *uprobe;

1526

1527

uprobe = utask->active_uprobe;

1527

uprobe = utask->active_uprobe;

1528

if (utask->state == UTASK_SSTEP_ACK)

1528

if (utask->state == UTASK_SSTEP_ACK)

1529

arch_uprobe_post_xol(&uprobe->arch, regs);

1529

arch_uprobe_post_xol(&uprobe->arch, regs);

1530

else if (utask->state == UTASK_SSTEP_TRAPPED)

1530

else if (utask->state == UTASK_SSTEP_TRAPPED)

1531

arch_uprobe_abort_xol(&uprobe->arch, regs);

1531

arch_uprobe_abort_xol(&uprobe->arch, regs);

1532

else

1532

else

1533

WARN_ON_ONCE(1);

1533

WARN_ON_ONCE(1);

1534

1535

arch_uprobe_disable_step(&uprobe->arch);

1535

arch_uprobe_disable_step(&uprobe->arch);

1536

put_uprobe(uprobe);

1536

put_uprobe(uprobe);

1537

utask->active_uprobe = NULL;

1537

utask->active_uprobe = NULL;

1538

utask->state = UTASK_RUNNING;

1538

utask->state = UTASK_RUNNING;

1539

xol_free_insn_slot(current);

1539

xol_free_insn_slot(current);

1540

1541

spin_lock_irq(&current->sighand->siglock);

1541

spin_lock_irq(&current->sighand->siglock);

1542

recalc_sigpending(); /* see uprobe_deny_signal() */

1542

recalc_sigpending(); /* see uprobe_deny_signal() */

1543

spin_unlock_irq(&current->sighand->siglock);

1543

spin_unlock_irq(&current->sighand->siglock);

1544

}

1544

}

1545

1546

/*

1546

/*

1547

* On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and

1547

* On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and

1548

* allows the thread to return from interrupt. After that handle_swbp()

1548

* allows the thread to return from interrupt. After that handle_swbp()

1549

* sets utask->active_uprobe.

1549

* sets utask->active_uprobe.

1550

*

1550

*

1551

* On singlestep exception, singlestep notifier sets the TIF_UPROBE flag

1551

* On singlestep exception, singlestep notifier sets the TIF_UPROBE flag

1552

* and allows the thread to return from interrupt.

1552

* and allows the thread to return from interrupt.

1553

*

1553

*

1554

* While returning to userspace, thread notices the TIF_UPROBE flag and calls

1554

* While returning to userspace, thread notices the TIF_UPROBE flag and calls

1555

* uprobe_notify_resume().

1555

* uprobe_notify_resume().

1556

*/

1556

*/

1557

void uprobe_notify_resume(struct pt_regs *regs)

1557

void uprobe_notify_resume(struct pt_regs *regs)

1558

{

1558

{

1559

struct uprobe_task *utask;

1559

struct uprobe_task *utask;

1560

1561

clear_thread_flag(TIF_UPROBE);

1562

1561

utask = current->utask;

1563

utask = current->utask;

1562

if (utask && utask->active_uprobe)

1564

if (utask && utask->active_uprobe)

1563

handle_singlestep(utask, regs);

1565

handle_singlestep(utask, regs);

1564

else

1566

else

1565

handle_swbp(regs);

1567

handle_swbp(regs);

1566

}

1568

}

1567

1569

1568

/*

1570

/*

1569

* uprobe_pre_sstep_notifier gets called from interrupt context as part of

1571

* uprobe_pre_sstep_notifier gets called from interrupt context as part of

1570

* notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.

1572

* notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.

1571

*/

1573

*/

1572

int uprobe_pre_sstep_notifier(struct pt_regs *regs)

1574

int uprobe_pre_sstep_notifier(struct pt_regs *regs)

1573

{

1575

{

1574

if (!current->mm || !test_bit(MMF_HAS_UPROBES, &current->mm->flags))

1576

if (!current->mm || !test_bit(MMF_HAS_UPROBES, &current->mm->flags))

1575

return 0;

1577

return 0;

1576

1578

1577

set_thread_flag(TIF_UPROBE);

1579

set_thread_flag(TIF_UPROBE);

1578

return 1;

1580

return 1;

1579

}

1581

}

1580

1582

1581

/*

1583

/*

1582

* uprobe_post_sstep_notifier gets called in interrupt context as part of notifier

1584

* uprobe_post_sstep_notifier gets called in interrupt context as part of notifier

1583

* mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.

1585

* mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.

1584

*/

1586

*/

1585

int uprobe_post_sstep_notifier(struct pt_regs *regs)

1587

int uprobe_post_sstep_notifier(struct pt_regs *regs)

1586

{

1588

{

1587

struct uprobe_task *utask = current->utask;

1589

struct uprobe_task *utask = current->utask;

1588

1590

1589

if (!current->mm || !utask || !utask->active_uprobe)

1591

if (!current->mm || !utask || !utask->active_uprobe)

1590

/* task is currently not uprobed */

1592

/* task is currently not uprobed */

1591

return 0;

1593

return 0;

1592

1594

1593

utask->state = UTASK_SSTEP_ACK;

1595

utask->state = UTASK_SSTEP_ACK;

1594

set_thread_flag(TIF_UPROBE);

1596

set_thread_flag(TIF_UPROBE);

1595

return 1;

1597

return 1;

1596

}

1598

}

1597

1599

1598

static struct notifier_block uprobe_exception_nb = {

1600

static struct notifier_block uprobe_exception_nb = {

1599

.notifier_call = arch_uprobe_exception_notify,

1601

.notifier_call = arch_uprobe_exception_notify,

1600

.priority = INT_MAX-1, /* notified after kprobes, kgdb */

1602

.priority = INT_MAX-1, /* notified after kprobes, kgdb */

1601

};

1603

};

1602

1604

1603

static int __init init_uprobes(void)

1605

static int __init init_uprobes(void)

1604

{

1606

{

1605

int i;

1607

int i;

1606

1608

1607

for (i = 0; i < UPROBES_HASH_SZ; i++) {

1609

for (i = 0; i < UPROBES_HASH_SZ; i++) {

1608

mutex_init(&uprobes_mutex[i]);

1610

mutex_init(&uprobes_mutex[i]);

1609

mutex_init(&uprobes_mmap_mutex[i]);

1611

mutex_init(&uprobes_mmap_mutex[i]);

1610

}

1612

}

1611

1613

1612

return register_die_notifier(&uprobe_exception_nb);

1614

return register_die_notifier(&uprobe_exception_nb);

1613

}

1615

}

1614

module_init(init_uprobes);

1616

module_init(init_uprobes);

1615

1617

1616

static void __exit exit_uprobes(void)

1618

static void __exit exit_uprobes(void)

1617

{

1619

{

1618

}

1620

}

1619

module_exit(exit_uprobes);

1621

module_exit(exit_uprobes);

1620

1622

GITLAB

uprobes: Move clear_thread_flag(TIF_UPROBE) to uprobe_notify_resume()

 /*
  *  Copyright (C) 1991, 1992  Linus Torvalds
  *  Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
  *
  *  1997-11-28  Modified for POSIX.1b signals by Richard Henderson
  *  2000-06-20  Pentium III FXSR, SSE support by Gareth Hughes
  *  2000-2002   x86-64 support by Andi Kleen
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/smp.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/wait.h>
 #include <linux/tracehook.h>
 #include <linux/unistd.h>
 #include <linux/stddef.h>
 #include <linux/personality.h>
 #include <linux/uaccess.h>
 #include <linux/user-return-notifier.h>
 #include <linux/uprobes.h>
 #include <asm/processor.h>
 #include <asm/ucontext.h>
 #include <asm/i387.h>
 #include <asm/fpu-internal.h>
 #include <asm/vdso.h>
 #include <asm/mce.h>
 #include <asm/sighandling.h>
 #ifdef CONFIG_X86_64
 #include <asm/proto.h>
 #include <asm/ia32_unistd.h>
 #include <asm/sys_ia32.h>
 #endif /* CONFIG_X86_64 */
 #include <asm/syscall.h>
 #include <asm/syscalls.h>
 #include <asm/sigframe.h>
 #ifdef CONFIG_X86_32
 # define FIX_EFLAGS	(__FIX_EFLAGS | X86_EFLAGS_RF)
 #else
 # define FIX_EFLAGS	__FIX_EFLAGS
 #endif
 #define COPY(x)			do {			\
 	get_user_ex(regs->x, &sc->x);			\
 } while (0)
 #define GET_SEG(seg)		({			\
 	unsigned short tmp;				\
 	get_user_ex(tmp, &sc->seg);			\
 	tmp;						\
 })
 #define COPY_SEG(seg)		do {			\
 	regs->seg = GET_SEG(seg);			\
 } while (0)
 #define COPY_SEG_CPL3(seg)	do {			\
 	regs->seg = GET_SEG(seg) | 3;			\
 } while (0)
 int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
 		       unsigned long *pax)
 {
 	void __user *buf;
 	unsigned int tmpflags;
 	unsigned int err = 0;
 	/* Always make any pending restarted system calls return -EINTR */
 	current_thread_info()->restart_block.fn = do_no_restart_syscall;
 	get_user_try {
 #ifdef CONFIG_X86_32
 		set_user_gs(regs, GET_SEG(gs));
 		COPY_SEG(fs);
 		COPY_SEG(es);
 		COPY_SEG(ds);
 #endif /* CONFIG_X86_32 */
 		COPY(di); COPY(si); COPY(bp); COPY(sp); COPY(bx);
 		COPY(dx); COPY(cx); COPY(ip);
 #ifdef CONFIG_X86_64
 		COPY(r8);
 		COPY(r9);
 		COPY(r10);
 		COPY(r11);
 		COPY(r12);
 		COPY(r13);
 		COPY(r14);
 		COPY(r15);
 #endif /* CONFIG_X86_64 */
 #ifdef CONFIG_X86_32
 		COPY_SEG_CPL3(cs);
 		COPY_SEG_CPL3(ss);
 #else /* !CONFIG_X86_32 */
 		/* Kernel saves and restores only the CS segment register on signals,
 		 * which is the bare minimum needed to allow mixed 32/64-bit code.
 		 * App's signal handler can save/restore other segments if needed. */
 		COPY_SEG_CPL3(cs);
 #endif /* CONFIG_X86_32 */
 		get_user_ex(tmpflags, &sc->flags);
 		regs->flags = (regs->flags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS);
 		regs->orig_ax = -1;		/* disable syscall checks */
 		get_user_ex(buf, &sc->fpstate);
 		err |= restore_i387_xstate(buf);
 		get_user_ex(*pax, &sc->ax);
 	} get_user_catch(err);
 	return err;
 }
 int setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
 		     struct pt_regs *regs, unsigned long mask)
 {
 	int err = 0;
 	put_user_try {
 #ifdef CONFIG_X86_32
 		put_user_ex(get_user_gs(regs), (unsigned int __user *)&sc->gs);
 		put_user_ex(regs->fs, (unsigned int __user *)&sc->fs);
 		put_user_ex(regs->es, (unsigned int __user *)&sc->es);
 		put_user_ex(regs->ds, (unsigned int __user *)&sc->ds);
 #endif /* CONFIG_X86_32 */
 		put_user_ex(regs->di, &sc->di);
 		put_user_ex(regs->si, &sc->si);
 		put_user_ex(regs->bp, &sc->bp);
 		put_user_ex(regs->sp, &sc->sp);
 		put_user_ex(regs->bx, &sc->bx);
 		put_user_ex(regs->dx, &sc->dx);
 		put_user_ex(regs->cx, &sc->cx);
 		put_user_ex(regs->ax, &sc->ax);
 #ifdef CONFIG_X86_64
 		put_user_ex(regs->r8, &sc->r8);
 		put_user_ex(regs->r9, &sc->r9);
 		put_user_ex(regs->r10, &sc->r10);
 		put_user_ex(regs->r11, &sc->r11);
 		put_user_ex(regs->r12, &sc->r12);
 		put_user_ex(regs->r13, &sc->r13);
 		put_user_ex(regs->r14, &sc->r14);
 		put_user_ex(regs->r15, &sc->r15);
 #endif /* CONFIG_X86_64 */
 		put_user_ex(current->thread.trap_nr, &sc->trapno);
 		put_user_ex(current->thread.error_code, &sc->err);
 		put_user_ex(regs->ip, &sc->ip);
 #ifdef CONFIG_X86_32
 		put_user_ex(regs->cs, (unsigned int __user *)&sc->cs);
 		put_user_ex(regs->flags, &sc->flags);
 		put_user_ex(regs->sp, &sc->sp_at_signal);
 		put_user_ex(regs->ss, (unsigned int __user *)&sc->ss);
 #else /* !CONFIG_X86_32 */
 		put_user_ex(regs->flags, &sc->flags);
 		put_user_ex(regs->cs, &sc->cs);
 		put_user_ex(0, &sc->gs);
 		put_user_ex(0, &sc->fs);
 #endif /* CONFIG_X86_32 */
 		put_user_ex(fpstate, &sc->fpstate);
 		/* non-iBCS2 extensions.. */
 		put_user_ex(mask, &sc->oldmask);
 		put_user_ex(current->thread.cr2, &sc->cr2);
 	} put_user_catch(err);
 	return err;
 }
 /*
  * Set up a signal frame.
  */
 /*
  * Determine which stack to use..
  */
 static unsigned long align_sigframe(unsigned long sp)
 {
 #ifdef CONFIG_X86_32
 	/*
 	 * Align the stack pointer according to the i386 ABI,
 	 * i.e. so that on function entry ((sp + 4) & 15) == 0.
 	 */
 	sp = ((sp + 4) & -16ul) - 4;
 #else /* !CONFIG_X86_32 */
 	sp = round_down(sp, 16) - 8;
 #endif
 	return sp;
 }
 static inline void __user *
 get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
 	     void __user **fpstate)
 {
 	/* Default to using normal stack */
 	unsigned long sp = regs->sp;
 	int onsigstack = on_sig_stack(sp);
 #ifdef CONFIG_X86_64
 	/* redzone */
 	sp -= 128;
 #endif /* CONFIG_X86_64 */
 	if (!onsigstack) {
 		/* This is the X/Open sanctioned signal stack switching.  */
 		if (ka->sa.sa_flags & SA_ONSTACK) {
 			if (current->sas_ss_size)
 				sp = current->sas_ss_sp + current->sas_ss_size;
 		} else {
 #ifdef CONFIG_X86_32
 			/* This is the legacy signal stack switching. */
 			if ((regs->ss & 0xffff) != __USER_DS &&
 				!(ka->sa.sa_flags & SA_RESTORER) &&
 					ka->sa.sa_restorer)
 				sp = (unsigned long) ka->sa.sa_restorer;
 #endif /* CONFIG_X86_32 */
 		}
 	}
 	if (used_math()) {
 		sp -= sig_xstate_size;
 #ifdef CONFIG_X86_64
 		sp = round_down(sp, 64);
 #endif /* CONFIG_X86_64 */
 		*fpstate = (void __user *)sp;
 	}
 	sp = align_sigframe(sp - frame_size);
 	/*
 	 * If we are on the alternate signal stack and would overflow it, don't.
 	 * Return an always-bogus address instead so we will die with SIGSEGV.
 	 */
 	if (onsigstack && !likely(on_sig_stack(sp)))
 		return (void __user *)-1L;
 	/* save i387 state */
 	if (used_math() && save_i387_xstate(*fpstate) < 0)
 		return (void __user *)-1L;
 	return (void __user *)sp;
 }
 #ifdef CONFIG_X86_32
 static const struct {
 	u16 poplmovl;
 	u32 val;
 	u16 int80;
 } __attribute__((packed)) retcode = {
 	0xb858,		/* popl %eax; movl $..., %eax */
 	__NR_sigreturn,
 	0x80cd,		/* int $0x80 */
 };
 static const struct {
 	u8  movl;
 	u32 val;
 	u16 int80;
 	u8  pad;
 } __attribute__((packed)) rt_retcode = {
 	0xb8,		/* movl $..., %eax */
 	__NR_rt_sigreturn,
 	0x80cd,		/* int $0x80 */
 	0
 };
 static int
 __setup_frame(int sig, struct k_sigaction *ka, sigset_t *set,
 	      struct pt_regs *regs)
 {
 	struct sigframe __user *frame;
 	void __user *restorer;
 	int err = 0;
 	void __user *fpstate = NULL;
 	frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
 	if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
 		return -EFAULT;
 	if (__put_user(sig, &frame->sig))
 		return -EFAULT;
 	if (setup_sigcontext(&frame->sc, fpstate, regs, set->sig[0]))
 		return -EFAULT;
 	if (_NSIG_WORDS > 1) {
 		if (__copy_to_user(&frame->extramask, &set->sig[1],
 				   sizeof(frame->extramask)))
 			return -EFAULT;
 	}
 	if (current->mm->context.vdso)
 		restorer = VDSO32_SYMBOL(current->mm->context.vdso, sigreturn);
 	else
 		restorer = &frame->retcode;
 	if (ka->sa.sa_flags & SA_RESTORER)
 		restorer = ka->sa.sa_restorer;
 	/* Set up to return from userspace.  */
 	err |= __put_user(restorer, &frame->pretcode);
 	/*
 	 * This is popl %eax ; movl $__NR_sigreturn, %eax ; int $0x80
 	 *
 	 * WE DO NOT USE IT ANY MORE! It's only left here for historical
 	 * reasons and because gdb uses it as a signature to notice
 	 * signal handler stack frames.
 	 */
 	err |= __put_user(*((u64 *)&retcode), (u64 *)frame->retcode);
 	if (err)
 		return -EFAULT;
 	/* Set up registers for signal handler */
 	regs->sp = (unsigned long)frame;
 	regs->ip = (unsigned long)ka->sa.sa_handler;
 	regs->ax = (unsigned long)sig;
 	regs->dx = 0;
 	regs->cx = 0;
 	regs->ds = __USER_DS;
 	regs->es = __USER_DS;
 	regs->ss = __USER_DS;
 	regs->cs = __USER_CS;
 	return 0;
 }
 static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 			    sigset_t *set, struct pt_regs *regs)
 {
 	struct rt_sigframe __user *frame;
 	void __user *restorer;
 	int err = 0;
 	void __user *fpstate = NULL;
 	frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
 	if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
 		return -EFAULT;
 	put_user_try {
 		put_user_ex(sig, &frame->sig);
 		put_user_ex(&frame->info, &frame->pinfo);
 		put_user_ex(&frame->uc, &frame->puc);
 		err |= copy_siginfo_to_user(&frame->info, info);
 		/* Create the ucontext.  */
 		if (cpu_has_xsave)
 			put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
 		else
 			put_user_ex(0, &frame->uc.uc_flags);
 		put_user_ex(0, &frame->uc.uc_link);
 		put_user_ex(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
 		put_user_ex(sas_ss_flags(regs->sp),
 			    &frame->uc.uc_stack.ss_flags);
 		put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
 		err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
 					regs, set->sig[0]);
 		err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
 		/* Set up to return from userspace.  */
 		restorer = VDSO32_SYMBOL(current->mm->context.vdso, rt_sigreturn);
 		if (ka->sa.sa_flags & SA_RESTORER)
 			restorer = ka->sa.sa_restorer;
 		put_user_ex(restorer, &frame->pretcode);
 		/*
 		 * This is movl $__NR_rt_sigreturn, %ax ; int $0x80
 		 *
 		 * WE DO NOT USE IT ANY MORE! It's only left here for historical
 		 * reasons and because gdb uses it as a signature to notice
 		 * signal handler stack frames.
 		 */
 		put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode);
 	} put_user_catch(err);
 	if (err)
 		return -EFAULT;
 	/* Set up registers for signal handler */
 	regs->sp = (unsigned long)frame;
 	regs->ip = (unsigned long)ka->sa.sa_handler;
 	regs->ax = (unsigned long)sig;
 	regs->dx = (unsigned long)&frame->info;
 	regs->cx = (unsigned long)&frame->uc;
 	regs->ds = __USER_DS;
 	regs->es = __USER_DS;
 	regs->ss = __USER_DS;
 	regs->cs = __USER_CS;
 	return 0;
 }
 #else /* !CONFIG_X86_32 */
 static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 			    sigset_t *set, struct pt_regs *regs)
 {
 	struct rt_sigframe __user *frame;
 	void __user *fp = NULL;
 	int err = 0;
 	struct task_struct *me = current;
 	frame = get_sigframe(ka, regs, sizeof(struct rt_sigframe), &fp);
 	if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
 		return -EFAULT;
 	if (ka->sa.sa_flags & SA_SIGINFO) {
 		if (copy_siginfo_to_user(&frame->info, info))
 			return -EFAULT;
 	}
 	put_user_try {
 		/* Create the ucontext.  */
 		if (cpu_has_xsave)
 			put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
 		else
 			put_user_ex(0, &frame->uc.uc_flags);
 		put_user_ex(0, &frame->uc.uc_link);
 		put_user_ex(me->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
 		put_user_ex(sas_ss_flags(regs->sp),
 			    &frame->uc.uc_stack.ss_flags);
 		put_user_ex(me->sas_ss_size, &frame->uc.uc_stack.ss_size);
 		err |= setup_sigcontext(&frame->uc.uc_mcontext, fp, regs, set->sig[0]);
 		err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
 		/* Set up to return from userspace.  If provided, use a stub
 		   already in userspace.  */
 		/* x86-64 should always use SA_RESTORER. */
 		if (ka->sa.sa_flags & SA_RESTORER) {
 			put_user_ex(ka->sa.sa_restorer, &frame->pretcode);
 		} else {
 			/* could use a vstub here */
 			err |= -EFAULT;
 		}
 	} put_user_catch(err);
 	if (err)
 		return -EFAULT;
 	/* Set up registers for signal handler */
 	regs->di = sig;
 	/* In case the signal handler was declared without prototypes */
 	regs->ax = 0;
 	/* This also works for non SA_SIGINFO handlers because they expect the
 	   next argument after the signal number on the stack. */
 	regs->si = (unsigned long)&frame->info;
 	regs->dx = (unsigned long)&frame->uc;
 	regs->ip = (unsigned long) ka->sa.sa_handler;
 	regs->sp = (unsigned long)frame;
 	/* Set up the CS register to run signal handlers in 64-bit mode,
 	   even if the handler happens to be interrupting 32-bit code. */
 	regs->cs = __USER_CS;
 	return 0;
 }
 #endif /* CONFIG_X86_32 */
 #ifdef CONFIG_X86_32
 /*
  * Atomically swap in the new signal mask, and wait for a signal.
  */
 asmlinkage int
 sys_sigsuspend(int history0, int history1, old_sigset_t mask)
 {
 	sigset_t blocked;
 	siginitset(&blocked, mask);
 	return sigsuspend(&blocked);
 }
 asmlinkage int
 sys_sigaction(int sig, const struct old_sigaction __user *act,
 	      struct old_sigaction __user *oact)
 {
 	struct k_sigaction new_ka, old_ka;
 	int ret = 0;
 	if (act) {
 		old_sigset_t mask;
 		if (!access_ok(VERIFY_READ, act, sizeof(*act)))
 			return -EFAULT;
 		get_user_try {
 			get_user_ex(new_ka.sa.sa_handler, &act->sa_handler);
 			get_user_ex(new_ka.sa.sa_flags, &act->sa_flags);
 			get_user_ex(mask, &act->sa_mask);
 			get_user_ex(new_ka.sa.sa_restorer, &act->sa_restorer);
 		} get_user_catch(ret);
 		if (ret)
 			return -EFAULT;
 		siginitset(&new_ka.sa.sa_mask, mask);
 	}
 	ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
 	if (!ret && oact) {
 		if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)))
 			return -EFAULT;
 		put_user_try {
 			put_user_ex(old_ka.sa.sa_handler, &oact->sa_handler);
 			put_user_ex(old_ka.sa.sa_flags, &oact->sa_flags);
 			put_user_ex(old_ka.sa.sa_mask.sig[0], &oact->sa_mask);
 			put_user_ex(old_ka.sa.sa_restorer, &oact->sa_restorer);
 		} put_user_catch(ret);
 		if (ret)
 			return -EFAULT;
 	}
 	return ret;
 }
 #endif /* CONFIG_X86_32 */
 long
 sys_sigaltstack(const stack_t __user *uss, stack_t __user *uoss,
 		struct pt_regs *regs)
 {
 	return do_sigaltstack(uss, uoss, regs->sp);
 }
 /*
  * Do a signal return; undo the signal stack.
  */
 #ifdef CONFIG_X86_32
 unsigned long sys_sigreturn(struct pt_regs *regs)
 {
 	struct sigframe __user *frame;
 	unsigned long ax;
 	sigset_t set;
 	frame = (struct sigframe __user *)(regs->sp - 8);
 	if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
 		goto badframe;
 	if (__get_user(set.sig[0], &frame->sc.oldmask) || (_NSIG_WORDS > 1
 		&& __copy_from_user(&set.sig[1], &frame->extramask,
 				    sizeof(frame->extramask))))
 		goto badframe;
 	set_current_blocked(&set);
 	if (restore_sigcontext(regs, &frame->sc, &ax))
 		goto badframe;
 	return ax;
 badframe:
 	signal_fault(regs, frame, "sigreturn");
 	return 0;
 }
 #endif /* CONFIG_X86_32 */
 long sys_rt_sigreturn(struct pt_regs *regs)
 {
 	struct rt_sigframe __user *frame;
 	unsigned long ax;
 	sigset_t set;
 	frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long));
 	if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
 		goto badframe;
 	if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
 		goto badframe;
 	set_current_blocked(&set);
 	if (restore_sigcontext(regs, &frame->uc.uc_mcontext, &ax))
 		goto badframe;
 	if (do_sigaltstack(&frame->uc.uc_stack, NULL, regs->sp) == -EFAULT)
 		goto badframe;
 	return ax;
 badframe:
 	signal_fault(regs, frame, "rt_sigreturn");
 	return 0;
 }
 /*
  * OK, we're invoking a handler:
  */
 static int signr_convert(int sig)
 {
 #ifdef CONFIG_X86_32
 	struct thread_info *info = current_thread_info();
 	if (info->exec_domain && info->exec_domain->signal_invmap && sig < 32)
 		return info->exec_domain->signal_invmap[sig];
 #endif /* CONFIG_X86_32 */
 	return sig;
 }
 #ifdef CONFIG_X86_32
 #define is_ia32	1
 #define ia32_setup_frame	__setup_frame
 #define ia32_setup_rt_frame	__setup_rt_frame
 #else /* !CONFIG_X86_32 */
 #ifdef CONFIG_IA32_EMULATION
 #define is_ia32	test_thread_flag(TIF_IA32)
 #else /* !CONFIG_IA32_EMULATION */
 #define is_ia32	0
 #endif /* CONFIG_IA32_EMULATION */
 #ifdef CONFIG_X86_X32_ABI
 #define is_x32	test_thread_flag(TIF_X32)
 static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
 			      siginfo_t *info, compat_sigset_t *set,
 			      struct pt_regs *regs);
 #else /* !CONFIG_X86_X32_ABI */
 #define is_x32	0
 #endif /* CONFIG_X86_X32_ABI */
 int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 		sigset_t *set, struct pt_regs *regs);
 int ia32_setup_frame(int sig, struct k_sigaction *ka,
 		sigset_t *set, struct pt_regs *regs);
 #endif /* CONFIG_X86_32 */
 static int
 setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
 		struct pt_regs *regs)
 {
 	int usig = signr_convert(sig);
 	sigset_t *set = sigmask_to_save();
 	/* Set up the stack frame */
 	if (is_ia32) {
 		if (ka->sa.sa_flags & SA_SIGINFO)
 			return ia32_setup_rt_frame(usig, ka, info, set, regs);
 		else
 			return ia32_setup_frame(usig, ka, set, regs);
 #ifdef CONFIG_X86_X32_ABI
 	} else if (is_x32) {
 		return x32_setup_rt_frame(usig, ka, info,
 					 (compat_sigset_t *)set, regs);
 #endif
 	} else {
 		return __setup_rt_frame(sig, ka, info, set, regs);
 	}
 }
 static void
 handle_signal(unsigned long sig, siginfo_t *info, struct k_sigaction *ka,
 		struct pt_regs *regs)
 {
 	/* Are we from a system call? */
 	if (syscall_get_nr(current, regs) >= 0) {
 		/* If so, check system call restarting.. */
 		switch (syscall_get_error(current, regs)) {
 		case -ERESTART_RESTARTBLOCK:
 		case -ERESTARTNOHAND:
 			regs->ax = -EINTR;
 			break;
 		case -ERESTARTSYS:
 			if (!(ka->sa.sa_flags & SA_RESTART)) {
 				regs->ax = -EINTR;
 				break;
 			}
 		/* fallthrough */
 		case -ERESTARTNOINTR:
 			regs->ax = regs->orig_ax;
 			regs->ip -= 2;
 			break;
 		}
 	}
 	/*
 	 * If TF is set due to a debugger (TIF_FORCED_TF), clear the TF
 	 * flag so that register information in the sigcontext is correct.
 	 */
 	if (unlikely(regs->flags & X86_EFLAGS_TF) &&
 	    likely(test_and_clear_thread_flag(TIF_FORCED_TF)))
 		regs->flags &= ~X86_EFLAGS_TF;
 	if (setup_rt_frame(sig, ka, info, regs) < 0) {
 		force_sigsegv(sig, current);
 		return;
 	}
 	/*
 	 * Clear the direction flag as per the ABI for function entry.
 	 */
 	regs->flags &= ~X86_EFLAGS_DF;
 	/*
 	 * Clear TF when entering the signal handler, but
 	 * notify any tracer that was single-stepping it.
 	 * The tracer may want to single-step inside the
 	 * handler too.
 	 */
 	regs->flags &= ~X86_EFLAGS_TF;
 	signal_delivered(sig, info, ka, regs,
 			 test_thread_flag(TIF_SINGLESTEP));
 }
 #ifdef CONFIG_X86_32
 #define NR_restart_syscall	__NR_restart_syscall
 #else /* !CONFIG_X86_32 */
 #define NR_restart_syscall	\
 	test_thread_flag(TIF_IA32) ? __NR_ia32_restart_syscall : __NR_restart_syscall
 #endif /* CONFIG_X86_32 */
 /*
  * Note that 'init' is a special process: it doesn't get signals it doesn't
  * want to handle. Thus you cannot kill init even with a SIGKILL even by
  * mistake.
  */
 static void do_signal(struct pt_regs *regs)
 {
 	struct k_sigaction ka;
 	siginfo_t info;
 	int signr;
 	signr = get_signal_to_deliver(&info, &ka, regs, NULL);
 	if (signr > 0) {
 		/* Whee! Actually deliver the signal.  */
 		handle_signal(signr, &info, &ka, regs);
 		return;
 	}
 	/* Did we come from a system call? */
 	if (syscall_get_nr(current, regs) >= 0) {
 		/* Restart the system call - no handlers present */
 		switch (syscall_get_error(current, regs)) {
 		case -ERESTARTNOHAND:
 		case -ERESTARTSYS:
 		case -ERESTARTNOINTR:
 			regs->ax = regs->orig_ax;
 			regs->ip -= 2;
 			break;
 		case -ERESTART_RESTARTBLOCK:
 			regs->ax = NR_restart_syscall;
 			regs->ip -= 2;
 			break;
 		}
 	}
 	/*
 	 * If there's no signal to deliver, we just put the saved sigmask
 	 * back.
 	 */
 	restore_saved_sigmask();
 }
 /*
  * notification of userspace execution resumption
  * - triggered by the TIF_WORK_MASK flags
  */
 void
 do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
 {
 #ifdef CONFIG_X86_MCE
 	/* notify userspace of pending MCEs */
 	if (thread_info_flags & _TIF_MCE_NOTIFY)
 		mce_notify_process();
 #endif /* CONFIG_X86_64 && CONFIG_X86_MCE */
-	if (thread_info_flags & _TIF_UPROBE) {
+	if (thread_info_flags & _TIF_UPROBE)
-		clear_thread_flag(TIF_UPROBE);
 		uprobe_notify_resume(regs);
-	}
 	/* deal with pending signal delivery */
 	if (thread_info_flags & _TIF_SIGPENDING)
 		do_signal(regs);
 	if (thread_info_flags & _TIF_NOTIFY_RESUME) {
 		clear_thread_flag(TIF_NOTIFY_RESUME);
 		tracehook_notify_resume(regs);
 	}
 	if (thread_info_flags & _TIF_USER_RETURN_NOTIFY)
 		fire_user_return_notifiers();
 #ifdef CONFIG_X86_32
 	clear_thread_flag(TIF_IRET);
 #endif /* CONFIG_X86_32 */
 }
 void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
 {
 	struct task_struct *me = current;
 	if (show_unhandled_signals && printk_ratelimit()) {
 		printk("%s"
 		       "%s[%d] bad frame in %s frame:%p ip:%lx sp:%lx orax:%lx",
 		       task_pid_nr(current) > 1 ? KERN_INFO : KERN_EMERG,
 		       me->comm, me->pid, where, frame,
 		       regs->ip, regs->sp, regs->orig_ax);
 		print_vma_addr(" in ", regs->ip);
 		pr_cont("\n");
 	}
 	force_sig(SIGSEGV, me);
 }
 #ifdef CONFIG_X86_X32_ABI
 static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
 			      siginfo_t *info, compat_sigset_t *set,
 			      struct pt_regs *regs)
 {
 	struct rt_sigframe_x32 __user *frame;
 	void __user *restorer;
 	int err = 0;
 	void __user *fpstate = NULL;
 	frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
 	if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
 		return -EFAULT;
 	if (ka->sa.sa_flags & SA_SIGINFO) {
 		if (copy_siginfo_to_user32(&frame->info, info))
 			return -EFAULT;
 	}
 	put_user_try {
 		/* Create the ucontext.  */
 		if (cpu_has_xsave)
 			put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
 		else
 			put_user_ex(0, &frame->uc.uc_flags);
 		put_user_ex(0, &frame->uc.uc_link);
 		put_user_ex(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
 		put_user_ex(sas_ss_flags(regs->sp),
 			    &frame->uc.uc_stack.ss_flags);
 		put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
 		put_user_ex(0, &frame->uc.uc__pad0);
 		err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
 					regs, set->sig[0]);
 		err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
 		if (ka->sa.sa_flags & SA_RESTORER) {
 			restorer = ka->sa.sa_restorer;
 		} else {
 			/* could use a vstub here */
 			restorer = NULL;
 			err |= -EFAULT;
 		}
 		put_user_ex(restorer, &frame->pretcode);
 	} put_user_catch(err);
 	if (err)
 		return -EFAULT;
 	/* Set up registers for signal handler */
 	regs->sp = (unsigned long) frame;
 	regs->ip = (unsigned long) ka->sa.sa_handler;
 	/* We use the x32 calling convention here... */
 	regs->di = sig;
 	regs->si = (unsigned long) &frame->info;
 	regs->dx = (unsigned long) &frame->uc;
 	loadsegment(ds, __USER_DS);
 	loadsegment(es, __USER_DS);
 	regs->cs = __USER_CS;
 	regs->ss = __USER_DS;
 	return 0;
 }
 asmlinkage long sys32_x32_rt_sigreturn(struct pt_regs *regs)
 {
 	struct rt_sigframe_x32 __user *frame;
 	sigset_t set;
 	unsigned long ax;
 	struct pt_regs tregs;
 	frame = (struct rt_sigframe_x32 __user *)(regs->sp - 8);
 	if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
 		goto badframe;
 	if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
 		goto badframe;
 	set_current_blocked(&set);
 	if (restore_sigcontext(regs, &frame->uc.uc_mcontext, &ax))
 		goto badframe;
 	tregs = *regs;
 	if (sys32_sigaltstack(&frame->uc.uc_stack, NULL, &tregs) == -EFAULT)
 		goto badframe;
 	return ax;
 badframe:
 	signal_fault(regs, frame, "x32 rt_sigreturn");
 	return 0;
 }
 #endif

 /*
  * User-space Probes (UProbes)
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  *
  * Copyright (C) IBM Corporation, 2008-2012
  * Authors:
  *	Srikar Dronamraju
  *	Jim Keniston
  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  */
 #include <linux/kernel.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>	/* read_mapping_page */
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/rmap.h>		/* anon_vma_prepare */
 #include <linux/mmu_notifier.h>	/* set_pte_at_notify */
 #include <linux/swap.h>		/* try_to_free_swap */
 #include <linux/ptrace.h>	/* user_enable_single_step */
 #include <linux/kdebug.h>	/* notifier mechanism */
 #include "../../mm/internal.h"	/* munlock_vma_page */
 #include <linux/uprobes.h>
 #define UINSNS_PER_PAGE			(PAGE_SIZE/UPROBE_XOL_SLOT_BYTES)
 #define MAX_UPROBE_XOL_SLOTS		UINSNS_PER_PAGE
 static struct rb_root uprobes_tree = RB_ROOT;
 static DEFINE_SPINLOCK(uprobes_treelock);	/* serialize rbtree access */
 #define UPROBES_HASH_SZ	13
 /*
  * We need separate register/unregister and mmap/munmap lock hashes because
  * of mmap_sem nesting.
  *
  * uprobe_register() needs to install probes on (potentially) all processes
  * and thus needs to acquire multiple mmap_sems (consequtively, not
  * concurrently), whereas uprobe_mmap() is called while holding mmap_sem
  * for the particular process doing the mmap.
  *
  * uprobe_register()->register_for_each_vma() needs to drop/acquire mmap_sem
  * because of lock order against i_mmap_mutex. This means there's a hole in
  * the register vma iteration where a mmap() can happen.
  *
  * Thus uprobe_register() can race with uprobe_mmap() and we can try and
  * install a probe where one is already installed.
  */
 /* serialize (un)register */
 static struct mutex uprobes_mutex[UPROBES_HASH_SZ];
 #define uprobes_hash(v)		(&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
 /* serialize uprobe->pending_list */
 static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ];
 #define uprobes_mmap_hash(v)	(&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
 /*
  * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe
  * events active at this time.  Probably a fine grained per inode count is
  * better?
  */
 static atomic_t uprobe_events = ATOMIC_INIT(0);
 struct uprobe {
 	struct rb_node		rb_node;	/* node in the rb tree */
 	atomic_t		ref;
 	struct rw_semaphore	consumer_rwsem;
 	struct list_head	pending_list;
 	struct uprobe_consumer	*consumers;
 	struct inode		*inode;		/* Also hold a ref to inode */
 	loff_t			offset;
 	int			flags;
 	struct arch_uprobe	arch;
 };
 /*
  * valid_vma: Verify if the specified vma is an executable vma
  * Relax restrictions while unregistering: vm_flags might have
  * changed after breakpoint was inserted.
  *	- is_register: indicates if we are in register context.
  *	- Return 1 if the specified virtual address is in an
  *	  executable vma.
  */
 static bool valid_vma(struct vm_area_struct *vma, bool is_register)
 {
 	if (!vma->vm_file)
 		return false;
 	if (!is_register)
 		return true;
 	if ((vma->vm_flags & (VM_HUGETLB|VM_READ|VM_WRITE|VM_EXEC|VM_SHARED))
 				== (VM_READ|VM_EXEC))
 		return true;
 	return false;
 }
 static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset)
 {
 	return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
 }
 static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr)
 {
 	return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start);
 }
 /**
  * __replace_page - replace page in vma by new page.
  * based on replace_page in mm/ksm.c
  *
  * @vma:      vma that holds the pte pointing to page
  * @addr:     address the old @page is mapped at
  * @page:     the cowed page we are replacing by kpage
  * @kpage:    the modified page we replace page by
  *
  * Returns 0 on success, -EFAULT on failure.
  */
 static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 				struct page *page, struct page *kpage)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	spinlock_t *ptl;
 	pte_t *ptep;
 	int err;
 	/* For try_to_free_swap() and munlock_vma_page() below */
 	lock_page(page);
 	err = -EAGAIN;
 	ptep = page_check_address(page, mm, addr, &ptl, 0);
 	if (!ptep)
 		goto unlock;
 	get_page(kpage);
 	page_add_new_anon_rmap(kpage, vma, addr);
 	if (!PageAnon(page)) {
 		dec_mm_counter(mm, MM_FILEPAGES);
 		inc_mm_counter(mm, MM_ANONPAGES);
 	}
 	flush_cache_page(vma, addr, pte_pfn(*ptep));
 	ptep_clear_flush(vma, addr, ptep);
 	set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
 	page_remove_rmap(page);
 	if (!page_mapped(page))
 		try_to_free_swap(page);
 	pte_unmap_unlock(ptep, ptl);
 	if (vma->vm_flags & VM_LOCKED)
 		munlock_vma_page(page);
 	put_page(page);
 	err = 0;
  unlock:
 	unlock_page(page);
 	return err;
 }
 /**
  * is_swbp_insn - check if instruction is breakpoint instruction.
  * @insn: instruction to be checked.
  * Default implementation of is_swbp_insn
  * Returns true if @insn is a breakpoint instruction.
  */
 bool __weak is_swbp_insn(uprobe_opcode_t *insn)
 {
 	return *insn == UPROBE_SWBP_INSN;
 }
 /*
  * NOTE:
  * Expect the breakpoint instruction to be the smallest size instruction for
  * the architecture. If an arch has variable length instruction and the
  * breakpoint instruction is not of the smallest length instruction
  * supported by that architecture then we need to modify read_opcode /
  * write_opcode accordingly. This would never be a problem for archs that
  * have fixed length instructions.
  */
 /*
  * write_opcode - write the opcode at a given virtual address.
  * @auprobe: arch breakpointing information.
  * @mm: the probed process address space.
  * @vaddr: the virtual address to store the opcode.
  * @opcode: opcode to be written at @vaddr.
  *
  * Called with mm->mmap_sem held (for read and with a reference to
  * mm).
  *
  * For mm @mm, write the opcode at @vaddr.
  * Return 0 (success) or a negative errno.
  */
 static int write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
 			unsigned long vaddr, uprobe_opcode_t opcode)
 {
 	struct page *old_page, *new_page;
 	void *vaddr_old, *vaddr_new;
 	struct vm_area_struct *vma;
 	int ret;
 retry:
 	/* Read the page with vaddr into memory */
 	ret = get_user_pages(NULL, mm, vaddr, 1, 0, 0, &old_page, &vma);
 	if (ret <= 0)
 		return ret;
 	ret = -ENOMEM;
 	new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
 	if (!new_page)
 		goto put_old;
 	__SetPageUptodate(new_page);
 	/* copy the page now that we've got it stable */
 	vaddr_old = kmap_atomic(old_page);
 	vaddr_new = kmap_atomic(new_page);
 	memcpy(vaddr_new, vaddr_old, PAGE_SIZE);
 	memcpy(vaddr_new + (vaddr & ~PAGE_MASK), &opcode, UPROBE_SWBP_INSN_SIZE);
 	kunmap_atomic(vaddr_new);
 	kunmap_atomic(vaddr_old);
 	ret = anon_vma_prepare(vma);
 	if (ret)
 		goto put_new;
 	ret = __replace_page(vma, vaddr, old_page, new_page);
 put_new:
 	page_cache_release(new_page);
 put_old:
 	put_page(old_page);
 	if (unlikely(ret == -EAGAIN))
 		goto retry;
 	return ret;
 }
 /**
  * read_opcode - read the opcode at a given virtual address.
  * @mm: the probed process address space.
  * @vaddr: the virtual address to read the opcode.
  * @opcode: location to store the read opcode.
  *
  * Called with mm->mmap_sem held (for read and with a reference to
  * mm.
  *
  * For mm @mm, read the opcode at @vaddr and store it in @opcode.
  * Return 0 (success) or a negative errno.
  */
 static int read_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t *opcode)
 {
 	struct page *page;
 	void *vaddr_new;
 	int ret;
 	ret = get_user_pages(NULL, mm, vaddr, 1, 0, 1, &page, NULL);
 	if (ret <= 0)
 		return ret;
 	vaddr_new = kmap_atomic(page);
 	vaddr &= ~PAGE_MASK;
 	memcpy(opcode, vaddr_new + vaddr, UPROBE_SWBP_INSN_SIZE);
 	kunmap_atomic(vaddr_new);
 	put_page(page);
 	return 0;
 }
 static int is_swbp_at_addr(struct mm_struct *mm, unsigned long vaddr)
 {
 	uprobe_opcode_t opcode;
 	int result;
 	if (current->mm == mm) {
 		pagefault_disable();
 		result = __copy_from_user_inatomic(&opcode, (void __user*)vaddr,
 								sizeof(opcode));
 		pagefault_enable();
 		if (likely(result == 0))
 			goto out;
 	}
 	result = read_opcode(mm, vaddr, &opcode);
 	if (result)
 		return result;
 out:
 	if (is_swbp_insn(&opcode))
 		return 1;
 	return 0;
 }
 /**
  * set_swbp - store breakpoint at a given address.
  * @auprobe: arch specific probepoint information.
  * @mm: the probed process address space.
  * @vaddr: the virtual address to insert the opcode.
  *
  * For mm @mm, store the breakpoint instruction at @vaddr.
  * Return 0 (success) or a negative errno.
  */
 int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 {
 	int result;
 	/*
 	 * See the comment near uprobes_hash().
 	 */
 	result = is_swbp_at_addr(mm, vaddr);
 	if (result == 1)
 		return 0;
 	if (result)
 		return result;
 	return write_opcode(auprobe, mm, vaddr, UPROBE_SWBP_INSN);
 }
 /**
  * set_orig_insn - Restore the original instruction.
  * @mm: the probed process address space.
  * @auprobe: arch specific probepoint information.
  * @vaddr: the virtual address to insert the opcode.
  *
  * For mm @mm, restore the original opcode (opcode) at @vaddr.
  * Return 0 (success) or a negative errno.
  */
 int __weak
 set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
 {
 	int result;
 	result = is_swbp_at_addr(mm, vaddr);
 	if (!result)
 		return -EINVAL;
 	if (result != 1)
 		return result;
 	return write_opcode(auprobe, mm, vaddr, *(uprobe_opcode_t *)auprobe->insn);
 }
 static int match_uprobe(struct uprobe *l, struct uprobe *r)
 {
 	if (l->inode < r->inode)
 		return -1;
 	if (l->inode > r->inode)
 		return 1;
 	if (l->offset < r->offset)
 		return -1;
 	if (l->offset > r->offset)
 		return 1;
 	return 0;
 }
 static struct uprobe *__find_uprobe(struct inode *inode, loff_t offset)
 {
 	struct uprobe u = { .inode = inode, .offset = offset };
 	struct rb_node *n = uprobes_tree.rb_node;
 	struct uprobe *uprobe;
 	int match;
 	while (n) {
 		uprobe = rb_entry(n, struct uprobe, rb_node);
 		match = match_uprobe(&u, uprobe);
 		if (!match) {
 			atomic_inc(&uprobe->ref);
 			return uprobe;
 		}
 		if (match < 0)
 			n = n->rb_left;
 		else
 			n = n->rb_right;
 	}
 	return NULL;
 }
 /*
  * Find a uprobe corresponding to a given inode:offset
  * Acquires uprobes_treelock
  */
 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
 {
 	struct uprobe *uprobe;
 	spin_lock(&uprobes_treelock);
 	uprobe = __find_uprobe(inode, offset);
 	spin_unlock(&uprobes_treelock);
 	return uprobe;
 }
 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
 {
 	struct rb_node **p = &uprobes_tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct uprobe *u;
 	int match;
 	while (*p) {
 		parent = *p;
 		u = rb_entry(parent, struct uprobe, rb_node);
 		match = match_uprobe(uprobe, u);
 		if (!match) {
 			atomic_inc(&u->ref);
 			return u;
 		}
 		if (match < 0)
 			p = &parent->rb_left;
 		else
 			p = &parent->rb_right;
 	}
 	u = NULL;
 	rb_link_node(&uprobe->rb_node, parent, p);
 	rb_insert_color(&uprobe->rb_node, &uprobes_tree);
 	/* get access + creation ref */
 	atomic_set(&uprobe->ref, 2);
 	return u;
 }
 /*
  * Acquire uprobes_treelock.
  * Matching uprobe already exists in rbtree;
  *	increment (access refcount) and return the matching uprobe.
  *
  * No matching uprobe; insert the uprobe in rb_tree;
  *	get a double refcount (access + creation) and return NULL.
  */
 static struct uprobe *insert_uprobe(struct uprobe *uprobe)
 {
 	struct uprobe *u;
 	spin_lock(&uprobes_treelock);
 	u = __insert_uprobe(uprobe);
 	spin_unlock(&uprobes_treelock);
 	/* For now assume that the instruction need not be single-stepped */
 	uprobe->flags |= UPROBE_SKIP_SSTEP;
 	return u;
 }
 static void put_uprobe(struct uprobe *uprobe)
 {
 	if (atomic_dec_and_test(&uprobe->ref))
 		kfree(uprobe);
 }
 static struct uprobe *alloc_uprobe(struct inode *inode, loff_t offset)
 {
 	struct uprobe *uprobe, *cur_uprobe;
 	uprobe = kzalloc(sizeof(struct uprobe), GFP_KERNEL);
 	if (!uprobe)
 		return NULL;
 	uprobe->inode = igrab(inode);
 	uprobe->offset = offset;
 	init_rwsem(&uprobe->consumer_rwsem);
 	/* add to uprobes_tree, sorted on inode:offset */
 	cur_uprobe = insert_uprobe(uprobe);
 	/* a uprobe exists for this inode:offset combination */
 	if (cur_uprobe) {
 		kfree(uprobe);
 		uprobe = cur_uprobe;
 		iput(inode);
 	} else {
 		atomic_inc(&uprobe_events);
 	}
 	return uprobe;
 }
 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
 {
 	struct uprobe_consumer *uc;
 	if (!(uprobe->flags & UPROBE_RUN_HANDLER))
 		return;
 	down_read(&uprobe->consumer_rwsem);
 	for (uc = uprobe->consumers; uc; uc = uc->next) {
 		if (!uc->filter || uc->filter(uc, current))
 			uc->handler(uc, regs);
 	}
 	up_read(&uprobe->consumer_rwsem);
 }
 /* Returns the previous consumer */
 static struct uprobe_consumer *
 consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
 {
 	down_write(&uprobe->consumer_rwsem);
 	uc->next = uprobe->consumers;
 	uprobe->consumers = uc;
 	up_write(&uprobe->consumer_rwsem);
 	return uc->next;
 }
 /*
  * For uprobe @uprobe, delete the consumer @uc.
  * Return true if the @uc is deleted successfully
  * or return false.
  */
 static bool consumer_del(struct uprobe *uprobe, struct uprobe_consumer *uc)
 {
 	struct uprobe_consumer **con;
 	bool ret = false;
 	down_write(&uprobe->consumer_rwsem);
 	for (con = &uprobe->consumers; *con; con = &(*con)->next) {
 		if (*con == uc) {
 			*con = uc->next;
 			ret = true;
 			break;
 		}
 	}
 	up_write(&uprobe->consumer_rwsem);
 	return ret;
 }
 static int
 __copy_insn(struct address_space *mapping, struct file *filp, char *insn,
 			unsigned long nbytes, loff_t offset)
 {
 	struct page *page;
 	void *vaddr;
 	unsigned long off;
 	pgoff_t idx;
 	if (!filp)
 		return -EINVAL;
 	if (!mapping->a_ops->readpage)
 		return -EIO;
 	idx = offset >> PAGE_CACHE_SHIFT;
 	off = offset & ~PAGE_MASK;
 	/*
 	 * Ensure that the page that has the original instruction is
 	 * populated and in page-cache.
 	 */
 	page = read_mapping_page(mapping, idx, filp);
 	if (IS_ERR(page))
 		return PTR_ERR(page);
 	vaddr = kmap_atomic(page);
 	memcpy(insn, vaddr + off, nbytes);
 	kunmap_atomic(vaddr);
 	page_cache_release(page);
 	return 0;
 }
 static int copy_insn(struct uprobe *uprobe, struct file *filp)
 {
 	struct address_space *mapping;
 	unsigned long nbytes;
 	int bytes;
 	nbytes = PAGE_SIZE - (uprobe->offset & ~PAGE_MASK);
 	mapping = uprobe->inode->i_mapping;
 	/* Instruction at end of binary; copy only available bytes */
 	if (uprobe->offset + MAX_UINSN_BYTES > uprobe->inode->i_size)
 		bytes = uprobe->inode->i_size - uprobe->offset;
 	else
 		bytes = MAX_UINSN_BYTES;
 	/* Instruction at the page-boundary; copy bytes in second page */
 	if (nbytes < bytes) {
 		int err = __copy_insn(mapping, filp, uprobe->arch.insn + nbytes,
 				bytes - nbytes, uprobe->offset + nbytes);
 		if (err)
 			return err;
 		bytes = nbytes;
 	}
 	return __copy_insn(mapping, filp, uprobe->arch.insn, bytes, uprobe->offset);
 }
 /*
  * How mm->uprobes_state.count gets updated
  * uprobe_mmap() increments the count if
  * 	- it successfully adds a breakpoint.
  * 	- it cannot add a breakpoint, but sees that there is a underlying
  * 	  breakpoint (via a is_swbp_at_addr()).
  *
  * uprobe_munmap() decrements the count if
  * 	- it sees a underlying breakpoint, (via is_swbp_at_addr)
  * 	  (Subsequent uprobe_unregister wouldnt find the breakpoint
  * 	  unless a uprobe_mmap kicks in, since the old vma would be
  * 	  dropped just after uprobe_munmap.)
  *
  * uprobe_register increments the count if:
  * 	- it successfully adds a breakpoint.
  *
  * uprobe_unregister decrements the count if:
  * 	- it sees a underlying breakpoint and removes successfully.
  * 	  (via is_swbp_at_addr)
  * 	  (Subsequent uprobe_munmap wouldnt find the breakpoint
  * 	  since there is no underlying breakpoint after the
  * 	  breakpoint removal.)
  */
 static int
 install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
 			struct vm_area_struct *vma, unsigned long vaddr)
 {
 	bool first_uprobe;
 	int ret;
 	/*
 	 * If probe is being deleted, unregister thread could be done with
 	 * the vma-rmap-walk through. Adding a probe now can be fatal since
 	 * nobody will be able to cleanup. Also we could be from fork or
 	 * mremap path, where the probe might have already been inserted.
 	 * Hence behave as if probe already existed.
 	 */
 	if (!uprobe->consumers)
 		return 0;
 	if (!(uprobe->flags & UPROBE_COPY_INSN)) {
 		ret = copy_insn(uprobe, vma->vm_file);
 		if (ret)
 			return ret;
 		if (is_swbp_insn((uprobe_opcode_t *)uprobe->arch.insn))
 			return -ENOTSUPP;
 		ret = arch_uprobe_analyze_insn(&uprobe->arch, mm, vaddr);
 		if (ret)
 			return ret;
 		/* write_opcode() assumes we don't cross page boundary */
 		BUG_ON((uprobe->offset & ~PAGE_MASK) +
 				UPROBE_SWBP_INSN_SIZE > PAGE_SIZE);
 		uprobe->flags |= UPROBE_COPY_INSN;
 	}
 	/*
 	 * set MMF_HAS_UPROBES in advance for uprobe_pre_sstep_notifier(),
 	 * the task can hit this breakpoint right after __replace_page().
 	 */
 	first_uprobe = !test_bit(MMF_HAS_UPROBES, &mm->flags);
 	if (first_uprobe)
 		set_bit(MMF_HAS_UPROBES, &mm->flags);
 	ret = set_swbp(&uprobe->arch, mm, vaddr);
 	if (!ret)
 		clear_bit(MMF_RECALC_UPROBES, &mm->flags);
 	else if (first_uprobe)
 		clear_bit(MMF_HAS_UPROBES, &mm->flags);
 	return ret;
 }
 static void
 remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
 {
 	/* can happen if uprobe_register() fails */
 	if (!test_bit(MMF_HAS_UPROBES, &mm->flags))
 		return;
 	set_bit(MMF_RECALC_UPROBES, &mm->flags);
 	set_orig_insn(&uprobe->arch, mm, vaddr);
 }
 /*
  * There could be threads that have already hit the breakpoint. They
  * will recheck the current insn and restart if find_uprobe() fails.
  * See find_active_uprobe().
  */
 static void delete_uprobe(struct uprobe *uprobe)
 {
 	spin_lock(&uprobes_treelock);
 	rb_erase(&uprobe->rb_node, &uprobes_tree);
 	spin_unlock(&uprobes_treelock);
 	iput(uprobe->inode);
 	put_uprobe(uprobe);
 	atomic_dec(&uprobe_events);
 }
 struct map_info {
 	struct map_info *next;
 	struct mm_struct *mm;
 	unsigned long vaddr;
 };
 static inline struct map_info *free_map_info(struct map_info *info)
 {
 	struct map_info *next = info->next;
 	kfree(info);
 	return next;
 }
 static struct map_info *
 build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
 {
 	unsigned long pgoff = offset >> PAGE_SHIFT;
 	struct prio_tree_iter iter;
 	struct vm_area_struct *vma;
 	struct map_info *curr = NULL;
 	struct map_info *prev = NULL;
 	struct map_info *info;
 	int more = 0;
  again:
 	mutex_lock(&mapping->i_mmap_mutex);
 	vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
 		if (!valid_vma(vma, is_register))
 			continue;
 		if (!prev && !more) {
 			/*
 			 * Needs GFP_NOWAIT to avoid i_mmap_mutex recursion through
 			 * reclaim. This is optimistic, no harm done if it fails.
 			 */
 			prev = kmalloc(sizeof(struct map_info),
 					GFP_NOWAIT | __GFP_NOMEMALLOC | __GFP_NOWARN);
 			if (prev)
 				prev->next = NULL;
 		}
 		if (!prev) {
 			more++;
 			continue;
 		}
 		if (!atomic_inc_not_zero(&vma->vm_mm->mm_users))
 			continue;
 		info = prev;
 		prev = prev->next;
 		info->next = curr;
 		curr = info;
 		info->mm = vma->vm_mm;
 		info->vaddr = offset_to_vaddr(vma, offset);
 	}
 	mutex_unlock(&mapping->i_mmap_mutex);
 	if (!more)
 		goto out;
 	prev = curr;
 	while (curr) {
 		mmput(curr->mm);
 		curr = curr->next;
 	}
 	do {
 		info = kmalloc(sizeof(struct map_info), GFP_KERNEL);
 		if (!info) {
 			curr = ERR_PTR(-ENOMEM);
 			goto out;
 		}
 		info->next = prev;
 		prev = info;
 	} while (--more);
 	goto again;
  out:
 	while (prev)
 		prev = free_map_info(prev);
 	return curr;
 }
 static int register_for_each_vma(struct uprobe *uprobe, bool is_register)
 {
 	struct map_info *info;
 	int err = 0;
 	info = build_map_info(uprobe->inode->i_mapping,
 					uprobe->offset, is_register);
 	if (IS_ERR(info))
 		return PTR_ERR(info);
 	while (info) {
 		struct mm_struct *mm = info->mm;
 		struct vm_area_struct *vma;
 		if (err)
 			goto free;
 		down_write(&mm->mmap_sem);
 		vma = find_vma(mm, info->vaddr);
 		if (!vma || !valid_vma(vma, is_register) ||
 		    vma->vm_file->f_mapping->host != uprobe->inode)
 			goto unlock;
 		if (vma->vm_start > info->vaddr ||
 		    vaddr_to_offset(vma, info->vaddr) != uprobe->offset)
 			goto unlock;
 		if (is_register)
 			err = install_breakpoint(uprobe, mm, vma, info->vaddr);
 		else
 			remove_breakpoint(uprobe, mm, info->vaddr);
  unlock:
 		up_write(&mm->mmap_sem);
  free:
 		mmput(mm);
 		info = free_map_info(info);
 	}
 	return err;
 }
 static int __uprobe_register(struct uprobe *uprobe)
 {
 	return register_for_each_vma(uprobe, true);
 }
 static void __uprobe_unregister(struct uprobe *uprobe)
 {
 	if (!register_for_each_vma(uprobe, false))
 		delete_uprobe(uprobe);
 	/* TODO : cant unregister? schedule a worker thread */
 }
 /*
  * uprobe_register - register a probe
  * @inode: the file in which the probe has to be placed.
  * @offset: offset from the start of the file.
  * @uc: information on howto handle the probe..
  *
  * Apart from the access refcount, uprobe_register() takes a creation
  * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
  * inserted into the rbtree (i.e first consumer for a @inode:@offset
  * tuple).  Creation refcount stops uprobe_unregister from freeing the
  * @uprobe even before the register operation is complete. Creation
  * refcount is released when the last @uc for the @uprobe
  * unregisters.
  *
  * Return errno if it cannot successully install probes
  * else return 0 (success)
  */
 int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 {
 	struct uprobe *uprobe;
 	int ret;
 	if (!inode || !uc || uc->next)
 		return -EINVAL;
 	if (offset > i_size_read(inode))
 		return -EINVAL;
 	ret = 0;
 	mutex_lock(uprobes_hash(inode));
 	uprobe = alloc_uprobe(inode, offset);
 	if (uprobe && !consumer_add(uprobe, uc)) {
 		ret = __uprobe_register(uprobe);
 		if (ret) {
 			uprobe->consumers = NULL;
 			__uprobe_unregister(uprobe);
 		} else {
 			uprobe->flags |= UPROBE_RUN_HANDLER;
 		}
 	}
 	mutex_unlock(uprobes_hash(inode));
 	if (uprobe)
 		put_uprobe(uprobe);
 	return ret;
 }
 /*
  * uprobe_unregister - unregister a already registered probe.
  * @inode: the file in which the probe has to be removed.
  * @offset: offset from the start of the file.
  * @uc: identify which probe if multiple probes are colocated.
  */
 void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 {
 	struct uprobe *uprobe;
 	if (!inode || !uc)
 		return;
 	uprobe = find_uprobe(inode, offset);
 	if (!uprobe)
 		return;
 	mutex_lock(uprobes_hash(inode));
 	if (consumer_del(uprobe, uc)) {
 		if (!uprobe->consumers) {
 			__uprobe_unregister(uprobe);
 			uprobe->flags &= ~UPROBE_RUN_HANDLER;
 		}
 	}
 	mutex_unlock(uprobes_hash(inode));
 	if (uprobe)
 		put_uprobe(uprobe);
 }
 static struct rb_node *
 find_node_in_range(struct inode *inode, loff_t min, loff_t max)
 {
 	struct rb_node *n = uprobes_tree.rb_node;
 	while (n) {
 		struct uprobe *u = rb_entry(n, struct uprobe, rb_node);
 		if (inode < u->inode) {
 			n = n->rb_left;
 		} else if (inode > u->inode) {
 			n = n->rb_right;
 		} else {
 			if (max < u->offset)
 				n = n->rb_left;
 			else if (min > u->offset)
 				n = n->rb_right;
 			else
 				break;
 		}
 	}
 	return n;
 }
 /*
  * For a given range in vma, build a list of probes that need to be inserted.
  */
 static void build_probe_list(struct inode *inode,
 				struct vm_area_struct *vma,
 				unsigned long start, unsigned long end,
 				struct list_head *head)
 {
 	loff_t min, max;
 	struct rb_node *n, *t;
 	struct uprobe *u;
 	INIT_LIST_HEAD(head);
 	min = vaddr_to_offset(vma, start);
 	max = min + (end - start) - 1;
 	spin_lock(&uprobes_treelock);
 	n = find_node_in_range(inode, min, max);
 	if (n) {
 		for (t = n; t; t = rb_prev(t)) {
 			u = rb_entry(t, struct uprobe, rb_node);
 			if (u->inode != inode || u->offset < min)
 				break;
 			list_add(&u->pending_list, head);
 			atomic_inc(&u->ref);
 		}
 		for (t = n; (t = rb_next(t)); ) {
 			u = rb_entry(t, struct uprobe, rb_node);
 			if (u->inode != inode || u->offset > max)
 				break;
 			list_add(&u->pending_list, head);
 			atomic_inc(&u->ref);
 		}
 	}
 	spin_unlock(&uprobes_treelock);
 }
 /*
  * Called from mmap_region/vma_adjust with mm->mmap_sem acquired.
  *
  * Currently we ignore all errors and always return 0, the callers
  * can't handle the failure anyway.
  */
 int uprobe_mmap(struct vm_area_struct *vma)
 {
 	struct list_head tmp_list;
 	struct uprobe *uprobe, *u;
 	struct inode *inode;
 	if (!atomic_read(&uprobe_events) || !valid_vma(vma, true))
 		return 0;
 	inode = vma->vm_file->f_mapping->host;
 	if (!inode)
 		return 0;
 	mutex_lock(uprobes_mmap_hash(inode));
 	build_probe_list(inode, vma, vma->vm_start, vma->vm_end, &tmp_list);
 	list_for_each_entry_safe(uprobe, u, &tmp_list, pending_list) {
 		if (!fatal_signal_pending(current)) {
 			unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
 			install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
 		}
 		put_uprobe(uprobe);
 	}
 	mutex_unlock(uprobes_mmap_hash(inode));
 	return 0;
 }
 static bool
 vma_has_uprobes(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
 	loff_t min, max;
 	struct inode *inode;
 	struct rb_node *n;
 	inode = vma->vm_file->f_mapping->host;
 	min = vaddr_to_offset(vma, start);
 	max = min + (end - start) - 1;
 	spin_lock(&uprobes_treelock);
 	n = find_node_in_range(inode, min, max);
 	spin_unlock(&uprobes_treelock);
 	return !!n;
 }
 /*
  * Called in context of a munmap of a vma.
  */
 void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
 	if (!atomic_read(&uprobe_events) || !valid_vma(vma, false))
 		return;
 	if (!atomic_read(&vma->vm_mm->mm_users)) /* called by mmput() ? */
 		return;
 	if (!test_bit(MMF_HAS_UPROBES, &vma->vm_mm->flags) ||
 	     test_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags))
 		return;
 	if (vma_has_uprobes(vma, start, end))
 		set_bit(MMF_RECALC_UPROBES, &vma->vm_mm->flags);
 }
 /* Slot allocation for XOL */
 static int xol_add_vma(struct xol_area *area)
 {
 	struct mm_struct *mm;
 	int ret;
 	area->page = alloc_page(GFP_HIGHUSER);
 	if (!area->page)
 		return -ENOMEM;
 	ret = -EALREADY;
 	mm = current->mm;
 	down_write(&mm->mmap_sem);
 	if (mm->uprobes_state.xol_area)
 		goto fail;
 	ret = -ENOMEM;
 	/* Try to map as high as possible, this is only a hint. */
 	area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0);
 	if (area->vaddr & ~PAGE_MASK) {
 		ret = area->vaddr;
 		goto fail;
 	}
 	ret = install_special_mapping(mm, area->vaddr, PAGE_SIZE,
 				VM_EXEC|VM_MAYEXEC|VM_DONTCOPY|VM_IO, &area->page);
 	if (ret)
 		goto fail;
 	smp_wmb();	/* pairs with get_xol_area() */
 	mm->uprobes_state.xol_area = area;
 	ret = 0;
 fail:
 	up_write(&mm->mmap_sem);
 	if (ret)
 		__free_page(area->page);
 	return ret;
 }
 static struct xol_area *get_xol_area(struct mm_struct *mm)
 {
 	struct xol_area *area;
 	area = mm->uprobes_state.xol_area;
 	smp_read_barrier_depends();	/* pairs with wmb in xol_add_vma() */
 	return area;
 }
 /*
  * xol_alloc_area - Allocate process's xol_area.
  * This area will be used for storing instructions for execution out of
  * line.
  *
  * Returns the allocated area or NULL.
  */
 static struct xol_area *xol_alloc_area(void)
 {
 	struct xol_area *area;
 	area = kzalloc(sizeof(*area), GFP_KERNEL);
 	if (unlikely(!area))
 		return NULL;
 	area->bitmap = kzalloc(BITS_TO_LONGS(UINSNS_PER_PAGE) * sizeof(long), GFP_KERNEL);
 	if (!area->bitmap)
 		goto fail;
 	init_waitqueue_head(&area->wq);
 	if (!xol_add_vma(area))
 		return area;
 fail:
 	kfree(area->bitmap);
 	kfree(area);
 	return get_xol_area(current->mm);
 }
 /*
  * uprobe_clear_state - Free the area allocated for slots.
  */
 void uprobe_clear_state(struct mm_struct *mm)
 {
 	struct xol_area *area = mm->uprobes_state.xol_area;
 	if (!area)
 		return;
 	put_page(area->page);
 	kfree(area->bitmap);
 	kfree(area);
 }
 void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm)
 {
 	newmm->uprobes_state.xol_area = NULL;
 	if (test_bit(MMF_HAS_UPROBES, &oldmm->flags)) {
 		set_bit(MMF_HAS_UPROBES, &newmm->flags);
 		/* unconditionally, dup_mmap() skips VM_DONTCOPY vmas */
 		set_bit(MMF_RECALC_UPROBES, &newmm->flags);
 	}
 }
 /*
  *  - search for a free slot.
  */
 static unsigned long xol_take_insn_slot(struct xol_area *area)
 {
 	unsigned long slot_addr;
 	int slot_nr;
 	do {
 		slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);
 		if (slot_nr < UINSNS_PER_PAGE) {
 			if (!test_and_set_bit(slot_nr, area->bitmap))
 				break;
 			slot_nr = UINSNS_PER_PAGE;
 			continue;
 		}
 		wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));
 	} while (slot_nr >= UINSNS_PER_PAGE);
 	slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);
 	atomic_inc(&area->slot_count);
 	return slot_addr;
 }
 /*
  * xol_get_insn_slot - If was not allocated a slot, then
  * allocate a slot.
  * Returns the allocated slot address or 0.
  */
 static unsigned long xol_get_insn_slot(struct uprobe *uprobe, unsigned long slot_addr)
 {
 	struct xol_area *area;
 	unsigned long offset;
 	void *vaddr;
 	area = get_xol_area(current->mm);
 	if (!area) {
 		area = xol_alloc_area();
 		if (!area)
 			return 0;
 	}
 	current->utask->xol_vaddr = xol_take_insn_slot(area);
 	/*
 	 * Initialize the slot if xol_vaddr points to valid
 	 * instruction slot.
 	 */
 	if (unlikely(!current->utask->xol_vaddr))
 		return 0;
 	current->utask->vaddr = slot_addr;
 	offset = current->utask->xol_vaddr & ~PAGE_MASK;
 	vaddr = kmap_atomic(area->page);
 	memcpy(vaddr + offset, uprobe->arch.insn, MAX_UINSN_BYTES);
 	kunmap_atomic(vaddr);
 	return current->utask->xol_vaddr;
 }
 /*
  * xol_free_insn_slot - If slot was earlier allocated by
  * @xol_get_insn_slot(), make the slot available for
  * subsequent requests.
  */
 static void xol_free_insn_slot(struct task_struct *tsk)
 {
 	struct xol_area *area;
 	unsigned long vma_end;
 	unsigned long slot_addr;
 	if (!tsk->mm || !tsk->mm->uprobes_state.xol_area || !tsk->utask)
 		return;
 	slot_addr = tsk->utask->xol_vaddr;
 	if (unlikely(!slot_addr || IS_ERR_VALUE(slot_addr)))
 		return;
 	area = tsk->mm->uprobes_state.xol_area;
 	vma_end = area->vaddr + PAGE_SIZE;
 	if (area->vaddr <= slot_addr && slot_addr < vma_end) {
 		unsigned long offset;
 		int slot_nr;
 		offset = slot_addr - area->vaddr;
 		slot_nr = offset / UPROBE_XOL_SLOT_BYTES;
 		if (slot_nr >= UINSNS_PER_PAGE)
 			return;
 		clear_bit(slot_nr, area->bitmap);
 		atomic_dec(&area->slot_count);
 		if (waitqueue_active(&area->wq))
 			wake_up(&area->wq);
 		tsk->utask->xol_vaddr = 0;
 	}
 }
 /**
  * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs
  * @regs: Reflects the saved state of the task after it has hit a breakpoint
  * instruction.
  * Return the address of the breakpoint instruction.
  */
 unsigned long __weak uprobe_get_swbp_addr(struct pt_regs *regs)
 {
 	return instruction_pointer(regs) - UPROBE_SWBP_INSN_SIZE;
 }
 /*
  * Called with no locks held.
  * Called in context of a exiting or a exec-ing thread.
  */
 void uprobe_free_utask(struct task_struct *t)
 {
 	struct uprobe_task *utask = t->utask;
 	if (!utask)
 		return;
 	if (utask->active_uprobe)
 		put_uprobe(utask->active_uprobe);
 	xol_free_insn_slot(t);
 	kfree(utask);
 	t->utask = NULL;
 }
 /*
  * Called in context of a new clone/fork from copy_process.
  */
 void uprobe_copy_process(struct task_struct *t)
 {
 	t->utask = NULL;
 }
 /*
  * Allocate a uprobe_task object for the task.
  * Called when the thread hits a breakpoint for the first time.
  *
  * Returns:
  * - pointer to new uprobe_task on success
  * - NULL otherwise
  */
 static struct uprobe_task *add_utask(void)
 {
 	struct uprobe_task *utask;
 	utask = kzalloc(sizeof *utask, GFP_KERNEL);
 	if (unlikely(!utask))
 		return NULL;
 	current->utask = utask;
 	return utask;
 }
 /* Prepare to single-step probed instruction out of line. */
 static int
 pre_ssout(struct uprobe *uprobe, struct pt_regs *regs, unsigned long vaddr)
 {
 	if (xol_get_insn_slot(uprobe, vaddr) && !arch_uprobe_pre_xol(&uprobe->arch, regs))
 		return 0;
 	return -EFAULT;
 }
 /*
  * If we are singlestepping, then ensure this thread is not connected to
  * non-fatal signals until completion of singlestep.  When xol insn itself
  * triggers the signal,  restart the original insn even if the task is
  * already SIGKILL'ed (since coredump should report the correct ip).  This
  * is even more important if the task has a handler for SIGSEGV/etc, The
  * _same_ instruction should be repeated again after return from the signal
  * handler, and SSTEP can never finish in this case.
  */
 bool uprobe_deny_signal(void)
 {
 	struct task_struct *t = current;
 	struct uprobe_task *utask = t->utask;
 	if (likely(!utask || !utask->active_uprobe))
 		return false;
 	WARN_ON_ONCE(utask->state != UTASK_SSTEP);
 	if (signal_pending(t)) {
 		spin_lock_irq(&t->sighand->siglock);
 		clear_tsk_thread_flag(t, TIF_SIGPENDING);
 		spin_unlock_irq(&t->sighand->siglock);
 		if (__fatal_signal_pending(t) || arch_uprobe_xol_was_trapped(t)) {
 			utask->state = UTASK_SSTEP_TRAPPED;
 			set_tsk_thread_flag(t, TIF_UPROBE);
 			set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
 		}
 	}
 	return true;
 }
 /*
  * Avoid singlestepping the original instruction if the original instruction
  * is a NOP or can be emulated.
  */
 static bool can_skip_sstep(struct uprobe *uprobe, struct pt_regs *regs)
 {
 	if (uprobe->flags & UPROBE_SKIP_SSTEP) {
 		if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
 			return true;
 		uprobe->flags &= ~UPROBE_SKIP_SSTEP;
 	}
 	return false;
 }
 static void mmf_recalc_uprobes(struct mm_struct *mm)
 {
 	struct vm_area_struct *vma;
 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
 		if (!valid_vma(vma, false))
 			continue;
 		/*
 		 * This is not strictly accurate, we can race with
 		 * uprobe_unregister() and see the already removed
 		 * uprobe if delete_uprobe() was not yet called.
 		 */
 		if (vma_has_uprobes(vma, vma->vm_start, vma->vm_end))
 			return;
 	}
 	clear_bit(MMF_HAS_UPROBES, &mm->flags);
 }
 static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
 {
 	struct mm_struct *mm = current->mm;
 	struct uprobe *uprobe = NULL;
 	struct vm_area_struct *vma;
 	down_read(&mm->mmap_sem);
 	vma = find_vma(mm, bp_vaddr);
 	if (vma && vma->vm_start <= bp_vaddr) {
 		if (valid_vma(vma, false)) {
 			struct inode *inode = vma->vm_file->f_mapping->host;
 			loff_t offset = vaddr_to_offset(vma, bp_vaddr);
 			uprobe = find_uprobe(inode, offset);
 		}
 		if (!uprobe)
 			*is_swbp = is_swbp_at_addr(mm, bp_vaddr);
 	} else {
 		*is_swbp = -EFAULT;
 	}
 	if (!uprobe && test_and_clear_bit(MMF_RECALC_UPROBES, &mm->flags))
 		mmf_recalc_uprobes(mm);
 	up_read(&mm->mmap_sem);
 	return uprobe;
 }
 void __weak arch_uprobe_enable_step(struct arch_uprobe *arch)
 {
 	user_enable_single_step(current);
 }
 void __weak arch_uprobe_disable_step(struct arch_uprobe *arch)
 {
 	user_disable_single_step(current);
 }
 /*
  * Run handler and ask thread to singlestep.
  * Ensure all non-fatal signals cannot interrupt thread while it singlesteps.
  */
 static void handle_swbp(struct pt_regs *regs)
 {
 	struct uprobe_task *utask;
 	struct uprobe *uprobe;
 	unsigned long bp_vaddr;
 	int uninitialized_var(is_swbp);
 	bp_vaddr = uprobe_get_swbp_addr(regs);
 	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
 	if (!uprobe) {
 		if (is_swbp > 0) {
 			/* No matching uprobe; signal SIGTRAP. */
 			send_sig(SIGTRAP, current, 0);
 		} else {
 			/*
 			 * Either we raced with uprobe_unregister() or we can't
 			 * access this memory. The latter is only possible if
 			 * another thread plays with our ->mm. In both cases
 			 * we can simply restart. If this vma was unmapped we
 			 * can pretend this insn was not executed yet and get
 			 * the (correct) SIGSEGV after restart.
 			 */
 			instruction_pointer_set(regs, bp_vaddr);
 		}
 		return;
 	}
 	utask = current->utask;
 	if (!utask) {
 		utask = add_utask();
 		/* Cannot allocate; re-execute the instruction. */
 		if (!utask)
 			goto restart;
 	}
 	handler_chain(uprobe, regs);
 	if (can_skip_sstep(uprobe, regs))
 		goto out;
 	if (!pre_ssout(uprobe, regs, bp_vaddr)) {
 		arch_uprobe_enable_step(&uprobe->arch);
 		utask->active_uprobe = uprobe;
 		utask->state = UTASK_SSTEP;
 		return;
 	}
 restart:
 	/*
 	 * cannot singlestep; cannot skip instruction;
 	 * re-execute the instruction.
 	 */
 	instruction_pointer_set(regs, bp_vaddr);
 out:
 	put_uprobe(uprobe);
 }
 /*
  * Perform required fix-ups and disable singlestep.
  * Allow pending signals to take effect.
  */
 static void handle_singlestep(struct uprobe_task *utask, struct pt_regs *regs)
 {
 	struct uprobe *uprobe;
 	uprobe = utask->active_uprobe;
 	if (utask->state == UTASK_SSTEP_ACK)
 		arch_uprobe_post_xol(&uprobe->arch, regs);
 	else if (utask->state == UTASK_SSTEP_TRAPPED)
 		arch_uprobe_abort_xol(&uprobe->arch, regs);
 	else
 		WARN_ON_ONCE(1);
 	arch_uprobe_disable_step(&uprobe->arch);
 	put_uprobe(uprobe);
 	utask->active_uprobe = NULL;
 	utask->state = UTASK_RUNNING;
 	xol_free_insn_slot(current);
 	spin_lock_irq(&current->sighand->siglock);
 	recalc_sigpending(); /* see uprobe_deny_signal() */
 	spin_unlock_irq(&current->sighand->siglock);
 }
 /*
  * On breakpoint hit, breakpoint notifier sets the TIF_UPROBE flag and
  * allows the thread to return from interrupt. After that handle_swbp()
  * sets utask->active_uprobe.
  *
  * On singlestep exception, singlestep notifier sets the TIF_UPROBE flag
  * and allows the thread to return from interrupt.
  *
  * While returning to userspace, thread notices the TIF_UPROBE flag and calls
  * uprobe_notify_resume().
  */
 void uprobe_notify_resume(struct pt_regs *regs)
 {
 	struct uprobe_task *utask;
+	clear_thread_flag(TIF_UPROBE);
 	utask = current->utask;
 	if (utask && utask->active_uprobe)
 		handle_singlestep(utask, regs);
 	else
 		handle_swbp(regs);
 }
 /*
  * uprobe_pre_sstep_notifier gets called from interrupt context as part of
  * notifier mechanism. Set TIF_UPROBE flag and indicate breakpoint hit.
  */
 int uprobe_pre_sstep_notifier(struct pt_regs *regs)
 {
 	if (!current->mm || !test_bit(MMF_HAS_UPROBES, &current->mm->flags))
 		return 0;
 	set_thread_flag(TIF_UPROBE);
 	return 1;
 }
 /*
  * uprobe_post_sstep_notifier gets called in interrupt context as part of notifier
  * mechanism. Set TIF_UPROBE flag and indicate completion of singlestep.
  */
 int uprobe_post_sstep_notifier(struct pt_regs *regs)
 {
 	struct uprobe_task *utask = current->utask;
 	if (!current->mm || !utask || !utask->active_uprobe)
 		/* task is currently not uprobed */
 		return 0;
 	utask->state = UTASK_SSTEP_ACK;
 	set_thread_flag(TIF_UPROBE);
 	return 1;
 }
 static struct notifier_block uprobe_exception_nb = {
 	.notifier_call		= arch_uprobe_exception_notify,
 	.priority		= INT_MAX-1,	/* notified after kprobes, kgdb */
 };
 static int __init init_uprobes(void)
 {
 	int i;
 	for (i = 0; i < UPROBES_HASH_SZ; i++) {
 		mutex_init(&uprobes_mutex[i]);
 		mutex_init(&uprobes_mmap_mutex[i]);
 	}
 	return register_die_notifier(&uprobe_exception_nb);
 }
 module_init(init_uprobes);
 static void __exit exit_uprobes(void)
 {
 }
 module_exit(exit_uprobes);