Eric Lee / smarc-ti-linux-kernel

Download as
Browse Code ยป

Commit ddb321a8dd158520d97ed1cbade1d4ac36b6af31

Authored by Linus Torvalds
2 parents 1e6c3e8f8f 5306c31c57
Exists in ti-lsk-linux-4.1.y and in 10 other branches dlt-processor-sdk-linux-03.00.00.04, processor-sdk-linux-02.00.01, smarc-ti-lsk-linux-4.1.y, smarct3x-processor-sdk-04.01.00.06, smarct3x-processor-sdk-linux-02.00.01, smarct3x-processor-sdk-linux-03.00.00.04, smarct4x-800-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-04.01.00.06, smarct4x-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-linux-03.00.00.04

Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull perf fixes from Ingo Molnar:
 "Mostly tooling fixes, but also some kernel side fixes: uncore PMU
  driver fix, user regs sampling fix and an instruction decoder fix that
  unbreaks PEBS precise sampling"

* 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  perf/x86/uncore/hsw-ep: Handle systems with only two SBOXes
  perf/x86_64: Improve user regs sampling
  perf: Move task_pt_regs sampling into arch code
  x86: Fix off-by-one in instruction decoder
  perf hists browser: Fix segfault when showing callchain
  perf callchain: Free callchains when hist entries are deleted
  perf hists: Fix children sort key behavior
  perf diff: Fix to sort by baseline field by default
  perf list: Fix --raw-dump option
  perf probe: Fix crash in dwarf_getcfi_elf
  perf probe: Fix to fall back to find probe point in symbols
  perf callchain: Append callchains only when requested
  perf ui/tui: Print backtrace symbols when segfault occurs
  perf report: Show progress bar for output resorting

Showing 26 changed files Inline Diff

arch/arm/kernel/perf_regs.c
Diff comments   View file @ ddb321a
1 1
2 #include <linux/errno.h> 2 #include <linux/errno.h>
3 #include <linux/kernel.h> 3 #include <linux/kernel.h>
4 #include <linux/perf_event.h> 4 #include <linux/perf_event.h>
5 #include <linux/bug.h> 5 #include <linux/bug.h>
6 #include <asm/perf_regs.h> 6 #include <asm/perf_regs.h>
7 #include <asm/ptrace.h> 7 #include <asm/ptrace.h>
8 8
9 u64 perf_reg_value(struct pt_regs *regs, int idx) 9 u64 perf_reg_value(struct pt_regs *regs, int idx)
10 { 10 {
11 if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM_MAX)) 11 if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM_MAX))
12 return 0; 12 return 0;
13 13
14 return regs->uregs[idx]; 14 return regs->uregs[idx];
15 } 15 }
16 16
17 #define REG_RESERVED (~((1ULL << PERF_REG_ARM_MAX) - 1)) 17 #define REG_RESERVED (~((1ULL << PERF_REG_ARM_MAX) - 1))
18 18
19 int perf_reg_validate(u64 mask) 19 int perf_reg_validate(u64 mask)
20 { 20 {
21 if (!mask || mask & REG_RESERVED) 21 if (!mask || mask & REG_RESERVED)
22 return -EINVAL; 22 return -EINVAL;
23 23
24 return 0; 24 return 0;
25 } 25 }
26 26
27 u64 perf_reg_abi(struct task_struct *task) 27 u64 perf_reg_abi(struct task_struct *task)
28 { 28 {
29 return PERF_SAMPLE_REGS_ABI_32; 29 return PERF_SAMPLE_REGS_ABI_32;
30 } 30 }
31
32 void perf_get_regs_user(struct perf_regs *regs_user,
33 struct pt_regs *regs,
34 struct pt_regs *regs_user_copy)
35 {
36 regs_user->regs = task_pt_regs(current);
37 regs_user->abi = perf_reg_abi(current);
38 }
31 39
arch/arm64/kernel/perf_regs.c
Diff comments   View file @ ddb321a
1 #include <linux/errno.h> 1 #include <linux/errno.h>
2 #include <linux/kernel.h> 2 #include <linux/kernel.h>
3 #include <linux/perf_event.h> 3 #include <linux/perf_event.h>
4 #include <linux/bug.h> 4 #include <linux/bug.h>
5 5
6 #include <asm/compat.h> 6 #include <asm/compat.h>
7 #include <asm/perf_regs.h> 7 #include <asm/perf_regs.h>
8 #include <asm/ptrace.h> 8 #include <asm/ptrace.h>
9 9
10 u64 perf_reg_value(struct pt_regs *regs, int idx) 10 u64 perf_reg_value(struct pt_regs *regs, int idx)
11 { 11 {
12 if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM64_MAX)) 12 if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM64_MAX))
13 return 0; 13 return 0;
14 14
15 /* 15 /*
16 * Compat (i.e. 32 bit) mode: 16 * Compat (i.e. 32 bit) mode:
17 * - PC has been set in the pt_regs struct in kernel_entry, 17 * - PC has been set in the pt_regs struct in kernel_entry,
18 * - Handle SP and LR here. 18 * - Handle SP and LR here.
19 */ 19 */
20 if (compat_user_mode(regs)) { 20 if (compat_user_mode(regs)) {
21 if ((u32)idx == PERF_REG_ARM64_SP) 21 if ((u32)idx == PERF_REG_ARM64_SP)
22 return regs->compat_sp; 22 return regs->compat_sp;
23 if ((u32)idx == PERF_REG_ARM64_LR) 23 if ((u32)idx == PERF_REG_ARM64_LR)
24 return regs->compat_lr; 24 return regs->compat_lr;
25 } 25 }
26 26
27 if ((u32)idx == PERF_REG_ARM64_SP) 27 if ((u32)idx == PERF_REG_ARM64_SP)
28 return regs->sp; 28 return regs->sp;
29 29
30 if ((u32)idx == PERF_REG_ARM64_PC) 30 if ((u32)idx == PERF_REG_ARM64_PC)
31 return regs->pc; 31 return regs->pc;
32 32
33 return regs->regs[idx]; 33 return regs->regs[idx];
34 } 34 }
35 35
36 #define REG_RESERVED (~((1ULL << PERF_REG_ARM64_MAX) - 1)) 36 #define REG_RESERVED (~((1ULL << PERF_REG_ARM64_MAX) - 1))
37 37
38 int perf_reg_validate(u64 mask) 38 int perf_reg_validate(u64 mask)
39 { 39 {
40 if (!mask || mask & REG_RESERVED) 40 if (!mask || mask & REG_RESERVED)
41 return -EINVAL; 41 return -EINVAL;
42 42
43 return 0; 43 return 0;
44 } 44 }
45 45
46 u64 perf_reg_abi(struct task_struct *task) 46 u64 perf_reg_abi(struct task_struct *task)
47 { 47 {
48 if (is_compat_thread(task_thread_info(task))) 48 if (is_compat_thread(task_thread_info(task)))
49 return PERF_SAMPLE_REGS_ABI_32; 49 return PERF_SAMPLE_REGS_ABI_32;
50 else 50 else
51 return PERF_SAMPLE_REGS_ABI_64; 51 return PERF_SAMPLE_REGS_ABI_64;
52 } 52 }
53
54 void perf_get_regs_user(struct perf_regs *regs_user,
55 struct pt_regs *regs,
56 struct pt_regs *regs_user_copy)
57 {
58 regs_user->regs = task_pt_regs(current);
59 regs_user->abi = perf_reg_abi(current);
60 }
53 61
arch/x86/kernel/cpu/perf_event_intel_uncore.h
Diff comments   View file @ ddb321a
1 #include <linux/module.h> 1 #include <linux/module.h>
2 #include <linux/slab.h> 2 #include <linux/slab.h>
3 #include <linux/pci.h> 3 #include <linux/pci.h>
4 #include <linux/perf_event.h> 4 #include <linux/perf_event.h>
5 #include "perf_event.h" 5 #include "perf_event.h"
6 6
7 #define UNCORE_PMU_NAME_LEN 32 7 #define UNCORE_PMU_NAME_LEN 32
8 #define UNCORE_PMU_HRTIMER_INTERVAL (60LL * NSEC_PER_SEC) 8 #define UNCORE_PMU_HRTIMER_INTERVAL (60LL * NSEC_PER_SEC)
9 #define UNCORE_SNB_IMC_HRTIMER_INTERVAL (5ULL * NSEC_PER_SEC) 9 #define UNCORE_SNB_IMC_HRTIMER_INTERVAL (5ULL * NSEC_PER_SEC)
10 10
11 #define UNCORE_FIXED_EVENT 0xff 11 #define UNCORE_FIXED_EVENT 0xff
12 #define UNCORE_PMC_IDX_MAX_GENERIC 8 12 #define UNCORE_PMC_IDX_MAX_GENERIC 8
13 #define UNCORE_PMC_IDX_FIXED UNCORE_PMC_IDX_MAX_GENERIC 13 #define UNCORE_PMC_IDX_FIXED UNCORE_PMC_IDX_MAX_GENERIC
14 #define UNCORE_PMC_IDX_MAX (UNCORE_PMC_IDX_FIXED + 1) 14 #define UNCORE_PMC_IDX_MAX (UNCORE_PMC_IDX_FIXED + 1)
15 15
16 #define UNCORE_PCI_DEV_DATA(type, idx) ((type << 8) | idx) 16 #define UNCORE_PCI_DEV_DATA(type, idx) ((type << 8) | idx)
17 #define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff) 17 #define UNCORE_PCI_DEV_TYPE(data) ((data >> 8) & 0xff)
18 #define UNCORE_PCI_DEV_IDX(data) (data & 0xff) 18 #define UNCORE_PCI_DEV_IDX(data) (data & 0xff)
19 #define UNCORE_EXTRA_PCI_DEV 0xff 19 #define UNCORE_EXTRA_PCI_DEV 0xff
20 #define UNCORE_EXTRA_PCI_DEV_MAX 2 20 #define UNCORE_EXTRA_PCI_DEV_MAX 3
21 21
22 /* support up to 8 sockets */ 22 /* support up to 8 sockets */
23 #define UNCORE_SOCKET_MAX 8 23 #define UNCORE_SOCKET_MAX 8
24 24
25 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff) 25 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff)
26 26
27 struct intel_uncore_ops; 27 struct intel_uncore_ops;
28 struct intel_uncore_pmu; 28 struct intel_uncore_pmu;
29 struct intel_uncore_box; 29 struct intel_uncore_box;
30 struct uncore_event_desc; 30 struct uncore_event_desc;
31 31
32 struct intel_uncore_type { 32 struct intel_uncore_type {
33 const char *name; 33 const char *name;
34 int num_counters; 34 int num_counters;
35 int num_boxes; 35 int num_boxes;
36 int perf_ctr_bits; 36 int perf_ctr_bits;
37 int fixed_ctr_bits; 37 int fixed_ctr_bits;
38 unsigned perf_ctr; 38 unsigned perf_ctr;
39 unsigned event_ctl; 39 unsigned event_ctl;
40 unsigned event_mask; 40 unsigned event_mask;
41 unsigned fixed_ctr; 41 unsigned fixed_ctr;
42 unsigned fixed_ctl; 42 unsigned fixed_ctl;
43 unsigned box_ctl; 43 unsigned box_ctl;
44 unsigned msr_offset; 44 unsigned msr_offset;
45 unsigned num_shared_regs:8; 45 unsigned num_shared_regs:8;
46 unsigned single_fixed:1; 46 unsigned single_fixed:1;
47 unsigned pair_ctr_ctl:1; 47 unsigned pair_ctr_ctl:1;
48 unsigned *msr_offsets; 48 unsigned *msr_offsets;
49 struct event_constraint unconstrainted; 49 struct event_constraint unconstrainted;
50 struct event_constraint *constraints; 50 struct event_constraint *constraints;
51 struct intel_uncore_pmu *pmus; 51 struct intel_uncore_pmu *pmus;
52 struct intel_uncore_ops *ops; 52 struct intel_uncore_ops *ops;
53 struct uncore_event_desc *event_descs; 53 struct uncore_event_desc *event_descs;
54 const struct attribute_group *attr_groups[4]; 54 const struct attribute_group *attr_groups[4];
55 struct pmu *pmu; /* for custom pmu ops */ 55 struct pmu *pmu; /* for custom pmu ops */
56 }; 56 };
57 57
58 #define pmu_group attr_groups[0] 58 #define pmu_group attr_groups[0]
59 #define format_group attr_groups[1] 59 #define format_group attr_groups[1]
60 #define events_group attr_groups[2] 60 #define events_group attr_groups[2]
61 61
62 struct intel_uncore_ops { 62 struct intel_uncore_ops {
63 void (*init_box)(struct intel_uncore_box *); 63 void (*init_box)(struct intel_uncore_box *);
64 void (*disable_box)(struct intel_uncore_box *); 64 void (*disable_box)(struct intel_uncore_box *);
65 void (*enable_box)(struct intel_uncore_box *); 65 void (*enable_box)(struct intel_uncore_box *);
66 void (*disable_event)(struct intel_uncore_box *, struct perf_event *); 66 void (*disable_event)(struct intel_uncore_box *, struct perf_event *);
67 void (*enable_event)(struct intel_uncore_box *, struct perf_event *); 67 void (*enable_event)(struct intel_uncore_box *, struct perf_event *);
68 u64 (*read_counter)(struct intel_uncore_box *, struct perf_event *); 68 u64 (*read_counter)(struct intel_uncore_box *, struct perf_event *);
69 int (*hw_config)(struct intel_uncore_box *, struct perf_event *); 69 int (*hw_config)(struct intel_uncore_box *, struct perf_event *);
70 struct event_constraint *(*get_constraint)(struct intel_uncore_box *, 70 struct event_constraint *(*get_constraint)(struct intel_uncore_box *,
71 struct perf_event *); 71 struct perf_event *);
72 void (*put_constraint)(struct intel_uncore_box *, struct perf_event *); 72 void (*put_constraint)(struct intel_uncore_box *, struct perf_event *);
73 }; 73 };
74 74
75 struct intel_uncore_pmu { 75 struct intel_uncore_pmu {
76 struct pmu pmu; 76 struct pmu pmu;
77 char name[UNCORE_PMU_NAME_LEN]; 77 char name[UNCORE_PMU_NAME_LEN];
78 int pmu_idx; 78 int pmu_idx;
79 int func_id; 79 int func_id;
80 struct intel_uncore_type *type; 80 struct intel_uncore_type *type;
81 struct intel_uncore_box ** __percpu box; 81 struct intel_uncore_box ** __percpu box;
82 struct list_head box_list; 82 struct list_head box_list;
83 }; 83 };
84 84
85 struct intel_uncore_extra_reg { 85 struct intel_uncore_extra_reg {
86 raw_spinlock_t lock; 86 raw_spinlock_t lock;
87 u64 config, config1, config2; 87 u64 config, config1, config2;
88 atomic_t ref; 88 atomic_t ref;
89 }; 89 };
90 90
91 struct intel_uncore_box { 91 struct intel_uncore_box {
92 int phys_id; 92 int phys_id;
93 int n_active; /* number of active events */ 93 int n_active; /* number of active events */
94 int n_events; 94 int n_events;
95 int cpu; /* cpu to collect events */ 95 int cpu; /* cpu to collect events */
96 unsigned long flags; 96 unsigned long flags;
97 atomic_t refcnt; 97 atomic_t refcnt;
98 struct perf_event *events[UNCORE_PMC_IDX_MAX]; 98 struct perf_event *events[UNCORE_PMC_IDX_MAX];
99 struct perf_event *event_list[UNCORE_PMC_IDX_MAX]; 99 struct perf_event *event_list[UNCORE_PMC_IDX_MAX];
100 unsigned long active_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)]; 100 unsigned long active_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
101 u64 tags[UNCORE_PMC_IDX_MAX]; 101 u64 tags[UNCORE_PMC_IDX_MAX];
102 struct pci_dev *pci_dev; 102 struct pci_dev *pci_dev;
103 struct intel_uncore_pmu *pmu; 103 struct intel_uncore_pmu *pmu;
104 u64 hrtimer_duration; /* hrtimer timeout for this box */ 104 u64 hrtimer_duration; /* hrtimer timeout for this box */
105 struct hrtimer hrtimer; 105 struct hrtimer hrtimer;
106 struct list_head list; 106 struct list_head list;
107 struct list_head active_list; 107 struct list_head active_list;
108 void *io_addr; 108 void *io_addr;
109 struct intel_uncore_extra_reg shared_regs[0]; 109 struct intel_uncore_extra_reg shared_regs[0];
110 }; 110 };
111 111
112 #define UNCORE_BOX_FLAG_INITIATED 0 112 #define UNCORE_BOX_FLAG_INITIATED 0
113 113
114 struct uncore_event_desc { 114 struct uncore_event_desc {
115 struct kobj_attribute attr; 115 struct kobj_attribute attr;
116 const char *config; 116 const char *config;
117 }; 117 };
118 118
119 ssize_t uncore_event_show(struct kobject *kobj, 119 ssize_t uncore_event_show(struct kobject *kobj,
120 struct kobj_attribute *attr, char *buf); 120 struct kobj_attribute *attr, char *buf);
121 121
122 #define INTEL_UNCORE_EVENT_DESC(_name, _config) \ 122 #define INTEL_UNCORE_EVENT_DESC(_name, _config) \
123 { \ 123 { \
124 .attr = __ATTR(_name, 0444, uncore_event_show, NULL), \ 124 .attr = __ATTR(_name, 0444, uncore_event_show, NULL), \
125 .config = _config, \ 125 .config = _config, \
126 } 126 }
127 127
128 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \ 128 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
129 static ssize_t __uncore_##_var##_show(struct kobject *kobj, \ 129 static ssize_t __uncore_##_var##_show(struct kobject *kobj, \
130 struct kobj_attribute *attr, \ 130 struct kobj_attribute *attr, \
131 char *page) \ 131 char *page) \
132 { \ 132 { \
133 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ 133 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
134 return sprintf(page, _format "\n"); \ 134 return sprintf(page, _format "\n"); \
135 } \ 135 } \
136 static struct kobj_attribute format_attr_##_var = \ 136 static struct kobj_attribute format_attr_##_var = \
137 __ATTR(_name, 0444, __uncore_##_var##_show, NULL) 137 __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
138 138
139 static inline unsigned uncore_pci_box_ctl(struct intel_uncore_box *box) 139 static inline unsigned uncore_pci_box_ctl(struct intel_uncore_box *box)
140 { 140 {
141 return box->pmu->type->box_ctl; 141 return box->pmu->type->box_ctl;
142 } 142 }
143 143
144 static inline unsigned uncore_pci_fixed_ctl(struct intel_uncore_box *box) 144 static inline unsigned uncore_pci_fixed_ctl(struct intel_uncore_box *box)
145 { 145 {
146 return box->pmu->type->fixed_ctl; 146 return box->pmu->type->fixed_ctl;
147 } 147 }
148 148
149 static inline unsigned uncore_pci_fixed_ctr(struct intel_uncore_box *box) 149 static inline unsigned uncore_pci_fixed_ctr(struct intel_uncore_box *box)
150 { 150 {
151 return box->pmu->type->fixed_ctr; 151 return box->pmu->type->fixed_ctr;
152 } 152 }
153 153
154 static inline 154 static inline
155 unsigned uncore_pci_event_ctl(struct intel_uncore_box *box, int idx) 155 unsigned uncore_pci_event_ctl(struct intel_uncore_box *box, int idx)
156 { 156 {
157 return idx * 4 + box->pmu->type->event_ctl; 157 return idx * 4 + box->pmu->type->event_ctl;
158 } 158 }
159 159
160 static inline 160 static inline
161 unsigned uncore_pci_perf_ctr(struct intel_uncore_box *box, int idx) 161 unsigned uncore_pci_perf_ctr(struct intel_uncore_box *box, int idx)
162 { 162 {
163 return idx * 8 + box->pmu->type->perf_ctr; 163 return idx * 8 + box->pmu->type->perf_ctr;
164 } 164 }
165 165
166 static inline unsigned uncore_msr_box_offset(struct intel_uncore_box *box) 166 static inline unsigned uncore_msr_box_offset(struct intel_uncore_box *box)
167 { 167 {
168 struct intel_uncore_pmu *pmu = box->pmu; 168 struct intel_uncore_pmu *pmu = box->pmu;
169 return pmu->type->msr_offsets ? 169 return pmu->type->msr_offsets ?
170 pmu->type->msr_offsets[pmu->pmu_idx] : 170 pmu->type->msr_offsets[pmu->pmu_idx] :
171 pmu->type->msr_offset * pmu->pmu_idx; 171 pmu->type->msr_offset * pmu->pmu_idx;
172 } 172 }
173 173
174 static inline unsigned uncore_msr_box_ctl(struct intel_uncore_box *box) 174 static inline unsigned uncore_msr_box_ctl(struct intel_uncore_box *box)
175 { 175 {
176 if (!box->pmu->type->box_ctl) 176 if (!box->pmu->type->box_ctl)
177 return 0; 177 return 0;
178 return box->pmu->type->box_ctl + uncore_msr_box_offset(box); 178 return box->pmu->type->box_ctl + uncore_msr_box_offset(box);
179 } 179 }
180 180
181 static inline unsigned uncore_msr_fixed_ctl(struct intel_uncore_box *box) 181 static inline unsigned uncore_msr_fixed_ctl(struct intel_uncore_box *box)
182 { 182 {
183 if (!box->pmu->type->fixed_ctl) 183 if (!box->pmu->type->fixed_ctl)
184 return 0; 184 return 0;
185 return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box); 185 return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box);
186 } 186 }
187 187
188 static inline unsigned uncore_msr_fixed_ctr(struct intel_uncore_box *box) 188 static inline unsigned uncore_msr_fixed_ctr(struct intel_uncore_box *box)
189 { 189 {
190 return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box); 190 return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box);
191 } 191 }
192 192
193 static inline 193 static inline
194 unsigned uncore_msr_event_ctl(struct intel_uncore_box *box, int idx) 194 unsigned uncore_msr_event_ctl(struct intel_uncore_box *box, int idx)
195 { 195 {
196 return box->pmu->type->event_ctl + 196 return box->pmu->type->event_ctl +
197 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) + 197 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
198 uncore_msr_box_offset(box); 198 uncore_msr_box_offset(box);
199 } 199 }
200 200
201 static inline 201 static inline
202 unsigned uncore_msr_perf_ctr(struct intel_uncore_box *box, int idx) 202 unsigned uncore_msr_perf_ctr(struct intel_uncore_box *box, int idx)
203 { 203 {
204 return box->pmu->type->perf_ctr + 204 return box->pmu->type->perf_ctr +
205 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) + 205 (box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
206 uncore_msr_box_offset(box); 206 uncore_msr_box_offset(box);
207 } 207 }
208 208
209 static inline 209 static inline
210 unsigned uncore_fixed_ctl(struct intel_uncore_box *box) 210 unsigned uncore_fixed_ctl(struct intel_uncore_box *box)
211 { 211 {
212 if (box->pci_dev) 212 if (box->pci_dev)
213 return uncore_pci_fixed_ctl(box); 213 return uncore_pci_fixed_ctl(box);
214 else 214 else
215 return uncore_msr_fixed_ctl(box); 215 return uncore_msr_fixed_ctl(box);
216 } 216 }
217 217
218 static inline 218 static inline
219 unsigned uncore_fixed_ctr(struct intel_uncore_box *box) 219 unsigned uncore_fixed_ctr(struct intel_uncore_box *box)
220 { 220 {
221 if (box->pci_dev) 221 if (box->pci_dev)
222 return uncore_pci_fixed_ctr(box); 222 return uncore_pci_fixed_ctr(box);
223 else 223 else
224 return uncore_msr_fixed_ctr(box); 224 return uncore_msr_fixed_ctr(box);
225 } 225 }
226 226
227 static inline 227 static inline
228 unsigned uncore_event_ctl(struct intel_uncore_box *box, int idx) 228 unsigned uncore_event_ctl(struct intel_uncore_box *box, int idx)
229 { 229 {
230 if (box->pci_dev) 230 if (box->pci_dev)
231 return uncore_pci_event_ctl(box, idx); 231 return uncore_pci_event_ctl(box, idx);
232 else 232 else
233 return uncore_msr_event_ctl(box, idx); 233 return uncore_msr_event_ctl(box, idx);
234 } 234 }
235 235
236 static inline 236 static inline
237 unsigned uncore_perf_ctr(struct intel_uncore_box *box, int idx) 237 unsigned uncore_perf_ctr(struct intel_uncore_box *box, int idx)
238 { 238 {
239 if (box->pci_dev) 239 if (box->pci_dev)
240 return uncore_pci_perf_ctr(box, idx); 240 return uncore_pci_perf_ctr(box, idx);
241 else 241 else
242 return uncore_msr_perf_ctr(box, idx); 242 return uncore_msr_perf_ctr(box, idx);
243 } 243 }
244 244
245 static inline int uncore_perf_ctr_bits(struct intel_uncore_box *box) 245 static inline int uncore_perf_ctr_bits(struct intel_uncore_box *box)
246 { 246 {
247 return box->pmu->type->perf_ctr_bits; 247 return box->pmu->type->perf_ctr_bits;
248 } 248 }
249 249
250 static inline int uncore_fixed_ctr_bits(struct intel_uncore_box *box) 250 static inline int uncore_fixed_ctr_bits(struct intel_uncore_box *box)
251 { 251 {
252 return box->pmu->type->fixed_ctr_bits; 252 return box->pmu->type->fixed_ctr_bits;
253 } 253 }
254 254
255 static inline int uncore_num_counters(struct intel_uncore_box *box) 255 static inline int uncore_num_counters(struct intel_uncore_box *box)
256 { 256 {
257 return box->pmu->type->num_counters; 257 return box->pmu->type->num_counters;
258 } 258 }
259 259
260 static inline void uncore_disable_box(struct intel_uncore_box *box) 260 static inline void uncore_disable_box(struct intel_uncore_box *box)
261 { 261 {
262 if (box->pmu->type->ops->disable_box) 262 if (box->pmu->type->ops->disable_box)
263 box->pmu->type->ops->disable_box(box); 263 box->pmu->type->ops->disable_box(box);
264 } 264 }
265 265
266 static inline void uncore_enable_box(struct intel_uncore_box *box) 266 static inline void uncore_enable_box(struct intel_uncore_box *box)
267 { 267 {
268 if (box->pmu->type->ops->enable_box) 268 if (box->pmu->type->ops->enable_box)
269 box->pmu->type->ops->enable_box(box); 269 box->pmu->type->ops->enable_box(box);
270 } 270 }
271 271
272 static inline void uncore_disable_event(struct intel_uncore_box *box, 272 static inline void uncore_disable_event(struct intel_uncore_box *box,
273 struct perf_event *event) 273 struct perf_event *event)
274 { 274 {
275 box->pmu->type->ops->disable_event(box, event); 275 box->pmu->type->ops->disable_event(box, event);
276 } 276 }
277 277
278 static inline void uncore_enable_event(struct intel_uncore_box *box, 278 static inline void uncore_enable_event(struct intel_uncore_box *box,
279 struct perf_event *event) 279 struct perf_event *event)
280 { 280 {
281 box->pmu->type->ops->enable_event(box, event); 281 box->pmu->type->ops->enable_event(box, event);
282 } 282 }
283 283
284 static inline u64 uncore_read_counter(struct intel_uncore_box *box, 284 static inline u64 uncore_read_counter(struct intel_uncore_box *box,
285 struct perf_event *event) 285 struct perf_event *event)
286 { 286 {
287 return box->pmu->type->ops->read_counter(box, event); 287 return box->pmu->type->ops->read_counter(box, event);
288 } 288 }
289 289
290 static inline void uncore_box_init(struct intel_uncore_box *box) 290 static inline void uncore_box_init(struct intel_uncore_box *box)
291 { 291 {
292 if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { 292 if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
293 if (box->pmu->type->ops->init_box) 293 if (box->pmu->type->ops->init_box)
294 box->pmu->type->ops->init_box(box); 294 box->pmu->type->ops->init_box(box);
295 } 295 }
296 } 296 }
297 297
298 static inline bool uncore_box_is_fake(struct intel_uncore_box *box) 298 static inline bool uncore_box_is_fake(struct intel_uncore_box *box)
299 { 299 {
300 return (box->phys_id < 0); 300 return (box->phys_id < 0);
301 } 301 }
302 302
303 struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event); 303 struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event);
304 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu); 304 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu);
305 struct intel_uncore_box *uncore_event_to_box(struct perf_event *event); 305 struct intel_uncore_box *uncore_event_to_box(struct perf_event *event);
306 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event); 306 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event);
307 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box); 307 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box);
308 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box); 308 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box);
309 void uncore_pmu_event_read(struct perf_event *event); 309 void uncore_pmu_event_read(struct perf_event *event);
310 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event); 310 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event);
311 struct event_constraint * 311 struct event_constraint *
312 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event); 312 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event);
313 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event); 313 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event);
314 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx); 314 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx);
315 315
316 extern struct intel_uncore_type **uncore_msr_uncores; 316 extern struct intel_uncore_type **uncore_msr_uncores;
317 extern struct intel_uncore_type **uncore_pci_uncores; 317 extern struct intel_uncore_type **uncore_pci_uncores;
318 extern struct pci_driver *uncore_pci_driver; 318 extern struct pci_driver *uncore_pci_driver;
319 extern int uncore_pcibus_to_physid[256]; 319 extern int uncore_pcibus_to_physid[256];
320 extern struct pci_dev *uncore_extra_pci_dev[UNCORE_SOCKET_MAX][UNCORE_EXTRA_PCI_DEV_MAX]; 320 extern struct pci_dev *uncore_extra_pci_dev[UNCORE_SOCKET_MAX][UNCORE_EXTRA_PCI_DEV_MAX];
321 extern struct event_constraint uncore_constraint_empty; 321 extern struct event_constraint uncore_constraint_empty;
322 322
323 /* perf_event_intel_uncore_snb.c */ 323 /* perf_event_intel_uncore_snb.c */
324 int snb_uncore_pci_init(void); 324 int snb_uncore_pci_init(void);
325 int ivb_uncore_pci_init(void); 325 int ivb_uncore_pci_init(void);
326 int hsw_uncore_pci_init(void); 326 int hsw_uncore_pci_init(void);
327 void snb_uncore_cpu_init(void); 327 void snb_uncore_cpu_init(void);
328 void nhm_uncore_cpu_init(void); 328 void nhm_uncore_cpu_init(void);
329 329
330 /* perf_event_intel_uncore_snbep.c */ 330 /* perf_event_intel_uncore_snbep.c */
331 int snbep_uncore_pci_init(void); 331 int snbep_uncore_pci_init(void);
332 void snbep_uncore_cpu_init(void); 332 void snbep_uncore_cpu_init(void);
333 int ivbep_uncore_pci_init(void); 333 int ivbep_uncore_pci_init(void);
334 void ivbep_uncore_cpu_init(void); 334 void ivbep_uncore_cpu_init(void);
335 int hswep_uncore_pci_init(void); 335 int hswep_uncore_pci_init(void);
336 void hswep_uncore_cpu_init(void); 336 void hswep_uncore_cpu_init(void);
337 337
338 /* perf_event_intel_uncore_nhmex.c */ 338 /* perf_event_intel_uncore_nhmex.c */
339 void nhmex_uncore_cpu_init(void); 339 void nhmex_uncore_cpu_init(void);
340 340
arch/x86/kernel/cpu/perf_event_intel_uncore_snbep.c
Diff comments   View file @ ddb321a
1 /* SandyBridge-EP/IvyTown uncore support */ 1 /* SandyBridge-EP/IvyTown uncore support */
2 #include "perf_event_intel_uncore.h" 2 #include "perf_event_intel_uncore.h"
3 3
4 4
5 /* SNB-EP Box level control */ 5 /* SNB-EP Box level control */
6 #define SNBEP_PMON_BOX_CTL_RST_CTRL (1 << 0) 6 #define SNBEP_PMON_BOX_CTL_RST_CTRL (1 << 0)
7 #define SNBEP_PMON_BOX_CTL_RST_CTRS (1 << 1) 7 #define SNBEP_PMON_BOX_CTL_RST_CTRS (1 << 1)
8 #define SNBEP_PMON_BOX_CTL_FRZ (1 << 8) 8 #define SNBEP_PMON_BOX_CTL_FRZ (1 << 8)
9 #define SNBEP_PMON_BOX_CTL_FRZ_EN (1 << 16) 9 #define SNBEP_PMON_BOX_CTL_FRZ_EN (1 << 16)
10 #define SNBEP_PMON_BOX_CTL_INT (SNBEP_PMON_BOX_CTL_RST_CTRL | \ 10 #define SNBEP_PMON_BOX_CTL_INT (SNBEP_PMON_BOX_CTL_RST_CTRL | \
11 SNBEP_PMON_BOX_CTL_RST_CTRS | \ 11 SNBEP_PMON_BOX_CTL_RST_CTRS | \
12 SNBEP_PMON_BOX_CTL_FRZ_EN) 12 SNBEP_PMON_BOX_CTL_FRZ_EN)
13 /* SNB-EP event control */ 13 /* SNB-EP event control */
14 #define SNBEP_PMON_CTL_EV_SEL_MASK 0x000000ff 14 #define SNBEP_PMON_CTL_EV_SEL_MASK 0x000000ff
15 #define SNBEP_PMON_CTL_UMASK_MASK 0x0000ff00 15 #define SNBEP_PMON_CTL_UMASK_MASK 0x0000ff00
16 #define SNBEP_PMON_CTL_RST (1 << 17) 16 #define SNBEP_PMON_CTL_RST (1 << 17)
17 #define SNBEP_PMON_CTL_EDGE_DET (1 << 18) 17 #define SNBEP_PMON_CTL_EDGE_DET (1 << 18)
18 #define SNBEP_PMON_CTL_EV_SEL_EXT (1 << 21) 18 #define SNBEP_PMON_CTL_EV_SEL_EXT (1 << 21)
19 #define SNBEP_PMON_CTL_EN (1 << 22) 19 #define SNBEP_PMON_CTL_EN (1 << 22)
20 #define SNBEP_PMON_CTL_INVERT (1 << 23) 20 #define SNBEP_PMON_CTL_INVERT (1 << 23)
21 #define SNBEP_PMON_CTL_TRESH_MASK 0xff000000 21 #define SNBEP_PMON_CTL_TRESH_MASK 0xff000000
22 #define SNBEP_PMON_RAW_EVENT_MASK (SNBEP_PMON_CTL_EV_SEL_MASK | \ 22 #define SNBEP_PMON_RAW_EVENT_MASK (SNBEP_PMON_CTL_EV_SEL_MASK | \
23 SNBEP_PMON_CTL_UMASK_MASK | \ 23 SNBEP_PMON_CTL_UMASK_MASK | \
24 SNBEP_PMON_CTL_EDGE_DET | \ 24 SNBEP_PMON_CTL_EDGE_DET | \
25 SNBEP_PMON_CTL_INVERT | \ 25 SNBEP_PMON_CTL_INVERT | \
26 SNBEP_PMON_CTL_TRESH_MASK) 26 SNBEP_PMON_CTL_TRESH_MASK)
27 27
28 /* SNB-EP Ubox event control */ 28 /* SNB-EP Ubox event control */
29 #define SNBEP_U_MSR_PMON_CTL_TRESH_MASK 0x1f000000 29 #define SNBEP_U_MSR_PMON_CTL_TRESH_MASK 0x1f000000
30 #define SNBEP_U_MSR_PMON_RAW_EVENT_MASK \ 30 #define SNBEP_U_MSR_PMON_RAW_EVENT_MASK \
31 (SNBEP_PMON_CTL_EV_SEL_MASK | \ 31 (SNBEP_PMON_CTL_EV_SEL_MASK | \
32 SNBEP_PMON_CTL_UMASK_MASK | \ 32 SNBEP_PMON_CTL_UMASK_MASK | \
33 SNBEP_PMON_CTL_EDGE_DET | \ 33 SNBEP_PMON_CTL_EDGE_DET | \
34 SNBEP_PMON_CTL_INVERT | \ 34 SNBEP_PMON_CTL_INVERT | \
35 SNBEP_U_MSR_PMON_CTL_TRESH_MASK) 35 SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
36 36
37 #define SNBEP_CBO_PMON_CTL_TID_EN (1 << 19) 37 #define SNBEP_CBO_PMON_CTL_TID_EN (1 << 19)
38 #define SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK (SNBEP_PMON_RAW_EVENT_MASK | \ 38 #define SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK (SNBEP_PMON_RAW_EVENT_MASK | \
39 SNBEP_CBO_PMON_CTL_TID_EN) 39 SNBEP_CBO_PMON_CTL_TID_EN)
40 40
41 /* SNB-EP PCU event control */ 41 /* SNB-EP PCU event control */
42 #define SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK 0x0000c000 42 #define SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK 0x0000c000
43 #define SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK 0x1f000000 43 #define SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK 0x1f000000
44 #define SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT (1 << 30) 44 #define SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT (1 << 30)
45 #define SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET (1 << 31) 45 #define SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET (1 << 31)
46 #define SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK \ 46 #define SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK \
47 (SNBEP_PMON_CTL_EV_SEL_MASK | \ 47 (SNBEP_PMON_CTL_EV_SEL_MASK | \
48 SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \ 48 SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
49 SNBEP_PMON_CTL_EDGE_DET | \ 49 SNBEP_PMON_CTL_EDGE_DET | \
50 SNBEP_PMON_CTL_EV_SEL_EXT | \ 50 SNBEP_PMON_CTL_EV_SEL_EXT | \
51 SNBEP_PMON_CTL_INVERT | \ 51 SNBEP_PMON_CTL_INVERT | \
52 SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \ 52 SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
53 SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \ 53 SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
54 SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET) 54 SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
55 55
56 #define SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK \ 56 #define SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK \
57 (SNBEP_PMON_RAW_EVENT_MASK | \ 57 (SNBEP_PMON_RAW_EVENT_MASK | \
58 SNBEP_PMON_CTL_EV_SEL_EXT) 58 SNBEP_PMON_CTL_EV_SEL_EXT)
59 59
60 /* SNB-EP pci control register */ 60 /* SNB-EP pci control register */
61 #define SNBEP_PCI_PMON_BOX_CTL 0xf4 61 #define SNBEP_PCI_PMON_BOX_CTL 0xf4
62 #define SNBEP_PCI_PMON_CTL0 0xd8 62 #define SNBEP_PCI_PMON_CTL0 0xd8
63 /* SNB-EP pci counter register */ 63 /* SNB-EP pci counter register */
64 #define SNBEP_PCI_PMON_CTR0 0xa0 64 #define SNBEP_PCI_PMON_CTR0 0xa0
65 65
66 /* SNB-EP home agent register */ 66 /* SNB-EP home agent register */
67 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH0 0x40 67 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH0 0x40
68 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH1 0x44 68 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH1 0x44
69 #define SNBEP_HA_PCI_PMON_BOX_OPCODEMATCH 0x48 69 #define SNBEP_HA_PCI_PMON_BOX_OPCODEMATCH 0x48
70 /* SNB-EP memory controller register */ 70 /* SNB-EP memory controller register */
71 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTL 0xf0 71 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTL 0xf0
72 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTR 0xd0 72 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTR 0xd0
73 /* SNB-EP QPI register */ 73 /* SNB-EP QPI register */
74 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH0 0x228 74 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH0 0x228
75 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH1 0x22c 75 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH1 0x22c
76 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK0 0x238 76 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK0 0x238
77 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK1 0x23c 77 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK1 0x23c
78 78
79 /* SNB-EP Ubox register */ 79 /* SNB-EP Ubox register */
80 #define SNBEP_U_MSR_PMON_CTR0 0xc16 80 #define SNBEP_U_MSR_PMON_CTR0 0xc16
81 #define SNBEP_U_MSR_PMON_CTL0 0xc10 81 #define SNBEP_U_MSR_PMON_CTL0 0xc10
82 82
83 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTL 0xc08 83 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTL 0xc08
84 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTR 0xc09 84 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTR 0xc09
85 85
86 /* SNB-EP Cbo register */ 86 /* SNB-EP Cbo register */
87 #define SNBEP_C0_MSR_PMON_CTR0 0xd16 87 #define SNBEP_C0_MSR_PMON_CTR0 0xd16
88 #define SNBEP_C0_MSR_PMON_CTL0 0xd10 88 #define SNBEP_C0_MSR_PMON_CTL0 0xd10
89 #define SNBEP_C0_MSR_PMON_BOX_CTL 0xd04 89 #define SNBEP_C0_MSR_PMON_BOX_CTL 0xd04
90 #define SNBEP_C0_MSR_PMON_BOX_FILTER 0xd14 90 #define SNBEP_C0_MSR_PMON_BOX_FILTER 0xd14
91 #define SNBEP_CBO_MSR_OFFSET 0x20 91 #define SNBEP_CBO_MSR_OFFSET 0x20
92 92
93 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_TID 0x1f 93 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_TID 0x1f
94 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_NID 0x3fc00 94 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_NID 0x3fc00
95 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE 0x7c0000 95 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE 0x7c0000
96 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC 0xff800000 96 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC 0xff800000
97 97
98 #define SNBEP_CBO_EVENT_EXTRA_REG(e, m, i) { \ 98 #define SNBEP_CBO_EVENT_EXTRA_REG(e, m, i) { \
99 .event = (e), \ 99 .event = (e), \
100 .msr = SNBEP_C0_MSR_PMON_BOX_FILTER, \ 100 .msr = SNBEP_C0_MSR_PMON_BOX_FILTER, \
101 .config_mask = (m), \ 101 .config_mask = (m), \
102 .idx = (i) \ 102 .idx = (i) \
103 } 103 }
104 104
105 /* SNB-EP PCU register */ 105 /* SNB-EP PCU register */
106 #define SNBEP_PCU_MSR_PMON_CTR0 0xc36 106 #define SNBEP_PCU_MSR_PMON_CTR0 0xc36
107 #define SNBEP_PCU_MSR_PMON_CTL0 0xc30 107 #define SNBEP_PCU_MSR_PMON_CTL0 0xc30
108 #define SNBEP_PCU_MSR_PMON_BOX_CTL 0xc24 108 #define SNBEP_PCU_MSR_PMON_BOX_CTL 0xc24
109 #define SNBEP_PCU_MSR_PMON_BOX_FILTER 0xc34 109 #define SNBEP_PCU_MSR_PMON_BOX_FILTER 0xc34
110 #define SNBEP_PCU_MSR_PMON_BOX_FILTER_MASK 0xffffffff 110 #define SNBEP_PCU_MSR_PMON_BOX_FILTER_MASK 0xffffffff
111 #define SNBEP_PCU_MSR_CORE_C3_CTR 0x3fc 111 #define SNBEP_PCU_MSR_CORE_C3_CTR 0x3fc
112 #define SNBEP_PCU_MSR_CORE_C6_CTR 0x3fd 112 #define SNBEP_PCU_MSR_CORE_C6_CTR 0x3fd
113 113
114 /* IVBEP event control */ 114 /* IVBEP event control */
115 #define IVBEP_PMON_BOX_CTL_INT (SNBEP_PMON_BOX_CTL_RST_CTRL | \ 115 #define IVBEP_PMON_BOX_CTL_INT (SNBEP_PMON_BOX_CTL_RST_CTRL | \
116 SNBEP_PMON_BOX_CTL_RST_CTRS) 116 SNBEP_PMON_BOX_CTL_RST_CTRS)
117 #define IVBEP_PMON_RAW_EVENT_MASK (SNBEP_PMON_CTL_EV_SEL_MASK | \ 117 #define IVBEP_PMON_RAW_EVENT_MASK (SNBEP_PMON_CTL_EV_SEL_MASK | \
118 SNBEP_PMON_CTL_UMASK_MASK | \ 118 SNBEP_PMON_CTL_UMASK_MASK | \
119 SNBEP_PMON_CTL_EDGE_DET | \ 119 SNBEP_PMON_CTL_EDGE_DET | \
120 SNBEP_PMON_CTL_TRESH_MASK) 120 SNBEP_PMON_CTL_TRESH_MASK)
121 /* IVBEP Ubox */ 121 /* IVBEP Ubox */
122 #define IVBEP_U_MSR_PMON_GLOBAL_CTL 0xc00 122 #define IVBEP_U_MSR_PMON_GLOBAL_CTL 0xc00
123 #define IVBEP_U_PMON_GLOBAL_FRZ_ALL (1 << 31) 123 #define IVBEP_U_PMON_GLOBAL_FRZ_ALL (1 << 31)
124 #define IVBEP_U_PMON_GLOBAL_UNFRZ_ALL (1 << 29) 124 #define IVBEP_U_PMON_GLOBAL_UNFRZ_ALL (1 << 29)
125 125
126 #define IVBEP_U_MSR_PMON_RAW_EVENT_MASK \ 126 #define IVBEP_U_MSR_PMON_RAW_EVENT_MASK \
127 (SNBEP_PMON_CTL_EV_SEL_MASK | \ 127 (SNBEP_PMON_CTL_EV_SEL_MASK | \
128 SNBEP_PMON_CTL_UMASK_MASK | \ 128 SNBEP_PMON_CTL_UMASK_MASK | \
129 SNBEP_PMON_CTL_EDGE_DET | \ 129 SNBEP_PMON_CTL_EDGE_DET | \
130 SNBEP_U_MSR_PMON_CTL_TRESH_MASK) 130 SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
131 /* IVBEP Cbo */ 131 /* IVBEP Cbo */
132 #define IVBEP_CBO_MSR_PMON_RAW_EVENT_MASK (IVBEP_PMON_RAW_EVENT_MASK | \ 132 #define IVBEP_CBO_MSR_PMON_RAW_EVENT_MASK (IVBEP_PMON_RAW_EVENT_MASK | \
133 SNBEP_CBO_PMON_CTL_TID_EN) 133 SNBEP_CBO_PMON_CTL_TID_EN)
134 134
135 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_TID (0x1fULL << 0) 135 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_TID (0x1fULL << 0)
136 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_LINK (0xfULL << 5) 136 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_LINK (0xfULL << 5)
137 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_STATE (0x3fULL << 17) 137 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_STATE (0x3fULL << 17)
138 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_NID (0xffffULL << 32) 138 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_NID (0xffffULL << 32)
139 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_OPC (0x1ffULL << 52) 139 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_OPC (0x1ffULL << 52)
140 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_C6 (0x1ULL << 61) 140 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_C6 (0x1ULL << 61)
141 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_NC (0x1ULL << 62) 141 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_NC (0x1ULL << 62)
142 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_ISOC (0x1ULL << 63) 142 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_ISOC (0x1ULL << 63)
143 143
144 /* IVBEP home agent */ 144 /* IVBEP home agent */
145 #define IVBEP_HA_PCI_PMON_CTL_Q_OCC_RST (1 << 16) 145 #define IVBEP_HA_PCI_PMON_CTL_Q_OCC_RST (1 << 16)
146 #define IVBEP_HA_PCI_PMON_RAW_EVENT_MASK \ 146 #define IVBEP_HA_PCI_PMON_RAW_EVENT_MASK \
147 (IVBEP_PMON_RAW_EVENT_MASK | \ 147 (IVBEP_PMON_RAW_EVENT_MASK | \
148 IVBEP_HA_PCI_PMON_CTL_Q_OCC_RST) 148 IVBEP_HA_PCI_PMON_CTL_Q_OCC_RST)
149 /* IVBEP PCU */ 149 /* IVBEP PCU */
150 #define IVBEP_PCU_MSR_PMON_RAW_EVENT_MASK \ 150 #define IVBEP_PCU_MSR_PMON_RAW_EVENT_MASK \
151 (SNBEP_PMON_CTL_EV_SEL_MASK | \ 151 (SNBEP_PMON_CTL_EV_SEL_MASK | \
152 SNBEP_PMON_CTL_EV_SEL_EXT | \ 152 SNBEP_PMON_CTL_EV_SEL_EXT | \
153 SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \ 153 SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
154 SNBEP_PMON_CTL_EDGE_DET | \ 154 SNBEP_PMON_CTL_EDGE_DET | \
155 SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \ 155 SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
156 SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \ 156 SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
157 SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET) 157 SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
158 /* IVBEP QPI */ 158 /* IVBEP QPI */
159 #define IVBEP_QPI_PCI_PMON_RAW_EVENT_MASK \ 159 #define IVBEP_QPI_PCI_PMON_RAW_EVENT_MASK \
160 (IVBEP_PMON_RAW_EVENT_MASK | \ 160 (IVBEP_PMON_RAW_EVENT_MASK | \
161 SNBEP_PMON_CTL_EV_SEL_EXT) 161 SNBEP_PMON_CTL_EV_SEL_EXT)
162 162
163 #define __BITS_VALUE(x, i, n) ((typeof(x))(((x) >> ((i) * (n))) & \ 163 #define __BITS_VALUE(x, i, n) ((typeof(x))(((x) >> ((i) * (n))) & \
164 ((1ULL << (n)) - 1))) 164 ((1ULL << (n)) - 1)))
165 165
166 /* Haswell-EP Ubox */ 166 /* Haswell-EP Ubox */
167 #define HSWEP_U_MSR_PMON_CTR0 0x705 167 #define HSWEP_U_MSR_PMON_CTR0 0x705
168 #define HSWEP_U_MSR_PMON_CTL0 0x709 168 #define HSWEP_U_MSR_PMON_CTL0 0x709
169 #define HSWEP_U_MSR_PMON_FILTER 0x707 169 #define HSWEP_U_MSR_PMON_FILTER 0x707
170 170
171 #define HSWEP_U_MSR_PMON_UCLK_FIXED_CTL 0x703 171 #define HSWEP_U_MSR_PMON_UCLK_FIXED_CTL 0x703
172 #define HSWEP_U_MSR_PMON_UCLK_FIXED_CTR 0x704 172 #define HSWEP_U_MSR_PMON_UCLK_FIXED_CTR 0x704
173 173
174 #define HSWEP_U_MSR_PMON_BOX_FILTER_TID (0x1 << 0) 174 #define HSWEP_U_MSR_PMON_BOX_FILTER_TID (0x1 << 0)
175 #define HSWEP_U_MSR_PMON_BOX_FILTER_CID (0x1fULL << 1) 175 #define HSWEP_U_MSR_PMON_BOX_FILTER_CID (0x1fULL << 1)
176 #define HSWEP_U_MSR_PMON_BOX_FILTER_MASK \ 176 #define HSWEP_U_MSR_PMON_BOX_FILTER_MASK \
177 (HSWEP_U_MSR_PMON_BOX_FILTER_TID | \ 177 (HSWEP_U_MSR_PMON_BOX_FILTER_TID | \
178 HSWEP_U_MSR_PMON_BOX_FILTER_CID) 178 HSWEP_U_MSR_PMON_BOX_FILTER_CID)
179 179
180 /* Haswell-EP CBo */ 180 /* Haswell-EP CBo */
181 #define HSWEP_C0_MSR_PMON_CTR0 0xe08 181 #define HSWEP_C0_MSR_PMON_CTR0 0xe08
182 #define HSWEP_C0_MSR_PMON_CTL0 0xe01 182 #define HSWEP_C0_MSR_PMON_CTL0 0xe01
183 #define HSWEP_C0_MSR_PMON_BOX_CTL 0xe00 183 #define HSWEP_C0_MSR_PMON_BOX_CTL 0xe00
184 #define HSWEP_C0_MSR_PMON_BOX_FILTER0 0xe05 184 #define HSWEP_C0_MSR_PMON_BOX_FILTER0 0xe05
185 #define HSWEP_CBO_MSR_OFFSET 0x10 185 #define HSWEP_CBO_MSR_OFFSET 0x10
186 186
187 187
188 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_TID (0x3fULL << 0) 188 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_TID (0x3fULL << 0)
189 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_LINK (0xfULL << 6) 189 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_LINK (0xfULL << 6)
190 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_STATE (0x7fULL << 17) 190 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_STATE (0x7fULL << 17)
191 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_NID (0xffffULL << 32) 191 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_NID (0xffffULL << 32)
192 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_OPC (0x1ffULL << 52) 192 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_OPC (0x1ffULL << 52)
193 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_C6 (0x1ULL << 61) 193 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_C6 (0x1ULL << 61)
194 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_NC (0x1ULL << 62) 194 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_NC (0x1ULL << 62)
195 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_ISOC (0x1ULL << 63) 195 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_ISOC (0x1ULL << 63)
196 196
197 197
198 /* Haswell-EP Sbox */ 198 /* Haswell-EP Sbox */
199 #define HSWEP_S0_MSR_PMON_CTR0 0x726 199 #define HSWEP_S0_MSR_PMON_CTR0 0x726
200 #define HSWEP_S0_MSR_PMON_CTL0 0x721 200 #define HSWEP_S0_MSR_PMON_CTL0 0x721
201 #define HSWEP_S0_MSR_PMON_BOX_CTL 0x720 201 #define HSWEP_S0_MSR_PMON_BOX_CTL 0x720
202 #define HSWEP_SBOX_MSR_OFFSET 0xa 202 #define HSWEP_SBOX_MSR_OFFSET 0xa
203 #define HSWEP_S_MSR_PMON_RAW_EVENT_MASK (SNBEP_PMON_RAW_EVENT_MASK | \ 203 #define HSWEP_S_MSR_PMON_RAW_EVENT_MASK (SNBEP_PMON_RAW_EVENT_MASK | \
204 SNBEP_CBO_PMON_CTL_TID_EN) 204 SNBEP_CBO_PMON_CTL_TID_EN)
205 205
206 /* Haswell-EP PCU */ 206 /* Haswell-EP PCU */
207 #define HSWEP_PCU_MSR_PMON_CTR0 0x717 207 #define HSWEP_PCU_MSR_PMON_CTR0 0x717
208 #define HSWEP_PCU_MSR_PMON_CTL0 0x711 208 #define HSWEP_PCU_MSR_PMON_CTL0 0x711
209 #define HSWEP_PCU_MSR_PMON_BOX_CTL 0x710 209 #define HSWEP_PCU_MSR_PMON_BOX_CTL 0x710
210 #define HSWEP_PCU_MSR_PMON_BOX_FILTER 0x715 210 #define HSWEP_PCU_MSR_PMON_BOX_FILTER 0x715
211 211
212 212
213 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); 213 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
214 DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21"); 214 DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
215 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); 215 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
216 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18"); 216 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
217 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19"); 217 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
218 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23"); 218 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
219 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31"); 219 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
220 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28"); 220 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
221 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15"); 221 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
222 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30"); 222 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
223 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51"); 223 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
224 DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4"); 224 DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4");
225 DEFINE_UNCORE_FORMAT_ATTR(filter_tid2, filter_tid, "config1:0"); 225 DEFINE_UNCORE_FORMAT_ATTR(filter_tid2, filter_tid, "config1:0");
226 DEFINE_UNCORE_FORMAT_ATTR(filter_tid3, filter_tid, "config1:0-5"); 226 DEFINE_UNCORE_FORMAT_ATTR(filter_tid3, filter_tid, "config1:0-5");
227 DEFINE_UNCORE_FORMAT_ATTR(filter_cid, filter_cid, "config1:5"); 227 DEFINE_UNCORE_FORMAT_ATTR(filter_cid, filter_cid, "config1:5");
228 DEFINE_UNCORE_FORMAT_ATTR(filter_link, filter_link, "config1:5-8"); 228 DEFINE_UNCORE_FORMAT_ATTR(filter_link, filter_link, "config1:5-8");
229 DEFINE_UNCORE_FORMAT_ATTR(filter_link2, filter_link, "config1:6-8"); 229 DEFINE_UNCORE_FORMAT_ATTR(filter_link2, filter_link, "config1:6-8");
230 DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17"); 230 DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17");
231 DEFINE_UNCORE_FORMAT_ATTR(filter_nid2, filter_nid, "config1:32-47"); 231 DEFINE_UNCORE_FORMAT_ATTR(filter_nid2, filter_nid, "config1:32-47");
232 DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22"); 232 DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22");
233 DEFINE_UNCORE_FORMAT_ATTR(filter_state2, filter_state, "config1:17-22"); 233 DEFINE_UNCORE_FORMAT_ATTR(filter_state2, filter_state, "config1:17-22");
234 DEFINE_UNCORE_FORMAT_ATTR(filter_state3, filter_state, "config1:17-23"); 234 DEFINE_UNCORE_FORMAT_ATTR(filter_state3, filter_state, "config1:17-23");
235 DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31"); 235 DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31");
236 DEFINE_UNCORE_FORMAT_ATTR(filter_opc2, filter_opc, "config1:52-60"); 236 DEFINE_UNCORE_FORMAT_ATTR(filter_opc2, filter_opc, "config1:52-60");
237 DEFINE_UNCORE_FORMAT_ATTR(filter_nc, filter_nc, "config1:62"); 237 DEFINE_UNCORE_FORMAT_ATTR(filter_nc, filter_nc, "config1:62");
238 DEFINE_UNCORE_FORMAT_ATTR(filter_c6, filter_c6, "config1:61"); 238 DEFINE_UNCORE_FORMAT_ATTR(filter_c6, filter_c6, "config1:61");
239 DEFINE_UNCORE_FORMAT_ATTR(filter_isoc, filter_isoc, "config1:63"); 239 DEFINE_UNCORE_FORMAT_ATTR(filter_isoc, filter_isoc, "config1:63");
240 DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7"); 240 DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7");
241 DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15"); 241 DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15");
242 DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23"); 242 DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23");
243 DEFINE_UNCORE_FORMAT_ATTR(filter_band3, filter_band3, "config1:24-31"); 243 DEFINE_UNCORE_FORMAT_ATTR(filter_band3, filter_band3, "config1:24-31");
244 DEFINE_UNCORE_FORMAT_ATTR(match_rds, match_rds, "config1:48-51"); 244 DEFINE_UNCORE_FORMAT_ATTR(match_rds, match_rds, "config1:48-51");
245 DEFINE_UNCORE_FORMAT_ATTR(match_rnid30, match_rnid30, "config1:32-35"); 245 DEFINE_UNCORE_FORMAT_ATTR(match_rnid30, match_rnid30, "config1:32-35");
246 DEFINE_UNCORE_FORMAT_ATTR(match_rnid4, match_rnid4, "config1:31"); 246 DEFINE_UNCORE_FORMAT_ATTR(match_rnid4, match_rnid4, "config1:31");
247 DEFINE_UNCORE_FORMAT_ATTR(match_dnid, match_dnid, "config1:13-17"); 247 DEFINE_UNCORE_FORMAT_ATTR(match_dnid, match_dnid, "config1:13-17");
248 DEFINE_UNCORE_FORMAT_ATTR(match_mc, match_mc, "config1:9-12"); 248 DEFINE_UNCORE_FORMAT_ATTR(match_mc, match_mc, "config1:9-12");
249 DEFINE_UNCORE_FORMAT_ATTR(match_opc, match_opc, "config1:5-8"); 249 DEFINE_UNCORE_FORMAT_ATTR(match_opc, match_opc, "config1:5-8");
250 DEFINE_UNCORE_FORMAT_ATTR(match_vnw, match_vnw, "config1:3-4"); 250 DEFINE_UNCORE_FORMAT_ATTR(match_vnw, match_vnw, "config1:3-4");
251 DEFINE_UNCORE_FORMAT_ATTR(match0, match0, "config1:0-31"); 251 DEFINE_UNCORE_FORMAT_ATTR(match0, match0, "config1:0-31");
252 DEFINE_UNCORE_FORMAT_ATTR(match1, match1, "config1:32-63"); 252 DEFINE_UNCORE_FORMAT_ATTR(match1, match1, "config1:32-63");
253 DEFINE_UNCORE_FORMAT_ATTR(mask_rds, mask_rds, "config2:48-51"); 253 DEFINE_UNCORE_FORMAT_ATTR(mask_rds, mask_rds, "config2:48-51");
254 DEFINE_UNCORE_FORMAT_ATTR(mask_rnid30, mask_rnid30, "config2:32-35"); 254 DEFINE_UNCORE_FORMAT_ATTR(mask_rnid30, mask_rnid30, "config2:32-35");
255 DEFINE_UNCORE_FORMAT_ATTR(mask_rnid4, mask_rnid4, "config2:31"); 255 DEFINE_UNCORE_FORMAT_ATTR(mask_rnid4, mask_rnid4, "config2:31");
256 DEFINE_UNCORE_FORMAT_ATTR(mask_dnid, mask_dnid, "config2:13-17"); 256 DEFINE_UNCORE_FORMAT_ATTR(mask_dnid, mask_dnid, "config2:13-17");
257 DEFINE_UNCORE_FORMAT_ATTR(mask_mc, mask_mc, "config2:9-12"); 257 DEFINE_UNCORE_FORMAT_ATTR(mask_mc, mask_mc, "config2:9-12");
258 DEFINE_UNCORE_FORMAT_ATTR(mask_opc, mask_opc, "config2:5-8"); 258 DEFINE_UNCORE_FORMAT_ATTR(mask_opc, mask_opc, "config2:5-8");
259 DEFINE_UNCORE_FORMAT_ATTR(mask_vnw, mask_vnw, "config2:3-4"); 259 DEFINE_UNCORE_FORMAT_ATTR(mask_vnw, mask_vnw, "config2:3-4");
260 DEFINE_UNCORE_FORMAT_ATTR(mask0, mask0, "config2:0-31"); 260 DEFINE_UNCORE_FORMAT_ATTR(mask0, mask0, "config2:0-31");
261 DEFINE_UNCORE_FORMAT_ATTR(mask1, mask1, "config2:32-63"); 261 DEFINE_UNCORE_FORMAT_ATTR(mask1, mask1, "config2:32-63");
262 262
263 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box) 263 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
264 { 264 {
265 struct pci_dev *pdev = box->pci_dev; 265 struct pci_dev *pdev = box->pci_dev;
266 int box_ctl = uncore_pci_box_ctl(box); 266 int box_ctl = uncore_pci_box_ctl(box);
267 u32 config = 0; 267 u32 config = 0;
268 268
269 if (!pci_read_config_dword(pdev, box_ctl, &config)) { 269 if (!pci_read_config_dword(pdev, box_ctl, &config)) {
270 config |= SNBEP_PMON_BOX_CTL_FRZ; 270 config |= SNBEP_PMON_BOX_CTL_FRZ;
271 pci_write_config_dword(pdev, box_ctl, config); 271 pci_write_config_dword(pdev, box_ctl, config);
272 } 272 }
273 } 273 }
274 274
275 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box) 275 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
276 { 276 {
277 struct pci_dev *pdev = box->pci_dev; 277 struct pci_dev *pdev = box->pci_dev;
278 int box_ctl = uncore_pci_box_ctl(box); 278 int box_ctl = uncore_pci_box_ctl(box);
279 u32 config = 0; 279 u32 config = 0;
280 280
281 if (!pci_read_config_dword(pdev, box_ctl, &config)) { 281 if (!pci_read_config_dword(pdev, box_ctl, &config)) {
282 config &= ~SNBEP_PMON_BOX_CTL_FRZ; 282 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
283 pci_write_config_dword(pdev, box_ctl, config); 283 pci_write_config_dword(pdev, box_ctl, config);
284 } 284 }
285 } 285 }
286 286
287 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event) 287 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event)
288 { 288 {
289 struct pci_dev *pdev = box->pci_dev; 289 struct pci_dev *pdev = box->pci_dev;
290 struct hw_perf_event *hwc = &event->hw; 290 struct hw_perf_event *hwc = &event->hw;
291 291
292 pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); 292 pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
293 } 293 }
294 294
295 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box, struct perf_event *event) 295 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box, struct perf_event *event)
296 { 296 {
297 struct pci_dev *pdev = box->pci_dev; 297 struct pci_dev *pdev = box->pci_dev;
298 struct hw_perf_event *hwc = &event->hw; 298 struct hw_perf_event *hwc = &event->hw;
299 299
300 pci_write_config_dword(pdev, hwc->config_base, hwc->config); 300 pci_write_config_dword(pdev, hwc->config_base, hwc->config);
301 } 301 }
302 302
303 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct perf_event *event) 303 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct perf_event *event)
304 { 304 {
305 struct pci_dev *pdev = box->pci_dev; 305 struct pci_dev *pdev = box->pci_dev;
306 struct hw_perf_event *hwc = &event->hw; 306 struct hw_perf_event *hwc = &event->hw;
307 u64 count = 0; 307 u64 count = 0;
308 308
309 pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count); 309 pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
310 pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1); 310 pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
311 311
312 return count; 312 return count;
313 } 313 }
314 314
315 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box) 315 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
316 { 316 {
317 struct pci_dev *pdev = box->pci_dev; 317 struct pci_dev *pdev = box->pci_dev;
318 318
319 pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, SNBEP_PMON_BOX_CTL_INT); 319 pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, SNBEP_PMON_BOX_CTL_INT);
320 } 320 }
321 321
322 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) 322 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
323 { 323 {
324 u64 config; 324 u64 config;
325 unsigned msr; 325 unsigned msr;
326 326
327 msr = uncore_msr_box_ctl(box); 327 msr = uncore_msr_box_ctl(box);
328 if (msr) { 328 if (msr) {
329 rdmsrl(msr, config); 329 rdmsrl(msr, config);
330 config |= SNBEP_PMON_BOX_CTL_FRZ; 330 config |= SNBEP_PMON_BOX_CTL_FRZ;
331 wrmsrl(msr, config); 331 wrmsrl(msr, config);
332 } 332 }
333 } 333 }
334 334
335 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box) 335 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
336 { 336 {
337 u64 config; 337 u64 config;
338 unsigned msr; 338 unsigned msr;
339 339
340 msr = uncore_msr_box_ctl(box); 340 msr = uncore_msr_box_ctl(box);
341 if (msr) { 341 if (msr) {
342 rdmsrl(msr, config); 342 rdmsrl(msr, config);
343 config &= ~SNBEP_PMON_BOX_CTL_FRZ; 343 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
344 wrmsrl(msr, config); 344 wrmsrl(msr, config);
345 } 345 }
346 } 346 }
347 347
348 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event) 348 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
349 { 349 {
350 struct hw_perf_event *hwc = &event->hw; 350 struct hw_perf_event *hwc = &event->hw;
351 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 351 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
352 352
353 if (reg1->idx != EXTRA_REG_NONE) 353 if (reg1->idx != EXTRA_REG_NONE)
354 wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0)); 354 wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0));
355 355
356 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); 356 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
357 } 357 }
358 358
359 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, 359 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
360 struct perf_event *event) 360 struct perf_event *event)
361 { 361 {
362 struct hw_perf_event *hwc = &event->hw; 362 struct hw_perf_event *hwc = &event->hw;
363 363
364 wrmsrl(hwc->config_base, hwc->config); 364 wrmsrl(hwc->config_base, hwc->config);
365 } 365 }
366 366
367 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) 367 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
368 { 368 {
369 unsigned msr = uncore_msr_box_ctl(box); 369 unsigned msr = uncore_msr_box_ctl(box);
370 370
371 if (msr) 371 if (msr)
372 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT); 372 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
373 } 373 }
374 374
375 static struct attribute *snbep_uncore_formats_attr[] = { 375 static struct attribute *snbep_uncore_formats_attr[] = {
376 &format_attr_event.attr, 376 &format_attr_event.attr,
377 &format_attr_umask.attr, 377 &format_attr_umask.attr,
378 &format_attr_edge.attr, 378 &format_attr_edge.attr,
379 &format_attr_inv.attr, 379 &format_attr_inv.attr,
380 &format_attr_thresh8.attr, 380 &format_attr_thresh8.attr,
381 NULL, 381 NULL,
382 }; 382 };
383 383
384 static struct attribute *snbep_uncore_ubox_formats_attr[] = { 384 static struct attribute *snbep_uncore_ubox_formats_attr[] = {
385 &format_attr_event.attr, 385 &format_attr_event.attr,
386 &format_attr_umask.attr, 386 &format_attr_umask.attr,
387 &format_attr_edge.attr, 387 &format_attr_edge.attr,
388 &format_attr_inv.attr, 388 &format_attr_inv.attr,
389 &format_attr_thresh5.attr, 389 &format_attr_thresh5.attr,
390 NULL, 390 NULL,
391 }; 391 };
392 392
393 static struct attribute *snbep_uncore_cbox_formats_attr[] = { 393 static struct attribute *snbep_uncore_cbox_formats_attr[] = {
394 &format_attr_event.attr, 394 &format_attr_event.attr,
395 &format_attr_umask.attr, 395 &format_attr_umask.attr,
396 &format_attr_edge.attr, 396 &format_attr_edge.attr,
397 &format_attr_tid_en.attr, 397 &format_attr_tid_en.attr,
398 &format_attr_inv.attr, 398 &format_attr_inv.attr,
399 &format_attr_thresh8.attr, 399 &format_attr_thresh8.attr,
400 &format_attr_filter_tid.attr, 400 &format_attr_filter_tid.attr,
401 &format_attr_filter_nid.attr, 401 &format_attr_filter_nid.attr,
402 &format_attr_filter_state.attr, 402 &format_attr_filter_state.attr,
403 &format_attr_filter_opc.attr, 403 &format_attr_filter_opc.attr,
404 NULL, 404 NULL,
405 }; 405 };
406 406
407 static struct attribute *snbep_uncore_pcu_formats_attr[] = { 407 static struct attribute *snbep_uncore_pcu_formats_attr[] = {
408 &format_attr_event_ext.attr, 408 &format_attr_event_ext.attr,
409 &format_attr_occ_sel.attr, 409 &format_attr_occ_sel.attr,
410 &format_attr_edge.attr, 410 &format_attr_edge.attr,
411 &format_attr_inv.attr, 411 &format_attr_inv.attr,
412 &format_attr_thresh5.attr, 412 &format_attr_thresh5.attr,
413 &format_attr_occ_invert.attr, 413 &format_attr_occ_invert.attr,
414 &format_attr_occ_edge.attr, 414 &format_attr_occ_edge.attr,
415 &format_attr_filter_band0.attr, 415 &format_attr_filter_band0.attr,
416 &format_attr_filter_band1.attr, 416 &format_attr_filter_band1.attr,
417 &format_attr_filter_band2.attr, 417 &format_attr_filter_band2.attr,
418 &format_attr_filter_band3.attr, 418 &format_attr_filter_band3.attr,
419 NULL, 419 NULL,
420 }; 420 };
421 421
422 static struct attribute *snbep_uncore_qpi_formats_attr[] = { 422 static struct attribute *snbep_uncore_qpi_formats_attr[] = {
423 &format_attr_event_ext.attr, 423 &format_attr_event_ext.attr,
424 &format_attr_umask.attr, 424 &format_attr_umask.attr,
425 &format_attr_edge.attr, 425 &format_attr_edge.attr,
426 &format_attr_inv.attr, 426 &format_attr_inv.attr,
427 &format_attr_thresh8.attr, 427 &format_attr_thresh8.attr,
428 &format_attr_match_rds.attr, 428 &format_attr_match_rds.attr,
429 &format_attr_match_rnid30.attr, 429 &format_attr_match_rnid30.attr,
430 &format_attr_match_rnid4.attr, 430 &format_attr_match_rnid4.attr,
431 &format_attr_match_dnid.attr, 431 &format_attr_match_dnid.attr,
432 &format_attr_match_mc.attr, 432 &format_attr_match_mc.attr,
433 &format_attr_match_opc.attr, 433 &format_attr_match_opc.attr,
434 &format_attr_match_vnw.attr, 434 &format_attr_match_vnw.attr,
435 &format_attr_match0.attr, 435 &format_attr_match0.attr,
436 &format_attr_match1.attr, 436 &format_attr_match1.attr,
437 &format_attr_mask_rds.attr, 437 &format_attr_mask_rds.attr,
438 &format_attr_mask_rnid30.attr, 438 &format_attr_mask_rnid30.attr,
439 &format_attr_mask_rnid4.attr, 439 &format_attr_mask_rnid4.attr,
440 &format_attr_mask_dnid.attr, 440 &format_attr_mask_dnid.attr,
441 &format_attr_mask_mc.attr, 441 &format_attr_mask_mc.attr,
442 &format_attr_mask_opc.attr, 442 &format_attr_mask_opc.attr,
443 &format_attr_mask_vnw.attr, 443 &format_attr_mask_vnw.attr,
444 &format_attr_mask0.attr, 444 &format_attr_mask0.attr,
445 &format_attr_mask1.attr, 445 &format_attr_mask1.attr,
446 NULL, 446 NULL,
447 }; 447 };
448 448
449 static struct uncore_event_desc snbep_uncore_imc_events[] = { 449 static struct uncore_event_desc snbep_uncore_imc_events[] = {
450 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"), 450 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"),
451 INTEL_UNCORE_EVENT_DESC(cas_count_read, "event=0x04,umask=0x03"), 451 INTEL_UNCORE_EVENT_DESC(cas_count_read, "event=0x04,umask=0x03"),
452 INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"), 452 INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"),
453 INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"), 453 INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"),
454 INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"), 454 INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
455 INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"), 455 INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"),
456 INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"), 456 INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"),
457 { /* end: all zeroes */ }, 457 { /* end: all zeroes */ },
458 }; 458 };
459 459
460 static struct uncore_event_desc snbep_uncore_qpi_events[] = { 460 static struct uncore_event_desc snbep_uncore_qpi_events[] = {
461 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x14"), 461 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x14"),
462 INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"), 462 INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
463 INTEL_UNCORE_EVENT_DESC(drs_data, "event=0x102,umask=0x08"), 463 INTEL_UNCORE_EVENT_DESC(drs_data, "event=0x102,umask=0x08"),
464 INTEL_UNCORE_EVENT_DESC(ncb_data, "event=0x103,umask=0x04"), 464 INTEL_UNCORE_EVENT_DESC(ncb_data, "event=0x103,umask=0x04"),
465 { /* end: all zeroes */ }, 465 { /* end: all zeroes */ },
466 }; 466 };
467 467
468 static struct attribute_group snbep_uncore_format_group = { 468 static struct attribute_group snbep_uncore_format_group = {
469 .name = "format", 469 .name = "format",
470 .attrs = snbep_uncore_formats_attr, 470 .attrs = snbep_uncore_formats_attr,
471 }; 471 };
472 472
473 static struct attribute_group snbep_uncore_ubox_format_group = { 473 static struct attribute_group snbep_uncore_ubox_format_group = {
474 .name = "format", 474 .name = "format",
475 .attrs = snbep_uncore_ubox_formats_attr, 475 .attrs = snbep_uncore_ubox_formats_attr,
476 }; 476 };
477 477
478 static struct attribute_group snbep_uncore_cbox_format_group = { 478 static struct attribute_group snbep_uncore_cbox_format_group = {
479 .name = "format", 479 .name = "format",
480 .attrs = snbep_uncore_cbox_formats_attr, 480 .attrs = snbep_uncore_cbox_formats_attr,
481 }; 481 };
482 482
483 static struct attribute_group snbep_uncore_pcu_format_group = { 483 static struct attribute_group snbep_uncore_pcu_format_group = {
484 .name = "format", 484 .name = "format",
485 .attrs = snbep_uncore_pcu_formats_attr, 485 .attrs = snbep_uncore_pcu_formats_attr,
486 }; 486 };
487 487
488 static struct attribute_group snbep_uncore_qpi_format_group = { 488 static struct attribute_group snbep_uncore_qpi_format_group = {
489 .name = "format", 489 .name = "format",
490 .attrs = snbep_uncore_qpi_formats_attr, 490 .attrs = snbep_uncore_qpi_formats_attr,
491 }; 491 };
492 492
493 #define __SNBEP_UNCORE_MSR_OPS_COMMON_INIT() \ 493 #define __SNBEP_UNCORE_MSR_OPS_COMMON_INIT() \
494 .disable_box = snbep_uncore_msr_disable_box, \ 494 .disable_box = snbep_uncore_msr_disable_box, \
495 .enable_box = snbep_uncore_msr_enable_box, \ 495 .enable_box = snbep_uncore_msr_enable_box, \
496 .disable_event = snbep_uncore_msr_disable_event, \ 496 .disable_event = snbep_uncore_msr_disable_event, \
497 .enable_event = snbep_uncore_msr_enable_event, \ 497 .enable_event = snbep_uncore_msr_enable_event, \
498 .read_counter = uncore_msr_read_counter 498 .read_counter = uncore_msr_read_counter
499 499
500 #define SNBEP_UNCORE_MSR_OPS_COMMON_INIT() \ 500 #define SNBEP_UNCORE_MSR_OPS_COMMON_INIT() \
501 __SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), \ 501 __SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), \
502 .init_box = snbep_uncore_msr_init_box \ 502 .init_box = snbep_uncore_msr_init_box \
503 503
504 static struct intel_uncore_ops snbep_uncore_msr_ops = { 504 static struct intel_uncore_ops snbep_uncore_msr_ops = {
505 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), 505 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
506 }; 506 };
507 507
508 #define SNBEP_UNCORE_PCI_OPS_COMMON_INIT() \ 508 #define SNBEP_UNCORE_PCI_OPS_COMMON_INIT() \
509 .init_box = snbep_uncore_pci_init_box, \ 509 .init_box = snbep_uncore_pci_init_box, \
510 .disable_box = snbep_uncore_pci_disable_box, \ 510 .disable_box = snbep_uncore_pci_disable_box, \
511 .enable_box = snbep_uncore_pci_enable_box, \ 511 .enable_box = snbep_uncore_pci_enable_box, \
512 .disable_event = snbep_uncore_pci_disable_event, \ 512 .disable_event = snbep_uncore_pci_disable_event, \
513 .read_counter = snbep_uncore_pci_read_counter 513 .read_counter = snbep_uncore_pci_read_counter
514 514
515 static struct intel_uncore_ops snbep_uncore_pci_ops = { 515 static struct intel_uncore_ops snbep_uncore_pci_ops = {
516 SNBEP_UNCORE_PCI_OPS_COMMON_INIT(), 516 SNBEP_UNCORE_PCI_OPS_COMMON_INIT(),
517 .enable_event = snbep_uncore_pci_enable_event, \ 517 .enable_event = snbep_uncore_pci_enable_event, \
518 }; 518 };
519 519
520 static struct event_constraint snbep_uncore_cbox_constraints[] = { 520 static struct event_constraint snbep_uncore_cbox_constraints[] = {
521 UNCORE_EVENT_CONSTRAINT(0x01, 0x1), 521 UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
522 UNCORE_EVENT_CONSTRAINT(0x02, 0x3), 522 UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
523 UNCORE_EVENT_CONSTRAINT(0x04, 0x3), 523 UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
524 UNCORE_EVENT_CONSTRAINT(0x05, 0x3), 524 UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
525 UNCORE_EVENT_CONSTRAINT(0x07, 0x3), 525 UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
526 UNCORE_EVENT_CONSTRAINT(0x09, 0x3), 526 UNCORE_EVENT_CONSTRAINT(0x09, 0x3),
527 UNCORE_EVENT_CONSTRAINT(0x11, 0x1), 527 UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
528 UNCORE_EVENT_CONSTRAINT(0x12, 0x3), 528 UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
529 UNCORE_EVENT_CONSTRAINT(0x13, 0x3), 529 UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
530 UNCORE_EVENT_CONSTRAINT(0x1b, 0xc), 530 UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
531 UNCORE_EVENT_CONSTRAINT(0x1c, 0xc), 531 UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
532 UNCORE_EVENT_CONSTRAINT(0x1d, 0xc), 532 UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
533 UNCORE_EVENT_CONSTRAINT(0x1e, 0xc), 533 UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
534 EVENT_CONSTRAINT_OVERLAP(0x1f, 0xe, 0xff), 534 EVENT_CONSTRAINT_OVERLAP(0x1f, 0xe, 0xff),
535 UNCORE_EVENT_CONSTRAINT(0x21, 0x3), 535 UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
536 UNCORE_EVENT_CONSTRAINT(0x23, 0x3), 536 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
537 UNCORE_EVENT_CONSTRAINT(0x31, 0x3), 537 UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
538 UNCORE_EVENT_CONSTRAINT(0x32, 0x3), 538 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
539 UNCORE_EVENT_CONSTRAINT(0x33, 0x3), 539 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
540 UNCORE_EVENT_CONSTRAINT(0x34, 0x3), 540 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
541 UNCORE_EVENT_CONSTRAINT(0x35, 0x3), 541 UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
542 UNCORE_EVENT_CONSTRAINT(0x36, 0x1), 542 UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
543 UNCORE_EVENT_CONSTRAINT(0x37, 0x3), 543 UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
544 UNCORE_EVENT_CONSTRAINT(0x38, 0x3), 544 UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
545 UNCORE_EVENT_CONSTRAINT(0x39, 0x3), 545 UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
546 UNCORE_EVENT_CONSTRAINT(0x3b, 0x1), 546 UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
547 EVENT_CONSTRAINT_END 547 EVENT_CONSTRAINT_END
548 }; 548 };
549 549
550 static struct event_constraint snbep_uncore_r2pcie_constraints[] = { 550 static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
551 UNCORE_EVENT_CONSTRAINT(0x10, 0x3), 551 UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
552 UNCORE_EVENT_CONSTRAINT(0x11, 0x3), 552 UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
553 UNCORE_EVENT_CONSTRAINT(0x12, 0x1), 553 UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
554 UNCORE_EVENT_CONSTRAINT(0x23, 0x3), 554 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
555 UNCORE_EVENT_CONSTRAINT(0x24, 0x3), 555 UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
556 UNCORE_EVENT_CONSTRAINT(0x25, 0x3), 556 UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
557 UNCORE_EVENT_CONSTRAINT(0x26, 0x3), 557 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
558 UNCORE_EVENT_CONSTRAINT(0x32, 0x3), 558 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
559 UNCORE_EVENT_CONSTRAINT(0x33, 0x3), 559 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
560 UNCORE_EVENT_CONSTRAINT(0x34, 0x3), 560 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
561 EVENT_CONSTRAINT_END 561 EVENT_CONSTRAINT_END
562 }; 562 };
563 563
564 static struct event_constraint snbep_uncore_r3qpi_constraints[] = { 564 static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
565 UNCORE_EVENT_CONSTRAINT(0x10, 0x3), 565 UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
566 UNCORE_EVENT_CONSTRAINT(0x11, 0x3), 566 UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
567 UNCORE_EVENT_CONSTRAINT(0x12, 0x3), 567 UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
568 UNCORE_EVENT_CONSTRAINT(0x13, 0x1), 568 UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
569 UNCORE_EVENT_CONSTRAINT(0x20, 0x3), 569 UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
570 UNCORE_EVENT_CONSTRAINT(0x21, 0x3), 570 UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
571 UNCORE_EVENT_CONSTRAINT(0x22, 0x3), 571 UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
572 UNCORE_EVENT_CONSTRAINT(0x23, 0x3), 572 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
573 UNCORE_EVENT_CONSTRAINT(0x24, 0x3), 573 UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
574 UNCORE_EVENT_CONSTRAINT(0x25, 0x3), 574 UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
575 UNCORE_EVENT_CONSTRAINT(0x26, 0x3), 575 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
576 UNCORE_EVENT_CONSTRAINT(0x28, 0x3), 576 UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
577 UNCORE_EVENT_CONSTRAINT(0x29, 0x3), 577 UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
578 UNCORE_EVENT_CONSTRAINT(0x2a, 0x3), 578 UNCORE_EVENT_CONSTRAINT(0x2a, 0x3),
579 UNCORE_EVENT_CONSTRAINT(0x2b, 0x3), 579 UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
580 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3), 580 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
581 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3), 581 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
582 UNCORE_EVENT_CONSTRAINT(0x2e, 0x3), 582 UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
583 UNCORE_EVENT_CONSTRAINT(0x2f, 0x3), 583 UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
584 UNCORE_EVENT_CONSTRAINT(0x30, 0x3), 584 UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
585 UNCORE_EVENT_CONSTRAINT(0x31, 0x3), 585 UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
586 UNCORE_EVENT_CONSTRAINT(0x32, 0x3), 586 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
587 UNCORE_EVENT_CONSTRAINT(0x33, 0x3), 587 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
588 UNCORE_EVENT_CONSTRAINT(0x34, 0x3), 588 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
589 UNCORE_EVENT_CONSTRAINT(0x36, 0x3), 589 UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
590 UNCORE_EVENT_CONSTRAINT(0x37, 0x3), 590 UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
591 UNCORE_EVENT_CONSTRAINT(0x38, 0x3), 591 UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
592 UNCORE_EVENT_CONSTRAINT(0x39, 0x3), 592 UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
593 EVENT_CONSTRAINT_END 593 EVENT_CONSTRAINT_END
594 }; 594 };
595 595
596 static struct intel_uncore_type snbep_uncore_ubox = { 596 static struct intel_uncore_type snbep_uncore_ubox = {
597 .name = "ubox", 597 .name = "ubox",
598 .num_counters = 2, 598 .num_counters = 2,
599 .num_boxes = 1, 599 .num_boxes = 1,
600 .perf_ctr_bits = 44, 600 .perf_ctr_bits = 44,
601 .fixed_ctr_bits = 48, 601 .fixed_ctr_bits = 48,
602 .perf_ctr = SNBEP_U_MSR_PMON_CTR0, 602 .perf_ctr = SNBEP_U_MSR_PMON_CTR0,
603 .event_ctl = SNBEP_U_MSR_PMON_CTL0, 603 .event_ctl = SNBEP_U_MSR_PMON_CTL0,
604 .event_mask = SNBEP_U_MSR_PMON_RAW_EVENT_MASK, 604 .event_mask = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
605 .fixed_ctr = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR, 605 .fixed_ctr = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
606 .fixed_ctl = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL, 606 .fixed_ctl = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
607 .ops = &snbep_uncore_msr_ops, 607 .ops = &snbep_uncore_msr_ops,
608 .format_group = &snbep_uncore_ubox_format_group, 608 .format_group = &snbep_uncore_ubox_format_group,
609 }; 609 };
610 610
611 static struct extra_reg snbep_uncore_cbox_extra_regs[] = { 611 static struct extra_reg snbep_uncore_cbox_extra_regs[] = {
612 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN, 612 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
613 SNBEP_CBO_PMON_CTL_TID_EN, 0x1), 613 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
614 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4), 614 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
615 SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0x6), 615 SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0x6),
616 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4), 616 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
617 SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0x6), 617 SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0x6),
618 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4), 618 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
619 SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0x6), 619 SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0x6),
620 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6), 620 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6),
621 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8), 621 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8),
622 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8), 622 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8),
623 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0xa), 623 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0xa),
624 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0xa), 624 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0xa),
625 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x2), 625 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x2),
626 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x2), 626 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x2),
627 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x2), 627 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x2),
628 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x2), 628 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x2),
629 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x8), 629 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x8),
630 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x8), 630 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x8),
631 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0xa), 631 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0xa),
632 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0xa), 632 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0xa),
633 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x2), 633 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x2),
634 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x2), 634 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x2),
635 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x2), 635 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x2),
636 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x2), 636 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x2),
637 EVENT_EXTRA_END 637 EVENT_EXTRA_END
638 }; 638 };
639 639
640 static void snbep_cbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event) 640 static void snbep_cbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
641 { 641 {
642 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 642 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
643 struct intel_uncore_extra_reg *er = &box->shared_regs[0]; 643 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
644 int i; 644 int i;
645 645
646 if (uncore_box_is_fake(box)) 646 if (uncore_box_is_fake(box))
647 return; 647 return;
648 648
649 for (i = 0; i < 5; i++) { 649 for (i = 0; i < 5; i++) {
650 if (reg1->alloc & (0x1 << i)) 650 if (reg1->alloc & (0x1 << i))
651 atomic_sub(1 << (i * 6), &er->ref); 651 atomic_sub(1 << (i * 6), &er->ref);
652 } 652 }
653 reg1->alloc = 0; 653 reg1->alloc = 0;
654 } 654 }
655 655
656 static struct event_constraint * 656 static struct event_constraint *
657 __snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event, 657 __snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event,
658 u64 (*cbox_filter_mask)(int fields)) 658 u64 (*cbox_filter_mask)(int fields))
659 { 659 {
660 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 660 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
661 struct intel_uncore_extra_reg *er = &box->shared_regs[0]; 661 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
662 int i, alloc = 0; 662 int i, alloc = 0;
663 unsigned long flags; 663 unsigned long flags;
664 u64 mask; 664 u64 mask;
665 665
666 if (reg1->idx == EXTRA_REG_NONE) 666 if (reg1->idx == EXTRA_REG_NONE)
667 return NULL; 667 return NULL;
668 668
669 raw_spin_lock_irqsave(&er->lock, flags); 669 raw_spin_lock_irqsave(&er->lock, flags);
670 for (i = 0; i < 5; i++) { 670 for (i = 0; i < 5; i++) {
671 if (!(reg1->idx & (0x1 << i))) 671 if (!(reg1->idx & (0x1 << i)))
672 continue; 672 continue;
673 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i))) 673 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
674 continue; 674 continue;
675 675
676 mask = cbox_filter_mask(0x1 << i); 676 mask = cbox_filter_mask(0x1 << i);
677 if (!__BITS_VALUE(atomic_read(&er->ref), i, 6) || 677 if (!__BITS_VALUE(atomic_read(&er->ref), i, 6) ||
678 !((reg1->config ^ er->config) & mask)) { 678 !((reg1->config ^ er->config) & mask)) {
679 atomic_add(1 << (i * 6), &er->ref); 679 atomic_add(1 << (i * 6), &er->ref);
680 er->config &= ~mask; 680 er->config &= ~mask;
681 er->config |= reg1->config & mask; 681 er->config |= reg1->config & mask;
682 alloc |= (0x1 << i); 682 alloc |= (0x1 << i);
683 } else { 683 } else {
684 break; 684 break;
685 } 685 }
686 } 686 }
687 raw_spin_unlock_irqrestore(&er->lock, flags); 687 raw_spin_unlock_irqrestore(&er->lock, flags);
688 if (i < 5) 688 if (i < 5)
689 goto fail; 689 goto fail;
690 690
691 if (!uncore_box_is_fake(box)) 691 if (!uncore_box_is_fake(box))
692 reg1->alloc |= alloc; 692 reg1->alloc |= alloc;
693 693
694 return NULL; 694 return NULL;
695 fail: 695 fail:
696 for (; i >= 0; i--) { 696 for (; i >= 0; i--) {
697 if (alloc & (0x1 << i)) 697 if (alloc & (0x1 << i))
698 atomic_sub(1 << (i * 6), &er->ref); 698 atomic_sub(1 << (i * 6), &er->ref);
699 } 699 }
700 return &uncore_constraint_empty; 700 return &uncore_constraint_empty;
701 } 701 }
702 702
703 static u64 snbep_cbox_filter_mask(int fields) 703 static u64 snbep_cbox_filter_mask(int fields)
704 { 704 {
705 u64 mask = 0; 705 u64 mask = 0;
706 706
707 if (fields & 0x1) 707 if (fields & 0x1)
708 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_TID; 708 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_TID;
709 if (fields & 0x2) 709 if (fields & 0x2)
710 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_NID; 710 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_NID;
711 if (fields & 0x4) 711 if (fields & 0x4)
712 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE; 712 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
713 if (fields & 0x8) 713 if (fields & 0x8)
714 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC; 714 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
715 715
716 return mask; 716 return mask;
717 } 717 }
718 718
719 static struct event_constraint * 719 static struct event_constraint *
720 snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 720 snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
721 { 721 {
722 return __snbep_cbox_get_constraint(box, event, snbep_cbox_filter_mask); 722 return __snbep_cbox_get_constraint(box, event, snbep_cbox_filter_mask);
723 } 723 }
724 724
725 static int snbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event) 725 static int snbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
726 { 726 {
727 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 727 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
728 struct extra_reg *er; 728 struct extra_reg *er;
729 int idx = 0; 729 int idx = 0;
730 730
731 for (er = snbep_uncore_cbox_extra_regs; er->msr; er++) { 731 for (er = snbep_uncore_cbox_extra_regs; er->msr; er++) {
732 if (er->event != (event->hw.config & er->config_mask)) 732 if (er->event != (event->hw.config & er->config_mask))
733 continue; 733 continue;
734 idx |= er->idx; 734 idx |= er->idx;
735 } 735 }
736 736
737 if (idx) { 737 if (idx) {
738 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER + 738 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
739 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx; 739 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
740 reg1->config = event->attr.config1 & snbep_cbox_filter_mask(idx); 740 reg1->config = event->attr.config1 & snbep_cbox_filter_mask(idx);
741 reg1->idx = idx; 741 reg1->idx = idx;
742 } 742 }
743 return 0; 743 return 0;
744 } 744 }
745 745
746 static struct intel_uncore_ops snbep_uncore_cbox_ops = { 746 static struct intel_uncore_ops snbep_uncore_cbox_ops = {
747 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), 747 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
748 .hw_config = snbep_cbox_hw_config, 748 .hw_config = snbep_cbox_hw_config,
749 .get_constraint = snbep_cbox_get_constraint, 749 .get_constraint = snbep_cbox_get_constraint,
750 .put_constraint = snbep_cbox_put_constraint, 750 .put_constraint = snbep_cbox_put_constraint,
751 }; 751 };
752 752
753 static struct intel_uncore_type snbep_uncore_cbox = { 753 static struct intel_uncore_type snbep_uncore_cbox = {
754 .name = "cbox", 754 .name = "cbox",
755 .num_counters = 4, 755 .num_counters = 4,
756 .num_boxes = 8, 756 .num_boxes = 8,
757 .perf_ctr_bits = 44, 757 .perf_ctr_bits = 44,
758 .event_ctl = SNBEP_C0_MSR_PMON_CTL0, 758 .event_ctl = SNBEP_C0_MSR_PMON_CTL0,
759 .perf_ctr = SNBEP_C0_MSR_PMON_CTR0, 759 .perf_ctr = SNBEP_C0_MSR_PMON_CTR0,
760 .event_mask = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK, 760 .event_mask = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
761 .box_ctl = SNBEP_C0_MSR_PMON_BOX_CTL, 761 .box_ctl = SNBEP_C0_MSR_PMON_BOX_CTL,
762 .msr_offset = SNBEP_CBO_MSR_OFFSET, 762 .msr_offset = SNBEP_CBO_MSR_OFFSET,
763 .num_shared_regs = 1, 763 .num_shared_regs = 1,
764 .constraints = snbep_uncore_cbox_constraints, 764 .constraints = snbep_uncore_cbox_constraints,
765 .ops = &snbep_uncore_cbox_ops, 765 .ops = &snbep_uncore_cbox_ops,
766 .format_group = &snbep_uncore_cbox_format_group, 766 .format_group = &snbep_uncore_cbox_format_group,
767 }; 767 };
768 768
769 static u64 snbep_pcu_alter_er(struct perf_event *event, int new_idx, bool modify) 769 static u64 snbep_pcu_alter_er(struct perf_event *event, int new_idx, bool modify)
770 { 770 {
771 struct hw_perf_event *hwc = &event->hw; 771 struct hw_perf_event *hwc = &event->hw;
772 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 772 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
773 u64 config = reg1->config; 773 u64 config = reg1->config;
774 774
775 if (new_idx > reg1->idx) 775 if (new_idx > reg1->idx)
776 config <<= 8 * (new_idx - reg1->idx); 776 config <<= 8 * (new_idx - reg1->idx);
777 else 777 else
778 config >>= 8 * (reg1->idx - new_idx); 778 config >>= 8 * (reg1->idx - new_idx);
779 779
780 if (modify) { 780 if (modify) {
781 hwc->config += new_idx - reg1->idx; 781 hwc->config += new_idx - reg1->idx;
782 reg1->config = config; 782 reg1->config = config;
783 reg1->idx = new_idx; 783 reg1->idx = new_idx;
784 } 784 }
785 return config; 785 return config;
786 } 786 }
787 787
788 static struct event_constraint * 788 static struct event_constraint *
789 snbep_pcu_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 789 snbep_pcu_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
790 { 790 {
791 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 791 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
792 struct intel_uncore_extra_reg *er = &box->shared_regs[0]; 792 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
793 unsigned long flags; 793 unsigned long flags;
794 int idx = reg1->idx; 794 int idx = reg1->idx;
795 u64 mask, config1 = reg1->config; 795 u64 mask, config1 = reg1->config;
796 bool ok = false; 796 bool ok = false;
797 797
798 if (reg1->idx == EXTRA_REG_NONE || 798 if (reg1->idx == EXTRA_REG_NONE ||
799 (!uncore_box_is_fake(box) && reg1->alloc)) 799 (!uncore_box_is_fake(box) && reg1->alloc))
800 return NULL; 800 return NULL;
801 again: 801 again:
802 mask = 0xffULL << (idx * 8); 802 mask = 0xffULL << (idx * 8);
803 raw_spin_lock_irqsave(&er->lock, flags); 803 raw_spin_lock_irqsave(&er->lock, flags);
804 if (!__BITS_VALUE(atomic_read(&er->ref), idx, 8) || 804 if (!__BITS_VALUE(atomic_read(&er->ref), idx, 8) ||
805 !((config1 ^ er->config) & mask)) { 805 !((config1 ^ er->config) & mask)) {
806 atomic_add(1 << (idx * 8), &er->ref); 806 atomic_add(1 << (idx * 8), &er->ref);
807 er->config &= ~mask; 807 er->config &= ~mask;
808 er->config |= config1 & mask; 808 er->config |= config1 & mask;
809 ok = true; 809 ok = true;
810 } 810 }
811 raw_spin_unlock_irqrestore(&er->lock, flags); 811 raw_spin_unlock_irqrestore(&er->lock, flags);
812 812
813 if (!ok) { 813 if (!ok) {
814 idx = (idx + 1) % 4; 814 idx = (idx + 1) % 4;
815 if (idx != reg1->idx) { 815 if (idx != reg1->idx) {
816 config1 = snbep_pcu_alter_er(event, idx, false); 816 config1 = snbep_pcu_alter_er(event, idx, false);
817 goto again; 817 goto again;
818 } 818 }
819 return &uncore_constraint_empty; 819 return &uncore_constraint_empty;
820 } 820 }
821 821
822 if (!uncore_box_is_fake(box)) { 822 if (!uncore_box_is_fake(box)) {
823 if (idx != reg1->idx) 823 if (idx != reg1->idx)
824 snbep_pcu_alter_er(event, idx, true); 824 snbep_pcu_alter_er(event, idx, true);
825 reg1->alloc = 1; 825 reg1->alloc = 1;
826 } 826 }
827 return NULL; 827 return NULL;
828 } 828 }
829 829
830 static void snbep_pcu_put_constraint(struct intel_uncore_box *box, struct perf_event *event) 830 static void snbep_pcu_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
831 { 831 {
832 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 832 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
833 struct intel_uncore_extra_reg *er = &box->shared_regs[0]; 833 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
834 834
835 if (uncore_box_is_fake(box) || !reg1->alloc) 835 if (uncore_box_is_fake(box) || !reg1->alloc)
836 return; 836 return;
837 837
838 atomic_sub(1 << (reg1->idx * 8), &er->ref); 838 atomic_sub(1 << (reg1->idx * 8), &er->ref);
839 reg1->alloc = 0; 839 reg1->alloc = 0;
840 } 840 }
841 841
842 static int snbep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event) 842 static int snbep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
843 { 843 {
844 struct hw_perf_event *hwc = &event->hw; 844 struct hw_perf_event *hwc = &event->hw;
845 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 845 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
846 int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK; 846 int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
847 847
848 if (ev_sel >= 0xb && ev_sel <= 0xe) { 848 if (ev_sel >= 0xb && ev_sel <= 0xe) {
849 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER; 849 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
850 reg1->idx = ev_sel - 0xb; 850 reg1->idx = ev_sel - 0xb;
851 reg1->config = event->attr.config1 & (0xff << (reg1->idx * 8)); 851 reg1->config = event->attr.config1 & (0xff << (reg1->idx * 8));
852 } 852 }
853 return 0; 853 return 0;
854 } 854 }
855 855
856 static struct intel_uncore_ops snbep_uncore_pcu_ops = { 856 static struct intel_uncore_ops snbep_uncore_pcu_ops = {
857 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), 857 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
858 .hw_config = snbep_pcu_hw_config, 858 .hw_config = snbep_pcu_hw_config,
859 .get_constraint = snbep_pcu_get_constraint, 859 .get_constraint = snbep_pcu_get_constraint,
860 .put_constraint = snbep_pcu_put_constraint, 860 .put_constraint = snbep_pcu_put_constraint,
861 }; 861 };
862 862
863 static struct intel_uncore_type snbep_uncore_pcu = { 863 static struct intel_uncore_type snbep_uncore_pcu = {
864 .name = "pcu", 864 .name = "pcu",
865 .num_counters = 4, 865 .num_counters = 4,
866 .num_boxes = 1, 866 .num_boxes = 1,
867 .perf_ctr_bits = 48, 867 .perf_ctr_bits = 48,
868 .perf_ctr = SNBEP_PCU_MSR_PMON_CTR0, 868 .perf_ctr = SNBEP_PCU_MSR_PMON_CTR0,
869 .event_ctl = SNBEP_PCU_MSR_PMON_CTL0, 869 .event_ctl = SNBEP_PCU_MSR_PMON_CTL0,
870 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK, 870 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
871 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL, 871 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL,
872 .num_shared_regs = 1, 872 .num_shared_regs = 1,
873 .ops = &snbep_uncore_pcu_ops, 873 .ops = &snbep_uncore_pcu_ops,
874 .format_group = &snbep_uncore_pcu_format_group, 874 .format_group = &snbep_uncore_pcu_format_group,
875 }; 875 };
876 876
877 static struct intel_uncore_type *snbep_msr_uncores[] = { 877 static struct intel_uncore_type *snbep_msr_uncores[] = {
878 &snbep_uncore_ubox, 878 &snbep_uncore_ubox,
879 &snbep_uncore_cbox, 879 &snbep_uncore_cbox,
880 &snbep_uncore_pcu, 880 &snbep_uncore_pcu,
881 NULL, 881 NULL,
882 }; 882 };
883 883
884 void snbep_uncore_cpu_init(void) 884 void snbep_uncore_cpu_init(void)
885 { 885 {
886 if (snbep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores) 886 if (snbep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
887 snbep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores; 887 snbep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
888 uncore_msr_uncores = snbep_msr_uncores; 888 uncore_msr_uncores = snbep_msr_uncores;
889 } 889 }
890 890
891 enum { 891 enum {
892 SNBEP_PCI_QPI_PORT0_FILTER, 892 SNBEP_PCI_QPI_PORT0_FILTER,
893 SNBEP_PCI_QPI_PORT1_FILTER, 893 SNBEP_PCI_QPI_PORT1_FILTER,
894 HSWEP_PCI_PCU_3,
894 }; 895 };
895 896
896 static int snbep_qpi_hw_config(struct intel_uncore_box *box, struct perf_event *event) 897 static int snbep_qpi_hw_config(struct intel_uncore_box *box, struct perf_event *event)
897 { 898 {
898 struct hw_perf_event *hwc = &event->hw; 899 struct hw_perf_event *hwc = &event->hw;
899 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 900 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
900 struct hw_perf_event_extra *reg2 = &hwc->branch_reg; 901 struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
901 902
902 if ((hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK) == 0x38) { 903 if ((hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK) == 0x38) {
903 reg1->idx = 0; 904 reg1->idx = 0;
904 reg1->reg = SNBEP_Q_Py_PCI_PMON_PKT_MATCH0; 905 reg1->reg = SNBEP_Q_Py_PCI_PMON_PKT_MATCH0;
905 reg1->config = event->attr.config1; 906 reg1->config = event->attr.config1;
906 reg2->reg = SNBEP_Q_Py_PCI_PMON_PKT_MASK0; 907 reg2->reg = SNBEP_Q_Py_PCI_PMON_PKT_MASK0;
907 reg2->config = event->attr.config2; 908 reg2->config = event->attr.config2;
908 } 909 }
909 return 0; 910 return 0;
910 } 911 }
911 912
912 static void snbep_qpi_enable_event(struct intel_uncore_box *box, struct perf_event *event) 913 static void snbep_qpi_enable_event(struct intel_uncore_box *box, struct perf_event *event)
913 { 914 {
914 struct pci_dev *pdev = box->pci_dev; 915 struct pci_dev *pdev = box->pci_dev;
915 struct hw_perf_event *hwc = &event->hw; 916 struct hw_perf_event *hwc = &event->hw;
916 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 917 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
917 struct hw_perf_event_extra *reg2 = &hwc->branch_reg; 918 struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
918 919
919 if (reg1->idx != EXTRA_REG_NONE) { 920 if (reg1->idx != EXTRA_REG_NONE) {
920 int idx = box->pmu->pmu_idx + SNBEP_PCI_QPI_PORT0_FILTER; 921 int idx = box->pmu->pmu_idx + SNBEP_PCI_QPI_PORT0_FILTER;
921 struct pci_dev *filter_pdev = uncore_extra_pci_dev[box->phys_id][idx]; 922 struct pci_dev *filter_pdev = uncore_extra_pci_dev[box->phys_id][idx];
922 if (filter_pdev) { 923 if (filter_pdev) {
923 pci_write_config_dword(filter_pdev, reg1->reg, 924 pci_write_config_dword(filter_pdev, reg1->reg,
924 (u32)reg1->config); 925 (u32)reg1->config);
925 pci_write_config_dword(filter_pdev, reg1->reg + 4, 926 pci_write_config_dword(filter_pdev, reg1->reg + 4,
926 (u32)(reg1->config >> 32)); 927 (u32)(reg1->config >> 32));
927 pci_write_config_dword(filter_pdev, reg2->reg, 928 pci_write_config_dword(filter_pdev, reg2->reg,
928 (u32)reg2->config); 929 (u32)reg2->config);
929 pci_write_config_dword(filter_pdev, reg2->reg + 4, 930 pci_write_config_dword(filter_pdev, reg2->reg + 4,
930 (u32)(reg2->config >> 32)); 931 (u32)(reg2->config >> 32));
931 } 932 }
932 } 933 }
933 934
934 pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); 935 pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
935 } 936 }
936 937
937 static struct intel_uncore_ops snbep_uncore_qpi_ops = { 938 static struct intel_uncore_ops snbep_uncore_qpi_ops = {
938 SNBEP_UNCORE_PCI_OPS_COMMON_INIT(), 939 SNBEP_UNCORE_PCI_OPS_COMMON_INIT(),
939 .enable_event = snbep_qpi_enable_event, 940 .enable_event = snbep_qpi_enable_event,
940 .hw_config = snbep_qpi_hw_config, 941 .hw_config = snbep_qpi_hw_config,
941 .get_constraint = uncore_get_constraint, 942 .get_constraint = uncore_get_constraint,
942 .put_constraint = uncore_put_constraint, 943 .put_constraint = uncore_put_constraint,
943 }; 944 };
944 945
945 #define SNBEP_UNCORE_PCI_COMMON_INIT() \ 946 #define SNBEP_UNCORE_PCI_COMMON_INIT() \
946 .perf_ctr = SNBEP_PCI_PMON_CTR0, \ 947 .perf_ctr = SNBEP_PCI_PMON_CTR0, \
947 .event_ctl = SNBEP_PCI_PMON_CTL0, \ 948 .event_ctl = SNBEP_PCI_PMON_CTL0, \
948 .event_mask = SNBEP_PMON_RAW_EVENT_MASK, \ 949 .event_mask = SNBEP_PMON_RAW_EVENT_MASK, \
949 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, \ 950 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, \
950 .ops = &snbep_uncore_pci_ops, \ 951 .ops = &snbep_uncore_pci_ops, \
951 .format_group = &snbep_uncore_format_group 952 .format_group = &snbep_uncore_format_group
952 953
953 static struct intel_uncore_type snbep_uncore_ha = { 954 static struct intel_uncore_type snbep_uncore_ha = {
954 .name = "ha", 955 .name = "ha",
955 .num_counters = 4, 956 .num_counters = 4,
956 .num_boxes = 1, 957 .num_boxes = 1,
957 .perf_ctr_bits = 48, 958 .perf_ctr_bits = 48,
958 SNBEP_UNCORE_PCI_COMMON_INIT(), 959 SNBEP_UNCORE_PCI_COMMON_INIT(),
959 }; 960 };
960 961
961 static struct intel_uncore_type snbep_uncore_imc = { 962 static struct intel_uncore_type snbep_uncore_imc = {
962 .name = "imc", 963 .name = "imc",
963 .num_counters = 4, 964 .num_counters = 4,
964 .num_boxes = 4, 965 .num_boxes = 4,
965 .perf_ctr_bits = 48, 966 .perf_ctr_bits = 48,
966 .fixed_ctr_bits = 48, 967 .fixed_ctr_bits = 48,
967 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR, 968 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
968 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL, 969 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
969 .event_descs = snbep_uncore_imc_events, 970 .event_descs = snbep_uncore_imc_events,
970 SNBEP_UNCORE_PCI_COMMON_INIT(), 971 SNBEP_UNCORE_PCI_COMMON_INIT(),
971 }; 972 };
972 973
973 static struct intel_uncore_type snbep_uncore_qpi = { 974 static struct intel_uncore_type snbep_uncore_qpi = {
974 .name = "qpi", 975 .name = "qpi",
975 .num_counters = 4, 976 .num_counters = 4,
976 .num_boxes = 2, 977 .num_boxes = 2,
977 .perf_ctr_bits = 48, 978 .perf_ctr_bits = 48,
978 .perf_ctr = SNBEP_PCI_PMON_CTR0, 979 .perf_ctr = SNBEP_PCI_PMON_CTR0,
979 .event_ctl = SNBEP_PCI_PMON_CTL0, 980 .event_ctl = SNBEP_PCI_PMON_CTL0,
980 .event_mask = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK, 981 .event_mask = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
981 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, 982 .box_ctl = SNBEP_PCI_PMON_BOX_CTL,
982 .num_shared_regs = 1, 983 .num_shared_regs = 1,
983 .ops = &snbep_uncore_qpi_ops, 984 .ops = &snbep_uncore_qpi_ops,
984 .event_descs = snbep_uncore_qpi_events, 985 .event_descs = snbep_uncore_qpi_events,
985 .format_group = &snbep_uncore_qpi_format_group, 986 .format_group = &snbep_uncore_qpi_format_group,
986 }; 987 };
987 988
988 989
989 static struct intel_uncore_type snbep_uncore_r2pcie = { 990 static struct intel_uncore_type snbep_uncore_r2pcie = {
990 .name = "r2pcie", 991 .name = "r2pcie",
991 .num_counters = 4, 992 .num_counters = 4,
992 .num_boxes = 1, 993 .num_boxes = 1,
993 .perf_ctr_bits = 44, 994 .perf_ctr_bits = 44,
994 .constraints = snbep_uncore_r2pcie_constraints, 995 .constraints = snbep_uncore_r2pcie_constraints,
995 SNBEP_UNCORE_PCI_COMMON_INIT(), 996 SNBEP_UNCORE_PCI_COMMON_INIT(),
996 }; 997 };
997 998
998 static struct intel_uncore_type snbep_uncore_r3qpi = { 999 static struct intel_uncore_type snbep_uncore_r3qpi = {
999 .name = "r3qpi", 1000 .name = "r3qpi",
1000 .num_counters = 3, 1001 .num_counters = 3,
1001 .num_boxes = 2, 1002 .num_boxes = 2,
1002 .perf_ctr_bits = 44, 1003 .perf_ctr_bits = 44,
1003 .constraints = snbep_uncore_r3qpi_constraints, 1004 .constraints = snbep_uncore_r3qpi_constraints,
1004 SNBEP_UNCORE_PCI_COMMON_INIT(), 1005 SNBEP_UNCORE_PCI_COMMON_INIT(),
1005 }; 1006 };
1006 1007
1007 enum { 1008 enum {
1008 SNBEP_PCI_UNCORE_HA, 1009 SNBEP_PCI_UNCORE_HA,
1009 SNBEP_PCI_UNCORE_IMC, 1010 SNBEP_PCI_UNCORE_IMC,
1010 SNBEP_PCI_UNCORE_QPI, 1011 SNBEP_PCI_UNCORE_QPI,
1011 SNBEP_PCI_UNCORE_R2PCIE, 1012 SNBEP_PCI_UNCORE_R2PCIE,
1012 SNBEP_PCI_UNCORE_R3QPI, 1013 SNBEP_PCI_UNCORE_R3QPI,
1013 }; 1014 };
1014 1015
1015 static struct intel_uncore_type *snbep_pci_uncores[] = { 1016 static struct intel_uncore_type *snbep_pci_uncores[] = {
1016 [SNBEP_PCI_UNCORE_HA] = &snbep_uncore_ha, 1017 [SNBEP_PCI_UNCORE_HA] = &snbep_uncore_ha,
1017 [SNBEP_PCI_UNCORE_IMC] = &snbep_uncore_imc, 1018 [SNBEP_PCI_UNCORE_IMC] = &snbep_uncore_imc,
1018 [SNBEP_PCI_UNCORE_QPI] = &snbep_uncore_qpi, 1019 [SNBEP_PCI_UNCORE_QPI] = &snbep_uncore_qpi,
1019 [SNBEP_PCI_UNCORE_R2PCIE] = &snbep_uncore_r2pcie, 1020 [SNBEP_PCI_UNCORE_R2PCIE] = &snbep_uncore_r2pcie,
1020 [SNBEP_PCI_UNCORE_R3QPI] = &snbep_uncore_r3qpi, 1021 [SNBEP_PCI_UNCORE_R3QPI] = &snbep_uncore_r3qpi,
1021 NULL, 1022 NULL,
1022 }; 1023 };
1023 1024
1024 static const struct pci_device_id snbep_uncore_pci_ids[] = { 1025 static const struct pci_device_id snbep_uncore_pci_ids[] = {
1025 { /* Home Agent */ 1026 { /* Home Agent */
1026 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA), 1027 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
1027 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_HA, 0), 1028 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_HA, 0),
1028 }, 1029 },
1029 { /* MC Channel 0 */ 1030 { /* MC Channel 0 */
1030 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0), 1031 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
1031 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 0), 1032 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 0),
1032 }, 1033 },
1033 { /* MC Channel 1 */ 1034 { /* MC Channel 1 */
1034 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1), 1035 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
1035 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 1), 1036 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 1),
1036 }, 1037 },
1037 { /* MC Channel 2 */ 1038 { /* MC Channel 2 */
1038 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2), 1039 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
1039 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 2), 1040 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 2),
1040 }, 1041 },
1041 { /* MC Channel 3 */ 1042 { /* MC Channel 3 */
1042 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3), 1043 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
1043 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 3), 1044 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 3),
1044 }, 1045 },
1045 { /* QPI Port 0 */ 1046 { /* QPI Port 0 */
1046 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0), 1047 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
1047 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 0), 1048 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 0),
1048 }, 1049 },
1049 { /* QPI Port 1 */ 1050 { /* QPI Port 1 */
1050 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1), 1051 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
1051 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 1), 1052 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 1),
1052 }, 1053 },
1053 { /* R2PCIe */ 1054 { /* R2PCIe */
1054 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE), 1055 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
1055 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R2PCIE, 0), 1056 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R2PCIE, 0),
1056 }, 1057 },
1057 { /* R3QPI Link 0 */ 1058 { /* R3QPI Link 0 */
1058 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0), 1059 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
1059 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 0), 1060 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 0),
1060 }, 1061 },
1061 { /* R3QPI Link 1 */ 1062 { /* R3QPI Link 1 */
1062 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1), 1063 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
1063 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 1), 1064 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 1),
1064 }, 1065 },
1065 { /* QPI Port 0 filter */ 1066 { /* QPI Port 0 filter */
1066 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c86), 1067 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c86),
1067 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 1068 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
1068 SNBEP_PCI_QPI_PORT0_FILTER), 1069 SNBEP_PCI_QPI_PORT0_FILTER),
1069 }, 1070 },
1070 { /* QPI Port 0 filter */ 1071 { /* QPI Port 0 filter */
1071 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c96), 1072 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c96),
1072 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 1073 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
1073 SNBEP_PCI_QPI_PORT1_FILTER), 1074 SNBEP_PCI_QPI_PORT1_FILTER),
1074 }, 1075 },
1075 { /* end: all zeroes */ } 1076 { /* end: all zeroes */ }
1076 }; 1077 };
1077 1078
1078 static struct pci_driver snbep_uncore_pci_driver = { 1079 static struct pci_driver snbep_uncore_pci_driver = {
1079 .name = "snbep_uncore", 1080 .name = "snbep_uncore",
1080 .id_table = snbep_uncore_pci_ids, 1081 .id_table = snbep_uncore_pci_ids,
1081 }; 1082 };
1082 1083
1083 /* 1084 /*
1084 * build pci bus to socket mapping 1085 * build pci bus to socket mapping
1085 */ 1086 */
1086 static int snbep_pci2phy_map_init(int devid) 1087 static int snbep_pci2phy_map_init(int devid)
1087 { 1088 {
1088 struct pci_dev *ubox_dev = NULL; 1089 struct pci_dev *ubox_dev = NULL;
1089 int i, bus, nodeid; 1090 int i, bus, nodeid;
1090 int err = 0; 1091 int err = 0;
1091 u32 config = 0; 1092 u32 config = 0;
1092 1093
1093 while (1) { 1094 while (1) {
1094 /* find the UBOX device */ 1095 /* find the UBOX device */
1095 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, ubox_dev); 1096 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, ubox_dev);
1096 if (!ubox_dev) 1097 if (!ubox_dev)
1097 break; 1098 break;
1098 bus = ubox_dev->bus->number; 1099 bus = ubox_dev->bus->number;
1099 /* get the Node ID of the local register */ 1100 /* get the Node ID of the local register */
1100 err = pci_read_config_dword(ubox_dev, 0x40, &config); 1101 err = pci_read_config_dword(ubox_dev, 0x40, &config);
1101 if (err) 1102 if (err)
1102 break; 1103 break;
1103 nodeid = config; 1104 nodeid = config;
1104 /* get the Node ID mapping */ 1105 /* get the Node ID mapping */
1105 err = pci_read_config_dword(ubox_dev, 0x54, &config); 1106 err = pci_read_config_dword(ubox_dev, 0x54, &config);
1106 if (err) 1107 if (err)
1107 break; 1108 break;
1108 /* 1109 /*
1109 * every three bits in the Node ID mapping register maps 1110 * every three bits in the Node ID mapping register maps
1110 * to a particular node. 1111 * to a particular node.
1111 */ 1112 */
1112 for (i = 0; i < 8; i++) { 1113 for (i = 0; i < 8; i++) {
1113 if (nodeid == ((config >> (3 * i)) & 0x7)) { 1114 if (nodeid == ((config >> (3 * i)) & 0x7)) {
1114 uncore_pcibus_to_physid[bus] = i; 1115 uncore_pcibus_to_physid[bus] = i;
1115 break; 1116 break;
1116 } 1117 }
1117 } 1118 }
1118 } 1119 }
1119 1120
1120 if (!err) { 1121 if (!err) {
1121 /* 1122 /*
1122 * For PCI bus with no UBOX device, find the next bus 1123 * For PCI bus with no UBOX device, find the next bus
1123 * that has UBOX device and use its mapping. 1124 * that has UBOX device and use its mapping.
1124 */ 1125 */
1125 i = -1; 1126 i = -1;
1126 for (bus = 255; bus >= 0; bus--) { 1127 for (bus = 255; bus >= 0; bus--) {
1127 if (uncore_pcibus_to_physid[bus] >= 0) 1128 if (uncore_pcibus_to_physid[bus] >= 0)
1128 i = uncore_pcibus_to_physid[bus]; 1129 i = uncore_pcibus_to_physid[bus];
1129 else 1130 else
1130 uncore_pcibus_to_physid[bus] = i; 1131 uncore_pcibus_to_physid[bus] = i;
1131 } 1132 }
1132 } 1133 }
1133 1134
1134 if (ubox_dev) 1135 if (ubox_dev)
1135 pci_dev_put(ubox_dev); 1136 pci_dev_put(ubox_dev);
1136 1137
1137 return err ? pcibios_err_to_errno(err) : 0; 1138 return err ? pcibios_err_to_errno(err) : 0;
1138 } 1139 }
1139 1140
1140 int snbep_uncore_pci_init(void) 1141 int snbep_uncore_pci_init(void)
1141 { 1142 {
1142 int ret = snbep_pci2phy_map_init(0x3ce0); 1143 int ret = snbep_pci2phy_map_init(0x3ce0);
1143 if (ret) 1144 if (ret)
1144 return ret; 1145 return ret;
1145 uncore_pci_uncores = snbep_pci_uncores; 1146 uncore_pci_uncores = snbep_pci_uncores;
1146 uncore_pci_driver = &snbep_uncore_pci_driver; 1147 uncore_pci_driver = &snbep_uncore_pci_driver;
1147 return 0; 1148 return 0;
1148 } 1149 }
1149 /* end of Sandy Bridge-EP uncore support */ 1150 /* end of Sandy Bridge-EP uncore support */
1150 1151
1151 /* IvyTown uncore support */ 1152 /* IvyTown uncore support */
1152 static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box) 1153 static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box)
1153 { 1154 {
1154 unsigned msr = uncore_msr_box_ctl(box); 1155 unsigned msr = uncore_msr_box_ctl(box);
1155 if (msr) 1156 if (msr)
1156 wrmsrl(msr, IVBEP_PMON_BOX_CTL_INT); 1157 wrmsrl(msr, IVBEP_PMON_BOX_CTL_INT);
1157 } 1158 }
1158 1159
1159 static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box) 1160 static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box)
1160 { 1161 {
1161 struct pci_dev *pdev = box->pci_dev; 1162 struct pci_dev *pdev = box->pci_dev;
1162 1163
1163 pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); 1164 pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
1164 } 1165 }
1165 1166
1166 #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT() \ 1167 #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT() \
1167 .init_box = ivbep_uncore_msr_init_box, \ 1168 .init_box = ivbep_uncore_msr_init_box, \
1168 .disable_box = snbep_uncore_msr_disable_box, \ 1169 .disable_box = snbep_uncore_msr_disable_box, \
1169 .enable_box = snbep_uncore_msr_enable_box, \ 1170 .enable_box = snbep_uncore_msr_enable_box, \
1170 .disable_event = snbep_uncore_msr_disable_event, \ 1171 .disable_event = snbep_uncore_msr_disable_event, \
1171 .enable_event = snbep_uncore_msr_enable_event, \ 1172 .enable_event = snbep_uncore_msr_enable_event, \
1172 .read_counter = uncore_msr_read_counter 1173 .read_counter = uncore_msr_read_counter
1173 1174
1174 static struct intel_uncore_ops ivbep_uncore_msr_ops = { 1175 static struct intel_uncore_ops ivbep_uncore_msr_ops = {
1175 IVBEP_UNCORE_MSR_OPS_COMMON_INIT(), 1176 IVBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1176 }; 1177 };
1177 1178
1178 static struct intel_uncore_ops ivbep_uncore_pci_ops = { 1179 static struct intel_uncore_ops ivbep_uncore_pci_ops = {
1179 .init_box = ivbep_uncore_pci_init_box, 1180 .init_box = ivbep_uncore_pci_init_box,
1180 .disable_box = snbep_uncore_pci_disable_box, 1181 .disable_box = snbep_uncore_pci_disable_box,
1181 .enable_box = snbep_uncore_pci_enable_box, 1182 .enable_box = snbep_uncore_pci_enable_box,
1182 .disable_event = snbep_uncore_pci_disable_event, 1183 .disable_event = snbep_uncore_pci_disable_event,
1183 .enable_event = snbep_uncore_pci_enable_event, 1184 .enable_event = snbep_uncore_pci_enable_event,
1184 .read_counter = snbep_uncore_pci_read_counter, 1185 .read_counter = snbep_uncore_pci_read_counter,
1185 }; 1186 };
1186 1187
1187 #define IVBEP_UNCORE_PCI_COMMON_INIT() \ 1188 #define IVBEP_UNCORE_PCI_COMMON_INIT() \
1188 .perf_ctr = SNBEP_PCI_PMON_CTR0, \ 1189 .perf_ctr = SNBEP_PCI_PMON_CTR0, \
1189 .event_ctl = SNBEP_PCI_PMON_CTL0, \ 1190 .event_ctl = SNBEP_PCI_PMON_CTL0, \
1190 .event_mask = IVBEP_PMON_RAW_EVENT_MASK, \ 1191 .event_mask = IVBEP_PMON_RAW_EVENT_MASK, \
1191 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, \ 1192 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, \
1192 .ops = &ivbep_uncore_pci_ops, \ 1193 .ops = &ivbep_uncore_pci_ops, \
1193 .format_group = &ivbep_uncore_format_group 1194 .format_group = &ivbep_uncore_format_group
1194 1195
1195 static struct attribute *ivbep_uncore_formats_attr[] = { 1196 static struct attribute *ivbep_uncore_formats_attr[] = {
1196 &format_attr_event.attr, 1197 &format_attr_event.attr,
1197 &format_attr_umask.attr, 1198 &format_attr_umask.attr,
1198 &format_attr_edge.attr, 1199 &format_attr_edge.attr,
1199 &format_attr_inv.attr, 1200 &format_attr_inv.attr,
1200 &format_attr_thresh8.attr, 1201 &format_attr_thresh8.attr,
1201 NULL, 1202 NULL,
1202 }; 1203 };
1203 1204
1204 static struct attribute *ivbep_uncore_ubox_formats_attr[] = { 1205 static struct attribute *ivbep_uncore_ubox_formats_attr[] = {
1205 &format_attr_event.attr, 1206 &format_attr_event.attr,
1206 &format_attr_umask.attr, 1207 &format_attr_umask.attr,
1207 &format_attr_edge.attr, 1208 &format_attr_edge.attr,
1208 &format_attr_inv.attr, 1209 &format_attr_inv.attr,
1209 &format_attr_thresh5.attr, 1210 &format_attr_thresh5.attr,
1210 NULL, 1211 NULL,
1211 }; 1212 };
1212 1213
1213 static struct attribute *ivbep_uncore_cbox_formats_attr[] = { 1214 static struct attribute *ivbep_uncore_cbox_formats_attr[] = {
1214 &format_attr_event.attr, 1215 &format_attr_event.attr,
1215 &format_attr_umask.attr, 1216 &format_attr_umask.attr,
1216 &format_attr_edge.attr, 1217 &format_attr_edge.attr,
1217 &format_attr_tid_en.attr, 1218 &format_attr_tid_en.attr,
1218 &format_attr_thresh8.attr, 1219 &format_attr_thresh8.attr,
1219 &format_attr_filter_tid.attr, 1220 &format_attr_filter_tid.attr,
1220 &format_attr_filter_link.attr, 1221 &format_attr_filter_link.attr,
1221 &format_attr_filter_state2.attr, 1222 &format_attr_filter_state2.attr,
1222 &format_attr_filter_nid2.attr, 1223 &format_attr_filter_nid2.attr,
1223 &format_attr_filter_opc2.attr, 1224 &format_attr_filter_opc2.attr,
1224 &format_attr_filter_nc.attr, 1225 &format_attr_filter_nc.attr,
1225 &format_attr_filter_c6.attr, 1226 &format_attr_filter_c6.attr,
1226 &format_attr_filter_isoc.attr, 1227 &format_attr_filter_isoc.attr,
1227 NULL, 1228 NULL,
1228 }; 1229 };
1229 1230
1230 static struct attribute *ivbep_uncore_pcu_formats_attr[] = { 1231 static struct attribute *ivbep_uncore_pcu_formats_attr[] = {
1231 &format_attr_event_ext.attr, 1232 &format_attr_event_ext.attr,
1232 &format_attr_occ_sel.attr, 1233 &format_attr_occ_sel.attr,
1233 &format_attr_edge.attr, 1234 &format_attr_edge.attr,
1234 &format_attr_thresh5.attr, 1235 &format_attr_thresh5.attr,
1235 &format_attr_occ_invert.attr, 1236 &format_attr_occ_invert.attr,
1236 &format_attr_occ_edge.attr, 1237 &format_attr_occ_edge.attr,
1237 &format_attr_filter_band0.attr, 1238 &format_attr_filter_band0.attr,
1238 &format_attr_filter_band1.attr, 1239 &format_attr_filter_band1.attr,
1239 &format_attr_filter_band2.attr, 1240 &format_attr_filter_band2.attr,
1240 &format_attr_filter_band3.attr, 1241 &format_attr_filter_band3.attr,
1241 NULL, 1242 NULL,
1242 }; 1243 };
1243 1244
1244 static struct attribute *ivbep_uncore_qpi_formats_attr[] = { 1245 static struct attribute *ivbep_uncore_qpi_formats_attr[] = {
1245 &format_attr_event_ext.attr, 1246 &format_attr_event_ext.attr,
1246 &format_attr_umask.attr, 1247 &format_attr_umask.attr,
1247 &format_attr_edge.attr, 1248 &format_attr_edge.attr,
1248 &format_attr_thresh8.attr, 1249 &format_attr_thresh8.attr,
1249 &format_attr_match_rds.attr, 1250 &format_attr_match_rds.attr,
1250 &format_attr_match_rnid30.attr, 1251 &format_attr_match_rnid30.attr,
1251 &format_attr_match_rnid4.attr, 1252 &format_attr_match_rnid4.attr,
1252 &format_attr_match_dnid.attr, 1253 &format_attr_match_dnid.attr,
1253 &format_attr_match_mc.attr, 1254 &format_attr_match_mc.attr,
1254 &format_attr_match_opc.attr, 1255 &format_attr_match_opc.attr,
1255 &format_attr_match_vnw.attr, 1256 &format_attr_match_vnw.attr,
1256 &format_attr_match0.attr, 1257 &format_attr_match0.attr,
1257 &format_attr_match1.attr, 1258 &format_attr_match1.attr,
1258 &format_attr_mask_rds.attr, 1259 &format_attr_mask_rds.attr,
1259 &format_attr_mask_rnid30.attr, 1260 &format_attr_mask_rnid30.attr,
1260 &format_attr_mask_rnid4.attr, 1261 &format_attr_mask_rnid4.attr,
1261 &format_attr_mask_dnid.attr, 1262 &format_attr_mask_dnid.attr,
1262 &format_attr_mask_mc.attr, 1263 &format_attr_mask_mc.attr,
1263 &format_attr_mask_opc.attr, 1264 &format_attr_mask_opc.attr,
1264 &format_attr_mask_vnw.attr, 1265 &format_attr_mask_vnw.attr,
1265 &format_attr_mask0.attr, 1266 &format_attr_mask0.attr,
1266 &format_attr_mask1.attr, 1267 &format_attr_mask1.attr,
1267 NULL, 1268 NULL,
1268 }; 1269 };
1269 1270
1270 static struct attribute_group ivbep_uncore_format_group = { 1271 static struct attribute_group ivbep_uncore_format_group = {
1271 .name = "format", 1272 .name = "format",
1272 .attrs = ivbep_uncore_formats_attr, 1273 .attrs = ivbep_uncore_formats_attr,
1273 }; 1274 };
1274 1275
1275 static struct attribute_group ivbep_uncore_ubox_format_group = { 1276 static struct attribute_group ivbep_uncore_ubox_format_group = {
1276 .name = "format", 1277 .name = "format",
1277 .attrs = ivbep_uncore_ubox_formats_attr, 1278 .attrs = ivbep_uncore_ubox_formats_attr,
1278 }; 1279 };
1279 1280
1280 static struct attribute_group ivbep_uncore_cbox_format_group = { 1281 static struct attribute_group ivbep_uncore_cbox_format_group = {
1281 .name = "format", 1282 .name = "format",
1282 .attrs = ivbep_uncore_cbox_formats_attr, 1283 .attrs = ivbep_uncore_cbox_formats_attr,
1283 }; 1284 };
1284 1285
1285 static struct attribute_group ivbep_uncore_pcu_format_group = { 1286 static struct attribute_group ivbep_uncore_pcu_format_group = {
1286 .name = "format", 1287 .name = "format",
1287 .attrs = ivbep_uncore_pcu_formats_attr, 1288 .attrs = ivbep_uncore_pcu_formats_attr,
1288 }; 1289 };
1289 1290
1290 static struct attribute_group ivbep_uncore_qpi_format_group = { 1291 static struct attribute_group ivbep_uncore_qpi_format_group = {
1291 .name = "format", 1292 .name = "format",
1292 .attrs = ivbep_uncore_qpi_formats_attr, 1293 .attrs = ivbep_uncore_qpi_formats_attr,
1293 }; 1294 };
1294 1295
1295 static struct intel_uncore_type ivbep_uncore_ubox = { 1296 static struct intel_uncore_type ivbep_uncore_ubox = {
1296 .name = "ubox", 1297 .name = "ubox",
1297 .num_counters = 2, 1298 .num_counters = 2,
1298 .num_boxes = 1, 1299 .num_boxes = 1,
1299 .perf_ctr_bits = 44, 1300 .perf_ctr_bits = 44,
1300 .fixed_ctr_bits = 48, 1301 .fixed_ctr_bits = 48,
1301 .perf_ctr = SNBEP_U_MSR_PMON_CTR0, 1302 .perf_ctr = SNBEP_U_MSR_PMON_CTR0,
1302 .event_ctl = SNBEP_U_MSR_PMON_CTL0, 1303 .event_ctl = SNBEP_U_MSR_PMON_CTL0,
1303 .event_mask = IVBEP_U_MSR_PMON_RAW_EVENT_MASK, 1304 .event_mask = IVBEP_U_MSR_PMON_RAW_EVENT_MASK,
1304 .fixed_ctr = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR, 1305 .fixed_ctr = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
1305 .fixed_ctl = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL, 1306 .fixed_ctl = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
1306 .ops = &ivbep_uncore_msr_ops, 1307 .ops = &ivbep_uncore_msr_ops,
1307 .format_group = &ivbep_uncore_ubox_format_group, 1308 .format_group = &ivbep_uncore_ubox_format_group,
1308 }; 1309 };
1309 1310
1310 static struct extra_reg ivbep_uncore_cbox_extra_regs[] = { 1311 static struct extra_reg ivbep_uncore_cbox_extra_regs[] = {
1311 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN, 1312 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1312 SNBEP_CBO_PMON_CTL_TID_EN, 0x1), 1313 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1313 SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2), 1314 SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2),
1314 SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4), 1315 SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4),
1315 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc), 1316 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc),
1316 SNBEP_CBO_EVENT_EXTRA_REG(0x5134, 0xffff, 0xc), 1317 SNBEP_CBO_EVENT_EXTRA_REG(0x5134, 0xffff, 0xc),
1317 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4), 1318 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1318 SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0xc), 1319 SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0xc),
1319 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4), 1320 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1320 SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0xc), 1321 SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0xc),
1321 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4), 1322 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1322 SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0xc), 1323 SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0xc),
1323 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10), 1324 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10),
1324 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10), 1325 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1325 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10), 1326 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
1326 SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10), 1327 SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
1327 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18), 1328 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
1328 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18), 1329 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
1329 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8), 1330 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
1330 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8), 1331 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
1331 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8), 1332 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
1332 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8), 1333 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
1333 SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10), 1334 SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
1334 SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10), 1335 SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
1335 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10), 1336 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
1336 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10), 1337 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
1337 SNBEP_CBO_EVENT_EXTRA_REG(0x2136, 0xffff, 0x10), 1338 SNBEP_CBO_EVENT_EXTRA_REG(0x2136, 0xffff, 0x10),
1338 SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10), 1339 SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1339 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18), 1340 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
1340 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18), 1341 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
1341 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8), 1342 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
1342 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8), 1343 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
1343 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8), 1344 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
1344 SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8), 1345 SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
1345 SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10), 1346 SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
1346 SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10), 1347 SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
1347 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8), 1348 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
1348 EVENT_EXTRA_END 1349 EVENT_EXTRA_END
1349 }; 1350 };
1350 1351
1351 static u64 ivbep_cbox_filter_mask(int fields) 1352 static u64 ivbep_cbox_filter_mask(int fields)
1352 { 1353 {
1353 u64 mask = 0; 1354 u64 mask = 0;
1354 1355
1355 if (fields & 0x1) 1356 if (fields & 0x1)
1356 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_TID; 1357 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_TID;
1357 if (fields & 0x2) 1358 if (fields & 0x2)
1358 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_LINK; 1359 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_LINK;
1359 if (fields & 0x4) 1360 if (fields & 0x4)
1360 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_STATE; 1361 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
1361 if (fields & 0x8) 1362 if (fields & 0x8)
1362 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_NID; 1363 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_NID;
1363 if (fields & 0x10) { 1364 if (fields & 0x10) {
1364 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_OPC; 1365 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
1365 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_NC; 1366 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_NC;
1366 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_C6; 1367 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_C6;
1367 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_ISOC; 1368 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_ISOC;
1368 } 1369 }
1369 1370
1370 return mask; 1371 return mask;
1371 } 1372 }
1372 1373
1373 static struct event_constraint * 1374 static struct event_constraint *
1374 ivbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 1375 ivbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1375 { 1376 {
1376 return __snbep_cbox_get_constraint(box, event, ivbep_cbox_filter_mask); 1377 return __snbep_cbox_get_constraint(box, event, ivbep_cbox_filter_mask);
1377 } 1378 }
1378 1379
1379 static int ivbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event) 1380 static int ivbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1380 { 1381 {
1381 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 1382 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1382 struct extra_reg *er; 1383 struct extra_reg *er;
1383 int idx = 0; 1384 int idx = 0;
1384 1385
1385 for (er = ivbep_uncore_cbox_extra_regs; er->msr; er++) { 1386 for (er = ivbep_uncore_cbox_extra_regs; er->msr; er++) {
1386 if (er->event != (event->hw.config & er->config_mask)) 1387 if (er->event != (event->hw.config & er->config_mask))
1387 continue; 1388 continue;
1388 idx |= er->idx; 1389 idx |= er->idx;
1389 } 1390 }
1390 1391
1391 if (idx) { 1392 if (idx) {
1392 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER + 1393 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
1393 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx; 1394 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1394 reg1->config = event->attr.config1 & ivbep_cbox_filter_mask(idx); 1395 reg1->config = event->attr.config1 & ivbep_cbox_filter_mask(idx);
1395 reg1->idx = idx; 1396 reg1->idx = idx;
1396 } 1397 }
1397 return 0; 1398 return 0;
1398 } 1399 }
1399 1400
1400 static void ivbep_cbox_enable_event(struct intel_uncore_box *box, struct perf_event *event) 1401 static void ivbep_cbox_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1401 { 1402 {
1402 struct hw_perf_event *hwc = &event->hw; 1403 struct hw_perf_event *hwc = &event->hw;
1403 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 1404 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1404 1405
1405 if (reg1->idx != EXTRA_REG_NONE) { 1406 if (reg1->idx != EXTRA_REG_NONE) {
1406 u64 filter = uncore_shared_reg_config(box, 0); 1407 u64 filter = uncore_shared_reg_config(box, 0);
1407 wrmsrl(reg1->reg, filter & 0xffffffff); 1408 wrmsrl(reg1->reg, filter & 0xffffffff);
1408 wrmsrl(reg1->reg + 6, filter >> 32); 1409 wrmsrl(reg1->reg + 6, filter >> 32);
1409 } 1410 }
1410 1411
1411 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); 1412 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1412 } 1413 }
1413 1414
1414 static struct intel_uncore_ops ivbep_uncore_cbox_ops = { 1415 static struct intel_uncore_ops ivbep_uncore_cbox_ops = {
1415 .init_box = ivbep_uncore_msr_init_box, 1416 .init_box = ivbep_uncore_msr_init_box,
1416 .disable_box = snbep_uncore_msr_disable_box, 1417 .disable_box = snbep_uncore_msr_disable_box,
1417 .enable_box = snbep_uncore_msr_enable_box, 1418 .enable_box = snbep_uncore_msr_enable_box,
1418 .disable_event = snbep_uncore_msr_disable_event, 1419 .disable_event = snbep_uncore_msr_disable_event,
1419 .enable_event = ivbep_cbox_enable_event, 1420 .enable_event = ivbep_cbox_enable_event,
1420 .read_counter = uncore_msr_read_counter, 1421 .read_counter = uncore_msr_read_counter,
1421 .hw_config = ivbep_cbox_hw_config, 1422 .hw_config = ivbep_cbox_hw_config,
1422 .get_constraint = ivbep_cbox_get_constraint, 1423 .get_constraint = ivbep_cbox_get_constraint,
1423 .put_constraint = snbep_cbox_put_constraint, 1424 .put_constraint = snbep_cbox_put_constraint,
1424 }; 1425 };
1425 1426
1426 static struct intel_uncore_type ivbep_uncore_cbox = { 1427 static struct intel_uncore_type ivbep_uncore_cbox = {
1427 .name = "cbox", 1428 .name = "cbox",
1428 .num_counters = 4, 1429 .num_counters = 4,
1429 .num_boxes = 15, 1430 .num_boxes = 15,
1430 .perf_ctr_bits = 44, 1431 .perf_ctr_bits = 44,
1431 .event_ctl = SNBEP_C0_MSR_PMON_CTL0, 1432 .event_ctl = SNBEP_C0_MSR_PMON_CTL0,
1432 .perf_ctr = SNBEP_C0_MSR_PMON_CTR0, 1433 .perf_ctr = SNBEP_C0_MSR_PMON_CTR0,
1433 .event_mask = IVBEP_CBO_MSR_PMON_RAW_EVENT_MASK, 1434 .event_mask = IVBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
1434 .box_ctl = SNBEP_C0_MSR_PMON_BOX_CTL, 1435 .box_ctl = SNBEP_C0_MSR_PMON_BOX_CTL,
1435 .msr_offset = SNBEP_CBO_MSR_OFFSET, 1436 .msr_offset = SNBEP_CBO_MSR_OFFSET,
1436 .num_shared_regs = 1, 1437 .num_shared_regs = 1,
1437 .constraints = snbep_uncore_cbox_constraints, 1438 .constraints = snbep_uncore_cbox_constraints,
1438 .ops = &ivbep_uncore_cbox_ops, 1439 .ops = &ivbep_uncore_cbox_ops,
1439 .format_group = &ivbep_uncore_cbox_format_group, 1440 .format_group = &ivbep_uncore_cbox_format_group,
1440 }; 1441 };
1441 1442
1442 static struct intel_uncore_ops ivbep_uncore_pcu_ops = { 1443 static struct intel_uncore_ops ivbep_uncore_pcu_ops = {
1443 IVBEP_UNCORE_MSR_OPS_COMMON_INIT(), 1444 IVBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1444 .hw_config = snbep_pcu_hw_config, 1445 .hw_config = snbep_pcu_hw_config,
1445 .get_constraint = snbep_pcu_get_constraint, 1446 .get_constraint = snbep_pcu_get_constraint,
1446 .put_constraint = snbep_pcu_put_constraint, 1447 .put_constraint = snbep_pcu_put_constraint,
1447 }; 1448 };
1448 1449
1449 static struct intel_uncore_type ivbep_uncore_pcu = { 1450 static struct intel_uncore_type ivbep_uncore_pcu = {
1450 .name = "pcu", 1451 .name = "pcu",
1451 .num_counters = 4, 1452 .num_counters = 4,
1452 .num_boxes = 1, 1453 .num_boxes = 1,
1453 .perf_ctr_bits = 48, 1454 .perf_ctr_bits = 48,
1454 .perf_ctr = SNBEP_PCU_MSR_PMON_CTR0, 1455 .perf_ctr = SNBEP_PCU_MSR_PMON_CTR0,
1455 .event_ctl = SNBEP_PCU_MSR_PMON_CTL0, 1456 .event_ctl = SNBEP_PCU_MSR_PMON_CTL0,
1456 .event_mask = IVBEP_PCU_MSR_PMON_RAW_EVENT_MASK, 1457 .event_mask = IVBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
1457 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL, 1458 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL,
1458 .num_shared_regs = 1, 1459 .num_shared_regs = 1,
1459 .ops = &ivbep_uncore_pcu_ops, 1460 .ops = &ivbep_uncore_pcu_ops,
1460 .format_group = &ivbep_uncore_pcu_format_group, 1461 .format_group = &ivbep_uncore_pcu_format_group,
1461 }; 1462 };
1462 1463
1463 static struct intel_uncore_type *ivbep_msr_uncores[] = { 1464 static struct intel_uncore_type *ivbep_msr_uncores[] = {
1464 &ivbep_uncore_ubox, 1465 &ivbep_uncore_ubox,
1465 &ivbep_uncore_cbox, 1466 &ivbep_uncore_cbox,
1466 &ivbep_uncore_pcu, 1467 &ivbep_uncore_pcu,
1467 NULL, 1468 NULL,
1468 }; 1469 };
1469 1470
1470 void ivbep_uncore_cpu_init(void) 1471 void ivbep_uncore_cpu_init(void)
1471 { 1472 {
1472 if (ivbep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores) 1473 if (ivbep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
1473 ivbep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores; 1474 ivbep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
1474 uncore_msr_uncores = ivbep_msr_uncores; 1475 uncore_msr_uncores = ivbep_msr_uncores;
1475 } 1476 }
1476 1477
1477 static struct intel_uncore_type ivbep_uncore_ha = { 1478 static struct intel_uncore_type ivbep_uncore_ha = {
1478 .name = "ha", 1479 .name = "ha",
1479 .num_counters = 4, 1480 .num_counters = 4,
1480 .num_boxes = 2, 1481 .num_boxes = 2,
1481 .perf_ctr_bits = 48, 1482 .perf_ctr_bits = 48,
1482 IVBEP_UNCORE_PCI_COMMON_INIT(), 1483 IVBEP_UNCORE_PCI_COMMON_INIT(),
1483 }; 1484 };
1484 1485
1485 static struct intel_uncore_type ivbep_uncore_imc = { 1486 static struct intel_uncore_type ivbep_uncore_imc = {
1486 .name = "imc", 1487 .name = "imc",
1487 .num_counters = 4, 1488 .num_counters = 4,
1488 .num_boxes = 8, 1489 .num_boxes = 8,
1489 .perf_ctr_bits = 48, 1490 .perf_ctr_bits = 48,
1490 .fixed_ctr_bits = 48, 1491 .fixed_ctr_bits = 48,
1491 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR, 1492 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
1492 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL, 1493 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
1493 .event_descs = snbep_uncore_imc_events, 1494 .event_descs = snbep_uncore_imc_events,
1494 IVBEP_UNCORE_PCI_COMMON_INIT(), 1495 IVBEP_UNCORE_PCI_COMMON_INIT(),
1495 }; 1496 };
1496 1497
1497 /* registers in IRP boxes are not properly aligned */ 1498 /* registers in IRP boxes are not properly aligned */
1498 static unsigned ivbep_uncore_irp_ctls[] = {0xd8, 0xdc, 0xe0, 0xe4}; 1499 static unsigned ivbep_uncore_irp_ctls[] = {0xd8, 0xdc, 0xe0, 0xe4};
1499 static unsigned ivbep_uncore_irp_ctrs[] = {0xa0, 0xb0, 0xb8, 0xc0}; 1500 static unsigned ivbep_uncore_irp_ctrs[] = {0xa0, 0xb0, 0xb8, 0xc0};
1500 1501
1501 static void ivbep_uncore_irp_enable_event(struct intel_uncore_box *box, struct perf_event *event) 1502 static void ivbep_uncore_irp_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1502 { 1503 {
1503 struct pci_dev *pdev = box->pci_dev; 1504 struct pci_dev *pdev = box->pci_dev;
1504 struct hw_perf_event *hwc = &event->hw; 1505 struct hw_perf_event *hwc = &event->hw;
1505 1506
1506 pci_write_config_dword(pdev, ivbep_uncore_irp_ctls[hwc->idx], 1507 pci_write_config_dword(pdev, ivbep_uncore_irp_ctls[hwc->idx],
1507 hwc->config | SNBEP_PMON_CTL_EN); 1508 hwc->config | SNBEP_PMON_CTL_EN);
1508 } 1509 }
1509 1510
1510 static void ivbep_uncore_irp_disable_event(struct intel_uncore_box *box, struct perf_event *event) 1511 static void ivbep_uncore_irp_disable_event(struct intel_uncore_box *box, struct perf_event *event)
1511 { 1512 {
1512 struct pci_dev *pdev = box->pci_dev; 1513 struct pci_dev *pdev = box->pci_dev;
1513 struct hw_perf_event *hwc = &event->hw; 1514 struct hw_perf_event *hwc = &event->hw;
1514 1515
1515 pci_write_config_dword(pdev, ivbep_uncore_irp_ctls[hwc->idx], hwc->config); 1516 pci_write_config_dword(pdev, ivbep_uncore_irp_ctls[hwc->idx], hwc->config);
1516 } 1517 }
1517 1518
1518 static u64 ivbep_uncore_irp_read_counter(struct intel_uncore_box *box, struct perf_event *event) 1519 static u64 ivbep_uncore_irp_read_counter(struct intel_uncore_box *box, struct perf_event *event)
1519 { 1520 {
1520 struct pci_dev *pdev = box->pci_dev; 1521 struct pci_dev *pdev = box->pci_dev;
1521 struct hw_perf_event *hwc = &event->hw; 1522 struct hw_perf_event *hwc = &event->hw;
1522 u64 count = 0; 1523 u64 count = 0;
1523 1524
1524 pci_read_config_dword(pdev, ivbep_uncore_irp_ctrs[hwc->idx], (u32 *)&count); 1525 pci_read_config_dword(pdev, ivbep_uncore_irp_ctrs[hwc->idx], (u32 *)&count);
1525 pci_read_config_dword(pdev, ivbep_uncore_irp_ctrs[hwc->idx] + 4, (u32 *)&count + 1); 1526 pci_read_config_dword(pdev, ivbep_uncore_irp_ctrs[hwc->idx] + 4, (u32 *)&count + 1);
1526 1527
1527 return count; 1528 return count;
1528 } 1529 }
1529 1530
1530 static struct intel_uncore_ops ivbep_uncore_irp_ops = { 1531 static struct intel_uncore_ops ivbep_uncore_irp_ops = {
1531 .init_box = ivbep_uncore_pci_init_box, 1532 .init_box = ivbep_uncore_pci_init_box,
1532 .disable_box = snbep_uncore_pci_disable_box, 1533 .disable_box = snbep_uncore_pci_disable_box,
1533 .enable_box = snbep_uncore_pci_enable_box, 1534 .enable_box = snbep_uncore_pci_enable_box,
1534 .disable_event = ivbep_uncore_irp_disable_event, 1535 .disable_event = ivbep_uncore_irp_disable_event,
1535 .enable_event = ivbep_uncore_irp_enable_event, 1536 .enable_event = ivbep_uncore_irp_enable_event,
1536 .read_counter = ivbep_uncore_irp_read_counter, 1537 .read_counter = ivbep_uncore_irp_read_counter,
1537 }; 1538 };
1538 1539
1539 static struct intel_uncore_type ivbep_uncore_irp = { 1540 static struct intel_uncore_type ivbep_uncore_irp = {
1540 .name = "irp", 1541 .name = "irp",
1541 .num_counters = 4, 1542 .num_counters = 4,
1542 .num_boxes = 1, 1543 .num_boxes = 1,
1543 .perf_ctr_bits = 48, 1544 .perf_ctr_bits = 48,
1544 .event_mask = IVBEP_PMON_RAW_EVENT_MASK, 1545 .event_mask = IVBEP_PMON_RAW_EVENT_MASK,
1545 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, 1546 .box_ctl = SNBEP_PCI_PMON_BOX_CTL,
1546 .ops = &ivbep_uncore_irp_ops, 1547 .ops = &ivbep_uncore_irp_ops,
1547 .format_group = &ivbep_uncore_format_group, 1548 .format_group = &ivbep_uncore_format_group,
1548 }; 1549 };
1549 1550
1550 static struct intel_uncore_ops ivbep_uncore_qpi_ops = { 1551 static struct intel_uncore_ops ivbep_uncore_qpi_ops = {
1551 .init_box = ivbep_uncore_pci_init_box, 1552 .init_box = ivbep_uncore_pci_init_box,
1552 .disable_box = snbep_uncore_pci_disable_box, 1553 .disable_box = snbep_uncore_pci_disable_box,
1553 .enable_box = snbep_uncore_pci_enable_box, 1554 .enable_box = snbep_uncore_pci_enable_box,
1554 .disable_event = snbep_uncore_pci_disable_event, 1555 .disable_event = snbep_uncore_pci_disable_event,
1555 .enable_event = snbep_qpi_enable_event, 1556 .enable_event = snbep_qpi_enable_event,
1556 .read_counter = snbep_uncore_pci_read_counter, 1557 .read_counter = snbep_uncore_pci_read_counter,
1557 .hw_config = snbep_qpi_hw_config, 1558 .hw_config = snbep_qpi_hw_config,
1558 .get_constraint = uncore_get_constraint, 1559 .get_constraint = uncore_get_constraint,
1559 .put_constraint = uncore_put_constraint, 1560 .put_constraint = uncore_put_constraint,
1560 }; 1561 };
1561 1562
1562 static struct intel_uncore_type ivbep_uncore_qpi = { 1563 static struct intel_uncore_type ivbep_uncore_qpi = {
1563 .name = "qpi", 1564 .name = "qpi",
1564 .num_counters = 4, 1565 .num_counters = 4,
1565 .num_boxes = 3, 1566 .num_boxes = 3,
1566 .perf_ctr_bits = 48, 1567 .perf_ctr_bits = 48,
1567 .perf_ctr = SNBEP_PCI_PMON_CTR0, 1568 .perf_ctr = SNBEP_PCI_PMON_CTR0,
1568 .event_ctl = SNBEP_PCI_PMON_CTL0, 1569 .event_ctl = SNBEP_PCI_PMON_CTL0,
1569 .event_mask = IVBEP_QPI_PCI_PMON_RAW_EVENT_MASK, 1570 .event_mask = IVBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
1570 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, 1571 .box_ctl = SNBEP_PCI_PMON_BOX_CTL,
1571 .num_shared_regs = 1, 1572 .num_shared_regs = 1,
1572 .ops = &ivbep_uncore_qpi_ops, 1573 .ops = &ivbep_uncore_qpi_ops,
1573 .format_group = &ivbep_uncore_qpi_format_group, 1574 .format_group = &ivbep_uncore_qpi_format_group,
1574 }; 1575 };
1575 1576
1576 static struct intel_uncore_type ivbep_uncore_r2pcie = { 1577 static struct intel_uncore_type ivbep_uncore_r2pcie = {
1577 .name = "r2pcie", 1578 .name = "r2pcie",
1578 .num_counters = 4, 1579 .num_counters = 4,
1579 .num_boxes = 1, 1580 .num_boxes = 1,
1580 .perf_ctr_bits = 44, 1581 .perf_ctr_bits = 44,
1581 .constraints = snbep_uncore_r2pcie_constraints, 1582 .constraints = snbep_uncore_r2pcie_constraints,
1582 IVBEP_UNCORE_PCI_COMMON_INIT(), 1583 IVBEP_UNCORE_PCI_COMMON_INIT(),
1583 }; 1584 };
1584 1585
1585 static struct intel_uncore_type ivbep_uncore_r3qpi = { 1586 static struct intel_uncore_type ivbep_uncore_r3qpi = {
1586 .name = "r3qpi", 1587 .name = "r3qpi",
1587 .num_counters = 3, 1588 .num_counters = 3,
1588 .num_boxes = 2, 1589 .num_boxes = 2,
1589 .perf_ctr_bits = 44, 1590 .perf_ctr_bits = 44,
1590 .constraints = snbep_uncore_r3qpi_constraints, 1591 .constraints = snbep_uncore_r3qpi_constraints,
1591 IVBEP_UNCORE_PCI_COMMON_INIT(), 1592 IVBEP_UNCORE_PCI_COMMON_INIT(),
1592 }; 1593 };
1593 1594
1594 enum { 1595 enum {
1595 IVBEP_PCI_UNCORE_HA, 1596 IVBEP_PCI_UNCORE_HA,
1596 IVBEP_PCI_UNCORE_IMC, 1597 IVBEP_PCI_UNCORE_IMC,
1597 IVBEP_PCI_UNCORE_IRP, 1598 IVBEP_PCI_UNCORE_IRP,
1598 IVBEP_PCI_UNCORE_QPI, 1599 IVBEP_PCI_UNCORE_QPI,
1599 IVBEP_PCI_UNCORE_R2PCIE, 1600 IVBEP_PCI_UNCORE_R2PCIE,
1600 IVBEP_PCI_UNCORE_R3QPI, 1601 IVBEP_PCI_UNCORE_R3QPI,
1601 }; 1602 };
1602 1603
1603 static struct intel_uncore_type *ivbep_pci_uncores[] = { 1604 static struct intel_uncore_type *ivbep_pci_uncores[] = {
1604 [IVBEP_PCI_UNCORE_HA] = &ivbep_uncore_ha, 1605 [IVBEP_PCI_UNCORE_HA] = &ivbep_uncore_ha,
1605 [IVBEP_PCI_UNCORE_IMC] = &ivbep_uncore_imc, 1606 [IVBEP_PCI_UNCORE_IMC] = &ivbep_uncore_imc,
1606 [IVBEP_PCI_UNCORE_IRP] = &ivbep_uncore_irp, 1607 [IVBEP_PCI_UNCORE_IRP] = &ivbep_uncore_irp,
1607 [IVBEP_PCI_UNCORE_QPI] = &ivbep_uncore_qpi, 1608 [IVBEP_PCI_UNCORE_QPI] = &ivbep_uncore_qpi,
1608 [IVBEP_PCI_UNCORE_R2PCIE] = &ivbep_uncore_r2pcie, 1609 [IVBEP_PCI_UNCORE_R2PCIE] = &ivbep_uncore_r2pcie,
1609 [IVBEP_PCI_UNCORE_R3QPI] = &ivbep_uncore_r3qpi, 1610 [IVBEP_PCI_UNCORE_R3QPI] = &ivbep_uncore_r3qpi,
1610 NULL, 1611 NULL,
1611 }; 1612 };
1612 1613
1613 static const struct pci_device_id ivbep_uncore_pci_ids[] = { 1614 static const struct pci_device_id ivbep_uncore_pci_ids[] = {
1614 { /* Home Agent 0 */ 1615 { /* Home Agent 0 */
1615 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30), 1616 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30),
1616 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_HA, 0), 1617 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_HA, 0),
1617 }, 1618 },
1618 { /* Home Agent 1 */ 1619 { /* Home Agent 1 */
1619 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38), 1620 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38),
1620 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_HA, 1), 1621 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_HA, 1),
1621 }, 1622 },
1622 { /* MC0 Channel 0 */ 1623 { /* MC0 Channel 0 */
1623 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4), 1624 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4),
1624 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 0), 1625 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 0),
1625 }, 1626 },
1626 { /* MC0 Channel 1 */ 1627 { /* MC0 Channel 1 */
1627 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5), 1628 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5),
1628 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 1), 1629 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 1),
1629 }, 1630 },
1630 { /* MC0 Channel 3 */ 1631 { /* MC0 Channel 3 */
1631 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0), 1632 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0),
1632 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 2), 1633 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 2),
1633 }, 1634 },
1634 { /* MC0 Channel 4 */ 1635 { /* MC0 Channel 4 */
1635 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1), 1636 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1),
1636 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 3), 1637 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 3),
1637 }, 1638 },
1638 { /* MC1 Channel 0 */ 1639 { /* MC1 Channel 0 */
1639 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4), 1640 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4),
1640 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 4), 1641 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 4),
1641 }, 1642 },
1642 { /* MC1 Channel 1 */ 1643 { /* MC1 Channel 1 */
1643 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5), 1644 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5),
1644 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 5), 1645 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 5),
1645 }, 1646 },
1646 { /* MC1 Channel 3 */ 1647 { /* MC1 Channel 3 */
1647 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0), 1648 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0),
1648 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 6), 1649 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 6),
1649 }, 1650 },
1650 { /* MC1 Channel 4 */ 1651 { /* MC1 Channel 4 */
1651 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1), 1652 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1),
1652 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 7), 1653 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 7),
1653 }, 1654 },
1654 { /* IRP */ 1655 { /* IRP */
1655 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe39), 1656 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe39),
1656 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IRP, 0), 1657 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IRP, 0),
1657 }, 1658 },
1658 { /* QPI0 Port 0 */ 1659 { /* QPI0 Port 0 */
1659 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32), 1660 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32),
1660 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 0), 1661 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 0),
1661 }, 1662 },
1662 { /* QPI0 Port 1 */ 1663 { /* QPI0 Port 1 */
1663 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33), 1664 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33),
1664 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 1), 1665 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 1),
1665 }, 1666 },
1666 { /* QPI1 Port 2 */ 1667 { /* QPI1 Port 2 */
1667 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a), 1668 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a),
1668 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 2), 1669 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 2),
1669 }, 1670 },
1670 { /* R2PCIe */ 1671 { /* R2PCIe */
1671 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34), 1672 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34),
1672 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R2PCIE, 0), 1673 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R2PCIE, 0),
1673 }, 1674 },
1674 { /* R3QPI0 Link 0 */ 1675 { /* R3QPI0 Link 0 */
1675 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36), 1676 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36),
1676 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 0), 1677 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 0),
1677 }, 1678 },
1678 { /* R3QPI0 Link 1 */ 1679 { /* R3QPI0 Link 1 */
1679 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37), 1680 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37),
1680 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 1), 1681 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 1),
1681 }, 1682 },
1682 { /* R3QPI1 Link 2 */ 1683 { /* R3QPI1 Link 2 */
1683 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e), 1684 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e),
1684 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 2), 1685 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 2),
1685 }, 1686 },
1686 { /* QPI Port 0 filter */ 1687 { /* QPI Port 0 filter */
1687 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe86), 1688 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe86),
1688 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 1689 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
1689 SNBEP_PCI_QPI_PORT0_FILTER), 1690 SNBEP_PCI_QPI_PORT0_FILTER),
1690 }, 1691 },
1691 { /* QPI Port 0 filter */ 1692 { /* QPI Port 0 filter */
1692 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe96), 1693 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe96),
1693 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 1694 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
1694 SNBEP_PCI_QPI_PORT1_FILTER), 1695 SNBEP_PCI_QPI_PORT1_FILTER),
1695 }, 1696 },
1696 { /* end: all zeroes */ } 1697 { /* end: all zeroes */ }
1697 }; 1698 };
1698 1699
1699 static struct pci_driver ivbep_uncore_pci_driver = { 1700 static struct pci_driver ivbep_uncore_pci_driver = {
1700 .name = "ivbep_uncore", 1701 .name = "ivbep_uncore",
1701 .id_table = ivbep_uncore_pci_ids, 1702 .id_table = ivbep_uncore_pci_ids,
1702 }; 1703 };
1703 1704
1704 int ivbep_uncore_pci_init(void) 1705 int ivbep_uncore_pci_init(void)
1705 { 1706 {
1706 int ret = snbep_pci2phy_map_init(0x0e1e); 1707 int ret = snbep_pci2phy_map_init(0x0e1e);
1707 if (ret) 1708 if (ret)
1708 return ret; 1709 return ret;
1709 uncore_pci_uncores = ivbep_pci_uncores; 1710 uncore_pci_uncores = ivbep_pci_uncores;
1710 uncore_pci_driver = &ivbep_uncore_pci_driver; 1711 uncore_pci_driver = &ivbep_uncore_pci_driver;
1711 return 0; 1712 return 0;
1712 } 1713 }
1713 /* end of IvyTown uncore support */ 1714 /* end of IvyTown uncore support */
1714 1715
1715 /* Haswell-EP uncore support */ 1716 /* Haswell-EP uncore support */
1716 static struct attribute *hswep_uncore_ubox_formats_attr[] = { 1717 static struct attribute *hswep_uncore_ubox_formats_attr[] = {
1717 &format_attr_event.attr, 1718 &format_attr_event.attr,
1718 &format_attr_umask.attr, 1719 &format_attr_umask.attr,
1719 &format_attr_edge.attr, 1720 &format_attr_edge.attr,
1720 &format_attr_inv.attr, 1721 &format_attr_inv.attr,
1721 &format_attr_thresh5.attr, 1722 &format_attr_thresh5.attr,
1722 &format_attr_filter_tid2.attr, 1723 &format_attr_filter_tid2.attr,
1723 &format_attr_filter_cid.attr, 1724 &format_attr_filter_cid.attr,
1724 NULL, 1725 NULL,
1725 }; 1726 };
1726 1727
1727 static struct attribute_group hswep_uncore_ubox_format_group = { 1728 static struct attribute_group hswep_uncore_ubox_format_group = {
1728 .name = "format", 1729 .name = "format",
1729 .attrs = hswep_uncore_ubox_formats_attr, 1730 .attrs = hswep_uncore_ubox_formats_attr,
1730 }; 1731 };
1731 1732
1732 static int hswep_ubox_hw_config(struct intel_uncore_box *box, struct perf_event *event) 1733 static int hswep_ubox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1733 { 1734 {
1734 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 1735 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1735 reg1->reg = HSWEP_U_MSR_PMON_FILTER; 1736 reg1->reg = HSWEP_U_MSR_PMON_FILTER;
1736 reg1->config = event->attr.config1 & HSWEP_U_MSR_PMON_BOX_FILTER_MASK; 1737 reg1->config = event->attr.config1 & HSWEP_U_MSR_PMON_BOX_FILTER_MASK;
1737 reg1->idx = 0; 1738 reg1->idx = 0;
1738 return 0; 1739 return 0;
1739 } 1740 }
1740 1741
1741 static struct intel_uncore_ops hswep_uncore_ubox_ops = { 1742 static struct intel_uncore_ops hswep_uncore_ubox_ops = {
1742 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), 1743 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1743 .hw_config = hswep_ubox_hw_config, 1744 .hw_config = hswep_ubox_hw_config,
1744 .get_constraint = uncore_get_constraint, 1745 .get_constraint = uncore_get_constraint,
1745 .put_constraint = uncore_put_constraint, 1746 .put_constraint = uncore_put_constraint,
1746 }; 1747 };
1747 1748
1748 static struct intel_uncore_type hswep_uncore_ubox = { 1749 static struct intel_uncore_type hswep_uncore_ubox = {
1749 .name = "ubox", 1750 .name = "ubox",
1750 .num_counters = 2, 1751 .num_counters = 2,
1751 .num_boxes = 1, 1752 .num_boxes = 1,
1752 .perf_ctr_bits = 44, 1753 .perf_ctr_bits = 44,
1753 .fixed_ctr_bits = 48, 1754 .fixed_ctr_bits = 48,
1754 .perf_ctr = HSWEP_U_MSR_PMON_CTR0, 1755 .perf_ctr = HSWEP_U_MSR_PMON_CTR0,
1755 .event_ctl = HSWEP_U_MSR_PMON_CTL0, 1756 .event_ctl = HSWEP_U_MSR_PMON_CTL0,
1756 .event_mask = SNBEP_U_MSR_PMON_RAW_EVENT_MASK, 1757 .event_mask = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
1757 .fixed_ctr = HSWEP_U_MSR_PMON_UCLK_FIXED_CTR, 1758 .fixed_ctr = HSWEP_U_MSR_PMON_UCLK_FIXED_CTR,
1758 .fixed_ctl = HSWEP_U_MSR_PMON_UCLK_FIXED_CTL, 1759 .fixed_ctl = HSWEP_U_MSR_PMON_UCLK_FIXED_CTL,
1759 .num_shared_regs = 1, 1760 .num_shared_regs = 1,
1760 .ops = &hswep_uncore_ubox_ops, 1761 .ops = &hswep_uncore_ubox_ops,
1761 .format_group = &hswep_uncore_ubox_format_group, 1762 .format_group = &hswep_uncore_ubox_format_group,
1762 }; 1763 };
1763 1764
1764 static struct attribute *hswep_uncore_cbox_formats_attr[] = { 1765 static struct attribute *hswep_uncore_cbox_formats_attr[] = {
1765 &format_attr_event.attr, 1766 &format_attr_event.attr,
1766 &format_attr_umask.attr, 1767 &format_attr_umask.attr,
1767 &format_attr_edge.attr, 1768 &format_attr_edge.attr,
1768 &format_attr_tid_en.attr, 1769 &format_attr_tid_en.attr,
1769 &format_attr_thresh8.attr, 1770 &format_attr_thresh8.attr,
1770 &format_attr_filter_tid3.attr, 1771 &format_attr_filter_tid3.attr,
1771 &format_attr_filter_link2.attr, 1772 &format_attr_filter_link2.attr,
1772 &format_attr_filter_state3.attr, 1773 &format_attr_filter_state3.attr,
1773 &format_attr_filter_nid2.attr, 1774 &format_attr_filter_nid2.attr,
1774 &format_attr_filter_opc2.attr, 1775 &format_attr_filter_opc2.attr,
1775 &format_attr_filter_nc.attr, 1776 &format_attr_filter_nc.attr,
1776 &format_attr_filter_c6.attr, 1777 &format_attr_filter_c6.attr,
1777 &format_attr_filter_isoc.attr, 1778 &format_attr_filter_isoc.attr,
1778 NULL, 1779 NULL,
1779 }; 1780 };
1780 1781
1781 static struct attribute_group hswep_uncore_cbox_format_group = { 1782 static struct attribute_group hswep_uncore_cbox_format_group = {
1782 .name = "format", 1783 .name = "format",
1783 .attrs = hswep_uncore_cbox_formats_attr, 1784 .attrs = hswep_uncore_cbox_formats_attr,
1784 }; 1785 };
1785 1786
1786 static struct event_constraint hswep_uncore_cbox_constraints[] = { 1787 static struct event_constraint hswep_uncore_cbox_constraints[] = {
1787 UNCORE_EVENT_CONSTRAINT(0x01, 0x1), 1788 UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
1788 UNCORE_EVENT_CONSTRAINT(0x09, 0x1), 1789 UNCORE_EVENT_CONSTRAINT(0x09, 0x1),
1789 UNCORE_EVENT_CONSTRAINT(0x11, 0x1), 1790 UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
1790 UNCORE_EVENT_CONSTRAINT(0x36, 0x1), 1791 UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
1791 UNCORE_EVENT_CONSTRAINT(0x38, 0x3), 1792 UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
1792 UNCORE_EVENT_CONSTRAINT(0x3b, 0x1), 1793 UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
1793 UNCORE_EVENT_CONSTRAINT(0x3e, 0x1), 1794 UNCORE_EVENT_CONSTRAINT(0x3e, 0x1),
1794 EVENT_CONSTRAINT_END 1795 EVENT_CONSTRAINT_END
1795 }; 1796 };
1796 1797
1797 static struct extra_reg hswep_uncore_cbox_extra_regs[] = { 1798 static struct extra_reg hswep_uncore_cbox_extra_regs[] = {
1798 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN, 1799 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1799 SNBEP_CBO_PMON_CTL_TID_EN, 0x1), 1800 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1800 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4), 1801 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1801 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4), 1802 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1802 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4), 1803 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1803 SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4), 1804 SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4),
1804 SNBEP_CBO_EVENT_EXTRA_REG(0x2134, 0xffff, 0x4), 1805 SNBEP_CBO_EVENT_EXTRA_REG(0x2134, 0xffff, 0x4),
1805 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x4), 1806 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x4),
1806 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8), 1807 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
1807 SNBEP_CBO_EVENT_EXTRA_REG(0x4028, 0x40ff, 0x8), 1808 SNBEP_CBO_EVENT_EXTRA_REG(0x4028, 0x40ff, 0x8),
1808 SNBEP_CBO_EVENT_EXTRA_REG(0x4032, 0x40ff, 0x8), 1809 SNBEP_CBO_EVENT_EXTRA_REG(0x4032, 0x40ff, 0x8),
1809 SNBEP_CBO_EVENT_EXTRA_REG(0x4029, 0x40ff, 0x8), 1810 SNBEP_CBO_EVENT_EXTRA_REG(0x4029, 0x40ff, 0x8),
1810 SNBEP_CBO_EVENT_EXTRA_REG(0x4033, 0x40ff, 0x8), 1811 SNBEP_CBO_EVENT_EXTRA_REG(0x4033, 0x40ff, 0x8),
1811 SNBEP_CBO_EVENT_EXTRA_REG(0x402A, 0x40ff, 0x8), 1812 SNBEP_CBO_EVENT_EXTRA_REG(0x402A, 0x40ff, 0x8),
1812 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x12), 1813 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x12),
1813 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10), 1814 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1814 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18), 1815 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
1815 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8), 1816 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
1816 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8), 1817 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
1817 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8), 1818 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
1818 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18), 1819 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
1819 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8), 1820 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
1820 SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10), 1821 SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
1821 SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10), 1822 SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
1822 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10), 1823 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
1823 SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10), 1824 SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
1824 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10), 1825 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
1825 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10), 1826 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
1826 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18), 1827 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
1827 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8), 1828 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
1828 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8), 1829 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
1829 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18), 1830 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
1830 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8), 1831 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
1831 SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10), 1832 SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1832 SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10), 1833 SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
1833 SNBEP_CBO_EVENT_EXTRA_REG(0x2136, 0xffff, 0x10), 1834 SNBEP_CBO_EVENT_EXTRA_REG(0x2136, 0xffff, 0x10),
1834 SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10), 1835 SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
1835 SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8), 1836 SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
1836 EVENT_EXTRA_END 1837 EVENT_EXTRA_END
1837 }; 1838 };
1838 1839
1839 static u64 hswep_cbox_filter_mask(int fields) 1840 static u64 hswep_cbox_filter_mask(int fields)
1840 { 1841 {
1841 u64 mask = 0; 1842 u64 mask = 0;
1842 if (fields & 0x1) 1843 if (fields & 0x1)
1843 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_TID; 1844 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_TID;
1844 if (fields & 0x2) 1845 if (fields & 0x2)
1845 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_LINK; 1846 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_LINK;
1846 if (fields & 0x4) 1847 if (fields & 0x4)
1847 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_STATE; 1848 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_STATE;
1848 if (fields & 0x8) 1849 if (fields & 0x8)
1849 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_NID; 1850 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_NID;
1850 if (fields & 0x10) { 1851 if (fields & 0x10) {
1851 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_OPC; 1852 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_OPC;
1852 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_NC; 1853 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_NC;
1853 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_C6; 1854 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_C6;
1854 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_ISOC; 1855 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_ISOC;
1855 } 1856 }
1856 return mask; 1857 return mask;
1857 } 1858 }
1858 1859
1859 static struct event_constraint * 1860 static struct event_constraint *
1860 hswep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 1861 hswep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1861 { 1862 {
1862 return __snbep_cbox_get_constraint(box, event, hswep_cbox_filter_mask); 1863 return __snbep_cbox_get_constraint(box, event, hswep_cbox_filter_mask);
1863 } 1864 }
1864 1865
1865 static int hswep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event) 1866 static int hswep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1866 { 1867 {
1867 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 1868 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1868 struct extra_reg *er; 1869 struct extra_reg *er;
1869 int idx = 0; 1870 int idx = 0;
1870 1871
1871 for (er = hswep_uncore_cbox_extra_regs; er->msr; er++) { 1872 for (er = hswep_uncore_cbox_extra_regs; er->msr; er++) {
1872 if (er->event != (event->hw.config & er->config_mask)) 1873 if (er->event != (event->hw.config & er->config_mask))
1873 continue; 1874 continue;
1874 idx |= er->idx; 1875 idx |= er->idx;
1875 } 1876 }
1876 1877
1877 if (idx) { 1878 if (idx) {
1878 reg1->reg = HSWEP_C0_MSR_PMON_BOX_FILTER0 + 1879 reg1->reg = HSWEP_C0_MSR_PMON_BOX_FILTER0 +
1879 HSWEP_CBO_MSR_OFFSET * box->pmu->pmu_idx; 1880 HSWEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1880 reg1->config = event->attr.config1 & hswep_cbox_filter_mask(idx); 1881 reg1->config = event->attr.config1 & hswep_cbox_filter_mask(idx);
1881 reg1->idx = idx; 1882 reg1->idx = idx;
1882 } 1883 }
1883 return 0; 1884 return 0;
1884 } 1885 }
1885 1886
1886 static void hswep_cbox_enable_event(struct intel_uncore_box *box, 1887 static void hswep_cbox_enable_event(struct intel_uncore_box *box,
1887 struct perf_event *event) 1888 struct perf_event *event)
1888 { 1889 {
1889 struct hw_perf_event *hwc = &event->hw; 1890 struct hw_perf_event *hwc = &event->hw;
1890 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 1891 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1891 1892
1892 if (reg1->idx != EXTRA_REG_NONE) { 1893 if (reg1->idx != EXTRA_REG_NONE) {
1893 u64 filter = uncore_shared_reg_config(box, 0); 1894 u64 filter = uncore_shared_reg_config(box, 0);
1894 wrmsrl(reg1->reg, filter & 0xffffffff); 1895 wrmsrl(reg1->reg, filter & 0xffffffff);
1895 wrmsrl(reg1->reg + 1, filter >> 32); 1896 wrmsrl(reg1->reg + 1, filter >> 32);
1896 } 1897 }
1897 1898
1898 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); 1899 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1899 } 1900 }
1900 1901
1901 static struct intel_uncore_ops hswep_uncore_cbox_ops = { 1902 static struct intel_uncore_ops hswep_uncore_cbox_ops = {
1902 .init_box = snbep_uncore_msr_init_box, 1903 .init_box = snbep_uncore_msr_init_box,
1903 .disable_box = snbep_uncore_msr_disable_box, 1904 .disable_box = snbep_uncore_msr_disable_box,
1904 .enable_box = snbep_uncore_msr_enable_box, 1905 .enable_box = snbep_uncore_msr_enable_box,
1905 .disable_event = snbep_uncore_msr_disable_event, 1906 .disable_event = snbep_uncore_msr_disable_event,
1906 .enable_event = hswep_cbox_enable_event, 1907 .enable_event = hswep_cbox_enable_event,
1907 .read_counter = uncore_msr_read_counter, 1908 .read_counter = uncore_msr_read_counter,
1908 .hw_config = hswep_cbox_hw_config, 1909 .hw_config = hswep_cbox_hw_config,
1909 .get_constraint = hswep_cbox_get_constraint, 1910 .get_constraint = hswep_cbox_get_constraint,
1910 .put_constraint = snbep_cbox_put_constraint, 1911 .put_constraint = snbep_cbox_put_constraint,
1911 }; 1912 };
1912 1913
1913 static struct intel_uncore_type hswep_uncore_cbox = { 1914 static struct intel_uncore_type hswep_uncore_cbox = {
1914 .name = "cbox", 1915 .name = "cbox",
1915 .num_counters = 4, 1916 .num_counters = 4,
1916 .num_boxes = 18, 1917 .num_boxes = 18,
1917 .perf_ctr_bits = 44, 1918 .perf_ctr_bits = 44,
1918 .event_ctl = HSWEP_C0_MSR_PMON_CTL0, 1919 .event_ctl = HSWEP_C0_MSR_PMON_CTL0,
1919 .perf_ctr = HSWEP_C0_MSR_PMON_CTR0, 1920 .perf_ctr = HSWEP_C0_MSR_PMON_CTR0,
1920 .event_mask = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK, 1921 .event_mask = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
1921 .box_ctl = HSWEP_C0_MSR_PMON_BOX_CTL, 1922 .box_ctl = HSWEP_C0_MSR_PMON_BOX_CTL,
1922 .msr_offset = HSWEP_CBO_MSR_OFFSET, 1923 .msr_offset = HSWEP_CBO_MSR_OFFSET,
1923 .num_shared_regs = 1, 1924 .num_shared_regs = 1,
1924 .constraints = hswep_uncore_cbox_constraints, 1925 .constraints = hswep_uncore_cbox_constraints,
1925 .ops = &hswep_uncore_cbox_ops, 1926 .ops = &hswep_uncore_cbox_ops,
1926 .format_group = &hswep_uncore_cbox_format_group, 1927 .format_group = &hswep_uncore_cbox_format_group,
1927 }; 1928 };
1928 1929
1929 /* 1930 /*
1930 * Write SBOX Initialization register bit by bit to avoid spurious #GPs 1931 * Write SBOX Initialization register bit by bit to avoid spurious #GPs
1931 */ 1932 */
1932 static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) 1933 static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box)
1933 { 1934 {
1934 unsigned msr = uncore_msr_box_ctl(box); 1935 unsigned msr = uncore_msr_box_ctl(box);
1935 1936
1936 if (msr) { 1937 if (msr) {
1937 u64 init = SNBEP_PMON_BOX_CTL_INT; 1938 u64 init = SNBEP_PMON_BOX_CTL_INT;
1938 u64 flags = 0; 1939 u64 flags = 0;
1939 int i; 1940 int i;
1940 1941
1941 for_each_set_bit(i, (unsigned long *)&init, 64) { 1942 for_each_set_bit(i, (unsigned long *)&init, 64) {
1942 flags |= (1ULL << i); 1943 flags |= (1ULL << i);
1943 wrmsrl(msr, flags); 1944 wrmsrl(msr, flags);
1944 } 1945 }
1945 } 1946 }
1946 } 1947 }
1947 1948
1948 static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = { 1949 static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = {
1949 __SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), 1950 __SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1950 .init_box = hswep_uncore_sbox_msr_init_box 1951 .init_box = hswep_uncore_sbox_msr_init_box
1951 }; 1952 };
1952 1953
1953 static struct attribute *hswep_uncore_sbox_formats_attr[] = { 1954 static struct attribute *hswep_uncore_sbox_formats_attr[] = {
1954 &format_attr_event.attr, 1955 &format_attr_event.attr,
1955 &format_attr_umask.attr, 1956 &format_attr_umask.attr,
1956 &format_attr_edge.attr, 1957 &format_attr_edge.attr,
1957 &format_attr_tid_en.attr, 1958 &format_attr_tid_en.attr,
1958 &format_attr_inv.attr, 1959 &format_attr_inv.attr,
1959 &format_attr_thresh8.attr, 1960 &format_attr_thresh8.attr,
1960 NULL, 1961 NULL,
1961 }; 1962 };
1962 1963
1963 static struct attribute_group hswep_uncore_sbox_format_group = { 1964 static struct attribute_group hswep_uncore_sbox_format_group = {
1964 .name = "format", 1965 .name = "format",
1965 .attrs = hswep_uncore_sbox_formats_attr, 1966 .attrs = hswep_uncore_sbox_formats_attr,
1966 }; 1967 };
1967 1968
1968 static struct intel_uncore_type hswep_uncore_sbox = { 1969 static struct intel_uncore_type hswep_uncore_sbox = {
1969 .name = "sbox", 1970 .name = "sbox",
1970 .num_counters = 4, 1971 .num_counters = 4,
1971 .num_boxes = 4, 1972 .num_boxes = 4,
1972 .perf_ctr_bits = 44, 1973 .perf_ctr_bits = 44,
1973 .event_ctl = HSWEP_S0_MSR_PMON_CTL0, 1974 .event_ctl = HSWEP_S0_MSR_PMON_CTL0,
1974 .perf_ctr = HSWEP_S0_MSR_PMON_CTR0, 1975 .perf_ctr = HSWEP_S0_MSR_PMON_CTR0,
1975 .event_mask = HSWEP_S_MSR_PMON_RAW_EVENT_MASK, 1976 .event_mask = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
1976 .box_ctl = HSWEP_S0_MSR_PMON_BOX_CTL, 1977 .box_ctl = HSWEP_S0_MSR_PMON_BOX_CTL,
1977 .msr_offset = HSWEP_SBOX_MSR_OFFSET, 1978 .msr_offset = HSWEP_SBOX_MSR_OFFSET,
1978 .ops = &hswep_uncore_sbox_msr_ops, 1979 .ops = &hswep_uncore_sbox_msr_ops,
1979 .format_group = &hswep_uncore_sbox_format_group, 1980 .format_group = &hswep_uncore_sbox_format_group,
1980 }; 1981 };
1981 1982
1982 static int hswep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event) 1983 static int hswep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1983 { 1984 {
1984 struct hw_perf_event *hwc = &event->hw; 1985 struct hw_perf_event *hwc = &event->hw;
1985 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 1986 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1986 int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK; 1987 int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
1987 1988
1988 if (ev_sel >= 0xb && ev_sel <= 0xe) { 1989 if (ev_sel >= 0xb && ev_sel <= 0xe) {
1989 reg1->reg = HSWEP_PCU_MSR_PMON_BOX_FILTER; 1990 reg1->reg = HSWEP_PCU_MSR_PMON_BOX_FILTER;
1990 reg1->idx = ev_sel - 0xb; 1991 reg1->idx = ev_sel - 0xb;
1991 reg1->config = event->attr.config1 & (0xff << reg1->idx); 1992 reg1->config = event->attr.config1 & (0xff << reg1->idx);
1992 } 1993 }
1993 return 0; 1994 return 0;
1994 } 1995 }
1995 1996
1996 static struct intel_uncore_ops hswep_uncore_pcu_ops = { 1997 static struct intel_uncore_ops hswep_uncore_pcu_ops = {
1997 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(), 1998 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1998 .hw_config = hswep_pcu_hw_config, 1999 .hw_config = hswep_pcu_hw_config,
1999 .get_constraint = snbep_pcu_get_constraint, 2000 .get_constraint = snbep_pcu_get_constraint,
2000 .put_constraint = snbep_pcu_put_constraint, 2001 .put_constraint = snbep_pcu_put_constraint,
2001 }; 2002 };
2002 2003
2003 static struct intel_uncore_type hswep_uncore_pcu = { 2004 static struct intel_uncore_type hswep_uncore_pcu = {
2004 .name = "pcu", 2005 .name = "pcu",
2005 .num_counters = 4, 2006 .num_counters = 4,
2006 .num_boxes = 1, 2007 .num_boxes = 1,
2007 .perf_ctr_bits = 48, 2008 .perf_ctr_bits = 48,
2008 .perf_ctr = HSWEP_PCU_MSR_PMON_CTR0, 2009 .perf_ctr = HSWEP_PCU_MSR_PMON_CTR0,
2009 .event_ctl = HSWEP_PCU_MSR_PMON_CTL0, 2010 .event_ctl = HSWEP_PCU_MSR_PMON_CTL0,
2010 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK, 2011 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
2011 .box_ctl = HSWEP_PCU_MSR_PMON_BOX_CTL, 2012 .box_ctl = HSWEP_PCU_MSR_PMON_BOX_CTL,
2012 .num_shared_regs = 1, 2013 .num_shared_regs = 1,
2013 .ops = &hswep_uncore_pcu_ops, 2014 .ops = &hswep_uncore_pcu_ops,
2014 .format_group = &snbep_uncore_pcu_format_group, 2015 .format_group = &snbep_uncore_pcu_format_group,
2015 }; 2016 };
2016 2017
2017 static struct intel_uncore_type *hswep_msr_uncores[] = { 2018 static struct intel_uncore_type *hswep_msr_uncores[] = {
2018 &hswep_uncore_ubox, 2019 &hswep_uncore_ubox,
2019 &hswep_uncore_cbox, 2020 &hswep_uncore_cbox,
2020 &hswep_uncore_sbox, 2021 &hswep_uncore_sbox,
2021 &hswep_uncore_pcu, 2022 &hswep_uncore_pcu,
2022 NULL, 2023 NULL,
2023 }; 2024 };
2024 2025
2025 void hswep_uncore_cpu_init(void) 2026 void hswep_uncore_cpu_init(void)
2026 { 2027 {
2027 if (hswep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores) 2028 if (hswep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
2028 hswep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores; 2029 hswep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
2030
2031 /* Detect 6-8 core systems with only two SBOXes */
2032 if (uncore_extra_pci_dev[0][HSWEP_PCI_PCU_3]) {
2033 u32 capid4;
2034
2035 pci_read_config_dword(uncore_extra_pci_dev[0][HSWEP_PCI_PCU_3],
2036 0x94, &capid4);
2037 if (((capid4 >> 6) & 0x3) == 0)
2038 hswep_uncore_sbox.num_boxes = 2;
2039 }
2040
2029 uncore_msr_uncores = hswep_msr_uncores; 2041 uncore_msr_uncores = hswep_msr_uncores;
2030 } 2042 }
2031 2043
2032 static struct intel_uncore_type hswep_uncore_ha = { 2044 static struct intel_uncore_type hswep_uncore_ha = {
2033 .name = "ha", 2045 .name = "ha",
2034 .num_counters = 5, 2046 .num_counters = 5,
2035 .num_boxes = 2, 2047 .num_boxes = 2,
2036 .perf_ctr_bits = 48, 2048 .perf_ctr_bits = 48,
2037 SNBEP_UNCORE_PCI_COMMON_INIT(), 2049 SNBEP_UNCORE_PCI_COMMON_INIT(),
2038 }; 2050 };
2039 2051
2040 static struct uncore_event_desc hswep_uncore_imc_events[] = { 2052 static struct uncore_event_desc hswep_uncore_imc_events[] = {
2041 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x00,umask=0x00"), 2053 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0x00,umask=0x00"),
2042 INTEL_UNCORE_EVENT_DESC(cas_count_read, "event=0x04,umask=0x03"), 2054 INTEL_UNCORE_EVENT_DESC(cas_count_read, "event=0x04,umask=0x03"),
2043 INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"), 2055 INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"),
2044 INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"), 2056 INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"),
2045 INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"), 2057 INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
2046 INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"), 2058 INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"),
2047 INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"), 2059 INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"),
2048 { /* end: all zeroes */ }, 2060 { /* end: all zeroes */ },
2049 }; 2061 };
2050 2062
2051 static struct intel_uncore_type hswep_uncore_imc = { 2063 static struct intel_uncore_type hswep_uncore_imc = {
2052 .name = "imc", 2064 .name = "imc",
2053 .num_counters = 5, 2065 .num_counters = 5,
2054 .num_boxes = 8, 2066 .num_boxes = 8,
2055 .perf_ctr_bits = 48, 2067 .perf_ctr_bits = 48,
2056 .fixed_ctr_bits = 48, 2068 .fixed_ctr_bits = 48,
2057 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR, 2069 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
2058 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL, 2070 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
2059 .event_descs = hswep_uncore_imc_events, 2071 .event_descs = hswep_uncore_imc_events,
2060 SNBEP_UNCORE_PCI_COMMON_INIT(), 2072 SNBEP_UNCORE_PCI_COMMON_INIT(),
2061 }; 2073 };
2062 2074
2063 static unsigned hswep_uncore_irp_ctrs[] = {0xa0, 0xa8, 0xb0, 0xb8}; 2075 static unsigned hswep_uncore_irp_ctrs[] = {0xa0, 0xa8, 0xb0, 0xb8};
2064 2076
2065 static u64 hswep_uncore_irp_read_counter(struct intel_uncore_box *box, struct perf_event *event) 2077 static u64 hswep_uncore_irp_read_counter(struct intel_uncore_box *box, struct perf_event *event)
2066 { 2078 {
2067 struct pci_dev *pdev = box->pci_dev; 2079 struct pci_dev *pdev = box->pci_dev;
2068 struct hw_perf_event *hwc = &event->hw; 2080 struct hw_perf_event *hwc = &event->hw;
2069 u64 count = 0; 2081 u64 count = 0;
2070 2082
2071 pci_read_config_dword(pdev, hswep_uncore_irp_ctrs[hwc->idx], (u32 *)&count); 2083 pci_read_config_dword(pdev, hswep_uncore_irp_ctrs[hwc->idx], (u32 *)&count);
2072 pci_read_config_dword(pdev, hswep_uncore_irp_ctrs[hwc->idx] + 4, (u32 *)&count + 1); 2084 pci_read_config_dword(pdev, hswep_uncore_irp_ctrs[hwc->idx] + 4, (u32 *)&count + 1);
2073 2085
2074 return count; 2086 return count;
2075 } 2087 }
2076 2088
2077 static struct intel_uncore_ops hswep_uncore_irp_ops = { 2089 static struct intel_uncore_ops hswep_uncore_irp_ops = {
2078 .init_box = snbep_uncore_pci_init_box, 2090 .init_box = snbep_uncore_pci_init_box,
2079 .disable_box = snbep_uncore_pci_disable_box, 2091 .disable_box = snbep_uncore_pci_disable_box,
2080 .enable_box = snbep_uncore_pci_enable_box, 2092 .enable_box = snbep_uncore_pci_enable_box,
2081 .disable_event = ivbep_uncore_irp_disable_event, 2093 .disable_event = ivbep_uncore_irp_disable_event,
2082 .enable_event = ivbep_uncore_irp_enable_event, 2094 .enable_event = ivbep_uncore_irp_enable_event,
2083 .read_counter = hswep_uncore_irp_read_counter, 2095 .read_counter = hswep_uncore_irp_read_counter,
2084 }; 2096 };
2085 2097
2086 static struct intel_uncore_type hswep_uncore_irp = { 2098 static struct intel_uncore_type hswep_uncore_irp = {
2087 .name = "irp", 2099 .name = "irp",
2088 .num_counters = 4, 2100 .num_counters = 4,
2089 .num_boxes = 1, 2101 .num_boxes = 1,
2090 .perf_ctr_bits = 48, 2102 .perf_ctr_bits = 48,
2091 .event_mask = SNBEP_PMON_RAW_EVENT_MASK, 2103 .event_mask = SNBEP_PMON_RAW_EVENT_MASK,
2092 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, 2104 .box_ctl = SNBEP_PCI_PMON_BOX_CTL,
2093 .ops = &hswep_uncore_irp_ops, 2105 .ops = &hswep_uncore_irp_ops,
2094 .format_group = &snbep_uncore_format_group, 2106 .format_group = &snbep_uncore_format_group,
2095 }; 2107 };
2096 2108
2097 static struct intel_uncore_type hswep_uncore_qpi = { 2109 static struct intel_uncore_type hswep_uncore_qpi = {
2098 .name = "qpi", 2110 .name = "qpi",
2099 .num_counters = 5, 2111 .num_counters = 5,
2100 .num_boxes = 3, 2112 .num_boxes = 3,
2101 .perf_ctr_bits = 48, 2113 .perf_ctr_bits = 48,
2102 .perf_ctr = SNBEP_PCI_PMON_CTR0, 2114 .perf_ctr = SNBEP_PCI_PMON_CTR0,
2103 .event_ctl = SNBEP_PCI_PMON_CTL0, 2115 .event_ctl = SNBEP_PCI_PMON_CTL0,
2104 .event_mask = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK, 2116 .event_mask = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
2105 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, 2117 .box_ctl = SNBEP_PCI_PMON_BOX_CTL,
2106 .num_shared_regs = 1, 2118 .num_shared_regs = 1,
2107 .ops = &snbep_uncore_qpi_ops, 2119 .ops = &snbep_uncore_qpi_ops,
2108 .format_group = &snbep_uncore_qpi_format_group, 2120 .format_group = &snbep_uncore_qpi_format_group,
2109 }; 2121 };
2110 2122
2111 static struct event_constraint hswep_uncore_r2pcie_constraints[] = { 2123 static struct event_constraint hswep_uncore_r2pcie_constraints[] = {
2112 UNCORE_EVENT_CONSTRAINT(0x10, 0x3), 2124 UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
2113 UNCORE_EVENT_CONSTRAINT(0x11, 0x3), 2125 UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
2114 UNCORE_EVENT_CONSTRAINT(0x13, 0x1), 2126 UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
2115 UNCORE_EVENT_CONSTRAINT(0x23, 0x1), 2127 UNCORE_EVENT_CONSTRAINT(0x23, 0x1),
2116 UNCORE_EVENT_CONSTRAINT(0x24, 0x1), 2128 UNCORE_EVENT_CONSTRAINT(0x24, 0x1),
2117 UNCORE_EVENT_CONSTRAINT(0x25, 0x1), 2129 UNCORE_EVENT_CONSTRAINT(0x25, 0x1),
2118 UNCORE_EVENT_CONSTRAINT(0x26, 0x3), 2130 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
2119 UNCORE_EVENT_CONSTRAINT(0x27, 0x1), 2131 UNCORE_EVENT_CONSTRAINT(0x27, 0x1),
2120 UNCORE_EVENT_CONSTRAINT(0x28, 0x3), 2132 UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
2121 UNCORE_EVENT_CONSTRAINT(0x29, 0x3), 2133 UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
2122 UNCORE_EVENT_CONSTRAINT(0x2a, 0x1), 2134 UNCORE_EVENT_CONSTRAINT(0x2a, 0x1),
2123 UNCORE_EVENT_CONSTRAINT(0x2b, 0x3), 2135 UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
2124 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3), 2136 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
2125 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3), 2137 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
2126 UNCORE_EVENT_CONSTRAINT(0x32, 0x3), 2138 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
2127 UNCORE_EVENT_CONSTRAINT(0x33, 0x3), 2139 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
2128 UNCORE_EVENT_CONSTRAINT(0x34, 0x3), 2140 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
2129 UNCORE_EVENT_CONSTRAINT(0x35, 0x3), 2141 UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
2130 EVENT_CONSTRAINT_END 2142 EVENT_CONSTRAINT_END
2131 }; 2143 };
2132 2144
2133 static struct intel_uncore_type hswep_uncore_r2pcie = { 2145 static struct intel_uncore_type hswep_uncore_r2pcie = {
2134 .name = "r2pcie", 2146 .name = "r2pcie",
2135 .num_counters = 4, 2147 .num_counters = 4,
2136 .num_boxes = 1, 2148 .num_boxes = 1,
2137 .perf_ctr_bits = 48, 2149 .perf_ctr_bits = 48,
2138 .constraints = hswep_uncore_r2pcie_constraints, 2150 .constraints = hswep_uncore_r2pcie_constraints,
2139 SNBEP_UNCORE_PCI_COMMON_INIT(), 2151 SNBEP_UNCORE_PCI_COMMON_INIT(),
2140 }; 2152 };
2141 2153
2142 static struct event_constraint hswep_uncore_r3qpi_constraints[] = { 2154 static struct event_constraint hswep_uncore_r3qpi_constraints[] = {
2143 UNCORE_EVENT_CONSTRAINT(0x01, 0x3), 2155 UNCORE_EVENT_CONSTRAINT(0x01, 0x3),
2144 UNCORE_EVENT_CONSTRAINT(0x07, 0x7), 2156 UNCORE_EVENT_CONSTRAINT(0x07, 0x7),
2145 UNCORE_EVENT_CONSTRAINT(0x08, 0x7), 2157 UNCORE_EVENT_CONSTRAINT(0x08, 0x7),
2146 UNCORE_EVENT_CONSTRAINT(0x09, 0x7), 2158 UNCORE_EVENT_CONSTRAINT(0x09, 0x7),
2147 UNCORE_EVENT_CONSTRAINT(0x0a, 0x7), 2159 UNCORE_EVENT_CONSTRAINT(0x0a, 0x7),
2148 UNCORE_EVENT_CONSTRAINT(0x0e, 0x7), 2160 UNCORE_EVENT_CONSTRAINT(0x0e, 0x7),
2149 UNCORE_EVENT_CONSTRAINT(0x10, 0x3), 2161 UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
2150 UNCORE_EVENT_CONSTRAINT(0x11, 0x3), 2162 UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
2151 UNCORE_EVENT_CONSTRAINT(0x12, 0x3), 2163 UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
2152 UNCORE_EVENT_CONSTRAINT(0x13, 0x1), 2164 UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
2153 UNCORE_EVENT_CONSTRAINT(0x14, 0x3), 2165 UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
2154 UNCORE_EVENT_CONSTRAINT(0x15, 0x3), 2166 UNCORE_EVENT_CONSTRAINT(0x15, 0x3),
2155 UNCORE_EVENT_CONSTRAINT(0x1f, 0x3), 2167 UNCORE_EVENT_CONSTRAINT(0x1f, 0x3),
2156 UNCORE_EVENT_CONSTRAINT(0x20, 0x3), 2168 UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
2157 UNCORE_EVENT_CONSTRAINT(0x21, 0x3), 2169 UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
2158 UNCORE_EVENT_CONSTRAINT(0x22, 0x3), 2170 UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
2159 UNCORE_EVENT_CONSTRAINT(0x23, 0x3), 2171 UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
2160 UNCORE_EVENT_CONSTRAINT(0x25, 0x3), 2172 UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
2161 UNCORE_EVENT_CONSTRAINT(0x26, 0x3), 2173 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
2162 UNCORE_EVENT_CONSTRAINT(0x28, 0x3), 2174 UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
2163 UNCORE_EVENT_CONSTRAINT(0x29, 0x3), 2175 UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
2164 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3), 2176 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
2165 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3), 2177 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
2166 UNCORE_EVENT_CONSTRAINT(0x2e, 0x3), 2178 UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
2167 UNCORE_EVENT_CONSTRAINT(0x2f, 0x3), 2179 UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
2168 UNCORE_EVENT_CONSTRAINT(0x31, 0x3), 2180 UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
2169 UNCORE_EVENT_CONSTRAINT(0x32, 0x3), 2181 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
2170 UNCORE_EVENT_CONSTRAINT(0x33, 0x3), 2182 UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
2171 UNCORE_EVENT_CONSTRAINT(0x34, 0x3), 2183 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
2172 UNCORE_EVENT_CONSTRAINT(0x36, 0x3), 2184 UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
2173 UNCORE_EVENT_CONSTRAINT(0x37, 0x3), 2185 UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
2174 UNCORE_EVENT_CONSTRAINT(0x38, 0x3), 2186 UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
2175 UNCORE_EVENT_CONSTRAINT(0x39, 0x3), 2187 UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
2176 EVENT_CONSTRAINT_END 2188 EVENT_CONSTRAINT_END
2177 }; 2189 };
2178 2190
2179 static struct intel_uncore_type hswep_uncore_r3qpi = { 2191 static struct intel_uncore_type hswep_uncore_r3qpi = {
2180 .name = "r3qpi", 2192 .name = "r3qpi",
2181 .num_counters = 4, 2193 .num_counters = 4,
2182 .num_boxes = 3, 2194 .num_boxes = 3,
2183 .perf_ctr_bits = 44, 2195 .perf_ctr_bits = 44,
2184 .constraints = hswep_uncore_r3qpi_constraints, 2196 .constraints = hswep_uncore_r3qpi_constraints,
2185 SNBEP_UNCORE_PCI_COMMON_INIT(), 2197 SNBEP_UNCORE_PCI_COMMON_INIT(),
2186 }; 2198 };
2187 2199
2188 enum { 2200 enum {
2189 HSWEP_PCI_UNCORE_HA, 2201 HSWEP_PCI_UNCORE_HA,
2190 HSWEP_PCI_UNCORE_IMC, 2202 HSWEP_PCI_UNCORE_IMC,
2191 HSWEP_PCI_UNCORE_IRP, 2203 HSWEP_PCI_UNCORE_IRP,
2192 HSWEP_PCI_UNCORE_QPI, 2204 HSWEP_PCI_UNCORE_QPI,
2193 HSWEP_PCI_UNCORE_R2PCIE, 2205 HSWEP_PCI_UNCORE_R2PCIE,
2194 HSWEP_PCI_UNCORE_R3QPI, 2206 HSWEP_PCI_UNCORE_R3QPI,
2195 }; 2207 };
2196 2208
2197 static struct intel_uncore_type *hswep_pci_uncores[] = { 2209 static struct intel_uncore_type *hswep_pci_uncores[] = {
2198 [HSWEP_PCI_UNCORE_HA] = &hswep_uncore_ha, 2210 [HSWEP_PCI_UNCORE_HA] = &hswep_uncore_ha,
2199 [HSWEP_PCI_UNCORE_IMC] = &hswep_uncore_imc, 2211 [HSWEP_PCI_UNCORE_IMC] = &hswep_uncore_imc,
2200 [HSWEP_PCI_UNCORE_IRP] = &hswep_uncore_irp, 2212 [HSWEP_PCI_UNCORE_IRP] = &hswep_uncore_irp,
2201 [HSWEP_PCI_UNCORE_QPI] = &hswep_uncore_qpi, 2213 [HSWEP_PCI_UNCORE_QPI] = &hswep_uncore_qpi,
2202 [HSWEP_PCI_UNCORE_R2PCIE] = &hswep_uncore_r2pcie, 2214 [HSWEP_PCI_UNCORE_R2PCIE] = &hswep_uncore_r2pcie,
2203 [HSWEP_PCI_UNCORE_R3QPI] = &hswep_uncore_r3qpi, 2215 [HSWEP_PCI_UNCORE_R3QPI] = &hswep_uncore_r3qpi,
2204 NULL, 2216 NULL,
2205 }; 2217 };
2206 2218
2207 static DEFINE_PCI_DEVICE_TABLE(hswep_uncore_pci_ids) = { 2219 static DEFINE_PCI_DEVICE_TABLE(hswep_uncore_pci_ids) = {
2208 { /* Home Agent 0 */ 2220 { /* Home Agent 0 */
2209 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f30), 2221 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f30),
2210 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_HA, 0), 2222 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_HA, 0),
2211 }, 2223 },
2212 { /* Home Agent 1 */ 2224 { /* Home Agent 1 */
2213 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f38), 2225 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f38),
2214 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_HA, 1), 2226 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_HA, 1),
2215 }, 2227 },
2216 { /* MC0 Channel 0 */ 2228 { /* MC0 Channel 0 */
2217 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb0), 2229 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb0),
2218 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 0), 2230 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 0),
2219 }, 2231 },
2220 { /* MC0 Channel 1 */ 2232 { /* MC0 Channel 1 */
2221 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb1), 2233 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb1),
2222 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 1), 2234 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 1),
2223 }, 2235 },
2224 { /* MC0 Channel 2 */ 2236 { /* MC0 Channel 2 */
2225 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb4), 2237 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb4),
2226 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 2), 2238 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 2),
2227 }, 2239 },
2228 { /* MC0 Channel 3 */ 2240 { /* MC0 Channel 3 */
2229 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb5), 2241 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb5),
2230 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 3), 2242 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 3),
2231 }, 2243 },
2232 { /* MC1 Channel 0 */ 2244 { /* MC1 Channel 0 */
2233 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd0), 2245 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd0),
2234 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 4), 2246 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 4),
2235 }, 2247 },
2236 { /* MC1 Channel 1 */ 2248 { /* MC1 Channel 1 */
2237 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd1), 2249 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd1),
2238 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 5), 2250 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 5),
2239 }, 2251 },
2240 { /* MC1 Channel 2 */ 2252 { /* MC1 Channel 2 */
2241 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd4), 2253 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd4),
2242 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 6), 2254 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 6),
2243 }, 2255 },
2244 { /* MC1 Channel 3 */ 2256 { /* MC1 Channel 3 */
2245 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd5), 2257 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd5),
2246 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 7), 2258 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 7),
2247 }, 2259 },
2248 { /* IRP */ 2260 { /* IRP */
2249 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f39), 2261 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f39),
2250 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IRP, 0), 2262 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IRP, 0),
2251 }, 2263 },
2252 { /* QPI0 Port 0 */ 2264 { /* QPI0 Port 0 */
2253 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f32), 2265 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f32),
2254 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 0), 2266 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 0),
2255 }, 2267 },
2256 { /* QPI0 Port 1 */ 2268 { /* QPI0 Port 1 */
2257 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f33), 2269 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f33),
2258 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 1), 2270 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 1),
2259 }, 2271 },
2260 { /* QPI1 Port 2 */ 2272 { /* QPI1 Port 2 */
2261 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f3a), 2273 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f3a),
2262 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 2), 2274 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 2),
2263 }, 2275 },
2264 { /* R2PCIe */ 2276 { /* R2PCIe */
2265 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f34), 2277 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f34),
2266 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R2PCIE, 0), 2278 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R2PCIE, 0),
2267 }, 2279 },
2268 { /* R3QPI0 Link 0 */ 2280 { /* R3QPI0 Link 0 */
2269 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f36), 2281 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f36),
2270 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 0), 2282 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 0),
2271 }, 2283 },
2272 { /* R3QPI0 Link 1 */ 2284 { /* R3QPI0 Link 1 */
2273 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f37), 2285 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f37),
2274 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 1), 2286 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 1),
2275 }, 2287 },
2276 { /* R3QPI1 Link 2 */ 2288 { /* R3QPI1 Link 2 */
2277 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f3e), 2289 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f3e),
2278 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 2), 2290 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 2),
2279 }, 2291 },
2280 { /* QPI Port 0 filter */ 2292 { /* QPI Port 0 filter */
2281 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f86), 2293 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f86),
2282 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 2294 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
2283 SNBEP_PCI_QPI_PORT0_FILTER), 2295 SNBEP_PCI_QPI_PORT0_FILTER),
2284 }, 2296 },
2285 { /* QPI Port 1 filter */ 2297 { /* QPI Port 1 filter */
2286 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f96), 2298 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f96),
2287 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV, 2299 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
2288 SNBEP_PCI_QPI_PORT1_FILTER), 2300 SNBEP_PCI_QPI_PORT1_FILTER),
2301 },
2302 { /* PCU.3 (for Capability registers) */
2303 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fc0),
2304 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
2305 HSWEP_PCI_PCU_3),
2289 }, 2306 },
2290 { /* end: all zeroes */ } 2307 { /* end: all zeroes */ }
2291 }; 2308 };
2292 2309
2293 static struct pci_driver hswep_uncore_pci_driver = { 2310 static struct pci_driver hswep_uncore_pci_driver = {
2294 .name = "hswep_uncore", 2311 .name = "hswep_uncore",
2295 .id_table = hswep_uncore_pci_ids, 2312 .id_table = hswep_uncore_pci_ids,
2296 }; 2313 };
2297 2314
2298 int hswep_uncore_pci_init(void) 2315 int hswep_uncore_pci_init(void)
2299 { 2316 {
2300 int ret = snbep_pci2phy_map_init(0x2f1e); 2317 int ret = snbep_pci2phy_map_init(0x2f1e);
2301 if (ret) 2318 if (ret)
2302 return ret; 2319 return ret;
2303 uncore_pci_uncores = hswep_pci_uncores; 2320 uncore_pci_uncores = hswep_pci_uncores;
2304 uncore_pci_driver = &hswep_uncore_pci_driver; 2321 uncore_pci_driver = &hswep_uncore_pci_driver;
2305 return 0; 2322 return 0;
2306 } 2323 }
2307 /* end of Haswell-EP uncore support */ 2324 /* end of Haswell-EP uncore support */
2308 2325
arch/x86/kernel/perf_regs.c
Diff comments   View file @ ddb321a
1 #include <linux/errno.h> 1 #include <linux/errno.h>
2 #include <linux/kernel.h> 2 #include <linux/kernel.h>
3 #include <linux/sched.h> 3 #include <linux/sched.h>
4 #include <linux/perf_event.h> 4 #include <linux/perf_event.h>
5 #include <linux/bug.h> 5 #include <linux/bug.h>
6 #include <linux/stddef.h> 6 #include <linux/stddef.h>
7 #include <asm/perf_regs.h> 7 #include <asm/perf_regs.h>
8 #include <asm/ptrace.h> 8 #include <asm/ptrace.h>
9 9
10 #ifdef CONFIG_X86_32 10 #ifdef CONFIG_X86_32
11 #define PERF_REG_X86_MAX PERF_REG_X86_32_MAX 11 #define PERF_REG_X86_MAX PERF_REG_X86_32_MAX
12 #else 12 #else
13 #define PERF_REG_X86_MAX PERF_REG_X86_64_MAX 13 #define PERF_REG_X86_MAX PERF_REG_X86_64_MAX
14 #endif 14 #endif
15 15
16 #define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r) 16 #define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r)
17 17
18 static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = { 18 static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = {
19 PT_REGS_OFFSET(PERF_REG_X86_AX, ax), 19 PT_REGS_OFFSET(PERF_REG_X86_AX, ax),
20 PT_REGS_OFFSET(PERF_REG_X86_BX, bx), 20 PT_REGS_OFFSET(PERF_REG_X86_BX, bx),
21 PT_REGS_OFFSET(PERF_REG_X86_CX, cx), 21 PT_REGS_OFFSET(PERF_REG_X86_CX, cx),
22 PT_REGS_OFFSET(PERF_REG_X86_DX, dx), 22 PT_REGS_OFFSET(PERF_REG_X86_DX, dx),
23 PT_REGS_OFFSET(PERF_REG_X86_SI, si), 23 PT_REGS_OFFSET(PERF_REG_X86_SI, si),
24 PT_REGS_OFFSET(PERF_REG_X86_DI, di), 24 PT_REGS_OFFSET(PERF_REG_X86_DI, di),
25 PT_REGS_OFFSET(PERF_REG_X86_BP, bp), 25 PT_REGS_OFFSET(PERF_REG_X86_BP, bp),
26 PT_REGS_OFFSET(PERF_REG_X86_SP, sp), 26 PT_REGS_OFFSET(PERF_REG_X86_SP, sp),
27 PT_REGS_OFFSET(PERF_REG_X86_IP, ip), 27 PT_REGS_OFFSET(PERF_REG_X86_IP, ip),
28 PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags), 28 PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags),
29 PT_REGS_OFFSET(PERF_REG_X86_CS, cs), 29 PT_REGS_OFFSET(PERF_REG_X86_CS, cs),
30 PT_REGS_OFFSET(PERF_REG_X86_SS, ss), 30 PT_REGS_OFFSET(PERF_REG_X86_SS, ss),
31 #ifdef CONFIG_X86_32 31 #ifdef CONFIG_X86_32
32 PT_REGS_OFFSET(PERF_REG_X86_DS, ds), 32 PT_REGS_OFFSET(PERF_REG_X86_DS, ds),
33 PT_REGS_OFFSET(PERF_REG_X86_ES, es), 33 PT_REGS_OFFSET(PERF_REG_X86_ES, es),
34 PT_REGS_OFFSET(PERF_REG_X86_FS, fs), 34 PT_REGS_OFFSET(PERF_REG_X86_FS, fs),
35 PT_REGS_OFFSET(PERF_REG_X86_GS, gs), 35 PT_REGS_OFFSET(PERF_REG_X86_GS, gs),
36 #else 36 #else
37 /* 37 /*
38 * The pt_regs struct does not store 38 * The pt_regs struct does not store
39 * ds, es, fs, gs in 64 bit mode. 39 * ds, es, fs, gs in 64 bit mode.
40 */ 40 */
41 (unsigned int) -1, 41 (unsigned int) -1,
42 (unsigned int) -1, 42 (unsigned int) -1,
43 (unsigned int) -1, 43 (unsigned int) -1,
44 (unsigned int) -1, 44 (unsigned int) -1,
45 #endif 45 #endif
46 #ifdef CONFIG_X86_64 46 #ifdef CONFIG_X86_64
47 PT_REGS_OFFSET(PERF_REG_X86_R8, r8), 47 PT_REGS_OFFSET(PERF_REG_X86_R8, r8),
48 PT_REGS_OFFSET(PERF_REG_X86_R9, r9), 48 PT_REGS_OFFSET(PERF_REG_X86_R9, r9),
49 PT_REGS_OFFSET(PERF_REG_X86_R10, r10), 49 PT_REGS_OFFSET(PERF_REG_X86_R10, r10),
50 PT_REGS_OFFSET(PERF_REG_X86_R11, r11), 50 PT_REGS_OFFSET(PERF_REG_X86_R11, r11),
51 PT_REGS_OFFSET(PERF_REG_X86_R12, r12), 51 PT_REGS_OFFSET(PERF_REG_X86_R12, r12),
52 PT_REGS_OFFSET(PERF_REG_X86_R13, r13), 52 PT_REGS_OFFSET(PERF_REG_X86_R13, r13),
53 PT_REGS_OFFSET(PERF_REG_X86_R14, r14), 53 PT_REGS_OFFSET(PERF_REG_X86_R14, r14),
54 PT_REGS_OFFSET(PERF_REG_X86_R15, r15), 54 PT_REGS_OFFSET(PERF_REG_X86_R15, r15),
55 #endif 55 #endif
56 }; 56 };
57 57
58 u64 perf_reg_value(struct pt_regs *regs, int idx) 58 u64 perf_reg_value(struct pt_regs *regs, int idx)
59 { 59 {
60 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset))) 60 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
61 return 0; 61 return 0;
62 62
63 return regs_get_register(regs, pt_regs_offset[idx]); 63 return regs_get_register(regs, pt_regs_offset[idx]);
64 } 64 }
65 65
66 #define REG_RESERVED (~((1ULL << PERF_REG_X86_MAX) - 1ULL)) 66 #define REG_RESERVED (~((1ULL << PERF_REG_X86_MAX) - 1ULL))
67 67
68 #ifdef CONFIG_X86_32 68 #ifdef CONFIG_X86_32
69 int perf_reg_validate(u64 mask) 69 int perf_reg_validate(u64 mask)
70 { 70 {
71 if (!mask || mask & REG_RESERVED) 71 if (!mask || mask & REG_RESERVED)
72 return -EINVAL; 72 return -EINVAL;
73 73
74 return 0; 74 return 0;
75 } 75 }
76 76
77 u64 perf_reg_abi(struct task_struct *task) 77 u64 perf_reg_abi(struct task_struct *task)
78 { 78 {
79 return PERF_SAMPLE_REGS_ABI_32; 79 return PERF_SAMPLE_REGS_ABI_32;
80 } 80 }
81
82 void perf_get_regs_user(struct perf_regs *regs_user,
83 struct pt_regs *regs,
84 struct pt_regs *regs_user_copy)
85 {
86 regs_user->regs = task_pt_regs(current);
87 regs_user->abi = perf_reg_abi(current);
88 }
81 #else /* CONFIG_X86_64 */ 89 #else /* CONFIG_X86_64 */
82 #define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \ 90 #define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \
83 (1ULL << PERF_REG_X86_ES) | \ 91 (1ULL << PERF_REG_X86_ES) | \
84 (1ULL << PERF_REG_X86_FS) | \ 92 (1ULL << PERF_REG_X86_FS) | \
85 (1ULL << PERF_REG_X86_GS)) 93 (1ULL << PERF_REG_X86_GS))
86 94
87 int perf_reg_validate(u64 mask) 95 int perf_reg_validate(u64 mask)
88 { 96 {
89 if (!mask || mask & REG_RESERVED) 97 if (!mask || mask & REG_RESERVED)
90 return -EINVAL; 98 return -EINVAL;
91 99
92 if (mask & REG_NOSUPPORT) 100 if (mask & REG_NOSUPPORT)
93 return -EINVAL; 101 return -EINVAL;
94 102
95 return 0; 103 return 0;
96 } 104 }
97 105
98 u64 perf_reg_abi(struct task_struct *task) 106 u64 perf_reg_abi(struct task_struct *task)
99 { 107 {
100 if (test_tsk_thread_flag(task, TIF_IA32)) 108 if (test_tsk_thread_flag(task, TIF_IA32))
101 return PERF_SAMPLE_REGS_ABI_32; 109 return PERF_SAMPLE_REGS_ABI_32;
102 else 110 else
103 return PERF_SAMPLE_REGS_ABI_64; 111 return PERF_SAMPLE_REGS_ABI_64;
112 }
113
114 void perf_get_regs_user(struct perf_regs *regs_user,
115 struct pt_regs *regs,
116 struct pt_regs *regs_user_copy)
117 {
118 struct pt_regs *user_regs = task_pt_regs(current);
119
120 /*
121 * If we're in an NMI that interrupted task_pt_regs setup, then
122 * we can't sample user regs at all. This check isn't really
123 * sufficient, though, as we could be in an NMI inside an interrupt
124 * that happened during task_pt_regs setup.
125 */
126 if (regs->sp > (unsigned long)&user_regs->r11 &&
127 regs->sp <= (unsigned long)(user_regs + 1)) {
128 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
129 regs_user->regs = NULL;
130 return;
131 }
132
133 /*
134 * RIP, flags, and the argument registers are usually saved.
135 * orig_ax is probably okay, too.
136 */
137 regs_user_copy->ip = user_regs->ip;
138 regs_user_copy->cx = user_regs->cx;
139 regs_user_copy->dx = user_regs->dx;
140 regs_user_copy->si = user_regs->si;
141 regs_user_copy->di = user_regs->di;
142 regs_user_copy->r8 = user_regs->r8;
143 regs_user_copy->r9 = user_regs->r9;
144 regs_user_copy->r10 = user_regs->r10;
145 regs_user_copy->r11 = user_regs->r11;
146 regs_user_copy->orig_ax = user_regs->orig_ax;
147 regs_user_copy->flags = user_regs->flags;
148
149 /*
150 * Don't even try to report the "rest" regs.
151 */
152 regs_user_copy->bx = -1;
153 regs_user_copy->bp = -1;
154 regs_user_copy->r12 = -1;
155 regs_user_copy->r13 = -1;
156 regs_user_copy->r14 = -1;
157 regs_user_copy->r15 = -1;
158
159 /*
160 * For this to be at all useful, we need a reasonable guess for
161 * sp and the ABI. Be careful: we're in NMI context, and we're
162 * considering current to be the current task, so we should
163 * be careful not to look at any other percpu variables that might
164 * change during context switches.
165 */
166 if (IS_ENABLED(CONFIG_IA32_EMULATION) &&
167 task_thread_info(current)->status & TS_COMPAT) {
168 /* Easy case: we're in a compat syscall. */
169 regs_user->abi = PERF_SAMPLE_REGS_ABI_32;
170 regs_user_copy->sp = user_regs->sp;
171 regs_user_copy->cs = user_regs->cs;
172 regs_user_copy->ss = user_regs->ss;
173 } else if (user_regs->orig_ax != -1) {
174 /*
175 * We're probably in a 64-bit syscall.
176 * Warning: this code is severely racy. At least it's better
177 * than just blindly copying user_regs.
178 */
179 regs_user->abi = PERF_SAMPLE_REGS_ABI_64;
180 regs_user_copy->sp = this_cpu_read(old_rsp);
181 regs_user_copy->cs = __USER_CS;
182 regs_user_copy->ss = __USER_DS;
183 regs_user_copy->cx = -1; /* usually contains garbage */
184 } else {
185 /* We're probably in an interrupt or exception. */
186 regs_user->abi = user_64bit_mode(user_regs) ?
187 PERF_SAMPLE_REGS_ABI_64 : PERF_SAMPLE_REGS_ABI_32;
188 regs_user_copy->sp = user_regs->sp;
189 regs_user_copy->cs = user_regs->cs;
190 regs_user_copy->ss = user_regs->ss;
191 }
192
193 regs_user->regs = regs_user_copy;
104 } 194 }
105 #endif /* CONFIG_X86_32 */ 195 #endif /* CONFIG_X86_32 */
106 196
1 /* 1 /*
2 * x86 instruction analysis 2 * x86 instruction analysis
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by 5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or 6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version. 7 * (at your option) any later version.
8 * 8 *
9 * This program is distributed in the hope that it will be useful, 9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 * 13 *
14 * You should have received a copy of the GNU General Public License 14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software 15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 * 17 *
18 * Copyright (C) IBM Corporation, 2002, 2004, 2009 18 * Copyright (C) IBM Corporation, 2002, 2004, 2009
19 */ 19 */
20 20
21 #ifdef __KERNEL__ 21 #ifdef __KERNEL__
22 #include <linux/string.h> 22 #include <linux/string.h>
23 #else 23 #else
24 #include <string.h> 24 #include <string.h>
25 #endif 25 #endif
26 #include <asm/inat.h> 26 #include <asm/inat.h>
27 #include <asm/insn.h> 27 #include <asm/insn.h>
28 28
29 /* Verify next sizeof(t) bytes can be on the same instruction */ 29 /* Verify next sizeof(t) bytes can be on the same instruction */
30 #define validate_next(t, insn, n) \ 30 #define validate_next(t, insn, n) \
31 ((insn)->next_byte + sizeof(t) + n < (insn)->end_kaddr) 31 ((insn)->next_byte + sizeof(t) + n <= (insn)->end_kaddr)
32 32
33 #define __get_next(t, insn) \ 33 #define __get_next(t, insn) \
34 ({ t r = *(t*)insn->next_byte; insn->next_byte += sizeof(t); r; }) 34 ({ t r = *(t*)insn->next_byte; insn->next_byte += sizeof(t); r; })
35 35
36 #define __peek_nbyte_next(t, insn, n) \ 36 #define __peek_nbyte_next(t, insn, n) \
37 ({ t r = *(t*)((insn)->next_byte + n); r; }) 37 ({ t r = *(t*)((insn)->next_byte + n); r; })
38 38
39 #define get_next(t, insn) \ 39 #define get_next(t, insn) \
40 ({ if (unlikely(!validate_next(t, insn, 0))) goto err_out; __get_next(t, insn); }) 40 ({ if (unlikely(!validate_next(t, insn, 0))) goto err_out; __get_next(t, insn); })
41 41
42 #define peek_nbyte_next(t, insn, n) \ 42 #define peek_nbyte_next(t, insn, n) \
43 ({ if (unlikely(!validate_next(t, insn, n))) goto err_out; __peek_nbyte_next(t, insn, n); }) 43 ({ if (unlikely(!validate_next(t, insn, n))) goto err_out; __peek_nbyte_next(t, insn, n); })
44 44
45 #define peek_next(t, insn) peek_nbyte_next(t, insn, 0) 45 #define peek_next(t, insn) peek_nbyte_next(t, insn, 0)
46 46
47 /** 47 /**
48 * insn_init() - initialize struct insn 48 * insn_init() - initialize struct insn
49 * @insn: &struct insn to be initialized 49 * @insn: &struct insn to be initialized
50 * @kaddr: address (in kernel memory) of instruction (or copy thereof) 50 * @kaddr: address (in kernel memory) of instruction (or copy thereof)
51 * @x86_64: !0 for 64-bit kernel or 64-bit app 51 * @x86_64: !0 for 64-bit kernel or 64-bit app
52 */ 52 */
53 void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64) 53 void insn_init(struct insn *insn, const void *kaddr, int buf_len, int x86_64)
54 { 54 {
55 memset(insn, 0, sizeof(*insn)); 55 memset(insn, 0, sizeof(*insn));
56 insn->kaddr = kaddr; 56 insn->kaddr = kaddr;
57 insn->end_kaddr = kaddr + buf_len; 57 insn->end_kaddr = kaddr + buf_len;
58 insn->next_byte = kaddr; 58 insn->next_byte = kaddr;
59 insn->x86_64 = x86_64 ? 1 : 0; 59 insn->x86_64 = x86_64 ? 1 : 0;
60 insn->opnd_bytes = 4; 60 insn->opnd_bytes = 4;
61 if (x86_64) 61 if (x86_64)
62 insn->addr_bytes = 8; 62 insn->addr_bytes = 8;
63 else 63 else
64 insn->addr_bytes = 4; 64 insn->addr_bytes = 4;
65 } 65 }
66 66
67 /** 67 /**
68 * insn_get_prefixes - scan x86 instruction prefix bytes 68 * insn_get_prefixes - scan x86 instruction prefix bytes
69 * @insn: &struct insn containing instruction 69 * @insn: &struct insn containing instruction
70 * 70 *
71 * Populates the @insn->prefixes bitmap, and updates @insn->next_byte 71 * Populates the @insn->prefixes bitmap, and updates @insn->next_byte
72 * to point to the (first) opcode. No effect if @insn->prefixes.got 72 * to point to the (first) opcode. No effect if @insn->prefixes.got
73 * is already set. 73 * is already set.
74 */ 74 */
75 void insn_get_prefixes(struct insn *insn) 75 void insn_get_prefixes(struct insn *insn)
76 { 76 {
77 struct insn_field *prefixes = &insn->prefixes; 77 struct insn_field *prefixes = &insn->prefixes;
78 insn_attr_t attr; 78 insn_attr_t attr;
79 insn_byte_t b, lb; 79 insn_byte_t b, lb;
80 int i, nb; 80 int i, nb;
81 81
82 if (prefixes->got) 82 if (prefixes->got)
83 return; 83 return;
84 84
85 nb = 0; 85 nb = 0;
86 lb = 0; 86 lb = 0;
87 b = peek_next(insn_byte_t, insn); 87 b = peek_next(insn_byte_t, insn);
88 attr = inat_get_opcode_attribute(b); 88 attr = inat_get_opcode_attribute(b);
89 while (inat_is_legacy_prefix(attr)) { 89 while (inat_is_legacy_prefix(attr)) {
90 /* Skip if same prefix */ 90 /* Skip if same prefix */
91 for (i = 0; i < nb; i++) 91 for (i = 0; i < nb; i++)
92 if (prefixes->bytes[i] == b) 92 if (prefixes->bytes[i] == b)
93 goto found; 93 goto found;
94 if (nb == 4) 94 if (nb == 4)
95 /* Invalid instruction */ 95 /* Invalid instruction */
96 break; 96 break;
97 prefixes->bytes[nb++] = b; 97 prefixes->bytes[nb++] = b;
98 if (inat_is_address_size_prefix(attr)) { 98 if (inat_is_address_size_prefix(attr)) {
99 /* address size switches 2/4 or 4/8 */ 99 /* address size switches 2/4 or 4/8 */
100 if (insn->x86_64) 100 if (insn->x86_64)
101 insn->addr_bytes ^= 12; 101 insn->addr_bytes ^= 12;
102 else 102 else
103 insn->addr_bytes ^= 6; 103 insn->addr_bytes ^= 6;
104 } else if (inat_is_operand_size_prefix(attr)) { 104 } else if (inat_is_operand_size_prefix(attr)) {
105 /* oprand size switches 2/4 */ 105 /* oprand size switches 2/4 */
106 insn->opnd_bytes ^= 6; 106 insn->opnd_bytes ^= 6;
107 } 107 }
108 found: 108 found:
109 prefixes->nbytes++; 109 prefixes->nbytes++;
110 insn->next_byte++; 110 insn->next_byte++;
111 lb = b; 111 lb = b;
112 b = peek_next(insn_byte_t, insn); 112 b = peek_next(insn_byte_t, insn);
113 attr = inat_get_opcode_attribute(b); 113 attr = inat_get_opcode_attribute(b);
114 } 114 }
115 /* Set the last prefix */ 115 /* Set the last prefix */
116 if (lb && lb != insn->prefixes.bytes[3]) { 116 if (lb && lb != insn->prefixes.bytes[3]) {
117 if (unlikely(insn->prefixes.bytes[3])) { 117 if (unlikely(insn->prefixes.bytes[3])) {
118 /* Swap the last prefix */ 118 /* Swap the last prefix */
119 b = insn->prefixes.bytes[3]; 119 b = insn->prefixes.bytes[3];
120 for (i = 0; i < nb; i++) 120 for (i = 0; i < nb; i++)
121 if (prefixes->bytes[i] == lb) 121 if (prefixes->bytes[i] == lb)
122 prefixes->bytes[i] = b; 122 prefixes->bytes[i] = b;
123 } 123 }
124 insn->prefixes.bytes[3] = lb; 124 insn->prefixes.bytes[3] = lb;
125 } 125 }
126 126
127 /* Decode REX prefix */ 127 /* Decode REX prefix */
128 if (insn->x86_64) { 128 if (insn->x86_64) {
129 b = peek_next(insn_byte_t, insn); 129 b = peek_next(insn_byte_t, insn);
130 attr = inat_get_opcode_attribute(b); 130 attr = inat_get_opcode_attribute(b);
131 if (inat_is_rex_prefix(attr)) { 131 if (inat_is_rex_prefix(attr)) {
132 insn->rex_prefix.value = b; 132 insn->rex_prefix.value = b;
133 insn->rex_prefix.nbytes = 1; 133 insn->rex_prefix.nbytes = 1;
134 insn->next_byte++; 134 insn->next_byte++;
135 if (X86_REX_W(b)) 135 if (X86_REX_W(b))
136 /* REX.W overrides opnd_size */ 136 /* REX.W overrides opnd_size */
137 insn->opnd_bytes = 8; 137 insn->opnd_bytes = 8;
138 } 138 }
139 } 139 }
140 insn->rex_prefix.got = 1; 140 insn->rex_prefix.got = 1;
141 141
142 /* Decode VEX prefix */ 142 /* Decode VEX prefix */
143 b = peek_next(insn_byte_t, insn); 143 b = peek_next(insn_byte_t, insn);
144 attr = inat_get_opcode_attribute(b); 144 attr = inat_get_opcode_attribute(b);
145 if (inat_is_vex_prefix(attr)) { 145 if (inat_is_vex_prefix(attr)) {
146 insn_byte_t b2 = peek_nbyte_next(insn_byte_t, insn, 1); 146 insn_byte_t b2 = peek_nbyte_next(insn_byte_t, insn, 1);
147 if (!insn->x86_64) { 147 if (!insn->x86_64) {
148 /* 148 /*
149 * In 32-bits mode, if the [7:6] bits (mod bits of 149 * In 32-bits mode, if the [7:6] bits (mod bits of
150 * ModRM) on the second byte are not 11b, it is 150 * ModRM) on the second byte are not 11b, it is
151 * LDS or LES. 151 * LDS or LES.
152 */ 152 */
153 if (X86_MODRM_MOD(b2) != 3) 153 if (X86_MODRM_MOD(b2) != 3)
154 goto vex_end; 154 goto vex_end;
155 } 155 }
156 insn->vex_prefix.bytes[0] = b; 156 insn->vex_prefix.bytes[0] = b;
157 insn->vex_prefix.bytes[1] = b2; 157 insn->vex_prefix.bytes[1] = b2;
158 if (inat_is_vex3_prefix(attr)) { 158 if (inat_is_vex3_prefix(attr)) {
159 b2 = peek_nbyte_next(insn_byte_t, insn, 2); 159 b2 = peek_nbyte_next(insn_byte_t, insn, 2);
160 insn->vex_prefix.bytes[2] = b2; 160 insn->vex_prefix.bytes[2] = b2;
161 insn->vex_prefix.nbytes = 3; 161 insn->vex_prefix.nbytes = 3;
162 insn->next_byte += 3; 162 insn->next_byte += 3;
163 if (insn->x86_64 && X86_VEX_W(b2)) 163 if (insn->x86_64 && X86_VEX_W(b2))
164 /* VEX.W overrides opnd_size */ 164 /* VEX.W overrides opnd_size */
165 insn->opnd_bytes = 8; 165 insn->opnd_bytes = 8;
166 } else { 166 } else {
167 insn->vex_prefix.nbytes = 2; 167 insn->vex_prefix.nbytes = 2;
168 insn->next_byte += 2; 168 insn->next_byte += 2;
169 } 169 }
170 } 170 }
171 vex_end: 171 vex_end:
172 insn->vex_prefix.got = 1; 172 insn->vex_prefix.got = 1;
173 173
174 prefixes->got = 1; 174 prefixes->got = 1;
175 175
176 err_out: 176 err_out:
177 return; 177 return;
178 } 178 }
179 179
180 /** 180 /**
181 * insn_get_opcode - collect opcode(s) 181 * insn_get_opcode - collect opcode(s)
182 * @insn: &struct insn containing instruction 182 * @insn: &struct insn containing instruction
183 * 183 *
184 * Populates @insn->opcode, updates @insn->next_byte to point past the 184 * Populates @insn->opcode, updates @insn->next_byte to point past the
185 * opcode byte(s), and set @insn->attr (except for groups). 185 * opcode byte(s), and set @insn->attr (except for groups).
186 * If necessary, first collects any preceding (prefix) bytes. 186 * If necessary, first collects any preceding (prefix) bytes.
187 * Sets @insn->opcode.value = opcode1. No effect if @insn->opcode.got 187 * Sets @insn->opcode.value = opcode1. No effect if @insn->opcode.got
188 * is already 1. 188 * is already 1.
189 */ 189 */
190 void insn_get_opcode(struct insn *insn) 190 void insn_get_opcode(struct insn *insn)
191 { 191 {
192 struct insn_field *opcode = &insn->opcode; 192 struct insn_field *opcode = &insn->opcode;
193 insn_byte_t op; 193 insn_byte_t op;
194 int pfx_id; 194 int pfx_id;
195 if (opcode->got) 195 if (opcode->got)
196 return; 196 return;
197 if (!insn->prefixes.got) 197 if (!insn->prefixes.got)
198 insn_get_prefixes(insn); 198 insn_get_prefixes(insn);
199 199
200 /* Get first opcode */ 200 /* Get first opcode */
201 op = get_next(insn_byte_t, insn); 201 op = get_next(insn_byte_t, insn);
202 opcode->bytes[0] = op; 202 opcode->bytes[0] = op;
203 opcode->nbytes = 1; 203 opcode->nbytes = 1;
204 204
205 /* Check if there is VEX prefix or not */ 205 /* Check if there is VEX prefix or not */
206 if (insn_is_avx(insn)) { 206 if (insn_is_avx(insn)) {
207 insn_byte_t m, p; 207 insn_byte_t m, p;
208 m = insn_vex_m_bits(insn); 208 m = insn_vex_m_bits(insn);
209 p = insn_vex_p_bits(insn); 209 p = insn_vex_p_bits(insn);
210 insn->attr = inat_get_avx_attribute(op, m, p); 210 insn->attr = inat_get_avx_attribute(op, m, p);
211 if (!inat_accept_vex(insn->attr) && !inat_is_group(insn->attr)) 211 if (!inat_accept_vex(insn->attr) && !inat_is_group(insn->attr))
212 insn->attr = 0; /* This instruction is bad */ 212 insn->attr = 0; /* This instruction is bad */
213 goto end; /* VEX has only 1 byte for opcode */ 213 goto end; /* VEX has only 1 byte for opcode */
214 } 214 }
215 215
216 insn->attr = inat_get_opcode_attribute(op); 216 insn->attr = inat_get_opcode_attribute(op);
217 while (inat_is_escape(insn->attr)) { 217 while (inat_is_escape(insn->attr)) {
218 /* Get escaped opcode */ 218 /* Get escaped opcode */
219 op = get_next(insn_byte_t, insn); 219 op = get_next(insn_byte_t, insn);
220 opcode->bytes[opcode->nbytes++] = op; 220 opcode->bytes[opcode->nbytes++] = op;
221 pfx_id = insn_last_prefix_id(insn); 221 pfx_id = insn_last_prefix_id(insn);
222 insn->attr = inat_get_escape_attribute(op, pfx_id, insn->attr); 222 insn->attr = inat_get_escape_attribute(op, pfx_id, insn->attr);
223 } 223 }
224 if (inat_must_vex(insn->attr)) 224 if (inat_must_vex(insn->attr))
225 insn->attr = 0; /* This instruction is bad */ 225 insn->attr = 0; /* This instruction is bad */
226 end: 226 end:
227 opcode->got = 1; 227 opcode->got = 1;
228 228
229 err_out: 229 err_out:
230 return; 230 return;
231 } 231 }
232 232
233 /** 233 /**
234 * insn_get_modrm - collect ModRM byte, if any 234 * insn_get_modrm - collect ModRM byte, if any
235 * @insn: &struct insn containing instruction 235 * @insn: &struct insn containing instruction
236 * 236 *
237 * Populates @insn->modrm and updates @insn->next_byte to point past the 237 * Populates @insn->modrm and updates @insn->next_byte to point past the
238 * ModRM byte, if any. If necessary, first collects the preceding bytes 238 * ModRM byte, if any. If necessary, first collects the preceding bytes
239 * (prefixes and opcode(s)). No effect if @insn->modrm.got is already 1. 239 * (prefixes and opcode(s)). No effect if @insn->modrm.got is already 1.
240 */ 240 */
241 void insn_get_modrm(struct insn *insn) 241 void insn_get_modrm(struct insn *insn)
242 { 242 {
243 struct insn_field *modrm = &insn->modrm; 243 struct insn_field *modrm = &insn->modrm;
244 insn_byte_t pfx_id, mod; 244 insn_byte_t pfx_id, mod;
245 if (modrm->got) 245 if (modrm->got)
246 return; 246 return;
247 if (!insn->opcode.got) 247 if (!insn->opcode.got)
248 insn_get_opcode(insn); 248 insn_get_opcode(insn);
249 249
250 if (inat_has_modrm(insn->attr)) { 250 if (inat_has_modrm(insn->attr)) {
251 mod = get_next(insn_byte_t, insn); 251 mod = get_next(insn_byte_t, insn);
252 modrm->value = mod; 252 modrm->value = mod;
253 modrm->nbytes = 1; 253 modrm->nbytes = 1;
254 if (inat_is_group(insn->attr)) { 254 if (inat_is_group(insn->attr)) {
255 pfx_id = insn_last_prefix_id(insn); 255 pfx_id = insn_last_prefix_id(insn);
256 insn->attr = inat_get_group_attribute(mod, pfx_id, 256 insn->attr = inat_get_group_attribute(mod, pfx_id,
257 insn->attr); 257 insn->attr);
258 if (insn_is_avx(insn) && !inat_accept_vex(insn->attr)) 258 if (insn_is_avx(insn) && !inat_accept_vex(insn->attr))
259 insn->attr = 0; /* This is bad */ 259 insn->attr = 0; /* This is bad */
260 } 260 }
261 } 261 }
262 262
263 if (insn->x86_64 && inat_is_force64(insn->attr)) 263 if (insn->x86_64 && inat_is_force64(insn->attr))
264 insn->opnd_bytes = 8; 264 insn->opnd_bytes = 8;
265 modrm->got = 1; 265 modrm->got = 1;
266 266
267 err_out: 267 err_out:
268 return; 268 return;
269 } 269 }
270 270
271 271
272 /** 272 /**
273 * insn_rip_relative() - Does instruction use RIP-relative addressing mode? 273 * insn_rip_relative() - Does instruction use RIP-relative addressing mode?
274 * @insn: &struct insn containing instruction 274 * @insn: &struct insn containing instruction
275 * 275 *
276 * If necessary, first collects the instruction up to and including the 276 * If necessary, first collects the instruction up to and including the
277 * ModRM byte. No effect if @insn->x86_64 is 0. 277 * ModRM byte. No effect if @insn->x86_64 is 0.
278 */ 278 */
279 int insn_rip_relative(struct insn *insn) 279 int insn_rip_relative(struct insn *insn)
280 { 280 {
281 struct insn_field *modrm = &insn->modrm; 281 struct insn_field *modrm = &insn->modrm;
282 282
283 if (!insn->x86_64) 283 if (!insn->x86_64)
284 return 0; 284 return 0;
285 if (!modrm->got) 285 if (!modrm->got)
286 insn_get_modrm(insn); 286 insn_get_modrm(insn);
287 /* 287 /*
288 * For rip-relative instructions, the mod field (top 2 bits) 288 * For rip-relative instructions, the mod field (top 2 bits)
289 * is zero and the r/m field (bottom 3 bits) is 0x5. 289 * is zero and the r/m field (bottom 3 bits) is 0x5.
290 */ 290 */
291 return (modrm->nbytes && (modrm->value & 0xc7) == 0x5); 291 return (modrm->nbytes && (modrm->value & 0xc7) == 0x5);
292 } 292 }
293 293
294 /** 294 /**
295 * insn_get_sib() - Get the SIB byte of instruction 295 * insn_get_sib() - Get the SIB byte of instruction
296 * @insn: &struct insn containing instruction 296 * @insn: &struct insn containing instruction
297 * 297 *
298 * If necessary, first collects the instruction up to and including the 298 * If necessary, first collects the instruction up to and including the
299 * ModRM byte. 299 * ModRM byte.
300 */ 300 */
301 void insn_get_sib(struct insn *insn) 301 void insn_get_sib(struct insn *insn)
302 { 302 {
303 insn_byte_t modrm; 303 insn_byte_t modrm;
304 304
305 if (insn->sib.got) 305 if (insn->sib.got)
306 return; 306 return;
307 if (!insn->modrm.got) 307 if (!insn->modrm.got)
308 insn_get_modrm(insn); 308 insn_get_modrm(insn);
309 if (insn->modrm.nbytes) { 309 if (insn->modrm.nbytes) {
310 modrm = (insn_byte_t)insn->modrm.value; 310 modrm = (insn_byte_t)insn->modrm.value;
311 if (insn->addr_bytes != 2 && 311 if (insn->addr_bytes != 2 &&
312 X86_MODRM_MOD(modrm) != 3 && X86_MODRM_RM(modrm) == 4) { 312 X86_MODRM_MOD(modrm) != 3 && X86_MODRM_RM(modrm) == 4) {
313 insn->sib.value = get_next(insn_byte_t, insn); 313 insn->sib.value = get_next(insn_byte_t, insn);
314 insn->sib.nbytes = 1; 314 insn->sib.nbytes = 1;
315 } 315 }
316 } 316 }
317 insn->sib.got = 1; 317 insn->sib.got = 1;
318 318
319 err_out: 319 err_out:
320 return; 320 return;
321 } 321 }
322 322
323 323
324 /** 324 /**
325 * insn_get_displacement() - Get the displacement of instruction 325 * insn_get_displacement() - Get the displacement of instruction
326 * @insn: &struct insn containing instruction 326 * @insn: &struct insn containing instruction
327 * 327 *
328 * If necessary, first collects the instruction up to and including the 328 * If necessary, first collects the instruction up to and including the
329 * SIB byte. 329 * SIB byte.
330 * Displacement value is sign-expanded. 330 * Displacement value is sign-expanded.
331 */ 331 */
332 void insn_get_displacement(struct insn *insn) 332 void insn_get_displacement(struct insn *insn)
333 { 333 {
334 insn_byte_t mod, rm, base; 334 insn_byte_t mod, rm, base;
335 335
336 if (insn->displacement.got) 336 if (insn->displacement.got)
337 return; 337 return;
338 if (!insn->sib.got) 338 if (!insn->sib.got)
339 insn_get_sib(insn); 339 insn_get_sib(insn);
340 if (insn->modrm.nbytes) { 340 if (insn->modrm.nbytes) {
341 /* 341 /*
342 * Interpreting the modrm byte: 342 * Interpreting the modrm byte:
343 * mod = 00 - no displacement fields (exceptions below) 343 * mod = 00 - no displacement fields (exceptions below)
344 * mod = 01 - 1-byte displacement field 344 * mod = 01 - 1-byte displacement field
345 * mod = 10 - displacement field is 4 bytes, or 2 bytes if 345 * mod = 10 - displacement field is 4 bytes, or 2 bytes if
346 * address size = 2 (0x67 prefix in 32-bit mode) 346 * address size = 2 (0x67 prefix in 32-bit mode)
347 * mod = 11 - no memory operand 347 * mod = 11 - no memory operand
348 * 348 *
349 * If address size = 2... 349 * If address size = 2...
350 * mod = 00, r/m = 110 - displacement field is 2 bytes 350 * mod = 00, r/m = 110 - displacement field is 2 bytes
351 * 351 *
352 * If address size != 2... 352 * If address size != 2...
353 * mod != 11, r/m = 100 - SIB byte exists 353 * mod != 11, r/m = 100 - SIB byte exists
354 * mod = 00, SIB base = 101 - displacement field is 4 bytes 354 * mod = 00, SIB base = 101 - displacement field is 4 bytes
355 * mod = 00, r/m = 101 - rip-relative addressing, displacement 355 * mod = 00, r/m = 101 - rip-relative addressing, displacement
356 * field is 4 bytes 356 * field is 4 bytes
357 */ 357 */
358 mod = X86_MODRM_MOD(insn->modrm.value); 358 mod = X86_MODRM_MOD(insn->modrm.value);
359 rm = X86_MODRM_RM(insn->modrm.value); 359 rm = X86_MODRM_RM(insn->modrm.value);
360 base = X86_SIB_BASE(insn->sib.value); 360 base = X86_SIB_BASE(insn->sib.value);
361 if (mod == 3) 361 if (mod == 3)
362 goto out; 362 goto out;
363 if (mod == 1) { 363 if (mod == 1) {
364 insn->displacement.value = get_next(char, insn); 364 insn->displacement.value = get_next(char, insn);
365 insn->displacement.nbytes = 1; 365 insn->displacement.nbytes = 1;
366 } else if (insn->addr_bytes == 2) { 366 } else if (insn->addr_bytes == 2) {
367 if ((mod == 0 && rm == 6) || mod == 2) { 367 if ((mod == 0 && rm == 6) || mod == 2) {
368 insn->displacement.value = 368 insn->displacement.value =
369 get_next(short, insn); 369 get_next(short, insn);
370 insn->displacement.nbytes = 2; 370 insn->displacement.nbytes = 2;
371 } 371 }
372 } else { 372 } else {
373 if ((mod == 0 && rm == 5) || mod == 2 || 373 if ((mod == 0 && rm == 5) || mod == 2 ||
374 (mod == 0 && base == 5)) { 374 (mod == 0 && base == 5)) {
375 insn->displacement.value = get_next(int, insn); 375 insn->displacement.value = get_next(int, insn);
376 insn->displacement.nbytes = 4; 376 insn->displacement.nbytes = 4;
377 } 377 }
378 } 378 }
379 } 379 }
380 out: 380 out:
381 insn->displacement.got = 1; 381 insn->displacement.got = 1;
382 382
383 err_out: 383 err_out:
384 return; 384 return;
385 } 385 }
386 386
387 /* Decode moffset16/32/64. Return 0 if failed */ 387 /* Decode moffset16/32/64. Return 0 if failed */
388 static int __get_moffset(struct insn *insn) 388 static int __get_moffset(struct insn *insn)
389 { 389 {
390 switch (insn->addr_bytes) { 390 switch (insn->addr_bytes) {
391 case 2: 391 case 2:
392 insn->moffset1.value = get_next(short, insn); 392 insn->moffset1.value = get_next(short, insn);
393 insn->moffset1.nbytes = 2; 393 insn->moffset1.nbytes = 2;
394 break; 394 break;
395 case 4: 395 case 4:
396 insn->moffset1.value = get_next(int, insn); 396 insn->moffset1.value = get_next(int, insn);
397 insn->moffset1.nbytes = 4; 397 insn->moffset1.nbytes = 4;
398 break; 398 break;
399 case 8: 399 case 8:
400 insn->moffset1.value = get_next(int, insn); 400 insn->moffset1.value = get_next(int, insn);
401 insn->moffset1.nbytes = 4; 401 insn->moffset1.nbytes = 4;
402 insn->moffset2.value = get_next(int, insn); 402 insn->moffset2.value = get_next(int, insn);
403 insn->moffset2.nbytes = 4; 403 insn->moffset2.nbytes = 4;
404 break; 404 break;
405 default: /* opnd_bytes must be modified manually */ 405 default: /* opnd_bytes must be modified manually */
406 goto err_out; 406 goto err_out;
407 } 407 }
408 insn->moffset1.got = insn->moffset2.got = 1; 408 insn->moffset1.got = insn->moffset2.got = 1;
409 409
410 return 1; 410 return 1;
411 411
412 err_out: 412 err_out:
413 return 0; 413 return 0;
414 } 414 }
415 415
416 /* Decode imm v32(Iz). Return 0 if failed */ 416 /* Decode imm v32(Iz). Return 0 if failed */
417 static int __get_immv32(struct insn *insn) 417 static int __get_immv32(struct insn *insn)
418 { 418 {
419 switch (insn->opnd_bytes) { 419 switch (insn->opnd_bytes) {
420 case 2: 420 case 2:
421 insn->immediate.value = get_next(short, insn); 421 insn->immediate.value = get_next(short, insn);
422 insn->immediate.nbytes = 2; 422 insn->immediate.nbytes = 2;
423 break; 423 break;
424 case 4: 424 case 4:
425 case 8: 425 case 8:
426 insn->immediate.value = get_next(int, insn); 426 insn->immediate.value = get_next(int, insn);
427 insn->immediate.nbytes = 4; 427 insn->immediate.nbytes = 4;
428 break; 428 break;
429 default: /* opnd_bytes must be modified manually */ 429 default: /* opnd_bytes must be modified manually */
430 goto err_out; 430 goto err_out;
431 } 431 }
432 432
433 return 1; 433 return 1;
434 434
435 err_out: 435 err_out:
436 return 0; 436 return 0;
437 } 437 }
438 438
439 /* Decode imm v64(Iv/Ov), Return 0 if failed */ 439 /* Decode imm v64(Iv/Ov), Return 0 if failed */
440 static int __get_immv(struct insn *insn) 440 static int __get_immv(struct insn *insn)
441 { 441 {
442 switch (insn->opnd_bytes) { 442 switch (insn->opnd_bytes) {
443 case 2: 443 case 2:
444 insn->immediate1.value = get_next(short, insn); 444 insn->immediate1.value = get_next(short, insn);
445 insn->immediate1.nbytes = 2; 445 insn->immediate1.nbytes = 2;
446 break; 446 break;
447 case 4: 447 case 4:
448 insn->immediate1.value = get_next(int, insn); 448 insn->immediate1.value = get_next(int, insn);
449 insn->immediate1.nbytes = 4; 449 insn->immediate1.nbytes = 4;
450 break; 450 break;
451 case 8: 451 case 8:
452 insn->immediate1.value = get_next(int, insn); 452 insn->immediate1.value = get_next(int, insn);
453 insn->immediate1.nbytes = 4; 453 insn->immediate1.nbytes = 4;
454 insn->immediate2.value = get_next(int, insn); 454 insn->immediate2.value = get_next(int, insn);
455 insn->immediate2.nbytes = 4; 455 insn->immediate2.nbytes = 4;
456 break; 456 break;
457 default: /* opnd_bytes must be modified manually */ 457 default: /* opnd_bytes must be modified manually */
458 goto err_out; 458 goto err_out;
459 } 459 }
460 insn->immediate1.got = insn->immediate2.got = 1; 460 insn->immediate1.got = insn->immediate2.got = 1;
461 461
462 return 1; 462 return 1;
463 err_out: 463 err_out:
464 return 0; 464 return 0;
465 } 465 }
466 466
467 /* Decode ptr16:16/32(Ap) */ 467 /* Decode ptr16:16/32(Ap) */
468 static int __get_immptr(struct insn *insn) 468 static int __get_immptr(struct insn *insn)
469 { 469 {
470 switch (insn->opnd_bytes) { 470 switch (insn->opnd_bytes) {
471 case 2: 471 case 2:
472 insn->immediate1.value = get_next(short, insn); 472 insn->immediate1.value = get_next(short, insn);
473 insn->immediate1.nbytes = 2; 473 insn->immediate1.nbytes = 2;
474 break; 474 break;
475 case 4: 475 case 4:
476 insn->immediate1.value = get_next(int, insn); 476 insn->immediate1.value = get_next(int, insn);
477 insn->immediate1.nbytes = 4; 477 insn->immediate1.nbytes = 4;
478 break; 478 break;
479 case 8: 479 case 8:
480 /* ptr16:64 is not exist (no segment) */ 480 /* ptr16:64 is not exist (no segment) */
481 return 0; 481 return 0;
482 default: /* opnd_bytes must be modified manually */ 482 default: /* opnd_bytes must be modified manually */
483 goto err_out; 483 goto err_out;
484 } 484 }
485 insn->immediate2.value = get_next(unsigned short, insn); 485 insn->immediate2.value = get_next(unsigned short, insn);
486 insn->immediate2.nbytes = 2; 486 insn->immediate2.nbytes = 2;
487 insn->immediate1.got = insn->immediate2.got = 1; 487 insn->immediate1.got = insn->immediate2.got = 1;
488 488
489 return 1; 489 return 1;
490 err_out: 490 err_out:
491 return 0; 491 return 0;
492 } 492 }
493 493
494 /** 494 /**
495 * insn_get_immediate() - Get the immediates of instruction 495 * insn_get_immediate() - Get the immediates of instruction
496 * @insn: &struct insn containing instruction 496 * @insn: &struct insn containing instruction
497 * 497 *
498 * If necessary, first collects the instruction up to and including the 498 * If necessary, first collects the instruction up to and including the
499 * displacement bytes. 499 * displacement bytes.
500 * Basically, most of immediates are sign-expanded. Unsigned-value can be 500 * Basically, most of immediates are sign-expanded. Unsigned-value can be
501 * get by bit masking with ((1 << (nbytes * 8)) - 1) 501 * get by bit masking with ((1 << (nbytes * 8)) - 1)
502 */ 502 */
503 void insn_get_immediate(struct insn *insn) 503 void insn_get_immediate(struct insn *insn)
504 { 504 {
505 if (insn->immediate.got) 505 if (insn->immediate.got)
506 return; 506 return;
507 if (!insn->displacement.got) 507 if (!insn->displacement.got)
508 insn_get_displacement(insn); 508 insn_get_displacement(insn);
509 509
510 if (inat_has_moffset(insn->attr)) { 510 if (inat_has_moffset(insn->attr)) {
511 if (!__get_moffset(insn)) 511 if (!__get_moffset(insn))
512 goto err_out; 512 goto err_out;
513 goto done; 513 goto done;
514 } 514 }
515 515
516 if (!inat_has_immediate(insn->attr)) 516 if (!inat_has_immediate(insn->attr))
517 /* no immediates */ 517 /* no immediates */
518 goto done; 518 goto done;
519 519
520 switch (inat_immediate_size(insn->attr)) { 520 switch (inat_immediate_size(insn->attr)) {
521 case INAT_IMM_BYTE: 521 case INAT_IMM_BYTE:
522 insn->immediate.value = get_next(char, insn); 522 insn->immediate.value = get_next(char, insn);
523 insn->immediate.nbytes = 1; 523 insn->immediate.nbytes = 1;
524 break; 524 break;
525 case INAT_IMM_WORD: 525 case INAT_IMM_WORD:
526 insn->immediate.value = get_next(short, insn); 526 insn->immediate.value = get_next(short, insn);
527 insn->immediate.nbytes = 2; 527 insn->immediate.nbytes = 2;
528 break; 528 break;
529 case INAT_IMM_DWORD: 529 case INAT_IMM_DWORD:
530 insn->immediate.value = get_next(int, insn); 530 insn->immediate.value = get_next(int, insn);
531 insn->immediate.nbytes = 4; 531 insn->immediate.nbytes = 4;
532 break; 532 break;
533 case INAT_IMM_QWORD: 533 case INAT_IMM_QWORD:
534 insn->immediate1.value = get_next(int, insn); 534 insn->immediate1.value = get_next(int, insn);
535 insn->immediate1.nbytes = 4; 535 insn->immediate1.nbytes = 4;
536 insn->immediate2.value = get_next(int, insn); 536 insn->immediate2.value = get_next(int, insn);
537 insn->immediate2.nbytes = 4; 537 insn->immediate2.nbytes = 4;
538 break; 538 break;
539 case INAT_IMM_PTR: 539 case INAT_IMM_PTR:
540 if (!__get_immptr(insn)) 540 if (!__get_immptr(insn))
541 goto err_out; 541 goto err_out;
542 break; 542 break;
543 case INAT_IMM_VWORD32: 543 case INAT_IMM_VWORD32:
544 if (!__get_immv32(insn)) 544 if (!__get_immv32(insn))
545 goto err_out; 545 goto err_out;
546 break; 546 break;
547 case INAT_IMM_VWORD: 547 case INAT_IMM_VWORD:
548 if (!__get_immv(insn)) 548 if (!__get_immv(insn))
549 goto err_out; 549 goto err_out;
550 break; 550 break;
551 default: 551 default:
552 /* Here, insn must have an immediate, but failed */ 552 /* Here, insn must have an immediate, but failed */
553 goto err_out; 553 goto err_out;
554 } 554 }
555 if (inat_has_second_immediate(insn->attr)) { 555 if (inat_has_second_immediate(insn->attr)) {
556 insn->immediate2.value = get_next(char, insn); 556 insn->immediate2.value = get_next(char, insn);
557 insn->immediate2.nbytes = 1; 557 insn->immediate2.nbytes = 1;
558 } 558 }
559 done: 559 done:
560 insn->immediate.got = 1; 560 insn->immediate.got = 1;
561 561
562 err_out: 562 err_out:
563 return; 563 return;
564 } 564 }
565 565
566 /** 566 /**
567 * insn_get_length() - Get the length of instruction 567 * insn_get_length() - Get the length of instruction
568 * @insn: &struct insn containing instruction 568 * @insn: &struct insn containing instruction
569 * 569 *
570 * If necessary, first collects the instruction up to and including the 570 * If necessary, first collects the instruction up to and including the
571 * immediates bytes. 571 * immediates bytes.
572 */ 572 */
573 void insn_get_length(struct insn *insn) 573 void insn_get_length(struct insn *insn)
574 { 574 {
575 if (insn->length) 575 if (insn->length)
576 return; 576 return;
577 if (!insn->immediate.got) 577 if (!insn->immediate.got)
578 insn_get_immediate(insn); 578 insn_get_immediate(insn);
579 insn->length = (unsigned char)((unsigned long)insn->next_byte 579 insn->length = (unsigned char)((unsigned long)insn->next_byte
580 - (unsigned long)insn->kaddr); 580 - (unsigned long)insn->kaddr);
581 } 581 }
582 582
include/linux/perf_event.h
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * Performance events: 2 * Performance events:
3 * 3 *
4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de> 4 * Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar 5 * Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra 6 * Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
7 * 7 *
8 * Data type definitions, declarations, prototypes. 8 * Data type definitions, declarations, prototypes.
9 * 9 *
10 * Started by: Thomas Gleixner and Ingo Molnar 10 * Started by: Thomas Gleixner and Ingo Molnar
11 * 11 *
12 * For licencing details see kernel-base/COPYING 12 * For licencing details see kernel-base/COPYING
13 */ 13 */
14 #ifndef _LINUX_PERF_EVENT_H 14 #ifndef _LINUX_PERF_EVENT_H
15 #define _LINUX_PERF_EVENT_H 15 #define _LINUX_PERF_EVENT_H
16 16
17 #include <uapi/linux/perf_event.h> 17 #include <uapi/linux/perf_event.h>
18 18
19 /* 19 /*
20 * Kernel-internal data types and definitions: 20 * Kernel-internal data types and definitions:
21 */ 21 */
22 22
23 #ifdef CONFIG_PERF_EVENTS 23 #ifdef CONFIG_PERF_EVENTS
24 # include <asm/perf_event.h> 24 # include <asm/perf_event.h>
25 # include <asm/local64.h> 25 # include <asm/local64.h>
26 #endif 26 #endif
27 27
28 struct perf_guest_info_callbacks { 28 struct perf_guest_info_callbacks {
29 int (*is_in_guest)(void); 29 int (*is_in_guest)(void);
30 int (*is_user_mode)(void); 30 int (*is_user_mode)(void);
31 unsigned long (*get_guest_ip)(void); 31 unsigned long (*get_guest_ip)(void);
32 }; 32 };
33 33
34 #ifdef CONFIG_HAVE_HW_BREAKPOINT 34 #ifdef CONFIG_HAVE_HW_BREAKPOINT
35 #include <asm/hw_breakpoint.h> 35 #include <asm/hw_breakpoint.h>
36 #endif 36 #endif
37 37
38 #include <linux/list.h> 38 #include <linux/list.h>
39 #include <linux/mutex.h> 39 #include <linux/mutex.h>
40 #include <linux/rculist.h> 40 #include <linux/rculist.h>
41 #include <linux/rcupdate.h> 41 #include <linux/rcupdate.h>
42 #include <linux/spinlock.h> 42 #include <linux/spinlock.h>
43 #include <linux/hrtimer.h> 43 #include <linux/hrtimer.h>
44 #include <linux/fs.h> 44 #include <linux/fs.h>
45 #include <linux/pid_namespace.h> 45 #include <linux/pid_namespace.h>
46 #include <linux/workqueue.h> 46 #include <linux/workqueue.h>
47 #include <linux/ftrace.h> 47 #include <linux/ftrace.h>
48 #include <linux/cpu.h> 48 #include <linux/cpu.h>
49 #include <linux/irq_work.h> 49 #include <linux/irq_work.h>
50 #include <linux/static_key.h> 50 #include <linux/static_key.h>
51 #include <linux/jump_label_ratelimit.h> 51 #include <linux/jump_label_ratelimit.h>
52 #include <linux/atomic.h> 52 #include <linux/atomic.h>
53 #include <linux/sysfs.h> 53 #include <linux/sysfs.h>
54 #include <linux/perf_regs.h> 54 #include <linux/perf_regs.h>
55 #include <linux/workqueue.h> 55 #include <linux/workqueue.h>
56 #include <asm/local.h> 56 #include <asm/local.h>
57 57
58 struct perf_callchain_entry { 58 struct perf_callchain_entry {
59 __u64 nr; 59 __u64 nr;
60 __u64 ip[PERF_MAX_STACK_DEPTH]; 60 __u64 ip[PERF_MAX_STACK_DEPTH];
61 }; 61 };
62 62
63 struct perf_raw_record { 63 struct perf_raw_record {
64 u32 size; 64 u32 size;
65 void *data; 65 void *data;
66 }; 66 };
67 67
68 /* 68 /*
69 * branch stack layout: 69 * branch stack layout:
70 * nr: number of taken branches stored in entries[] 70 * nr: number of taken branches stored in entries[]
71 * 71 *
72 * Note that nr can vary from sample to sample 72 * Note that nr can vary from sample to sample
73 * branches (to, from) are stored from most recent 73 * branches (to, from) are stored from most recent
74 * to least recent, i.e., entries[0] contains the most 74 * to least recent, i.e., entries[0] contains the most
75 * recent branch. 75 * recent branch.
76 */ 76 */
77 struct perf_branch_stack { 77 struct perf_branch_stack {
78 __u64 nr; 78 __u64 nr;
79 struct perf_branch_entry entries[0]; 79 struct perf_branch_entry entries[0];
80 }; 80 };
81 81
82 struct perf_regs {
83 __u64 abi;
84 struct pt_regs *regs;
85 };
86
87 struct task_struct; 82 struct task_struct;
88 83
89 /* 84 /*
90 * extra PMU register associated with an event 85 * extra PMU register associated with an event
91 */ 86 */
92 struct hw_perf_event_extra { 87 struct hw_perf_event_extra {
93 u64 config; /* register value */ 88 u64 config; /* register value */
94 unsigned int reg; /* register address or index */ 89 unsigned int reg; /* register address or index */
95 int alloc; /* extra register already allocated */ 90 int alloc; /* extra register already allocated */
96 int idx; /* index in shared_regs->regs[] */ 91 int idx; /* index in shared_regs->regs[] */
97 }; 92 };
98 93
99 struct event_constraint; 94 struct event_constraint;
100 95
101 /** 96 /**
102 * struct hw_perf_event - performance event hardware details: 97 * struct hw_perf_event - performance event hardware details:
103 */ 98 */
104 struct hw_perf_event { 99 struct hw_perf_event {
105 #ifdef CONFIG_PERF_EVENTS 100 #ifdef CONFIG_PERF_EVENTS
106 union { 101 union {
107 struct { /* hardware */ 102 struct { /* hardware */
108 u64 config; 103 u64 config;
109 u64 last_tag; 104 u64 last_tag;
110 unsigned long config_base; 105 unsigned long config_base;
111 unsigned long event_base; 106 unsigned long event_base;
112 int event_base_rdpmc; 107 int event_base_rdpmc;
113 int idx; 108 int idx;
114 int last_cpu; 109 int last_cpu;
115 int flags; 110 int flags;
116 111
117 struct hw_perf_event_extra extra_reg; 112 struct hw_perf_event_extra extra_reg;
118 struct hw_perf_event_extra branch_reg; 113 struct hw_perf_event_extra branch_reg;
119 114
120 struct event_constraint *constraint; 115 struct event_constraint *constraint;
121 }; 116 };
122 struct { /* software */ 117 struct { /* software */
123 struct hrtimer hrtimer; 118 struct hrtimer hrtimer;
124 }; 119 };
125 struct { /* tracepoint */ 120 struct { /* tracepoint */
126 struct task_struct *tp_target; 121 struct task_struct *tp_target;
127 /* for tp_event->class */ 122 /* for tp_event->class */
128 struct list_head tp_list; 123 struct list_head tp_list;
129 }; 124 };
130 #ifdef CONFIG_HAVE_HW_BREAKPOINT 125 #ifdef CONFIG_HAVE_HW_BREAKPOINT
131 struct { /* breakpoint */ 126 struct { /* breakpoint */
132 /* 127 /*
133 * Crufty hack to avoid the chicken and egg 128 * Crufty hack to avoid the chicken and egg
134 * problem hw_breakpoint has with context 129 * problem hw_breakpoint has with context
135 * creation and event initalization. 130 * creation and event initalization.
136 */ 131 */
137 struct task_struct *bp_target; 132 struct task_struct *bp_target;
138 struct arch_hw_breakpoint info; 133 struct arch_hw_breakpoint info;
139 struct list_head bp_list; 134 struct list_head bp_list;
140 }; 135 };
141 #endif 136 #endif
142 }; 137 };
143 int state; 138 int state;
144 local64_t prev_count; 139 local64_t prev_count;
145 u64 sample_period; 140 u64 sample_period;
146 u64 last_period; 141 u64 last_period;
147 local64_t period_left; 142 local64_t period_left;
148 u64 interrupts_seq; 143 u64 interrupts_seq;
149 u64 interrupts; 144 u64 interrupts;
150 145
151 u64 freq_time_stamp; 146 u64 freq_time_stamp;
152 u64 freq_count_stamp; 147 u64 freq_count_stamp;
153 #endif 148 #endif
154 }; 149 };
155 150
156 /* 151 /*
157 * hw_perf_event::state flags 152 * hw_perf_event::state flags
158 */ 153 */
159 #define PERF_HES_STOPPED 0x01 /* the counter is stopped */ 154 #define PERF_HES_STOPPED 0x01 /* the counter is stopped */
160 #define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */ 155 #define PERF_HES_UPTODATE 0x02 /* event->count up-to-date */
161 #define PERF_HES_ARCH 0x04 156 #define PERF_HES_ARCH 0x04
162 157
163 struct perf_event; 158 struct perf_event;
164 159
165 /* 160 /*
166 * Common implementation detail of pmu::{start,commit,cancel}_txn 161 * Common implementation detail of pmu::{start,commit,cancel}_txn
167 */ 162 */
168 #define PERF_EVENT_TXN 0x1 163 #define PERF_EVENT_TXN 0x1
169 164
170 /** 165 /**
171 * pmu::capabilities flags 166 * pmu::capabilities flags
172 */ 167 */
173 #define PERF_PMU_CAP_NO_INTERRUPT 0x01 168 #define PERF_PMU_CAP_NO_INTERRUPT 0x01
174 169
175 /** 170 /**
176 * struct pmu - generic performance monitoring unit 171 * struct pmu - generic performance monitoring unit
177 */ 172 */
178 struct pmu { 173 struct pmu {
179 struct list_head entry; 174 struct list_head entry;
180 175
181 struct module *module; 176 struct module *module;
182 struct device *dev; 177 struct device *dev;
183 const struct attribute_group **attr_groups; 178 const struct attribute_group **attr_groups;
184 const char *name; 179 const char *name;
185 int type; 180 int type;
186 181
187 /* 182 /*
188 * various common per-pmu feature flags 183 * various common per-pmu feature flags
189 */ 184 */
190 int capabilities; 185 int capabilities;
191 186
192 int * __percpu pmu_disable_count; 187 int * __percpu pmu_disable_count;
193 struct perf_cpu_context * __percpu pmu_cpu_context; 188 struct perf_cpu_context * __percpu pmu_cpu_context;
194 int task_ctx_nr; 189 int task_ctx_nr;
195 int hrtimer_interval_ms; 190 int hrtimer_interval_ms;
196 191
197 /* 192 /*
198 * Fully disable/enable this PMU, can be used to protect from the PMI 193 * Fully disable/enable this PMU, can be used to protect from the PMI
199 * as well as for lazy/batch writing of the MSRs. 194 * as well as for lazy/batch writing of the MSRs.
200 */ 195 */
201 void (*pmu_enable) (struct pmu *pmu); /* optional */ 196 void (*pmu_enable) (struct pmu *pmu); /* optional */
202 void (*pmu_disable) (struct pmu *pmu); /* optional */ 197 void (*pmu_disable) (struct pmu *pmu); /* optional */
203 198
204 /* 199 /*
205 * Try and initialize the event for this PMU. 200 * Try and initialize the event for this PMU.
206 * Should return -ENOENT when the @event doesn't match this PMU. 201 * Should return -ENOENT when the @event doesn't match this PMU.
207 */ 202 */
208 int (*event_init) (struct perf_event *event); 203 int (*event_init) (struct perf_event *event);
209 204
210 #define PERF_EF_START 0x01 /* start the counter when adding */ 205 #define PERF_EF_START 0x01 /* start the counter when adding */
211 #define PERF_EF_RELOAD 0x02 /* reload the counter when starting */ 206 #define PERF_EF_RELOAD 0x02 /* reload the counter when starting */
212 #define PERF_EF_UPDATE 0x04 /* update the counter when stopping */ 207 #define PERF_EF_UPDATE 0x04 /* update the counter when stopping */
213 208
214 /* 209 /*
215 * Adds/Removes a counter to/from the PMU, can be done inside 210 * Adds/Removes a counter to/from the PMU, can be done inside
216 * a transaction, see the ->*_txn() methods. 211 * a transaction, see the ->*_txn() methods.
217 */ 212 */
218 int (*add) (struct perf_event *event, int flags); 213 int (*add) (struct perf_event *event, int flags);
219 void (*del) (struct perf_event *event, int flags); 214 void (*del) (struct perf_event *event, int flags);
220 215
221 /* 216 /*
222 * Starts/Stops a counter present on the PMU. The PMI handler 217 * Starts/Stops a counter present on the PMU. The PMI handler
223 * should stop the counter when perf_event_overflow() returns 218 * should stop the counter when perf_event_overflow() returns
224 * !0. ->start() will be used to continue. 219 * !0. ->start() will be used to continue.
225 */ 220 */
226 void (*start) (struct perf_event *event, int flags); 221 void (*start) (struct perf_event *event, int flags);
227 void (*stop) (struct perf_event *event, int flags); 222 void (*stop) (struct perf_event *event, int flags);
228 223
229 /* 224 /*
230 * Updates the counter value of the event. 225 * Updates the counter value of the event.
231 */ 226 */
232 void (*read) (struct perf_event *event); 227 void (*read) (struct perf_event *event);
233 228
234 /* 229 /*
235 * Group events scheduling is treated as a transaction, add 230 * Group events scheduling is treated as a transaction, add
236 * group events as a whole and perform one schedulability test. 231 * group events as a whole and perform one schedulability test.
237 * If the test fails, roll back the whole group 232 * If the test fails, roll back the whole group
238 * 233 *
239 * Start the transaction, after this ->add() doesn't need to 234 * Start the transaction, after this ->add() doesn't need to
240 * do schedulability tests. 235 * do schedulability tests.
241 */ 236 */
242 void (*start_txn) (struct pmu *pmu); /* optional */ 237 void (*start_txn) (struct pmu *pmu); /* optional */
243 /* 238 /*
244 * If ->start_txn() disabled the ->add() schedulability test 239 * If ->start_txn() disabled the ->add() schedulability test
245 * then ->commit_txn() is required to perform one. On success 240 * then ->commit_txn() is required to perform one. On success
246 * the transaction is closed. On error the transaction is kept 241 * the transaction is closed. On error the transaction is kept
247 * open until ->cancel_txn() is called. 242 * open until ->cancel_txn() is called.
248 */ 243 */
249 int (*commit_txn) (struct pmu *pmu); /* optional */ 244 int (*commit_txn) (struct pmu *pmu); /* optional */
250 /* 245 /*
251 * Will cancel the transaction, assumes ->del() is called 246 * Will cancel the transaction, assumes ->del() is called
252 * for each successful ->add() during the transaction. 247 * for each successful ->add() during the transaction.
253 */ 248 */
254 void (*cancel_txn) (struct pmu *pmu); /* optional */ 249 void (*cancel_txn) (struct pmu *pmu); /* optional */
255 250
256 /* 251 /*
257 * Will return the value for perf_event_mmap_page::index for this event, 252 * Will return the value for perf_event_mmap_page::index for this event,
258 * if no implementation is provided it will default to: event->hw.idx + 1. 253 * if no implementation is provided it will default to: event->hw.idx + 1.
259 */ 254 */
260 int (*event_idx) (struct perf_event *event); /*optional */ 255 int (*event_idx) (struct perf_event *event); /*optional */
261 256
262 /* 257 /*
263 * flush branch stack on context-switches (needed in cpu-wide mode) 258 * flush branch stack on context-switches (needed in cpu-wide mode)
264 */ 259 */
265 void (*flush_branch_stack) (void); 260 void (*flush_branch_stack) (void);
266 }; 261 };
267 262
268 /** 263 /**
269 * enum perf_event_active_state - the states of a event 264 * enum perf_event_active_state - the states of a event
270 */ 265 */
271 enum perf_event_active_state { 266 enum perf_event_active_state {
272 PERF_EVENT_STATE_EXIT = -3, 267 PERF_EVENT_STATE_EXIT = -3,
273 PERF_EVENT_STATE_ERROR = -2, 268 PERF_EVENT_STATE_ERROR = -2,
274 PERF_EVENT_STATE_OFF = -1, 269 PERF_EVENT_STATE_OFF = -1,
275 PERF_EVENT_STATE_INACTIVE = 0, 270 PERF_EVENT_STATE_INACTIVE = 0,
276 PERF_EVENT_STATE_ACTIVE = 1, 271 PERF_EVENT_STATE_ACTIVE = 1,
277 }; 272 };
278 273
279 struct file; 274 struct file;
280 struct perf_sample_data; 275 struct perf_sample_data;
281 276
282 typedef void (*perf_overflow_handler_t)(struct perf_event *, 277 typedef void (*perf_overflow_handler_t)(struct perf_event *,
283 struct perf_sample_data *, 278 struct perf_sample_data *,
284 struct pt_regs *regs); 279 struct pt_regs *regs);
285 280
286 enum perf_group_flag { 281 enum perf_group_flag {
287 PERF_GROUP_SOFTWARE = 0x1, 282 PERF_GROUP_SOFTWARE = 0x1,
288 }; 283 };
289 284
290 #define SWEVENT_HLIST_BITS 8 285 #define SWEVENT_HLIST_BITS 8
291 #define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS) 286 #define SWEVENT_HLIST_SIZE (1 << SWEVENT_HLIST_BITS)
292 287
293 struct swevent_hlist { 288 struct swevent_hlist {
294 struct hlist_head heads[SWEVENT_HLIST_SIZE]; 289 struct hlist_head heads[SWEVENT_HLIST_SIZE];
295 struct rcu_head rcu_head; 290 struct rcu_head rcu_head;
296 }; 291 };
297 292
298 #define PERF_ATTACH_CONTEXT 0x01 293 #define PERF_ATTACH_CONTEXT 0x01
299 #define PERF_ATTACH_GROUP 0x02 294 #define PERF_ATTACH_GROUP 0x02
300 #define PERF_ATTACH_TASK 0x04 295 #define PERF_ATTACH_TASK 0x04
301 296
302 struct perf_cgroup; 297 struct perf_cgroup;
303 struct ring_buffer; 298 struct ring_buffer;
304 299
305 /** 300 /**
306 * struct perf_event - performance event kernel representation: 301 * struct perf_event - performance event kernel representation:
307 */ 302 */
308 struct perf_event { 303 struct perf_event {
309 #ifdef CONFIG_PERF_EVENTS 304 #ifdef CONFIG_PERF_EVENTS
310 /* 305 /*
311 * entry onto perf_event_context::event_list; 306 * entry onto perf_event_context::event_list;
312 * modifications require ctx->lock 307 * modifications require ctx->lock
313 * RCU safe iterations. 308 * RCU safe iterations.
314 */ 309 */
315 struct list_head event_entry; 310 struct list_head event_entry;
316 311
317 /* 312 /*
318 * XXX: group_entry and sibling_list should be mutually exclusive; 313 * XXX: group_entry and sibling_list should be mutually exclusive;
319 * either you're a sibling on a group, or you're the group leader. 314 * either you're a sibling on a group, or you're the group leader.
320 * Rework the code to always use the same list element. 315 * Rework the code to always use the same list element.
321 * 316 *
322 * Locked for modification by both ctx->mutex and ctx->lock; holding 317 * Locked for modification by both ctx->mutex and ctx->lock; holding
323 * either sufficies for read. 318 * either sufficies for read.
324 */ 319 */
325 struct list_head group_entry; 320 struct list_head group_entry;
326 struct list_head sibling_list; 321 struct list_head sibling_list;
327 322
328 /* 323 /*
329 * We need storage to track the entries in perf_pmu_migrate_context; we 324 * We need storage to track the entries in perf_pmu_migrate_context; we
330 * cannot use the event_entry because of RCU and we want to keep the 325 * cannot use the event_entry because of RCU and we want to keep the
331 * group in tact which avoids us using the other two entries. 326 * group in tact which avoids us using the other two entries.
332 */ 327 */
333 struct list_head migrate_entry; 328 struct list_head migrate_entry;
334 329
335 struct hlist_node hlist_entry; 330 struct hlist_node hlist_entry;
336 struct list_head active_entry; 331 struct list_head active_entry;
337 int nr_siblings; 332 int nr_siblings;
338 int group_flags; 333 int group_flags;
339 struct perf_event *group_leader; 334 struct perf_event *group_leader;
340 struct pmu *pmu; 335 struct pmu *pmu;
341 336
342 enum perf_event_active_state state; 337 enum perf_event_active_state state;
343 unsigned int attach_state; 338 unsigned int attach_state;
344 local64_t count; 339 local64_t count;
345 atomic64_t child_count; 340 atomic64_t child_count;
346 341
347 /* 342 /*
348 * These are the total time in nanoseconds that the event 343 * These are the total time in nanoseconds that the event
349 * has been enabled (i.e. eligible to run, and the task has 344 * has been enabled (i.e. eligible to run, and the task has
350 * been scheduled in, if this is a per-task event) 345 * been scheduled in, if this is a per-task event)
351 * and running (scheduled onto the CPU), respectively. 346 * and running (scheduled onto the CPU), respectively.
352 * 347 *
353 * They are computed from tstamp_enabled, tstamp_running and 348 * They are computed from tstamp_enabled, tstamp_running and
354 * tstamp_stopped when the event is in INACTIVE or ACTIVE state. 349 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
355 */ 350 */
356 u64 total_time_enabled; 351 u64 total_time_enabled;
357 u64 total_time_running; 352 u64 total_time_running;
358 353
359 /* 354 /*
360 * These are timestamps used for computing total_time_enabled 355 * These are timestamps used for computing total_time_enabled
361 * and total_time_running when the event is in INACTIVE or 356 * and total_time_running when the event is in INACTIVE or
362 * ACTIVE state, measured in nanoseconds from an arbitrary point 357 * ACTIVE state, measured in nanoseconds from an arbitrary point
363 * in time. 358 * in time.
364 * tstamp_enabled: the notional time when the event was enabled 359 * tstamp_enabled: the notional time when the event was enabled
365 * tstamp_running: the notional time when the event was scheduled on 360 * tstamp_running: the notional time when the event was scheduled on
366 * tstamp_stopped: in INACTIVE state, the notional time when the 361 * tstamp_stopped: in INACTIVE state, the notional time when the
367 * event was scheduled off. 362 * event was scheduled off.
368 */ 363 */
369 u64 tstamp_enabled; 364 u64 tstamp_enabled;
370 u64 tstamp_running; 365 u64 tstamp_running;
371 u64 tstamp_stopped; 366 u64 tstamp_stopped;
372 367
373 /* 368 /*
374 * timestamp shadows the actual context timing but it can 369 * timestamp shadows the actual context timing but it can
375 * be safely used in NMI interrupt context. It reflects the 370 * be safely used in NMI interrupt context. It reflects the
376 * context time as it was when the event was last scheduled in. 371 * context time as it was when the event was last scheduled in.
377 * 372 *
378 * ctx_time already accounts for ctx->timestamp. Therefore to 373 * ctx_time already accounts for ctx->timestamp. Therefore to
379 * compute ctx_time for a sample, simply add perf_clock(). 374 * compute ctx_time for a sample, simply add perf_clock().
380 */ 375 */
381 u64 shadow_ctx_time; 376 u64 shadow_ctx_time;
382 377
383 struct perf_event_attr attr; 378 struct perf_event_attr attr;
384 u16 header_size; 379 u16 header_size;
385 u16 id_header_size; 380 u16 id_header_size;
386 u16 read_size; 381 u16 read_size;
387 struct hw_perf_event hw; 382 struct hw_perf_event hw;
388 383
389 struct perf_event_context *ctx; 384 struct perf_event_context *ctx;
390 atomic_long_t refcount; 385 atomic_long_t refcount;
391 386
392 /* 387 /*
393 * These accumulate total time (in nanoseconds) that children 388 * These accumulate total time (in nanoseconds) that children
394 * events have been enabled and running, respectively. 389 * events have been enabled and running, respectively.
395 */ 390 */
396 atomic64_t child_total_time_enabled; 391 atomic64_t child_total_time_enabled;
397 atomic64_t child_total_time_running; 392 atomic64_t child_total_time_running;
398 393
399 /* 394 /*
400 * Protect attach/detach and child_list: 395 * Protect attach/detach and child_list:
401 */ 396 */
402 struct mutex child_mutex; 397 struct mutex child_mutex;
403 struct list_head child_list; 398 struct list_head child_list;
404 struct perf_event *parent; 399 struct perf_event *parent;
405 400
406 int oncpu; 401 int oncpu;
407 int cpu; 402 int cpu;
408 403
409 struct list_head owner_entry; 404 struct list_head owner_entry;
410 struct task_struct *owner; 405 struct task_struct *owner;
411 406
412 /* mmap bits */ 407 /* mmap bits */
413 struct mutex mmap_mutex; 408 struct mutex mmap_mutex;
414 atomic_t mmap_count; 409 atomic_t mmap_count;
415 410
416 struct ring_buffer *rb; 411 struct ring_buffer *rb;
417 struct list_head rb_entry; 412 struct list_head rb_entry;
418 unsigned long rcu_batches; 413 unsigned long rcu_batches;
419 int rcu_pending; 414 int rcu_pending;
420 415
421 /* poll related */ 416 /* poll related */
422 wait_queue_head_t waitq; 417 wait_queue_head_t waitq;
423 struct fasync_struct *fasync; 418 struct fasync_struct *fasync;
424 419
425 /* delayed work for NMIs and such */ 420 /* delayed work for NMIs and such */
426 int pending_wakeup; 421 int pending_wakeup;
427 int pending_kill; 422 int pending_kill;
428 int pending_disable; 423 int pending_disable;
429 struct irq_work pending; 424 struct irq_work pending;
430 425
431 atomic_t event_limit; 426 atomic_t event_limit;
432 427
433 void (*destroy)(struct perf_event *); 428 void (*destroy)(struct perf_event *);
434 struct rcu_head rcu_head; 429 struct rcu_head rcu_head;
435 430
436 struct pid_namespace *ns; 431 struct pid_namespace *ns;
437 u64 id; 432 u64 id;
438 433
439 perf_overflow_handler_t overflow_handler; 434 perf_overflow_handler_t overflow_handler;
440 void *overflow_handler_context; 435 void *overflow_handler_context;
441 436
442 #ifdef CONFIG_EVENT_TRACING 437 #ifdef CONFIG_EVENT_TRACING
443 struct ftrace_event_call *tp_event; 438 struct ftrace_event_call *tp_event;
444 struct event_filter *filter; 439 struct event_filter *filter;
445 #ifdef CONFIG_FUNCTION_TRACER 440 #ifdef CONFIG_FUNCTION_TRACER
446 struct ftrace_ops ftrace_ops; 441 struct ftrace_ops ftrace_ops;
447 #endif 442 #endif
448 #endif 443 #endif
449 444
450 #ifdef CONFIG_CGROUP_PERF 445 #ifdef CONFIG_CGROUP_PERF
451 struct perf_cgroup *cgrp; /* cgroup event is attach to */ 446 struct perf_cgroup *cgrp; /* cgroup event is attach to */
452 int cgrp_defer_enabled; 447 int cgrp_defer_enabled;
453 #endif 448 #endif
454 449
455 #endif /* CONFIG_PERF_EVENTS */ 450 #endif /* CONFIG_PERF_EVENTS */
456 }; 451 };
457 452
458 enum perf_event_context_type { 453 enum perf_event_context_type {
459 task_context, 454 task_context,
460 cpu_context, 455 cpu_context,
461 }; 456 };
462 457
463 /** 458 /**
464 * struct perf_event_context - event context structure 459 * struct perf_event_context - event context structure
465 * 460 *
466 * Used as a container for task events and CPU events as well: 461 * Used as a container for task events and CPU events as well:
467 */ 462 */
468 struct perf_event_context { 463 struct perf_event_context {
469 struct pmu *pmu; 464 struct pmu *pmu;
470 enum perf_event_context_type type; 465 enum perf_event_context_type type;
471 /* 466 /*
472 * Protect the states of the events in the list, 467 * Protect the states of the events in the list,
473 * nr_active, and the list: 468 * nr_active, and the list:
474 */ 469 */
475 raw_spinlock_t lock; 470 raw_spinlock_t lock;
476 /* 471 /*
477 * Protect the list of events. Locking either mutex or lock 472 * Protect the list of events. Locking either mutex or lock
478 * is sufficient to ensure the list doesn't change; to change 473 * is sufficient to ensure the list doesn't change; to change
479 * the list you need to lock both the mutex and the spinlock. 474 * the list you need to lock both the mutex and the spinlock.
480 */ 475 */
481 struct mutex mutex; 476 struct mutex mutex;
482 477
483 struct list_head pinned_groups; 478 struct list_head pinned_groups;
484 struct list_head flexible_groups; 479 struct list_head flexible_groups;
485 struct list_head event_list; 480 struct list_head event_list;
486 int nr_events; 481 int nr_events;
487 int nr_active; 482 int nr_active;
488 int is_active; 483 int is_active;
489 int nr_stat; 484 int nr_stat;
490 int nr_freq; 485 int nr_freq;
491 int rotate_disable; 486 int rotate_disable;
492 atomic_t refcount; 487 atomic_t refcount;
493 struct task_struct *task; 488 struct task_struct *task;
494 489
495 /* 490 /*
496 * Context clock, runs when context enabled. 491 * Context clock, runs when context enabled.
497 */ 492 */
498 u64 time; 493 u64 time;
499 u64 timestamp; 494 u64 timestamp;
500 495
501 /* 496 /*
502 * These fields let us detect when two contexts have both 497 * These fields let us detect when two contexts have both
503 * been cloned (inherited) from a common ancestor. 498 * been cloned (inherited) from a common ancestor.
504 */ 499 */
505 struct perf_event_context *parent_ctx; 500 struct perf_event_context *parent_ctx;
506 u64 parent_gen; 501 u64 parent_gen;
507 u64 generation; 502 u64 generation;
508 int pin_count; 503 int pin_count;
509 int nr_cgroups; /* cgroup evts */ 504 int nr_cgroups; /* cgroup evts */
510 int nr_branch_stack; /* branch_stack evt */ 505 int nr_branch_stack; /* branch_stack evt */
511 struct rcu_head rcu_head; 506 struct rcu_head rcu_head;
512 507
513 struct delayed_work orphans_remove; 508 struct delayed_work orphans_remove;
514 bool orphans_remove_sched; 509 bool orphans_remove_sched;
515 }; 510 };
516 511
517 /* 512 /*
518 * Number of contexts where an event can trigger: 513 * Number of contexts where an event can trigger:
519 * task, softirq, hardirq, nmi. 514 * task, softirq, hardirq, nmi.
520 */ 515 */
521 #define PERF_NR_CONTEXTS 4 516 #define PERF_NR_CONTEXTS 4
522 517
523 /** 518 /**
524 * struct perf_event_cpu_context - per cpu event context structure 519 * struct perf_event_cpu_context - per cpu event context structure
525 */ 520 */
526 struct perf_cpu_context { 521 struct perf_cpu_context {
527 struct perf_event_context ctx; 522 struct perf_event_context ctx;
528 struct perf_event_context *task_ctx; 523 struct perf_event_context *task_ctx;
529 int active_oncpu; 524 int active_oncpu;
530 int exclusive; 525 int exclusive;
531 struct hrtimer hrtimer; 526 struct hrtimer hrtimer;
532 ktime_t hrtimer_interval; 527 ktime_t hrtimer_interval;
533 struct list_head rotation_list; 528 struct list_head rotation_list;
534 struct pmu *unique_pmu; 529 struct pmu *unique_pmu;
535 struct perf_cgroup *cgrp; 530 struct perf_cgroup *cgrp;
536 }; 531 };
537 532
538 struct perf_output_handle { 533 struct perf_output_handle {
539 struct perf_event *event; 534 struct perf_event *event;
540 struct ring_buffer *rb; 535 struct ring_buffer *rb;
541 unsigned long wakeup; 536 unsigned long wakeup;
542 unsigned long size; 537 unsigned long size;
543 void *addr; 538 void *addr;
544 int page; 539 int page;
545 }; 540 };
546 541
547 #ifdef CONFIG_PERF_EVENTS 542 #ifdef CONFIG_PERF_EVENTS
548 543
549 extern int perf_pmu_register(struct pmu *pmu, const char *name, int type); 544 extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
550 extern void perf_pmu_unregister(struct pmu *pmu); 545 extern void perf_pmu_unregister(struct pmu *pmu);
551 546
552 extern int perf_num_counters(void); 547 extern int perf_num_counters(void);
553 extern const char *perf_pmu_name(void); 548 extern const char *perf_pmu_name(void);
554 extern void __perf_event_task_sched_in(struct task_struct *prev, 549 extern void __perf_event_task_sched_in(struct task_struct *prev,
555 struct task_struct *task); 550 struct task_struct *task);
556 extern void __perf_event_task_sched_out(struct task_struct *prev, 551 extern void __perf_event_task_sched_out(struct task_struct *prev,
557 struct task_struct *next); 552 struct task_struct *next);
558 extern int perf_event_init_task(struct task_struct *child); 553 extern int perf_event_init_task(struct task_struct *child);
559 extern void perf_event_exit_task(struct task_struct *child); 554 extern void perf_event_exit_task(struct task_struct *child);
560 extern void perf_event_free_task(struct task_struct *task); 555 extern void perf_event_free_task(struct task_struct *task);
561 extern void perf_event_delayed_put(struct task_struct *task); 556 extern void perf_event_delayed_put(struct task_struct *task);
562 extern void perf_event_print_debug(void); 557 extern void perf_event_print_debug(void);
563 extern void perf_pmu_disable(struct pmu *pmu); 558 extern void perf_pmu_disable(struct pmu *pmu);
564 extern void perf_pmu_enable(struct pmu *pmu); 559 extern void perf_pmu_enable(struct pmu *pmu);
565 extern int perf_event_task_disable(void); 560 extern int perf_event_task_disable(void);
566 extern int perf_event_task_enable(void); 561 extern int perf_event_task_enable(void);
567 extern int perf_event_refresh(struct perf_event *event, int refresh); 562 extern int perf_event_refresh(struct perf_event *event, int refresh);
568 extern void perf_event_update_userpage(struct perf_event *event); 563 extern void perf_event_update_userpage(struct perf_event *event);
569 extern int perf_event_release_kernel(struct perf_event *event); 564 extern int perf_event_release_kernel(struct perf_event *event);
570 extern struct perf_event * 565 extern struct perf_event *
571 perf_event_create_kernel_counter(struct perf_event_attr *attr, 566 perf_event_create_kernel_counter(struct perf_event_attr *attr,
572 int cpu, 567 int cpu,
573 struct task_struct *task, 568 struct task_struct *task,
574 perf_overflow_handler_t callback, 569 perf_overflow_handler_t callback,
575 void *context); 570 void *context);
576 extern void perf_pmu_migrate_context(struct pmu *pmu, 571 extern void perf_pmu_migrate_context(struct pmu *pmu,
577 int src_cpu, int dst_cpu); 572 int src_cpu, int dst_cpu);
578 extern u64 perf_event_read_value(struct perf_event *event, 573 extern u64 perf_event_read_value(struct perf_event *event,
579 u64 *enabled, u64 *running); 574 u64 *enabled, u64 *running);
580 575
581 576
582 struct perf_sample_data { 577 struct perf_sample_data {
583 /* 578 /*
584 * Fields set by perf_sample_data_init(), group so as to 579 * Fields set by perf_sample_data_init(), group so as to
585 * minimize the cachelines touched. 580 * minimize the cachelines touched.
586 */ 581 */
587 u64 addr; 582 u64 addr;
588 struct perf_raw_record *raw; 583 struct perf_raw_record *raw;
589 struct perf_branch_stack *br_stack; 584 struct perf_branch_stack *br_stack;
590 u64 period; 585 u64 period;
591 u64 weight; 586 u64 weight;
592 u64 txn; 587 u64 txn;
593 union perf_mem_data_src data_src; 588 union perf_mem_data_src data_src;
594 589
595 /* 590 /*
596 * The other fields, optionally {set,used} by 591 * The other fields, optionally {set,used} by
597 * perf_{prepare,output}_sample(). 592 * perf_{prepare,output}_sample().
598 */ 593 */
599 u64 type; 594 u64 type;
600 u64 ip; 595 u64 ip;
601 struct { 596 struct {
602 u32 pid; 597 u32 pid;
603 u32 tid; 598 u32 tid;
604 } tid_entry; 599 } tid_entry;
605 u64 time; 600 u64 time;
606 u64 id; 601 u64 id;
607 u64 stream_id; 602 u64 stream_id;
608 struct { 603 struct {
609 u32 cpu; 604 u32 cpu;
610 u32 reserved; 605 u32 reserved;
611 } cpu_entry; 606 } cpu_entry;
612 struct perf_callchain_entry *callchain; 607 struct perf_callchain_entry *callchain;
608
609 /*
610 * regs_user may point to task_pt_regs or to regs_user_copy, depending
611 * on arch details.
612 */
613 struct perf_regs regs_user; 613 struct perf_regs regs_user;
614 struct pt_regs regs_user_copy;
615
614 struct perf_regs regs_intr; 616 struct perf_regs regs_intr;
615 u64 stack_user_size; 617 u64 stack_user_size;
616 } ____cacheline_aligned; 618 } ____cacheline_aligned;
617 619
618 /* default value for data source */ 620 /* default value for data source */
619 #define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\ 621 #define PERF_MEM_NA (PERF_MEM_S(OP, NA) |\
620 PERF_MEM_S(LVL, NA) |\ 622 PERF_MEM_S(LVL, NA) |\
621 PERF_MEM_S(SNOOP, NA) |\ 623 PERF_MEM_S(SNOOP, NA) |\
622 PERF_MEM_S(LOCK, NA) |\ 624 PERF_MEM_S(LOCK, NA) |\
623 PERF_MEM_S(TLB, NA)) 625 PERF_MEM_S(TLB, NA))
624 626
625 static inline void perf_sample_data_init(struct perf_sample_data *data, 627 static inline void perf_sample_data_init(struct perf_sample_data *data,
626 u64 addr, u64 period) 628 u64 addr, u64 period)
627 { 629 {
628 /* remaining struct members initialized in perf_prepare_sample() */ 630 /* remaining struct members initialized in perf_prepare_sample() */
629 data->addr = addr; 631 data->addr = addr;
630 data->raw = NULL; 632 data->raw = NULL;
631 data->br_stack = NULL; 633 data->br_stack = NULL;
632 data->period = period; 634 data->period = period;
633 data->weight = 0; 635 data->weight = 0;
634 data->data_src.val = PERF_MEM_NA; 636 data->data_src.val = PERF_MEM_NA;
635 data->txn = 0; 637 data->txn = 0;
636 } 638 }
637 639
638 extern void perf_output_sample(struct perf_output_handle *handle, 640 extern void perf_output_sample(struct perf_output_handle *handle,
639 struct perf_event_header *header, 641 struct perf_event_header *header,
640 struct perf_sample_data *data, 642 struct perf_sample_data *data,
641 struct perf_event *event); 643 struct perf_event *event);
642 extern void perf_prepare_sample(struct perf_event_header *header, 644 extern void perf_prepare_sample(struct perf_event_header *header,
643 struct perf_sample_data *data, 645 struct perf_sample_data *data,
644 struct perf_event *event, 646 struct perf_event *event,
645 struct pt_regs *regs); 647 struct pt_regs *regs);
646 648
647 extern int perf_event_overflow(struct perf_event *event, 649 extern int perf_event_overflow(struct perf_event *event,
648 struct perf_sample_data *data, 650 struct perf_sample_data *data,
649 struct pt_regs *regs); 651 struct pt_regs *regs);
650 652
651 static inline bool is_sampling_event(struct perf_event *event) 653 static inline bool is_sampling_event(struct perf_event *event)
652 { 654 {
653 return event->attr.sample_period != 0; 655 return event->attr.sample_period != 0;
654 } 656 }
655 657
656 /* 658 /*
657 * Return 1 for a software event, 0 for a hardware event 659 * Return 1 for a software event, 0 for a hardware event
658 */ 660 */
659 static inline int is_software_event(struct perf_event *event) 661 static inline int is_software_event(struct perf_event *event)
660 { 662 {
661 return event->pmu->task_ctx_nr == perf_sw_context; 663 return event->pmu->task_ctx_nr == perf_sw_context;
662 } 664 }
663 665
664 extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 666 extern struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
665 667
666 extern void __perf_sw_event(u32, u64, struct pt_regs *, u64); 668 extern void __perf_sw_event(u32, u64, struct pt_regs *, u64);
667 669
668 #ifndef perf_arch_fetch_caller_regs 670 #ifndef perf_arch_fetch_caller_regs
669 static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { } 671 static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
670 #endif 672 #endif
671 673
672 /* 674 /*
673 * Take a snapshot of the regs. Skip ip and frame pointer to 675 * Take a snapshot of the regs. Skip ip and frame pointer to
674 * the nth caller. We only need a few of the regs: 676 * the nth caller. We only need a few of the regs:
675 * - ip for PERF_SAMPLE_IP 677 * - ip for PERF_SAMPLE_IP
676 * - cs for user_mode() tests 678 * - cs for user_mode() tests
677 * - bp for callchains 679 * - bp for callchains
678 * - eflags, for future purposes, just in case 680 * - eflags, for future purposes, just in case
679 */ 681 */
680 static inline void perf_fetch_caller_regs(struct pt_regs *regs) 682 static inline void perf_fetch_caller_regs(struct pt_regs *regs)
681 { 683 {
682 memset(regs, 0, sizeof(*regs)); 684 memset(regs, 0, sizeof(*regs));
683 685
684 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0); 686 perf_arch_fetch_caller_regs(regs, CALLER_ADDR0);
685 } 687 }
686 688
687 static __always_inline void 689 static __always_inline void
688 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 690 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
689 { 691 {
690 struct pt_regs hot_regs; 692 struct pt_regs hot_regs;
691 693
692 if (static_key_false(&perf_swevent_enabled[event_id])) { 694 if (static_key_false(&perf_swevent_enabled[event_id])) {
693 if (!regs) { 695 if (!regs) {
694 perf_fetch_caller_regs(&hot_regs); 696 perf_fetch_caller_regs(&hot_regs);
695 regs = &hot_regs; 697 regs = &hot_regs;
696 } 698 }
697 __perf_sw_event(event_id, nr, regs, addr); 699 __perf_sw_event(event_id, nr, regs, addr);
698 } 700 }
699 } 701 }
700 702
701 extern struct static_key_deferred perf_sched_events; 703 extern struct static_key_deferred perf_sched_events;
702 704
703 static inline void perf_event_task_sched_in(struct task_struct *prev, 705 static inline void perf_event_task_sched_in(struct task_struct *prev,
704 struct task_struct *task) 706 struct task_struct *task)
705 { 707 {
706 if (static_key_false(&perf_sched_events.key)) 708 if (static_key_false(&perf_sched_events.key))
707 __perf_event_task_sched_in(prev, task); 709 __perf_event_task_sched_in(prev, task);
708 } 710 }
709 711
710 static inline void perf_event_task_sched_out(struct task_struct *prev, 712 static inline void perf_event_task_sched_out(struct task_struct *prev,
711 struct task_struct *next) 713 struct task_struct *next)
712 { 714 {
713 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0); 715 perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
714 716
715 if (static_key_false(&perf_sched_events.key)) 717 if (static_key_false(&perf_sched_events.key))
716 __perf_event_task_sched_out(prev, next); 718 __perf_event_task_sched_out(prev, next);
717 } 719 }
718 720
719 extern void perf_event_mmap(struct vm_area_struct *vma); 721 extern void perf_event_mmap(struct vm_area_struct *vma);
720 extern struct perf_guest_info_callbacks *perf_guest_cbs; 722 extern struct perf_guest_info_callbacks *perf_guest_cbs;
721 extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); 723 extern int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
722 extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks); 724 extern int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *callbacks);
723 725
724 extern void perf_event_exec(void); 726 extern void perf_event_exec(void);
725 extern void perf_event_comm(struct task_struct *tsk, bool exec); 727 extern void perf_event_comm(struct task_struct *tsk, bool exec);
726 extern void perf_event_fork(struct task_struct *tsk); 728 extern void perf_event_fork(struct task_struct *tsk);
727 729
728 /* Callchains */ 730 /* Callchains */
729 DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry); 731 DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
730 732
731 extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs); 733 extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
732 extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs); 734 extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
733 735
734 static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip) 736 static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
735 { 737 {
736 if (entry->nr < PERF_MAX_STACK_DEPTH) 738 if (entry->nr < PERF_MAX_STACK_DEPTH)
737 entry->ip[entry->nr++] = ip; 739 entry->ip[entry->nr++] = ip;
738 } 740 }
739 741
740 extern int sysctl_perf_event_paranoid; 742 extern int sysctl_perf_event_paranoid;
741 extern int sysctl_perf_event_mlock; 743 extern int sysctl_perf_event_mlock;
742 extern int sysctl_perf_event_sample_rate; 744 extern int sysctl_perf_event_sample_rate;
743 extern int sysctl_perf_cpu_time_max_percent; 745 extern int sysctl_perf_cpu_time_max_percent;
744 746
745 extern void perf_sample_event_took(u64 sample_len_ns); 747 extern void perf_sample_event_took(u64 sample_len_ns);
746 748
747 extern int perf_proc_update_handler(struct ctl_table *table, int write, 749 extern int perf_proc_update_handler(struct ctl_table *table, int write,
748 void __user *buffer, size_t *lenp, 750 void __user *buffer, size_t *lenp,
749 loff_t *ppos); 751 loff_t *ppos);
750 extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 752 extern int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
751 void __user *buffer, size_t *lenp, 753 void __user *buffer, size_t *lenp,
752 loff_t *ppos); 754 loff_t *ppos);
753 755
754 756
755 static inline bool perf_paranoid_tracepoint_raw(void) 757 static inline bool perf_paranoid_tracepoint_raw(void)
756 { 758 {
757 return sysctl_perf_event_paranoid > -1; 759 return sysctl_perf_event_paranoid > -1;
758 } 760 }
759 761
760 static inline bool perf_paranoid_cpu(void) 762 static inline bool perf_paranoid_cpu(void)
761 { 763 {
762 return sysctl_perf_event_paranoid > 0; 764 return sysctl_perf_event_paranoid > 0;
763 } 765 }
764 766
765 static inline bool perf_paranoid_kernel(void) 767 static inline bool perf_paranoid_kernel(void)
766 { 768 {
767 return sysctl_perf_event_paranoid > 1; 769 return sysctl_perf_event_paranoid > 1;
768 } 770 }
769 771
770 extern void perf_event_init(void); 772 extern void perf_event_init(void);
771 extern void perf_tp_event(u64 addr, u64 count, void *record, 773 extern void perf_tp_event(u64 addr, u64 count, void *record,
772 int entry_size, struct pt_regs *regs, 774 int entry_size, struct pt_regs *regs,
773 struct hlist_head *head, int rctx, 775 struct hlist_head *head, int rctx,
774 struct task_struct *task); 776 struct task_struct *task);
775 extern void perf_bp_event(struct perf_event *event, void *data); 777 extern void perf_bp_event(struct perf_event *event, void *data);
776 778
777 #ifndef perf_misc_flags 779 #ifndef perf_misc_flags
778 # define perf_misc_flags(regs) \ 780 # define perf_misc_flags(regs) \
779 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL) 781 (user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
780 # define perf_instruction_pointer(regs) instruction_pointer(regs) 782 # define perf_instruction_pointer(regs) instruction_pointer(regs)
781 #endif 783 #endif
782 784
783 static inline bool has_branch_stack(struct perf_event *event) 785 static inline bool has_branch_stack(struct perf_event *event)
784 { 786 {
785 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK; 787 return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
786 } 788 }
787 789
788 extern int perf_output_begin(struct perf_output_handle *handle, 790 extern int perf_output_begin(struct perf_output_handle *handle,
789 struct perf_event *event, unsigned int size); 791 struct perf_event *event, unsigned int size);
790 extern void perf_output_end(struct perf_output_handle *handle); 792 extern void perf_output_end(struct perf_output_handle *handle);
791 extern unsigned int perf_output_copy(struct perf_output_handle *handle, 793 extern unsigned int perf_output_copy(struct perf_output_handle *handle,
792 const void *buf, unsigned int len); 794 const void *buf, unsigned int len);
793 extern unsigned int perf_output_skip(struct perf_output_handle *handle, 795 extern unsigned int perf_output_skip(struct perf_output_handle *handle,
794 unsigned int len); 796 unsigned int len);
795 extern int perf_swevent_get_recursion_context(void); 797 extern int perf_swevent_get_recursion_context(void);
796 extern void perf_swevent_put_recursion_context(int rctx); 798 extern void perf_swevent_put_recursion_context(int rctx);
797 extern u64 perf_swevent_set_period(struct perf_event *event); 799 extern u64 perf_swevent_set_period(struct perf_event *event);
798 extern void perf_event_enable(struct perf_event *event); 800 extern void perf_event_enable(struct perf_event *event);
799 extern void perf_event_disable(struct perf_event *event); 801 extern void perf_event_disable(struct perf_event *event);
800 extern int __perf_event_disable(void *info); 802 extern int __perf_event_disable(void *info);
801 extern void perf_event_task_tick(void); 803 extern void perf_event_task_tick(void);
802 #else /* !CONFIG_PERF_EVENTS: */ 804 #else /* !CONFIG_PERF_EVENTS: */
803 static inline void 805 static inline void
804 perf_event_task_sched_in(struct task_struct *prev, 806 perf_event_task_sched_in(struct task_struct *prev,
805 struct task_struct *task) { } 807 struct task_struct *task) { }
806 static inline void 808 static inline void
807 perf_event_task_sched_out(struct task_struct *prev, 809 perf_event_task_sched_out(struct task_struct *prev,
808 struct task_struct *next) { } 810 struct task_struct *next) { }
809 static inline int perf_event_init_task(struct task_struct *child) { return 0; } 811 static inline int perf_event_init_task(struct task_struct *child) { return 0; }
810 static inline void perf_event_exit_task(struct task_struct *child) { } 812 static inline void perf_event_exit_task(struct task_struct *child) { }
811 static inline void perf_event_free_task(struct task_struct *task) { } 813 static inline void perf_event_free_task(struct task_struct *task) { }
812 static inline void perf_event_delayed_put(struct task_struct *task) { } 814 static inline void perf_event_delayed_put(struct task_struct *task) { }
813 static inline void perf_event_print_debug(void) { } 815 static inline void perf_event_print_debug(void) { }
814 static inline int perf_event_task_disable(void) { return -EINVAL; } 816 static inline int perf_event_task_disable(void) { return -EINVAL; }
815 static inline int perf_event_task_enable(void) { return -EINVAL; } 817 static inline int perf_event_task_enable(void) { return -EINVAL; }
816 static inline int perf_event_refresh(struct perf_event *event, int refresh) 818 static inline int perf_event_refresh(struct perf_event *event, int refresh)
817 { 819 {
818 return -EINVAL; 820 return -EINVAL;
819 } 821 }
820 822
821 static inline void 823 static inline void
822 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { } 824 perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) { }
823 static inline void 825 static inline void
824 perf_bp_event(struct perf_event *event, void *data) { } 826 perf_bp_event(struct perf_event *event, void *data) { }
825 827
826 static inline int perf_register_guest_info_callbacks 828 static inline int perf_register_guest_info_callbacks
827 (struct perf_guest_info_callbacks *callbacks) { return 0; } 829 (struct perf_guest_info_callbacks *callbacks) { return 0; }
828 static inline int perf_unregister_guest_info_callbacks 830 static inline int perf_unregister_guest_info_callbacks
829 (struct perf_guest_info_callbacks *callbacks) { return 0; } 831 (struct perf_guest_info_callbacks *callbacks) { return 0; }
830 832
831 static inline void perf_event_mmap(struct vm_area_struct *vma) { } 833 static inline void perf_event_mmap(struct vm_area_struct *vma) { }
832 static inline void perf_event_exec(void) { } 834 static inline void perf_event_exec(void) { }
833 static inline void perf_event_comm(struct task_struct *tsk, bool exec) { } 835 static inline void perf_event_comm(struct task_struct *tsk, bool exec) { }
834 static inline void perf_event_fork(struct task_struct *tsk) { } 836 static inline void perf_event_fork(struct task_struct *tsk) { }
835 static inline void perf_event_init(void) { } 837 static inline void perf_event_init(void) { }
836 static inline int perf_swevent_get_recursion_context(void) { return -1; } 838 static inline int perf_swevent_get_recursion_context(void) { return -1; }
837 static inline void perf_swevent_put_recursion_context(int rctx) { } 839 static inline void perf_swevent_put_recursion_context(int rctx) { }
838 static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; } 840 static inline u64 perf_swevent_set_period(struct perf_event *event) { return 0; }
839 static inline void perf_event_enable(struct perf_event *event) { } 841 static inline void perf_event_enable(struct perf_event *event) { }
840 static inline void perf_event_disable(struct perf_event *event) { } 842 static inline void perf_event_disable(struct perf_event *event) { }
841 static inline int __perf_event_disable(void *info) { return -1; } 843 static inline int __perf_event_disable(void *info) { return -1; }
842 static inline void perf_event_task_tick(void) { } 844 static inline void perf_event_task_tick(void) { }
843 #endif 845 #endif
844 846
845 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL) 847 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_NO_HZ_FULL)
846 extern bool perf_event_can_stop_tick(void); 848 extern bool perf_event_can_stop_tick(void);
847 #else 849 #else
848 static inline bool perf_event_can_stop_tick(void) { return true; } 850 static inline bool perf_event_can_stop_tick(void) { return true; }
849 #endif 851 #endif
850 852
851 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) 853 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL)
852 extern void perf_restore_debug_store(void); 854 extern void perf_restore_debug_store(void);
853 #else 855 #else
854 static inline void perf_restore_debug_store(void) { } 856 static inline void perf_restore_debug_store(void) { }
855 #endif 857 #endif
856 858
857 #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x)) 859 #define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
858 860
859 /* 861 /*
860 * This has to have a higher priority than migration_notifier in sched/core.c. 862 * This has to have a higher priority than migration_notifier in sched/core.c.
861 */ 863 */
862 #define perf_cpu_notifier(fn) \ 864 #define perf_cpu_notifier(fn) \
863 do { \ 865 do { \
864 static struct notifier_block fn##_nb = \ 866 static struct notifier_block fn##_nb = \
865 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ 867 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
866 unsigned long cpu = smp_processor_id(); \ 868 unsigned long cpu = smp_processor_id(); \
867 unsigned long flags; \ 869 unsigned long flags; \
868 \ 870 \
869 cpu_notifier_register_begin(); \ 871 cpu_notifier_register_begin(); \
870 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \ 872 fn(&fn##_nb, (unsigned long)CPU_UP_PREPARE, \
871 (void *)(unsigned long)cpu); \ 873 (void *)(unsigned long)cpu); \
872 local_irq_save(flags); \ 874 local_irq_save(flags); \
873 fn(&fn##_nb, (unsigned long)CPU_STARTING, \ 875 fn(&fn##_nb, (unsigned long)CPU_STARTING, \
874 (void *)(unsigned long)cpu); \ 876 (void *)(unsigned long)cpu); \
875 local_irq_restore(flags); \ 877 local_irq_restore(flags); \
876 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \ 878 fn(&fn##_nb, (unsigned long)CPU_ONLINE, \
877 (void *)(unsigned long)cpu); \ 879 (void *)(unsigned long)cpu); \
878 __register_cpu_notifier(&fn##_nb); \ 880 __register_cpu_notifier(&fn##_nb); \
879 cpu_notifier_register_done(); \ 881 cpu_notifier_register_done(); \
880 } while (0) 882 } while (0)
881 883
882 /* 884 /*
883 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the 885 * Bare-bones version of perf_cpu_notifier(), which doesn't invoke the
884 * callback for already online CPUs. 886 * callback for already online CPUs.
885 */ 887 */
886 #define __perf_cpu_notifier(fn) \ 888 #define __perf_cpu_notifier(fn) \
887 do { \ 889 do { \
888 static struct notifier_block fn##_nb = \ 890 static struct notifier_block fn##_nb = \
889 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \ 891 { .notifier_call = fn, .priority = CPU_PRI_PERF }; \
890 \ 892 \
891 __register_cpu_notifier(&fn##_nb); \ 893 __register_cpu_notifier(&fn##_nb); \
892 } while (0) 894 } while (0)
893 895
894 struct perf_pmu_events_attr { 896 struct perf_pmu_events_attr {
895 struct device_attribute attr; 897 struct device_attribute attr;
896 u64 id; 898 u64 id;
897 const char *event_str; 899 const char *event_str;
898 }; 900 };
899 901
900 #define PMU_EVENT_ATTR(_name, _var, _id, _show) \ 902 #define PMU_EVENT_ATTR(_name, _var, _id, _show) \
901 static struct perf_pmu_events_attr _var = { \ 903 static struct perf_pmu_events_attr _var = { \
902 .attr = __ATTR(_name, 0444, _show, NULL), \ 904 .attr = __ATTR(_name, 0444, _show, NULL), \
903 .id = _id, \ 905 .id = _id, \
904 }; 906 };
905 907
906 #define PMU_FORMAT_ATTR(_name, _format) \ 908 #define PMU_FORMAT_ATTR(_name, _format) \
907 static ssize_t \ 909 static ssize_t \
908 _name##_show(struct device *dev, \ 910 _name##_show(struct device *dev, \
909 struct device_attribute *attr, \ 911 struct device_attribute *attr, \
910 char *page) \ 912 char *page) \
911 { \ 913 { \
912 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ 914 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
913 return sprintf(page, _format "\n"); \ 915 return sprintf(page, _format "\n"); \
914 } \ 916 } \
include/linux/perf_regs.h
Diff comments   View file @ ddb321a
1 #ifndef _LINUX_PERF_REGS_H 1 #ifndef _LINUX_PERF_REGS_H
2 #define _LINUX_PERF_REGS_H 2 #define _LINUX_PERF_REGS_H
3 3
4 struct perf_regs {
5 __u64 abi;
6 struct pt_regs *regs;
7 };
8
4 #ifdef CONFIG_HAVE_PERF_REGS 9 #ifdef CONFIG_HAVE_PERF_REGS
5 #include <asm/perf_regs.h> 10 #include <asm/perf_regs.h>
6 u64 perf_reg_value(struct pt_regs *regs, int idx); 11 u64 perf_reg_value(struct pt_regs *regs, int idx);
7 int perf_reg_validate(u64 mask); 12 int perf_reg_validate(u64 mask);
8 u64 perf_reg_abi(struct task_struct *task); 13 u64 perf_reg_abi(struct task_struct *task);
14 void perf_get_regs_user(struct perf_regs *regs_user,
15 struct pt_regs *regs,
16 struct pt_regs *regs_user_copy);
9 #else 17 #else
10 static inline u64 perf_reg_value(struct pt_regs *regs, int idx) 18 static inline u64 perf_reg_value(struct pt_regs *regs, int idx)
11 { 19 {
12 return 0; 20 return 0;
13 } 21 }
14 22
15 static inline int perf_reg_validate(u64 mask) 23 static inline int perf_reg_validate(u64 mask)
16 { 24 {
17 return mask ? -ENOSYS : 0; 25 return mask ? -ENOSYS : 0;
18 } 26 }
19 27
20 static inline u64 perf_reg_abi(struct task_struct *task) 28 static inline u64 perf_reg_abi(struct task_struct *task)
21 { 29 {
22 return PERF_SAMPLE_REGS_ABI_NONE; 30 return PERF_SAMPLE_REGS_ABI_NONE;
31 }
32
33 static inline void perf_get_regs_user(struct perf_regs *regs_user,
34 struct pt_regs *regs,
35 struct pt_regs *regs_user_copy)
36 {
37 regs_user->regs = task_pt_regs(current);
38 regs_user->abi = perf_reg_abi(current);
23 } 39 }
24 #endif /* CONFIG_HAVE_PERF_REGS */ 40 #endif /* CONFIG_HAVE_PERF_REGS */
25 #endif /* _LINUX_PERF_REGS_H */ 41 #endif /* _LINUX_PERF_REGS_H */
26 42
kernel/events/core.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * Performance events core code: 2 * Performance events core code:
3 * 3 *
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 * Copyright ยฉ 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 7 * Copyright ยฉ 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 * 8 *
9 * For licensing details see kernel-base/COPYING 9 * For licensing details see kernel-base/COPYING
10 */ 10 */
11 11
12 #include <linux/fs.h> 12 #include <linux/fs.h>
13 #include <linux/mm.h> 13 #include <linux/mm.h>
14 #include <linux/cpu.h> 14 #include <linux/cpu.h>
15 #include <linux/smp.h> 15 #include <linux/smp.h>
16 #include <linux/idr.h> 16 #include <linux/idr.h>
17 #include <linux/file.h> 17 #include <linux/file.h>
18 #include <linux/poll.h> 18 #include <linux/poll.h>
19 #include <linux/slab.h> 19 #include <linux/slab.h>
20 #include <linux/hash.h> 20 #include <linux/hash.h>
21 #include <linux/tick.h> 21 #include <linux/tick.h>
22 #include <linux/sysfs.h> 22 #include <linux/sysfs.h>
23 #include <linux/dcache.h> 23 #include <linux/dcache.h>
24 #include <linux/percpu.h> 24 #include <linux/percpu.h>
25 #include <linux/ptrace.h> 25 #include <linux/ptrace.h>
26 #include <linux/reboot.h> 26 #include <linux/reboot.h>
27 #include <linux/vmstat.h> 27 #include <linux/vmstat.h>
28 #include <linux/device.h> 28 #include <linux/device.h>
29 #include <linux/export.h> 29 #include <linux/export.h>
30 #include <linux/vmalloc.h> 30 #include <linux/vmalloc.h>
31 #include <linux/hardirq.h> 31 #include <linux/hardirq.h>
32 #include <linux/rculist.h> 32 #include <linux/rculist.h>
33 #include <linux/uaccess.h> 33 #include <linux/uaccess.h>
34 #include <linux/syscalls.h> 34 #include <linux/syscalls.h>
35 #include <linux/anon_inodes.h> 35 #include <linux/anon_inodes.h>
36 #include <linux/kernel_stat.h> 36 #include <linux/kernel_stat.h>
37 #include <linux/perf_event.h> 37 #include <linux/perf_event.h>
38 #include <linux/ftrace_event.h> 38 #include <linux/ftrace_event.h>
39 #include <linux/hw_breakpoint.h> 39 #include <linux/hw_breakpoint.h>
40 #include <linux/mm_types.h> 40 #include <linux/mm_types.h>
41 #include <linux/cgroup.h> 41 #include <linux/cgroup.h>
42 #include <linux/module.h> 42 #include <linux/module.h>
43 #include <linux/mman.h> 43 #include <linux/mman.h>
44 #include <linux/compat.h> 44 #include <linux/compat.h>
45 45
46 #include "internal.h" 46 #include "internal.h"
47 47
48 #include <asm/irq_regs.h> 48 #include <asm/irq_regs.h>
49 49
50 static struct workqueue_struct *perf_wq; 50 static struct workqueue_struct *perf_wq;
51 51
52 struct remote_function_call { 52 struct remote_function_call {
53 struct task_struct *p; 53 struct task_struct *p;
54 int (*func)(void *info); 54 int (*func)(void *info);
55 void *info; 55 void *info;
56 int ret; 56 int ret;
57 }; 57 };
58 58
59 static void remote_function(void *data) 59 static void remote_function(void *data)
60 { 60 {
61 struct remote_function_call *tfc = data; 61 struct remote_function_call *tfc = data;
62 struct task_struct *p = tfc->p; 62 struct task_struct *p = tfc->p;
63 63
64 if (p) { 64 if (p) {
65 tfc->ret = -EAGAIN; 65 tfc->ret = -EAGAIN;
66 if (task_cpu(p) != smp_processor_id() || !task_curr(p)) 66 if (task_cpu(p) != smp_processor_id() || !task_curr(p))
67 return; 67 return;
68 } 68 }
69 69
70 tfc->ret = tfc->func(tfc->info); 70 tfc->ret = tfc->func(tfc->info);
71 } 71 }
72 72
73 /** 73 /**
74 * task_function_call - call a function on the cpu on which a task runs 74 * task_function_call - call a function on the cpu on which a task runs
75 * @p: the task to evaluate 75 * @p: the task to evaluate
76 * @func: the function to be called 76 * @func: the function to be called
77 * @info: the function call argument 77 * @info: the function call argument
78 * 78 *
79 * Calls the function @func when the task is currently running. This might 79 * Calls the function @func when the task is currently running. This might
80 * be on the current CPU, which just calls the function directly 80 * be on the current CPU, which just calls the function directly
81 * 81 *
82 * returns: @func return value, or 82 * returns: @func return value, or
83 * -ESRCH - when the process isn't running 83 * -ESRCH - when the process isn't running
84 * -EAGAIN - when the process moved away 84 * -EAGAIN - when the process moved away
85 */ 85 */
86 static int 86 static int
87 task_function_call(struct task_struct *p, int (*func) (void *info), void *info) 87 task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
88 { 88 {
89 struct remote_function_call data = { 89 struct remote_function_call data = {
90 .p = p, 90 .p = p,
91 .func = func, 91 .func = func,
92 .info = info, 92 .info = info,
93 .ret = -ESRCH, /* No such (running) process */ 93 .ret = -ESRCH, /* No such (running) process */
94 }; 94 };
95 95
96 if (task_curr(p)) 96 if (task_curr(p))
97 smp_call_function_single(task_cpu(p), remote_function, &data, 1); 97 smp_call_function_single(task_cpu(p), remote_function, &data, 1);
98 98
99 return data.ret; 99 return data.ret;
100 } 100 }
101 101
102 /** 102 /**
103 * cpu_function_call - call a function on the cpu 103 * cpu_function_call - call a function on the cpu
104 * @func: the function to be called 104 * @func: the function to be called
105 * @info: the function call argument 105 * @info: the function call argument
106 * 106 *
107 * Calls the function @func on the remote cpu. 107 * Calls the function @func on the remote cpu.
108 * 108 *
109 * returns: @func return value or -ENXIO when the cpu is offline 109 * returns: @func return value or -ENXIO when the cpu is offline
110 */ 110 */
111 static int cpu_function_call(int cpu, int (*func) (void *info), void *info) 111 static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
112 { 112 {
113 struct remote_function_call data = { 113 struct remote_function_call data = {
114 .p = NULL, 114 .p = NULL,
115 .func = func, 115 .func = func,
116 .info = info, 116 .info = info,
117 .ret = -ENXIO, /* No such CPU */ 117 .ret = -ENXIO, /* No such CPU */
118 }; 118 };
119 119
120 smp_call_function_single(cpu, remote_function, &data, 1); 120 smp_call_function_single(cpu, remote_function, &data, 1);
121 121
122 return data.ret; 122 return data.ret;
123 } 123 }
124 124
125 #define EVENT_OWNER_KERNEL ((void *) -1) 125 #define EVENT_OWNER_KERNEL ((void *) -1)
126 126
127 static bool is_kernel_event(struct perf_event *event) 127 static bool is_kernel_event(struct perf_event *event)
128 { 128 {
129 return event->owner == EVENT_OWNER_KERNEL; 129 return event->owner == EVENT_OWNER_KERNEL;
130 } 130 }
131 131
132 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 132 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
133 PERF_FLAG_FD_OUTPUT |\ 133 PERF_FLAG_FD_OUTPUT |\
134 PERF_FLAG_PID_CGROUP |\ 134 PERF_FLAG_PID_CGROUP |\
135 PERF_FLAG_FD_CLOEXEC) 135 PERF_FLAG_FD_CLOEXEC)
136 136
137 /* 137 /*
138 * branch priv levels that need permission checks 138 * branch priv levels that need permission checks
139 */ 139 */
140 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 140 #define PERF_SAMPLE_BRANCH_PERM_PLM \
141 (PERF_SAMPLE_BRANCH_KERNEL |\ 141 (PERF_SAMPLE_BRANCH_KERNEL |\
142 PERF_SAMPLE_BRANCH_HV) 142 PERF_SAMPLE_BRANCH_HV)
143 143
144 enum event_type_t { 144 enum event_type_t {
145 EVENT_FLEXIBLE = 0x1, 145 EVENT_FLEXIBLE = 0x1,
146 EVENT_PINNED = 0x2, 146 EVENT_PINNED = 0x2,
147 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 147 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
148 }; 148 };
149 149
150 /* 150 /*
151 * perf_sched_events : >0 events exist 151 * perf_sched_events : >0 events exist
152 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 152 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
153 */ 153 */
154 struct static_key_deferred perf_sched_events __read_mostly; 154 struct static_key_deferred perf_sched_events __read_mostly;
155 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 155 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
156 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events); 156 static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
157 157
158 static atomic_t nr_mmap_events __read_mostly; 158 static atomic_t nr_mmap_events __read_mostly;
159 static atomic_t nr_comm_events __read_mostly; 159 static atomic_t nr_comm_events __read_mostly;
160 static atomic_t nr_task_events __read_mostly; 160 static atomic_t nr_task_events __read_mostly;
161 static atomic_t nr_freq_events __read_mostly; 161 static atomic_t nr_freq_events __read_mostly;
162 162
163 static LIST_HEAD(pmus); 163 static LIST_HEAD(pmus);
164 static DEFINE_MUTEX(pmus_lock); 164 static DEFINE_MUTEX(pmus_lock);
165 static struct srcu_struct pmus_srcu; 165 static struct srcu_struct pmus_srcu;
166 166
167 /* 167 /*
168 * perf event paranoia level: 168 * perf event paranoia level:
169 * -1 - not paranoid at all 169 * -1 - not paranoid at all
170 * 0 - disallow raw tracepoint access for unpriv 170 * 0 - disallow raw tracepoint access for unpriv
171 * 1 - disallow cpu events for unpriv 171 * 1 - disallow cpu events for unpriv
172 * 2 - disallow kernel profiling for unpriv 172 * 2 - disallow kernel profiling for unpriv
173 */ 173 */
174 int sysctl_perf_event_paranoid __read_mostly = 1; 174 int sysctl_perf_event_paranoid __read_mostly = 1;
175 175
176 /* Minimum for 512 kiB + 1 user control page */ 176 /* Minimum for 512 kiB + 1 user control page */
177 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 177 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
178 178
179 /* 179 /*
180 * max perf event sample rate 180 * max perf event sample rate
181 */ 181 */
182 #define DEFAULT_MAX_SAMPLE_RATE 100000 182 #define DEFAULT_MAX_SAMPLE_RATE 100000
183 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 183 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
184 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 184 #define DEFAULT_CPU_TIME_MAX_PERCENT 25
185 185
186 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 186 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
187 187
188 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 188 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
189 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 189 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS;
190 190
191 static int perf_sample_allowed_ns __read_mostly = 191 static int perf_sample_allowed_ns __read_mostly =
192 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 192 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
193 193
194 void update_perf_cpu_limits(void) 194 void update_perf_cpu_limits(void)
195 { 195 {
196 u64 tmp = perf_sample_period_ns; 196 u64 tmp = perf_sample_period_ns;
197 197
198 tmp *= sysctl_perf_cpu_time_max_percent; 198 tmp *= sysctl_perf_cpu_time_max_percent;
199 do_div(tmp, 100); 199 do_div(tmp, 100);
200 ACCESS_ONCE(perf_sample_allowed_ns) = tmp; 200 ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
201 } 201 }
202 202
203 static int perf_rotate_context(struct perf_cpu_context *cpuctx); 203 static int perf_rotate_context(struct perf_cpu_context *cpuctx);
204 204
205 int perf_proc_update_handler(struct ctl_table *table, int write, 205 int perf_proc_update_handler(struct ctl_table *table, int write,
206 void __user *buffer, size_t *lenp, 206 void __user *buffer, size_t *lenp,
207 loff_t *ppos) 207 loff_t *ppos)
208 { 208 {
209 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 209 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
210 210
211 if (ret || !write) 211 if (ret || !write)
212 return ret; 212 return ret;
213 213
214 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 214 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
215 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 215 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
216 update_perf_cpu_limits(); 216 update_perf_cpu_limits();
217 217
218 return 0; 218 return 0;
219 } 219 }
220 220
221 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 221 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
222 222
223 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 223 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
224 void __user *buffer, size_t *lenp, 224 void __user *buffer, size_t *lenp,
225 loff_t *ppos) 225 loff_t *ppos)
226 { 226 {
227 int ret = proc_dointvec(table, write, buffer, lenp, ppos); 227 int ret = proc_dointvec(table, write, buffer, lenp, ppos);
228 228
229 if (ret || !write) 229 if (ret || !write)
230 return ret; 230 return ret;
231 231
232 update_perf_cpu_limits(); 232 update_perf_cpu_limits();
233 233
234 return 0; 234 return 0;
235 } 235 }
236 236
237 /* 237 /*
238 * perf samples are done in some very critical code paths (NMIs). 238 * perf samples are done in some very critical code paths (NMIs).
239 * If they take too much CPU time, the system can lock up and not 239 * If they take too much CPU time, the system can lock up and not
240 * get any real work done. This will drop the sample rate when 240 * get any real work done. This will drop the sample rate when
241 * we detect that events are taking too long. 241 * we detect that events are taking too long.
242 */ 242 */
243 #define NR_ACCUMULATED_SAMPLES 128 243 #define NR_ACCUMULATED_SAMPLES 128
244 static DEFINE_PER_CPU(u64, running_sample_length); 244 static DEFINE_PER_CPU(u64, running_sample_length);
245 245
246 static void perf_duration_warn(struct irq_work *w) 246 static void perf_duration_warn(struct irq_work *w)
247 { 247 {
248 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 248 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
249 u64 avg_local_sample_len; 249 u64 avg_local_sample_len;
250 u64 local_samples_len; 250 u64 local_samples_len;
251 251
252 local_samples_len = __this_cpu_read(running_sample_length); 252 local_samples_len = __this_cpu_read(running_sample_length);
253 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 253 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
254 254
255 printk_ratelimited(KERN_WARNING 255 printk_ratelimited(KERN_WARNING
256 "perf interrupt took too long (%lld > %lld), lowering " 256 "perf interrupt took too long (%lld > %lld), lowering "
257 "kernel.perf_event_max_sample_rate to %d\n", 257 "kernel.perf_event_max_sample_rate to %d\n",
258 avg_local_sample_len, allowed_ns >> 1, 258 avg_local_sample_len, allowed_ns >> 1,
259 sysctl_perf_event_sample_rate); 259 sysctl_perf_event_sample_rate);
260 } 260 }
261 261
262 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 262 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
263 263
264 void perf_sample_event_took(u64 sample_len_ns) 264 void perf_sample_event_took(u64 sample_len_ns)
265 { 265 {
266 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns); 266 u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
267 u64 avg_local_sample_len; 267 u64 avg_local_sample_len;
268 u64 local_samples_len; 268 u64 local_samples_len;
269 269
270 if (allowed_ns == 0) 270 if (allowed_ns == 0)
271 return; 271 return;
272 272
273 /* decay the counter by 1 average sample */ 273 /* decay the counter by 1 average sample */
274 local_samples_len = __this_cpu_read(running_sample_length); 274 local_samples_len = __this_cpu_read(running_sample_length);
275 local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES; 275 local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
276 local_samples_len += sample_len_ns; 276 local_samples_len += sample_len_ns;
277 __this_cpu_write(running_sample_length, local_samples_len); 277 __this_cpu_write(running_sample_length, local_samples_len);
278 278
279 /* 279 /*
280 * note: this will be biased artifically low until we have 280 * note: this will be biased artifically low until we have
281 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 281 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us
282 * from having to maintain a count. 282 * from having to maintain a count.
283 */ 283 */
284 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES; 284 avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
285 285
286 if (avg_local_sample_len <= allowed_ns) 286 if (avg_local_sample_len <= allowed_ns)
287 return; 287 return;
288 288
289 if (max_samples_per_tick <= 1) 289 if (max_samples_per_tick <= 1)
290 return; 290 return;
291 291
292 max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2); 292 max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
293 sysctl_perf_event_sample_rate = max_samples_per_tick * HZ; 293 sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
294 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 294 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
295 295
296 update_perf_cpu_limits(); 296 update_perf_cpu_limits();
297 297
298 if (!irq_work_queue(&perf_duration_work)) { 298 if (!irq_work_queue(&perf_duration_work)) {
299 early_printk("perf interrupt took too long (%lld > %lld), lowering " 299 early_printk("perf interrupt took too long (%lld > %lld), lowering "
300 "kernel.perf_event_max_sample_rate to %d\n", 300 "kernel.perf_event_max_sample_rate to %d\n",
301 avg_local_sample_len, allowed_ns >> 1, 301 avg_local_sample_len, allowed_ns >> 1,
302 sysctl_perf_event_sample_rate); 302 sysctl_perf_event_sample_rate);
303 } 303 }
304 } 304 }
305 305
306 static atomic64_t perf_event_id; 306 static atomic64_t perf_event_id;
307 307
308 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 308 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
309 enum event_type_t event_type); 309 enum event_type_t event_type);
310 310
311 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 311 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
312 enum event_type_t event_type, 312 enum event_type_t event_type,
313 struct task_struct *task); 313 struct task_struct *task);
314 314
315 static void update_context_time(struct perf_event_context *ctx); 315 static void update_context_time(struct perf_event_context *ctx);
316 static u64 perf_event_time(struct perf_event *event); 316 static u64 perf_event_time(struct perf_event *event);
317 317
318 void __weak perf_event_print_debug(void) { } 318 void __weak perf_event_print_debug(void) { }
319 319
320 extern __weak const char *perf_pmu_name(void) 320 extern __weak const char *perf_pmu_name(void)
321 { 321 {
322 return "pmu"; 322 return "pmu";
323 } 323 }
324 324
325 static inline u64 perf_clock(void) 325 static inline u64 perf_clock(void)
326 { 326 {
327 return local_clock(); 327 return local_clock();
328 } 328 }
329 329
330 static inline struct perf_cpu_context * 330 static inline struct perf_cpu_context *
331 __get_cpu_context(struct perf_event_context *ctx) 331 __get_cpu_context(struct perf_event_context *ctx)
332 { 332 {
333 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 333 return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
334 } 334 }
335 335
336 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 336 static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
337 struct perf_event_context *ctx) 337 struct perf_event_context *ctx)
338 { 338 {
339 raw_spin_lock(&cpuctx->ctx.lock); 339 raw_spin_lock(&cpuctx->ctx.lock);
340 if (ctx) 340 if (ctx)
341 raw_spin_lock(&ctx->lock); 341 raw_spin_lock(&ctx->lock);
342 } 342 }
343 343
344 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 344 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
345 struct perf_event_context *ctx) 345 struct perf_event_context *ctx)
346 { 346 {
347 if (ctx) 347 if (ctx)
348 raw_spin_unlock(&ctx->lock); 348 raw_spin_unlock(&ctx->lock);
349 raw_spin_unlock(&cpuctx->ctx.lock); 349 raw_spin_unlock(&cpuctx->ctx.lock);
350 } 350 }
351 351
352 #ifdef CONFIG_CGROUP_PERF 352 #ifdef CONFIG_CGROUP_PERF
353 353
354 /* 354 /*
355 * perf_cgroup_info keeps track of time_enabled for a cgroup. 355 * perf_cgroup_info keeps track of time_enabled for a cgroup.
356 * This is a per-cpu dynamically allocated data structure. 356 * This is a per-cpu dynamically allocated data structure.
357 */ 357 */
358 struct perf_cgroup_info { 358 struct perf_cgroup_info {
359 u64 time; 359 u64 time;
360 u64 timestamp; 360 u64 timestamp;
361 }; 361 };
362 362
363 struct perf_cgroup { 363 struct perf_cgroup {
364 struct cgroup_subsys_state css; 364 struct cgroup_subsys_state css;
365 struct perf_cgroup_info __percpu *info; 365 struct perf_cgroup_info __percpu *info;
366 }; 366 };
367 367
368 /* 368 /*
369 * Must ensure cgroup is pinned (css_get) before calling 369 * Must ensure cgroup is pinned (css_get) before calling
370 * this function. In other words, we cannot call this function 370 * this function. In other words, we cannot call this function
371 * if there is no cgroup event for the current CPU context. 371 * if there is no cgroup event for the current CPU context.
372 */ 372 */
373 static inline struct perf_cgroup * 373 static inline struct perf_cgroup *
374 perf_cgroup_from_task(struct task_struct *task) 374 perf_cgroup_from_task(struct task_struct *task)
375 { 375 {
376 return container_of(task_css(task, perf_event_cgrp_id), 376 return container_of(task_css(task, perf_event_cgrp_id),
377 struct perf_cgroup, css); 377 struct perf_cgroup, css);
378 } 378 }
379 379
380 static inline bool 380 static inline bool
381 perf_cgroup_match(struct perf_event *event) 381 perf_cgroup_match(struct perf_event *event)
382 { 382 {
383 struct perf_event_context *ctx = event->ctx; 383 struct perf_event_context *ctx = event->ctx;
384 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 384 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
385 385
386 /* @event doesn't care about cgroup */ 386 /* @event doesn't care about cgroup */
387 if (!event->cgrp) 387 if (!event->cgrp)
388 return true; 388 return true;
389 389
390 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 390 /* wants specific cgroup scope but @cpuctx isn't associated with any */
391 if (!cpuctx->cgrp) 391 if (!cpuctx->cgrp)
392 return false; 392 return false;
393 393
394 /* 394 /*
395 * Cgroup scoping is recursive. An event enabled for a cgroup is 395 * Cgroup scoping is recursive. An event enabled for a cgroup is
396 * also enabled for all its descendant cgroups. If @cpuctx's 396 * also enabled for all its descendant cgroups. If @cpuctx's
397 * cgroup is a descendant of @event's (the test covers identity 397 * cgroup is a descendant of @event's (the test covers identity
398 * case), it's a match. 398 * case), it's a match.
399 */ 399 */
400 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 400 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
401 event->cgrp->css.cgroup); 401 event->cgrp->css.cgroup);
402 } 402 }
403 403
404 static inline void perf_detach_cgroup(struct perf_event *event) 404 static inline void perf_detach_cgroup(struct perf_event *event)
405 { 405 {
406 css_put(&event->cgrp->css); 406 css_put(&event->cgrp->css);
407 event->cgrp = NULL; 407 event->cgrp = NULL;
408 } 408 }
409 409
410 static inline int is_cgroup_event(struct perf_event *event) 410 static inline int is_cgroup_event(struct perf_event *event)
411 { 411 {
412 return event->cgrp != NULL; 412 return event->cgrp != NULL;
413 } 413 }
414 414
415 static inline u64 perf_cgroup_event_time(struct perf_event *event) 415 static inline u64 perf_cgroup_event_time(struct perf_event *event)
416 { 416 {
417 struct perf_cgroup_info *t; 417 struct perf_cgroup_info *t;
418 418
419 t = per_cpu_ptr(event->cgrp->info, event->cpu); 419 t = per_cpu_ptr(event->cgrp->info, event->cpu);
420 return t->time; 420 return t->time;
421 } 421 }
422 422
423 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 423 static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
424 { 424 {
425 struct perf_cgroup_info *info; 425 struct perf_cgroup_info *info;
426 u64 now; 426 u64 now;
427 427
428 now = perf_clock(); 428 now = perf_clock();
429 429
430 info = this_cpu_ptr(cgrp->info); 430 info = this_cpu_ptr(cgrp->info);
431 431
432 info->time += now - info->timestamp; 432 info->time += now - info->timestamp;
433 info->timestamp = now; 433 info->timestamp = now;
434 } 434 }
435 435
436 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 436 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
437 { 437 {
438 struct perf_cgroup *cgrp_out = cpuctx->cgrp; 438 struct perf_cgroup *cgrp_out = cpuctx->cgrp;
439 if (cgrp_out) 439 if (cgrp_out)
440 __update_cgrp_time(cgrp_out); 440 __update_cgrp_time(cgrp_out);
441 } 441 }
442 442
443 static inline void update_cgrp_time_from_event(struct perf_event *event) 443 static inline void update_cgrp_time_from_event(struct perf_event *event)
444 { 444 {
445 struct perf_cgroup *cgrp; 445 struct perf_cgroup *cgrp;
446 446
447 /* 447 /*
448 * ensure we access cgroup data only when needed and 448 * ensure we access cgroup data only when needed and
449 * when we know the cgroup is pinned (css_get) 449 * when we know the cgroup is pinned (css_get)
450 */ 450 */
451 if (!is_cgroup_event(event)) 451 if (!is_cgroup_event(event))
452 return; 452 return;
453 453
454 cgrp = perf_cgroup_from_task(current); 454 cgrp = perf_cgroup_from_task(current);
455 /* 455 /*
456 * Do not update time when cgroup is not active 456 * Do not update time when cgroup is not active
457 */ 457 */
458 if (cgrp == event->cgrp) 458 if (cgrp == event->cgrp)
459 __update_cgrp_time(event->cgrp); 459 __update_cgrp_time(event->cgrp);
460 } 460 }
461 461
462 static inline void 462 static inline void
463 perf_cgroup_set_timestamp(struct task_struct *task, 463 perf_cgroup_set_timestamp(struct task_struct *task,
464 struct perf_event_context *ctx) 464 struct perf_event_context *ctx)
465 { 465 {
466 struct perf_cgroup *cgrp; 466 struct perf_cgroup *cgrp;
467 struct perf_cgroup_info *info; 467 struct perf_cgroup_info *info;
468 468
469 /* 469 /*
470 * ctx->lock held by caller 470 * ctx->lock held by caller
471 * ensure we do not access cgroup data 471 * ensure we do not access cgroup data
472 * unless we have the cgroup pinned (css_get) 472 * unless we have the cgroup pinned (css_get)
473 */ 473 */
474 if (!task || !ctx->nr_cgroups) 474 if (!task || !ctx->nr_cgroups)
475 return; 475 return;
476 476
477 cgrp = perf_cgroup_from_task(task); 477 cgrp = perf_cgroup_from_task(task);
478 info = this_cpu_ptr(cgrp->info); 478 info = this_cpu_ptr(cgrp->info);
479 info->timestamp = ctx->timestamp; 479 info->timestamp = ctx->timestamp;
480 } 480 }
481 481
482 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 482 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */
483 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 483 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */
484 484
485 /* 485 /*
486 * reschedule events based on the cgroup constraint of task. 486 * reschedule events based on the cgroup constraint of task.
487 * 487 *
488 * mode SWOUT : schedule out everything 488 * mode SWOUT : schedule out everything
489 * mode SWIN : schedule in based on cgroup for next 489 * mode SWIN : schedule in based on cgroup for next
490 */ 490 */
491 void perf_cgroup_switch(struct task_struct *task, int mode) 491 void perf_cgroup_switch(struct task_struct *task, int mode)
492 { 492 {
493 struct perf_cpu_context *cpuctx; 493 struct perf_cpu_context *cpuctx;
494 struct pmu *pmu; 494 struct pmu *pmu;
495 unsigned long flags; 495 unsigned long flags;
496 496
497 /* 497 /*
498 * disable interrupts to avoid geting nr_cgroup 498 * disable interrupts to avoid geting nr_cgroup
499 * changes via __perf_event_disable(). Also 499 * changes via __perf_event_disable(). Also
500 * avoids preemption. 500 * avoids preemption.
501 */ 501 */
502 local_irq_save(flags); 502 local_irq_save(flags);
503 503
504 /* 504 /*
505 * we reschedule only in the presence of cgroup 505 * we reschedule only in the presence of cgroup
506 * constrained events. 506 * constrained events.
507 */ 507 */
508 rcu_read_lock(); 508 rcu_read_lock();
509 509
510 list_for_each_entry_rcu(pmu, &pmus, entry) { 510 list_for_each_entry_rcu(pmu, &pmus, entry) {
511 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 511 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
512 if (cpuctx->unique_pmu != pmu) 512 if (cpuctx->unique_pmu != pmu)
513 continue; /* ensure we process each cpuctx once */ 513 continue; /* ensure we process each cpuctx once */
514 514
515 /* 515 /*
516 * perf_cgroup_events says at least one 516 * perf_cgroup_events says at least one
517 * context on this CPU has cgroup events. 517 * context on this CPU has cgroup events.
518 * 518 *
519 * ctx->nr_cgroups reports the number of cgroup 519 * ctx->nr_cgroups reports the number of cgroup
520 * events for a context. 520 * events for a context.
521 */ 521 */
522 if (cpuctx->ctx.nr_cgroups > 0) { 522 if (cpuctx->ctx.nr_cgroups > 0) {
523 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 523 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
524 perf_pmu_disable(cpuctx->ctx.pmu); 524 perf_pmu_disable(cpuctx->ctx.pmu);
525 525
526 if (mode & PERF_CGROUP_SWOUT) { 526 if (mode & PERF_CGROUP_SWOUT) {
527 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 527 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
528 /* 528 /*
529 * must not be done before ctxswout due 529 * must not be done before ctxswout due
530 * to event_filter_match() in event_sched_out() 530 * to event_filter_match() in event_sched_out()
531 */ 531 */
532 cpuctx->cgrp = NULL; 532 cpuctx->cgrp = NULL;
533 } 533 }
534 534
535 if (mode & PERF_CGROUP_SWIN) { 535 if (mode & PERF_CGROUP_SWIN) {
536 WARN_ON_ONCE(cpuctx->cgrp); 536 WARN_ON_ONCE(cpuctx->cgrp);
537 /* 537 /*
538 * set cgrp before ctxsw in to allow 538 * set cgrp before ctxsw in to allow
539 * event_filter_match() to not have to pass 539 * event_filter_match() to not have to pass
540 * task around 540 * task around
541 */ 541 */
542 cpuctx->cgrp = perf_cgroup_from_task(task); 542 cpuctx->cgrp = perf_cgroup_from_task(task);
543 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 543 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
544 } 544 }
545 perf_pmu_enable(cpuctx->ctx.pmu); 545 perf_pmu_enable(cpuctx->ctx.pmu);
546 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 546 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
547 } 547 }
548 } 548 }
549 549
550 rcu_read_unlock(); 550 rcu_read_unlock();
551 551
552 local_irq_restore(flags); 552 local_irq_restore(flags);
553 } 553 }
554 554
555 static inline void perf_cgroup_sched_out(struct task_struct *task, 555 static inline void perf_cgroup_sched_out(struct task_struct *task,
556 struct task_struct *next) 556 struct task_struct *next)
557 { 557 {
558 struct perf_cgroup *cgrp1; 558 struct perf_cgroup *cgrp1;
559 struct perf_cgroup *cgrp2 = NULL; 559 struct perf_cgroup *cgrp2 = NULL;
560 560
561 /* 561 /*
562 * we come here when we know perf_cgroup_events > 0 562 * we come here when we know perf_cgroup_events > 0
563 */ 563 */
564 cgrp1 = perf_cgroup_from_task(task); 564 cgrp1 = perf_cgroup_from_task(task);
565 565
566 /* 566 /*
567 * next is NULL when called from perf_event_enable_on_exec() 567 * next is NULL when called from perf_event_enable_on_exec()
568 * that will systematically cause a cgroup_switch() 568 * that will systematically cause a cgroup_switch()
569 */ 569 */
570 if (next) 570 if (next)
571 cgrp2 = perf_cgroup_from_task(next); 571 cgrp2 = perf_cgroup_from_task(next);
572 572
573 /* 573 /*
574 * only schedule out current cgroup events if we know 574 * only schedule out current cgroup events if we know
575 * that we are switching to a different cgroup. Otherwise, 575 * that we are switching to a different cgroup. Otherwise,
576 * do no touch the cgroup events. 576 * do no touch the cgroup events.
577 */ 577 */
578 if (cgrp1 != cgrp2) 578 if (cgrp1 != cgrp2)
579 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 579 perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
580 } 580 }
581 581
582 static inline void perf_cgroup_sched_in(struct task_struct *prev, 582 static inline void perf_cgroup_sched_in(struct task_struct *prev,
583 struct task_struct *task) 583 struct task_struct *task)
584 { 584 {
585 struct perf_cgroup *cgrp1; 585 struct perf_cgroup *cgrp1;
586 struct perf_cgroup *cgrp2 = NULL; 586 struct perf_cgroup *cgrp2 = NULL;
587 587
588 /* 588 /*
589 * we come here when we know perf_cgroup_events > 0 589 * we come here when we know perf_cgroup_events > 0
590 */ 590 */
591 cgrp1 = perf_cgroup_from_task(task); 591 cgrp1 = perf_cgroup_from_task(task);
592 592
593 /* prev can never be NULL */ 593 /* prev can never be NULL */
594 cgrp2 = perf_cgroup_from_task(prev); 594 cgrp2 = perf_cgroup_from_task(prev);
595 595
596 /* 596 /*
597 * only need to schedule in cgroup events if we are changing 597 * only need to schedule in cgroup events if we are changing
598 * cgroup during ctxsw. Cgroup events were not scheduled 598 * cgroup during ctxsw. Cgroup events were not scheduled
599 * out of ctxsw out if that was not the case. 599 * out of ctxsw out if that was not the case.
600 */ 600 */
601 if (cgrp1 != cgrp2) 601 if (cgrp1 != cgrp2)
602 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 602 perf_cgroup_switch(task, PERF_CGROUP_SWIN);
603 } 603 }
604 604
605 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 605 static inline int perf_cgroup_connect(int fd, struct perf_event *event,
606 struct perf_event_attr *attr, 606 struct perf_event_attr *attr,
607 struct perf_event *group_leader) 607 struct perf_event *group_leader)
608 { 608 {
609 struct perf_cgroup *cgrp; 609 struct perf_cgroup *cgrp;
610 struct cgroup_subsys_state *css; 610 struct cgroup_subsys_state *css;
611 struct fd f = fdget(fd); 611 struct fd f = fdget(fd);
612 int ret = 0; 612 int ret = 0;
613 613
614 if (!f.file) 614 if (!f.file)
615 return -EBADF; 615 return -EBADF;
616 616
617 css = css_tryget_online_from_dir(f.file->f_path.dentry, 617 css = css_tryget_online_from_dir(f.file->f_path.dentry,
618 &perf_event_cgrp_subsys); 618 &perf_event_cgrp_subsys);
619 if (IS_ERR(css)) { 619 if (IS_ERR(css)) {
620 ret = PTR_ERR(css); 620 ret = PTR_ERR(css);
621 goto out; 621 goto out;
622 } 622 }
623 623
624 cgrp = container_of(css, struct perf_cgroup, css); 624 cgrp = container_of(css, struct perf_cgroup, css);
625 event->cgrp = cgrp; 625 event->cgrp = cgrp;
626 626
627 /* 627 /*
628 * all events in a group must monitor 628 * all events in a group must monitor
629 * the same cgroup because a task belongs 629 * the same cgroup because a task belongs
630 * to only one perf cgroup at a time 630 * to only one perf cgroup at a time
631 */ 631 */
632 if (group_leader && group_leader->cgrp != cgrp) { 632 if (group_leader && group_leader->cgrp != cgrp) {
633 perf_detach_cgroup(event); 633 perf_detach_cgroup(event);
634 ret = -EINVAL; 634 ret = -EINVAL;
635 } 635 }
636 out: 636 out:
637 fdput(f); 637 fdput(f);
638 return ret; 638 return ret;
639 } 639 }
640 640
641 static inline void 641 static inline void
642 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 642 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
643 { 643 {
644 struct perf_cgroup_info *t; 644 struct perf_cgroup_info *t;
645 t = per_cpu_ptr(event->cgrp->info, event->cpu); 645 t = per_cpu_ptr(event->cgrp->info, event->cpu);
646 event->shadow_ctx_time = now - t->timestamp; 646 event->shadow_ctx_time = now - t->timestamp;
647 } 647 }
648 648
649 static inline void 649 static inline void
650 perf_cgroup_defer_enabled(struct perf_event *event) 650 perf_cgroup_defer_enabled(struct perf_event *event)
651 { 651 {
652 /* 652 /*
653 * when the current task's perf cgroup does not match 653 * when the current task's perf cgroup does not match
654 * the event's, we need to remember to call the 654 * the event's, we need to remember to call the
655 * perf_mark_enable() function the first time a task with 655 * perf_mark_enable() function the first time a task with
656 * a matching perf cgroup is scheduled in. 656 * a matching perf cgroup is scheduled in.
657 */ 657 */
658 if (is_cgroup_event(event) && !perf_cgroup_match(event)) 658 if (is_cgroup_event(event) && !perf_cgroup_match(event))
659 event->cgrp_defer_enabled = 1; 659 event->cgrp_defer_enabled = 1;
660 } 660 }
661 661
662 static inline void 662 static inline void
663 perf_cgroup_mark_enabled(struct perf_event *event, 663 perf_cgroup_mark_enabled(struct perf_event *event,
664 struct perf_event_context *ctx) 664 struct perf_event_context *ctx)
665 { 665 {
666 struct perf_event *sub; 666 struct perf_event *sub;
667 u64 tstamp = perf_event_time(event); 667 u64 tstamp = perf_event_time(event);
668 668
669 if (!event->cgrp_defer_enabled) 669 if (!event->cgrp_defer_enabled)
670 return; 670 return;
671 671
672 event->cgrp_defer_enabled = 0; 672 event->cgrp_defer_enabled = 0;
673 673
674 event->tstamp_enabled = tstamp - event->total_time_enabled; 674 event->tstamp_enabled = tstamp - event->total_time_enabled;
675 list_for_each_entry(sub, &event->sibling_list, group_entry) { 675 list_for_each_entry(sub, &event->sibling_list, group_entry) {
676 if (sub->state >= PERF_EVENT_STATE_INACTIVE) { 676 if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
677 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 677 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
678 sub->cgrp_defer_enabled = 0; 678 sub->cgrp_defer_enabled = 0;
679 } 679 }
680 } 680 }
681 } 681 }
682 #else /* !CONFIG_CGROUP_PERF */ 682 #else /* !CONFIG_CGROUP_PERF */
683 683
684 static inline bool 684 static inline bool
685 perf_cgroup_match(struct perf_event *event) 685 perf_cgroup_match(struct perf_event *event)
686 { 686 {
687 return true; 687 return true;
688 } 688 }
689 689
690 static inline void perf_detach_cgroup(struct perf_event *event) 690 static inline void perf_detach_cgroup(struct perf_event *event)
691 {} 691 {}
692 692
693 static inline int is_cgroup_event(struct perf_event *event) 693 static inline int is_cgroup_event(struct perf_event *event)
694 { 694 {
695 return 0; 695 return 0;
696 } 696 }
697 697
698 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event) 698 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
699 { 699 {
700 return 0; 700 return 0;
701 } 701 }
702 702
703 static inline void update_cgrp_time_from_event(struct perf_event *event) 703 static inline void update_cgrp_time_from_event(struct perf_event *event)
704 { 704 {
705 } 705 }
706 706
707 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 707 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
708 { 708 {
709 } 709 }
710 710
711 static inline void perf_cgroup_sched_out(struct task_struct *task, 711 static inline void perf_cgroup_sched_out(struct task_struct *task,
712 struct task_struct *next) 712 struct task_struct *next)
713 { 713 {
714 } 714 }
715 715
716 static inline void perf_cgroup_sched_in(struct task_struct *prev, 716 static inline void perf_cgroup_sched_in(struct task_struct *prev,
717 struct task_struct *task) 717 struct task_struct *task)
718 { 718 {
719 } 719 }
720 720
721 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 721 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
722 struct perf_event_attr *attr, 722 struct perf_event_attr *attr,
723 struct perf_event *group_leader) 723 struct perf_event *group_leader)
724 { 724 {
725 return -EINVAL; 725 return -EINVAL;
726 } 726 }
727 727
728 static inline void 728 static inline void
729 perf_cgroup_set_timestamp(struct task_struct *task, 729 perf_cgroup_set_timestamp(struct task_struct *task,
730 struct perf_event_context *ctx) 730 struct perf_event_context *ctx)
731 { 731 {
732 } 732 }
733 733
734 void 734 void
735 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 735 perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
736 { 736 {
737 } 737 }
738 738
739 static inline void 739 static inline void
740 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 740 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
741 { 741 {
742 } 742 }
743 743
744 static inline u64 perf_cgroup_event_time(struct perf_event *event) 744 static inline u64 perf_cgroup_event_time(struct perf_event *event)
745 { 745 {
746 return 0; 746 return 0;
747 } 747 }
748 748
749 static inline void 749 static inline void
750 perf_cgroup_defer_enabled(struct perf_event *event) 750 perf_cgroup_defer_enabled(struct perf_event *event)
751 { 751 {
752 } 752 }
753 753
754 static inline void 754 static inline void
755 perf_cgroup_mark_enabled(struct perf_event *event, 755 perf_cgroup_mark_enabled(struct perf_event *event,
756 struct perf_event_context *ctx) 756 struct perf_event_context *ctx)
757 { 757 {
758 } 758 }
759 #endif 759 #endif
760 760
761 /* 761 /*
762 * set default to be dependent on timer tick just 762 * set default to be dependent on timer tick just
763 * like original code 763 * like original code
764 */ 764 */
765 #define PERF_CPU_HRTIMER (1000 / HZ) 765 #define PERF_CPU_HRTIMER (1000 / HZ)
766 /* 766 /*
767 * function must be called with interrupts disbled 767 * function must be called with interrupts disbled
768 */ 768 */
769 static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr) 769 static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
770 { 770 {
771 struct perf_cpu_context *cpuctx; 771 struct perf_cpu_context *cpuctx;
772 enum hrtimer_restart ret = HRTIMER_NORESTART; 772 enum hrtimer_restart ret = HRTIMER_NORESTART;
773 int rotations = 0; 773 int rotations = 0;
774 774
775 WARN_ON(!irqs_disabled()); 775 WARN_ON(!irqs_disabled());
776 776
777 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 777 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
778 778
779 rotations = perf_rotate_context(cpuctx); 779 rotations = perf_rotate_context(cpuctx);
780 780
781 /* 781 /*
782 * arm timer if needed 782 * arm timer if needed
783 */ 783 */
784 if (rotations) { 784 if (rotations) {
785 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 785 hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
786 ret = HRTIMER_RESTART; 786 ret = HRTIMER_RESTART;
787 } 787 }
788 788
789 return ret; 789 return ret;
790 } 790 }
791 791
792 /* CPU is going down */ 792 /* CPU is going down */
793 void perf_cpu_hrtimer_cancel(int cpu) 793 void perf_cpu_hrtimer_cancel(int cpu)
794 { 794 {
795 struct perf_cpu_context *cpuctx; 795 struct perf_cpu_context *cpuctx;
796 struct pmu *pmu; 796 struct pmu *pmu;
797 unsigned long flags; 797 unsigned long flags;
798 798
799 if (WARN_ON(cpu != smp_processor_id())) 799 if (WARN_ON(cpu != smp_processor_id()))
800 return; 800 return;
801 801
802 local_irq_save(flags); 802 local_irq_save(flags);
803 803
804 rcu_read_lock(); 804 rcu_read_lock();
805 805
806 list_for_each_entry_rcu(pmu, &pmus, entry) { 806 list_for_each_entry_rcu(pmu, &pmus, entry) {
807 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 807 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
808 808
809 if (pmu->task_ctx_nr == perf_sw_context) 809 if (pmu->task_ctx_nr == perf_sw_context)
810 continue; 810 continue;
811 811
812 hrtimer_cancel(&cpuctx->hrtimer); 812 hrtimer_cancel(&cpuctx->hrtimer);
813 } 813 }
814 814
815 rcu_read_unlock(); 815 rcu_read_unlock();
816 816
817 local_irq_restore(flags); 817 local_irq_restore(flags);
818 } 818 }
819 819
820 static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 820 static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
821 { 821 {
822 struct hrtimer *hr = &cpuctx->hrtimer; 822 struct hrtimer *hr = &cpuctx->hrtimer;
823 struct pmu *pmu = cpuctx->ctx.pmu; 823 struct pmu *pmu = cpuctx->ctx.pmu;
824 int timer; 824 int timer;
825 825
826 /* no multiplexing needed for SW PMU */ 826 /* no multiplexing needed for SW PMU */
827 if (pmu->task_ctx_nr == perf_sw_context) 827 if (pmu->task_ctx_nr == perf_sw_context)
828 return; 828 return;
829 829
830 /* 830 /*
831 * check default is sane, if not set then force to 831 * check default is sane, if not set then force to
832 * default interval (1/tick) 832 * default interval (1/tick)
833 */ 833 */
834 timer = pmu->hrtimer_interval_ms; 834 timer = pmu->hrtimer_interval_ms;
835 if (timer < 1) 835 if (timer < 1)
836 timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 836 timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
837 837
838 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 838 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
839 839
840 hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 840 hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
841 hr->function = perf_cpu_hrtimer_handler; 841 hr->function = perf_cpu_hrtimer_handler;
842 } 842 }
843 843
844 static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx) 844 static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
845 { 845 {
846 struct hrtimer *hr = &cpuctx->hrtimer; 846 struct hrtimer *hr = &cpuctx->hrtimer;
847 struct pmu *pmu = cpuctx->ctx.pmu; 847 struct pmu *pmu = cpuctx->ctx.pmu;
848 848
849 /* not for SW PMU */ 849 /* not for SW PMU */
850 if (pmu->task_ctx_nr == perf_sw_context) 850 if (pmu->task_ctx_nr == perf_sw_context)
851 return; 851 return;
852 852
853 if (hrtimer_active(hr)) 853 if (hrtimer_active(hr))
854 return; 854 return;
855 855
856 if (!hrtimer_callback_running(hr)) 856 if (!hrtimer_callback_running(hr))
857 __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval, 857 __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
858 0, HRTIMER_MODE_REL_PINNED, 0); 858 0, HRTIMER_MODE_REL_PINNED, 0);
859 } 859 }
860 860
861 void perf_pmu_disable(struct pmu *pmu) 861 void perf_pmu_disable(struct pmu *pmu)
862 { 862 {
863 int *count = this_cpu_ptr(pmu->pmu_disable_count); 863 int *count = this_cpu_ptr(pmu->pmu_disable_count);
864 if (!(*count)++) 864 if (!(*count)++)
865 pmu->pmu_disable(pmu); 865 pmu->pmu_disable(pmu);
866 } 866 }
867 867
868 void perf_pmu_enable(struct pmu *pmu) 868 void perf_pmu_enable(struct pmu *pmu)
869 { 869 {
870 int *count = this_cpu_ptr(pmu->pmu_disable_count); 870 int *count = this_cpu_ptr(pmu->pmu_disable_count);
871 if (!--(*count)) 871 if (!--(*count))
872 pmu->pmu_enable(pmu); 872 pmu->pmu_enable(pmu);
873 } 873 }
874 874
875 static DEFINE_PER_CPU(struct list_head, rotation_list); 875 static DEFINE_PER_CPU(struct list_head, rotation_list);
876 876
877 /* 877 /*
878 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized 878 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
879 * because they're strictly cpu affine and rotate_start is called with IRQs 879 * because they're strictly cpu affine and rotate_start is called with IRQs
880 * disabled, while rotate_context is called from IRQ context. 880 * disabled, while rotate_context is called from IRQ context.
881 */ 881 */
882 static void perf_pmu_rotate_start(struct pmu *pmu) 882 static void perf_pmu_rotate_start(struct pmu *pmu)
883 { 883 {
884 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 884 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
885 struct list_head *head = this_cpu_ptr(&rotation_list); 885 struct list_head *head = this_cpu_ptr(&rotation_list);
886 886
887 WARN_ON(!irqs_disabled()); 887 WARN_ON(!irqs_disabled());
888 888
889 if (list_empty(&cpuctx->rotation_list)) 889 if (list_empty(&cpuctx->rotation_list))
890 list_add(&cpuctx->rotation_list, head); 890 list_add(&cpuctx->rotation_list, head);
891 } 891 }
892 892
893 static void get_ctx(struct perf_event_context *ctx) 893 static void get_ctx(struct perf_event_context *ctx)
894 { 894 {
895 WARN_ON(!atomic_inc_not_zero(&ctx->refcount)); 895 WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
896 } 896 }
897 897
898 static void put_ctx(struct perf_event_context *ctx) 898 static void put_ctx(struct perf_event_context *ctx)
899 { 899 {
900 if (atomic_dec_and_test(&ctx->refcount)) { 900 if (atomic_dec_and_test(&ctx->refcount)) {
901 if (ctx->parent_ctx) 901 if (ctx->parent_ctx)
902 put_ctx(ctx->parent_ctx); 902 put_ctx(ctx->parent_ctx);
903 if (ctx->task) 903 if (ctx->task)
904 put_task_struct(ctx->task); 904 put_task_struct(ctx->task);
905 kfree_rcu(ctx, rcu_head); 905 kfree_rcu(ctx, rcu_head);
906 } 906 }
907 } 907 }
908 908
909 /* 909 /*
910 * This must be done under the ctx->lock, such as to serialize against 910 * This must be done under the ctx->lock, such as to serialize against
911 * context_equiv(), therefore we cannot call put_ctx() since that might end up 911 * context_equiv(), therefore we cannot call put_ctx() since that might end up
912 * calling scheduler related locks and ctx->lock nests inside those. 912 * calling scheduler related locks and ctx->lock nests inside those.
913 */ 913 */
914 static __must_check struct perf_event_context * 914 static __must_check struct perf_event_context *
915 unclone_ctx(struct perf_event_context *ctx) 915 unclone_ctx(struct perf_event_context *ctx)
916 { 916 {
917 struct perf_event_context *parent_ctx = ctx->parent_ctx; 917 struct perf_event_context *parent_ctx = ctx->parent_ctx;
918 918
919 lockdep_assert_held(&ctx->lock); 919 lockdep_assert_held(&ctx->lock);
920 920
921 if (parent_ctx) 921 if (parent_ctx)
922 ctx->parent_ctx = NULL; 922 ctx->parent_ctx = NULL;
923 ctx->generation++; 923 ctx->generation++;
924 924
925 return parent_ctx; 925 return parent_ctx;
926 } 926 }
927 927
928 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 928 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
929 { 929 {
930 /* 930 /*
931 * only top level events have the pid namespace they were created in 931 * only top level events have the pid namespace they were created in
932 */ 932 */
933 if (event->parent) 933 if (event->parent)
934 event = event->parent; 934 event = event->parent;
935 935
936 return task_tgid_nr_ns(p, event->ns); 936 return task_tgid_nr_ns(p, event->ns);
937 } 937 }
938 938
939 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 939 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
940 { 940 {
941 /* 941 /*
942 * only top level events have the pid namespace they were created in 942 * only top level events have the pid namespace they were created in
943 */ 943 */
944 if (event->parent) 944 if (event->parent)
945 event = event->parent; 945 event = event->parent;
946 946
947 return task_pid_nr_ns(p, event->ns); 947 return task_pid_nr_ns(p, event->ns);
948 } 948 }
949 949
950 /* 950 /*
951 * If we inherit events we want to return the parent event id 951 * If we inherit events we want to return the parent event id
952 * to userspace. 952 * to userspace.
953 */ 953 */
954 static u64 primary_event_id(struct perf_event *event) 954 static u64 primary_event_id(struct perf_event *event)
955 { 955 {
956 u64 id = event->id; 956 u64 id = event->id;
957 957
958 if (event->parent) 958 if (event->parent)
959 id = event->parent->id; 959 id = event->parent->id;
960 960
961 return id; 961 return id;
962 } 962 }
963 963
964 /* 964 /*
965 * Get the perf_event_context for a task and lock it. 965 * Get the perf_event_context for a task and lock it.
966 * This has to cope with with the fact that until it is locked, 966 * This has to cope with with the fact that until it is locked,
967 * the context could get moved to another task. 967 * the context could get moved to another task.
968 */ 968 */
969 static struct perf_event_context * 969 static struct perf_event_context *
970 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 970 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
971 { 971 {
972 struct perf_event_context *ctx; 972 struct perf_event_context *ctx;
973 973
974 retry: 974 retry:
975 /* 975 /*
976 * One of the few rules of preemptible RCU is that one cannot do 976 * One of the few rules of preemptible RCU is that one cannot do
977 * rcu_read_unlock() while holding a scheduler (or nested) lock when 977 * rcu_read_unlock() while holding a scheduler (or nested) lock when
978 * part of the read side critical section was preemptible -- see 978 * part of the read side critical section was preemptible -- see
979 * rcu_read_unlock_special(). 979 * rcu_read_unlock_special().
980 * 980 *
981 * Since ctx->lock nests under rq->lock we must ensure the entire read 981 * Since ctx->lock nests under rq->lock we must ensure the entire read
982 * side critical section is non-preemptible. 982 * side critical section is non-preemptible.
983 */ 983 */
984 preempt_disable(); 984 preempt_disable();
985 rcu_read_lock(); 985 rcu_read_lock();
986 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 986 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
987 if (ctx) { 987 if (ctx) {
988 /* 988 /*
989 * If this context is a clone of another, it might 989 * If this context is a clone of another, it might
990 * get swapped for another underneath us by 990 * get swapped for another underneath us by
991 * perf_event_task_sched_out, though the 991 * perf_event_task_sched_out, though the
992 * rcu_read_lock() protects us from any context 992 * rcu_read_lock() protects us from any context
993 * getting freed. Lock the context and check if it 993 * getting freed. Lock the context and check if it
994 * got swapped before we could get the lock, and retry 994 * got swapped before we could get the lock, and retry
995 * if so. If we locked the right context, then it 995 * if so. If we locked the right context, then it
996 * can't get swapped on us any more. 996 * can't get swapped on us any more.
997 */ 997 */
998 raw_spin_lock_irqsave(&ctx->lock, *flags); 998 raw_spin_lock_irqsave(&ctx->lock, *flags);
999 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 999 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1000 raw_spin_unlock_irqrestore(&ctx->lock, *flags); 1000 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1001 rcu_read_unlock(); 1001 rcu_read_unlock();
1002 preempt_enable(); 1002 preempt_enable();
1003 goto retry; 1003 goto retry;
1004 } 1004 }
1005 1005
1006 if (!atomic_inc_not_zero(&ctx->refcount)) { 1006 if (!atomic_inc_not_zero(&ctx->refcount)) {
1007 raw_spin_unlock_irqrestore(&ctx->lock, *flags); 1007 raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1008 ctx = NULL; 1008 ctx = NULL;
1009 } 1009 }
1010 } 1010 }
1011 rcu_read_unlock(); 1011 rcu_read_unlock();
1012 preempt_enable(); 1012 preempt_enable();
1013 return ctx; 1013 return ctx;
1014 } 1014 }
1015 1015
1016 /* 1016 /*
1017 * Get the context for a task and increment its pin_count so it 1017 * Get the context for a task and increment its pin_count so it
1018 * can't get swapped to another task. This also increments its 1018 * can't get swapped to another task. This also increments its
1019 * reference count so that the context can't get freed. 1019 * reference count so that the context can't get freed.
1020 */ 1020 */
1021 static struct perf_event_context * 1021 static struct perf_event_context *
1022 perf_pin_task_context(struct task_struct *task, int ctxn) 1022 perf_pin_task_context(struct task_struct *task, int ctxn)
1023 { 1023 {
1024 struct perf_event_context *ctx; 1024 struct perf_event_context *ctx;
1025 unsigned long flags; 1025 unsigned long flags;
1026 1026
1027 ctx = perf_lock_task_context(task, ctxn, &flags); 1027 ctx = perf_lock_task_context(task, ctxn, &flags);
1028 if (ctx) { 1028 if (ctx) {
1029 ++ctx->pin_count; 1029 ++ctx->pin_count;
1030 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1030 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1031 } 1031 }
1032 return ctx; 1032 return ctx;
1033 } 1033 }
1034 1034
1035 static void perf_unpin_context(struct perf_event_context *ctx) 1035 static void perf_unpin_context(struct perf_event_context *ctx)
1036 { 1036 {
1037 unsigned long flags; 1037 unsigned long flags;
1038 1038
1039 raw_spin_lock_irqsave(&ctx->lock, flags); 1039 raw_spin_lock_irqsave(&ctx->lock, flags);
1040 --ctx->pin_count; 1040 --ctx->pin_count;
1041 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1041 raw_spin_unlock_irqrestore(&ctx->lock, flags);
1042 } 1042 }
1043 1043
1044 /* 1044 /*
1045 * Update the record of the current time in a context. 1045 * Update the record of the current time in a context.
1046 */ 1046 */
1047 static void update_context_time(struct perf_event_context *ctx) 1047 static void update_context_time(struct perf_event_context *ctx)
1048 { 1048 {
1049 u64 now = perf_clock(); 1049 u64 now = perf_clock();
1050 1050
1051 ctx->time += now - ctx->timestamp; 1051 ctx->time += now - ctx->timestamp;
1052 ctx->timestamp = now; 1052 ctx->timestamp = now;
1053 } 1053 }
1054 1054
1055 static u64 perf_event_time(struct perf_event *event) 1055 static u64 perf_event_time(struct perf_event *event)
1056 { 1056 {
1057 struct perf_event_context *ctx = event->ctx; 1057 struct perf_event_context *ctx = event->ctx;
1058 1058
1059 if (is_cgroup_event(event)) 1059 if (is_cgroup_event(event))
1060 return perf_cgroup_event_time(event); 1060 return perf_cgroup_event_time(event);
1061 1061
1062 return ctx ? ctx->time : 0; 1062 return ctx ? ctx->time : 0;
1063 } 1063 }
1064 1064
1065 /* 1065 /*
1066 * Update the total_time_enabled and total_time_running fields for a event. 1066 * Update the total_time_enabled and total_time_running fields for a event.
1067 * The caller of this function needs to hold the ctx->lock. 1067 * The caller of this function needs to hold the ctx->lock.
1068 */ 1068 */
1069 static void update_event_times(struct perf_event *event) 1069 static void update_event_times(struct perf_event *event)
1070 { 1070 {
1071 struct perf_event_context *ctx = event->ctx; 1071 struct perf_event_context *ctx = event->ctx;
1072 u64 run_end; 1072 u64 run_end;
1073 1073
1074 if (event->state < PERF_EVENT_STATE_INACTIVE || 1074 if (event->state < PERF_EVENT_STATE_INACTIVE ||
1075 event->group_leader->state < PERF_EVENT_STATE_INACTIVE) 1075 event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
1076 return; 1076 return;
1077 /* 1077 /*
1078 * in cgroup mode, time_enabled represents 1078 * in cgroup mode, time_enabled represents
1079 * the time the event was enabled AND active 1079 * the time the event was enabled AND active
1080 * tasks were in the monitored cgroup. This is 1080 * tasks were in the monitored cgroup. This is
1081 * independent of the activity of the context as 1081 * independent of the activity of the context as
1082 * there may be a mix of cgroup and non-cgroup events. 1082 * there may be a mix of cgroup and non-cgroup events.
1083 * 1083 *
1084 * That is why we treat cgroup events differently 1084 * That is why we treat cgroup events differently
1085 * here. 1085 * here.
1086 */ 1086 */
1087 if (is_cgroup_event(event)) 1087 if (is_cgroup_event(event))
1088 run_end = perf_cgroup_event_time(event); 1088 run_end = perf_cgroup_event_time(event);
1089 else if (ctx->is_active) 1089 else if (ctx->is_active)
1090 run_end = ctx->time; 1090 run_end = ctx->time;
1091 else 1091 else
1092 run_end = event->tstamp_stopped; 1092 run_end = event->tstamp_stopped;
1093 1093
1094 event->total_time_enabled = run_end - event->tstamp_enabled; 1094 event->total_time_enabled = run_end - event->tstamp_enabled;
1095 1095
1096 if (event->state == PERF_EVENT_STATE_INACTIVE) 1096 if (event->state == PERF_EVENT_STATE_INACTIVE)
1097 run_end = event->tstamp_stopped; 1097 run_end = event->tstamp_stopped;
1098 else 1098 else
1099 run_end = perf_event_time(event); 1099 run_end = perf_event_time(event);
1100 1100
1101 event->total_time_running = run_end - event->tstamp_running; 1101 event->total_time_running = run_end - event->tstamp_running;
1102 1102
1103 } 1103 }
1104 1104
1105 /* 1105 /*
1106 * Update total_time_enabled and total_time_running for all events in a group. 1106 * Update total_time_enabled and total_time_running for all events in a group.
1107 */ 1107 */
1108 static void update_group_times(struct perf_event *leader) 1108 static void update_group_times(struct perf_event *leader)
1109 { 1109 {
1110 struct perf_event *event; 1110 struct perf_event *event;
1111 1111
1112 update_event_times(leader); 1112 update_event_times(leader);
1113 list_for_each_entry(event, &leader->sibling_list, group_entry) 1113 list_for_each_entry(event, &leader->sibling_list, group_entry)
1114 update_event_times(event); 1114 update_event_times(event);
1115 } 1115 }
1116 1116
1117 static struct list_head * 1117 static struct list_head *
1118 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) 1118 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
1119 { 1119 {
1120 if (event->attr.pinned) 1120 if (event->attr.pinned)
1121 return &ctx->pinned_groups; 1121 return &ctx->pinned_groups;
1122 else 1122 else
1123 return &ctx->flexible_groups; 1123 return &ctx->flexible_groups;
1124 } 1124 }
1125 1125
1126 /* 1126 /*
1127 * Add a event from the lists for its context. 1127 * Add a event from the lists for its context.
1128 * Must be called with ctx->mutex and ctx->lock held. 1128 * Must be called with ctx->mutex and ctx->lock held.
1129 */ 1129 */
1130 static void 1130 static void
1131 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1131 list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1132 { 1132 {
1133 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1133 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1134 event->attach_state |= PERF_ATTACH_CONTEXT; 1134 event->attach_state |= PERF_ATTACH_CONTEXT;
1135 1135
1136 /* 1136 /*
1137 * If we're a stand alone event or group leader, we go to the context 1137 * If we're a stand alone event or group leader, we go to the context
1138 * list, group events are kept attached to the group so that 1138 * list, group events are kept attached to the group so that
1139 * perf_group_detach can, at all times, locate all siblings. 1139 * perf_group_detach can, at all times, locate all siblings.
1140 */ 1140 */
1141 if (event->group_leader == event) { 1141 if (event->group_leader == event) {
1142 struct list_head *list; 1142 struct list_head *list;
1143 1143
1144 if (is_software_event(event)) 1144 if (is_software_event(event))
1145 event->group_flags |= PERF_GROUP_SOFTWARE; 1145 event->group_flags |= PERF_GROUP_SOFTWARE;
1146 1146
1147 list = ctx_group_list(event, ctx); 1147 list = ctx_group_list(event, ctx);
1148 list_add_tail(&event->group_entry, list); 1148 list_add_tail(&event->group_entry, list);
1149 } 1149 }
1150 1150
1151 if (is_cgroup_event(event)) 1151 if (is_cgroup_event(event))
1152 ctx->nr_cgroups++; 1152 ctx->nr_cgroups++;
1153 1153
1154 if (has_branch_stack(event)) 1154 if (has_branch_stack(event))
1155 ctx->nr_branch_stack++; 1155 ctx->nr_branch_stack++;
1156 1156
1157 list_add_rcu(&event->event_entry, &ctx->event_list); 1157 list_add_rcu(&event->event_entry, &ctx->event_list);
1158 if (!ctx->nr_events) 1158 if (!ctx->nr_events)
1159 perf_pmu_rotate_start(ctx->pmu); 1159 perf_pmu_rotate_start(ctx->pmu);
1160 ctx->nr_events++; 1160 ctx->nr_events++;
1161 if (event->attr.inherit_stat) 1161 if (event->attr.inherit_stat)
1162 ctx->nr_stat++; 1162 ctx->nr_stat++;
1163 1163
1164 ctx->generation++; 1164 ctx->generation++;
1165 } 1165 }
1166 1166
1167 /* 1167 /*
1168 * Initialize event state based on the perf_event_attr::disabled. 1168 * Initialize event state based on the perf_event_attr::disabled.
1169 */ 1169 */
1170 static inline void perf_event__state_init(struct perf_event *event) 1170 static inline void perf_event__state_init(struct perf_event *event)
1171 { 1171 {
1172 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1172 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1173 PERF_EVENT_STATE_INACTIVE; 1173 PERF_EVENT_STATE_INACTIVE;
1174 } 1174 }
1175 1175
1176 /* 1176 /*
1177 * Called at perf_event creation and when events are attached/detached from a 1177 * Called at perf_event creation and when events are attached/detached from a
1178 * group. 1178 * group.
1179 */ 1179 */
1180 static void perf_event__read_size(struct perf_event *event) 1180 static void perf_event__read_size(struct perf_event *event)
1181 { 1181 {
1182 int entry = sizeof(u64); /* value */ 1182 int entry = sizeof(u64); /* value */
1183 int size = 0; 1183 int size = 0;
1184 int nr = 1; 1184 int nr = 1;
1185 1185
1186 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1186 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1187 size += sizeof(u64); 1187 size += sizeof(u64);
1188 1188
1189 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1189 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1190 size += sizeof(u64); 1190 size += sizeof(u64);
1191 1191
1192 if (event->attr.read_format & PERF_FORMAT_ID) 1192 if (event->attr.read_format & PERF_FORMAT_ID)
1193 entry += sizeof(u64); 1193 entry += sizeof(u64);
1194 1194
1195 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1195 if (event->attr.read_format & PERF_FORMAT_GROUP) {
1196 nr += event->group_leader->nr_siblings; 1196 nr += event->group_leader->nr_siblings;
1197 size += sizeof(u64); 1197 size += sizeof(u64);
1198 } 1198 }
1199 1199
1200 size += entry * nr; 1200 size += entry * nr;
1201 event->read_size = size; 1201 event->read_size = size;
1202 } 1202 }
1203 1203
1204 static void perf_event__header_size(struct perf_event *event) 1204 static void perf_event__header_size(struct perf_event *event)
1205 { 1205 {
1206 struct perf_sample_data *data; 1206 struct perf_sample_data *data;
1207 u64 sample_type = event->attr.sample_type; 1207 u64 sample_type = event->attr.sample_type;
1208 u16 size = 0; 1208 u16 size = 0;
1209 1209
1210 perf_event__read_size(event); 1210 perf_event__read_size(event);
1211 1211
1212 if (sample_type & PERF_SAMPLE_IP) 1212 if (sample_type & PERF_SAMPLE_IP)
1213 size += sizeof(data->ip); 1213 size += sizeof(data->ip);
1214 1214
1215 if (sample_type & PERF_SAMPLE_ADDR) 1215 if (sample_type & PERF_SAMPLE_ADDR)
1216 size += sizeof(data->addr); 1216 size += sizeof(data->addr);
1217 1217
1218 if (sample_type & PERF_SAMPLE_PERIOD) 1218 if (sample_type & PERF_SAMPLE_PERIOD)
1219 size += sizeof(data->period); 1219 size += sizeof(data->period);
1220 1220
1221 if (sample_type & PERF_SAMPLE_WEIGHT) 1221 if (sample_type & PERF_SAMPLE_WEIGHT)
1222 size += sizeof(data->weight); 1222 size += sizeof(data->weight);
1223 1223
1224 if (sample_type & PERF_SAMPLE_READ) 1224 if (sample_type & PERF_SAMPLE_READ)
1225 size += event->read_size; 1225 size += event->read_size;
1226 1226
1227 if (sample_type & PERF_SAMPLE_DATA_SRC) 1227 if (sample_type & PERF_SAMPLE_DATA_SRC)
1228 size += sizeof(data->data_src.val); 1228 size += sizeof(data->data_src.val);
1229 1229
1230 if (sample_type & PERF_SAMPLE_TRANSACTION) 1230 if (sample_type & PERF_SAMPLE_TRANSACTION)
1231 size += sizeof(data->txn); 1231 size += sizeof(data->txn);
1232 1232
1233 event->header_size = size; 1233 event->header_size = size;
1234 } 1234 }
1235 1235
1236 static void perf_event__id_header_size(struct perf_event *event) 1236 static void perf_event__id_header_size(struct perf_event *event)
1237 { 1237 {
1238 struct perf_sample_data *data; 1238 struct perf_sample_data *data;
1239 u64 sample_type = event->attr.sample_type; 1239 u64 sample_type = event->attr.sample_type;
1240 u16 size = 0; 1240 u16 size = 0;
1241 1241
1242 if (sample_type & PERF_SAMPLE_TID) 1242 if (sample_type & PERF_SAMPLE_TID)
1243 size += sizeof(data->tid_entry); 1243 size += sizeof(data->tid_entry);
1244 1244
1245 if (sample_type & PERF_SAMPLE_TIME) 1245 if (sample_type & PERF_SAMPLE_TIME)
1246 size += sizeof(data->time); 1246 size += sizeof(data->time);
1247 1247
1248 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1248 if (sample_type & PERF_SAMPLE_IDENTIFIER)
1249 size += sizeof(data->id); 1249 size += sizeof(data->id);
1250 1250
1251 if (sample_type & PERF_SAMPLE_ID) 1251 if (sample_type & PERF_SAMPLE_ID)
1252 size += sizeof(data->id); 1252 size += sizeof(data->id);
1253 1253
1254 if (sample_type & PERF_SAMPLE_STREAM_ID) 1254 if (sample_type & PERF_SAMPLE_STREAM_ID)
1255 size += sizeof(data->stream_id); 1255 size += sizeof(data->stream_id);
1256 1256
1257 if (sample_type & PERF_SAMPLE_CPU) 1257 if (sample_type & PERF_SAMPLE_CPU)
1258 size += sizeof(data->cpu_entry); 1258 size += sizeof(data->cpu_entry);
1259 1259
1260 event->id_header_size = size; 1260 event->id_header_size = size;
1261 } 1261 }
1262 1262
1263 static void perf_group_attach(struct perf_event *event) 1263 static void perf_group_attach(struct perf_event *event)
1264 { 1264 {
1265 struct perf_event *group_leader = event->group_leader, *pos; 1265 struct perf_event *group_leader = event->group_leader, *pos;
1266 1266
1267 /* 1267 /*
1268 * We can have double attach due to group movement in perf_event_open. 1268 * We can have double attach due to group movement in perf_event_open.
1269 */ 1269 */
1270 if (event->attach_state & PERF_ATTACH_GROUP) 1270 if (event->attach_state & PERF_ATTACH_GROUP)
1271 return; 1271 return;
1272 1272
1273 event->attach_state |= PERF_ATTACH_GROUP; 1273 event->attach_state |= PERF_ATTACH_GROUP;
1274 1274
1275 if (group_leader == event) 1275 if (group_leader == event)
1276 return; 1276 return;
1277 1277
1278 if (group_leader->group_flags & PERF_GROUP_SOFTWARE && 1278 if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1279 !is_software_event(event)) 1279 !is_software_event(event))
1280 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE; 1280 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1281 1281
1282 list_add_tail(&event->group_entry, &group_leader->sibling_list); 1282 list_add_tail(&event->group_entry, &group_leader->sibling_list);
1283 group_leader->nr_siblings++; 1283 group_leader->nr_siblings++;
1284 1284
1285 perf_event__header_size(group_leader); 1285 perf_event__header_size(group_leader);
1286 1286
1287 list_for_each_entry(pos, &group_leader->sibling_list, group_entry) 1287 list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1288 perf_event__header_size(pos); 1288 perf_event__header_size(pos);
1289 } 1289 }
1290 1290
1291 /* 1291 /*
1292 * Remove a event from the lists for its context. 1292 * Remove a event from the lists for its context.
1293 * Must be called with ctx->mutex and ctx->lock held. 1293 * Must be called with ctx->mutex and ctx->lock held.
1294 */ 1294 */
1295 static void 1295 static void
1296 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1296 list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1297 { 1297 {
1298 struct perf_cpu_context *cpuctx; 1298 struct perf_cpu_context *cpuctx;
1299 /* 1299 /*
1300 * We can have double detach due to exit/hot-unplug + close. 1300 * We can have double detach due to exit/hot-unplug + close.
1301 */ 1301 */
1302 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1302 if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1303 return; 1303 return;
1304 1304
1305 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1305 event->attach_state &= ~PERF_ATTACH_CONTEXT;
1306 1306
1307 if (is_cgroup_event(event)) { 1307 if (is_cgroup_event(event)) {
1308 ctx->nr_cgroups--; 1308 ctx->nr_cgroups--;
1309 cpuctx = __get_cpu_context(ctx); 1309 cpuctx = __get_cpu_context(ctx);
1310 /* 1310 /*
1311 * if there are no more cgroup events 1311 * if there are no more cgroup events
1312 * then cler cgrp to avoid stale pointer 1312 * then cler cgrp to avoid stale pointer
1313 * in update_cgrp_time_from_cpuctx() 1313 * in update_cgrp_time_from_cpuctx()
1314 */ 1314 */
1315 if (!ctx->nr_cgroups) 1315 if (!ctx->nr_cgroups)
1316 cpuctx->cgrp = NULL; 1316 cpuctx->cgrp = NULL;
1317 } 1317 }
1318 1318
1319 if (has_branch_stack(event)) 1319 if (has_branch_stack(event))
1320 ctx->nr_branch_stack--; 1320 ctx->nr_branch_stack--;
1321 1321
1322 ctx->nr_events--; 1322 ctx->nr_events--;
1323 if (event->attr.inherit_stat) 1323 if (event->attr.inherit_stat)
1324 ctx->nr_stat--; 1324 ctx->nr_stat--;
1325 1325
1326 list_del_rcu(&event->event_entry); 1326 list_del_rcu(&event->event_entry);
1327 1327
1328 if (event->group_leader == event) 1328 if (event->group_leader == event)
1329 list_del_init(&event->group_entry); 1329 list_del_init(&event->group_entry);
1330 1330
1331 update_group_times(event); 1331 update_group_times(event);
1332 1332
1333 /* 1333 /*
1334 * If event was in error state, then keep it 1334 * If event was in error state, then keep it
1335 * that way, otherwise bogus counts will be 1335 * that way, otherwise bogus counts will be
1336 * returned on read(). The only way to get out 1336 * returned on read(). The only way to get out
1337 * of error state is by explicit re-enabling 1337 * of error state is by explicit re-enabling
1338 * of the event 1338 * of the event
1339 */ 1339 */
1340 if (event->state > PERF_EVENT_STATE_OFF) 1340 if (event->state > PERF_EVENT_STATE_OFF)
1341 event->state = PERF_EVENT_STATE_OFF; 1341 event->state = PERF_EVENT_STATE_OFF;
1342 1342
1343 ctx->generation++; 1343 ctx->generation++;
1344 } 1344 }
1345 1345
1346 static void perf_group_detach(struct perf_event *event) 1346 static void perf_group_detach(struct perf_event *event)
1347 { 1347 {
1348 struct perf_event *sibling, *tmp; 1348 struct perf_event *sibling, *tmp;
1349 struct list_head *list = NULL; 1349 struct list_head *list = NULL;
1350 1350
1351 /* 1351 /*
1352 * We can have double detach due to exit/hot-unplug + close. 1352 * We can have double detach due to exit/hot-unplug + close.
1353 */ 1353 */
1354 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1354 if (!(event->attach_state & PERF_ATTACH_GROUP))
1355 return; 1355 return;
1356 1356
1357 event->attach_state &= ~PERF_ATTACH_GROUP; 1357 event->attach_state &= ~PERF_ATTACH_GROUP;
1358 1358
1359 /* 1359 /*
1360 * If this is a sibling, remove it from its group. 1360 * If this is a sibling, remove it from its group.
1361 */ 1361 */
1362 if (event->group_leader != event) { 1362 if (event->group_leader != event) {
1363 list_del_init(&event->group_entry); 1363 list_del_init(&event->group_entry);
1364 event->group_leader->nr_siblings--; 1364 event->group_leader->nr_siblings--;
1365 goto out; 1365 goto out;
1366 } 1366 }
1367 1367
1368 if (!list_empty(&event->group_entry)) 1368 if (!list_empty(&event->group_entry))
1369 list = &event->group_entry; 1369 list = &event->group_entry;
1370 1370
1371 /* 1371 /*
1372 * If this was a group event with sibling events then 1372 * If this was a group event with sibling events then
1373 * upgrade the siblings to singleton events by adding them 1373 * upgrade the siblings to singleton events by adding them
1374 * to whatever list we are on. 1374 * to whatever list we are on.
1375 */ 1375 */
1376 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { 1376 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1377 if (list) 1377 if (list)
1378 list_move_tail(&sibling->group_entry, list); 1378 list_move_tail(&sibling->group_entry, list);
1379 sibling->group_leader = sibling; 1379 sibling->group_leader = sibling;
1380 1380
1381 /* Inherit group flags from the previous leader */ 1381 /* Inherit group flags from the previous leader */
1382 sibling->group_flags = event->group_flags; 1382 sibling->group_flags = event->group_flags;
1383 } 1383 }
1384 1384
1385 out: 1385 out:
1386 perf_event__header_size(event->group_leader); 1386 perf_event__header_size(event->group_leader);
1387 1387
1388 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry) 1388 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1389 perf_event__header_size(tmp); 1389 perf_event__header_size(tmp);
1390 } 1390 }
1391 1391
1392 /* 1392 /*
1393 * User event without the task. 1393 * User event without the task.
1394 */ 1394 */
1395 static bool is_orphaned_event(struct perf_event *event) 1395 static bool is_orphaned_event(struct perf_event *event)
1396 { 1396 {
1397 return event && !is_kernel_event(event) && !event->owner; 1397 return event && !is_kernel_event(event) && !event->owner;
1398 } 1398 }
1399 1399
1400 /* 1400 /*
1401 * Event has a parent but parent's task finished and it's 1401 * Event has a parent but parent's task finished and it's
1402 * alive only because of children holding refference. 1402 * alive only because of children holding refference.
1403 */ 1403 */
1404 static bool is_orphaned_child(struct perf_event *event) 1404 static bool is_orphaned_child(struct perf_event *event)
1405 { 1405 {
1406 return is_orphaned_event(event->parent); 1406 return is_orphaned_event(event->parent);
1407 } 1407 }
1408 1408
1409 static void orphans_remove_work(struct work_struct *work); 1409 static void orphans_remove_work(struct work_struct *work);
1410 1410
1411 static void schedule_orphans_remove(struct perf_event_context *ctx) 1411 static void schedule_orphans_remove(struct perf_event_context *ctx)
1412 { 1412 {
1413 if (!ctx->task || ctx->orphans_remove_sched || !perf_wq) 1413 if (!ctx->task || ctx->orphans_remove_sched || !perf_wq)
1414 return; 1414 return;
1415 1415
1416 if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) { 1416 if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) {
1417 get_ctx(ctx); 1417 get_ctx(ctx);
1418 ctx->orphans_remove_sched = true; 1418 ctx->orphans_remove_sched = true;
1419 } 1419 }
1420 } 1420 }
1421 1421
1422 static int __init perf_workqueue_init(void) 1422 static int __init perf_workqueue_init(void)
1423 { 1423 {
1424 perf_wq = create_singlethread_workqueue("perf"); 1424 perf_wq = create_singlethread_workqueue("perf");
1425 WARN(!perf_wq, "failed to create perf workqueue\n"); 1425 WARN(!perf_wq, "failed to create perf workqueue\n");
1426 return perf_wq ? 0 : -1; 1426 return perf_wq ? 0 : -1;
1427 } 1427 }
1428 1428
1429 core_initcall(perf_workqueue_init); 1429 core_initcall(perf_workqueue_init);
1430 1430
1431 static inline int 1431 static inline int
1432 event_filter_match(struct perf_event *event) 1432 event_filter_match(struct perf_event *event)
1433 { 1433 {
1434 return (event->cpu == -1 || event->cpu == smp_processor_id()) 1434 return (event->cpu == -1 || event->cpu == smp_processor_id())
1435 && perf_cgroup_match(event); 1435 && perf_cgroup_match(event);
1436 } 1436 }
1437 1437
1438 static void 1438 static void
1439 event_sched_out(struct perf_event *event, 1439 event_sched_out(struct perf_event *event,
1440 struct perf_cpu_context *cpuctx, 1440 struct perf_cpu_context *cpuctx,
1441 struct perf_event_context *ctx) 1441 struct perf_event_context *ctx)
1442 { 1442 {
1443 u64 tstamp = perf_event_time(event); 1443 u64 tstamp = perf_event_time(event);
1444 u64 delta; 1444 u64 delta;
1445 /* 1445 /*
1446 * An event which could not be activated because of 1446 * An event which could not be activated because of
1447 * filter mismatch still needs to have its timings 1447 * filter mismatch still needs to have its timings
1448 * maintained, otherwise bogus information is return 1448 * maintained, otherwise bogus information is return
1449 * via read() for time_enabled, time_running: 1449 * via read() for time_enabled, time_running:
1450 */ 1450 */
1451 if (event->state == PERF_EVENT_STATE_INACTIVE 1451 if (event->state == PERF_EVENT_STATE_INACTIVE
1452 && !event_filter_match(event)) { 1452 && !event_filter_match(event)) {
1453 delta = tstamp - event->tstamp_stopped; 1453 delta = tstamp - event->tstamp_stopped;
1454 event->tstamp_running += delta; 1454 event->tstamp_running += delta;
1455 event->tstamp_stopped = tstamp; 1455 event->tstamp_stopped = tstamp;
1456 } 1456 }
1457 1457
1458 if (event->state != PERF_EVENT_STATE_ACTIVE) 1458 if (event->state != PERF_EVENT_STATE_ACTIVE)
1459 return; 1459 return;
1460 1460
1461 perf_pmu_disable(event->pmu); 1461 perf_pmu_disable(event->pmu);
1462 1462
1463 event->state = PERF_EVENT_STATE_INACTIVE; 1463 event->state = PERF_EVENT_STATE_INACTIVE;
1464 if (event->pending_disable) { 1464 if (event->pending_disable) {
1465 event->pending_disable = 0; 1465 event->pending_disable = 0;
1466 event->state = PERF_EVENT_STATE_OFF; 1466 event->state = PERF_EVENT_STATE_OFF;
1467 } 1467 }
1468 event->tstamp_stopped = tstamp; 1468 event->tstamp_stopped = tstamp;
1469 event->pmu->del(event, 0); 1469 event->pmu->del(event, 0);
1470 event->oncpu = -1; 1470 event->oncpu = -1;
1471 1471
1472 if (!is_software_event(event)) 1472 if (!is_software_event(event))
1473 cpuctx->active_oncpu--; 1473 cpuctx->active_oncpu--;
1474 ctx->nr_active--; 1474 ctx->nr_active--;
1475 if (event->attr.freq && event->attr.sample_freq) 1475 if (event->attr.freq && event->attr.sample_freq)
1476 ctx->nr_freq--; 1476 ctx->nr_freq--;
1477 if (event->attr.exclusive || !cpuctx->active_oncpu) 1477 if (event->attr.exclusive || !cpuctx->active_oncpu)
1478 cpuctx->exclusive = 0; 1478 cpuctx->exclusive = 0;
1479 1479
1480 if (is_orphaned_child(event)) 1480 if (is_orphaned_child(event))
1481 schedule_orphans_remove(ctx); 1481 schedule_orphans_remove(ctx);
1482 1482
1483 perf_pmu_enable(event->pmu); 1483 perf_pmu_enable(event->pmu);
1484 } 1484 }
1485 1485
1486 static void 1486 static void
1487 group_sched_out(struct perf_event *group_event, 1487 group_sched_out(struct perf_event *group_event,
1488 struct perf_cpu_context *cpuctx, 1488 struct perf_cpu_context *cpuctx,
1489 struct perf_event_context *ctx) 1489 struct perf_event_context *ctx)
1490 { 1490 {
1491 struct perf_event *event; 1491 struct perf_event *event;
1492 int state = group_event->state; 1492 int state = group_event->state;
1493 1493
1494 event_sched_out(group_event, cpuctx, ctx); 1494 event_sched_out(group_event, cpuctx, ctx);
1495 1495
1496 /* 1496 /*
1497 * Schedule out siblings (if any): 1497 * Schedule out siblings (if any):
1498 */ 1498 */
1499 list_for_each_entry(event, &group_event->sibling_list, group_entry) 1499 list_for_each_entry(event, &group_event->sibling_list, group_entry)
1500 event_sched_out(event, cpuctx, ctx); 1500 event_sched_out(event, cpuctx, ctx);
1501 1501
1502 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive) 1502 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1503 cpuctx->exclusive = 0; 1503 cpuctx->exclusive = 0;
1504 } 1504 }
1505 1505
1506 struct remove_event { 1506 struct remove_event {
1507 struct perf_event *event; 1507 struct perf_event *event;
1508 bool detach_group; 1508 bool detach_group;
1509 }; 1509 };
1510 1510
1511 /* 1511 /*
1512 * Cross CPU call to remove a performance event 1512 * Cross CPU call to remove a performance event
1513 * 1513 *
1514 * We disable the event on the hardware level first. After that we 1514 * We disable the event on the hardware level first. After that we
1515 * remove it from the context list. 1515 * remove it from the context list.
1516 */ 1516 */
1517 static int __perf_remove_from_context(void *info) 1517 static int __perf_remove_from_context(void *info)
1518 { 1518 {
1519 struct remove_event *re = info; 1519 struct remove_event *re = info;
1520 struct perf_event *event = re->event; 1520 struct perf_event *event = re->event;
1521 struct perf_event_context *ctx = event->ctx; 1521 struct perf_event_context *ctx = event->ctx;
1522 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1522 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1523 1523
1524 raw_spin_lock(&ctx->lock); 1524 raw_spin_lock(&ctx->lock);
1525 event_sched_out(event, cpuctx, ctx); 1525 event_sched_out(event, cpuctx, ctx);
1526 if (re->detach_group) 1526 if (re->detach_group)
1527 perf_group_detach(event); 1527 perf_group_detach(event);
1528 list_del_event(event, ctx); 1528 list_del_event(event, ctx);
1529 if (!ctx->nr_events && cpuctx->task_ctx == ctx) { 1529 if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1530 ctx->is_active = 0; 1530 ctx->is_active = 0;
1531 cpuctx->task_ctx = NULL; 1531 cpuctx->task_ctx = NULL;
1532 } 1532 }
1533 raw_spin_unlock(&ctx->lock); 1533 raw_spin_unlock(&ctx->lock);
1534 1534
1535 return 0; 1535 return 0;
1536 } 1536 }
1537 1537
1538 1538
1539 /* 1539 /*
1540 * Remove the event from a task's (or a CPU's) list of events. 1540 * Remove the event from a task's (or a CPU's) list of events.
1541 * 1541 *
1542 * CPU events are removed with a smp call. For task events we only 1542 * CPU events are removed with a smp call. For task events we only
1543 * call when the task is on a CPU. 1543 * call when the task is on a CPU.
1544 * 1544 *
1545 * If event->ctx is a cloned context, callers must make sure that 1545 * If event->ctx is a cloned context, callers must make sure that
1546 * every task struct that event->ctx->task could possibly point to 1546 * every task struct that event->ctx->task could possibly point to
1547 * remains valid. This is OK when called from perf_release since 1547 * remains valid. This is OK when called from perf_release since
1548 * that only calls us on the top-level context, which can't be a clone. 1548 * that only calls us on the top-level context, which can't be a clone.
1549 * When called from perf_event_exit_task, it's OK because the 1549 * When called from perf_event_exit_task, it's OK because the
1550 * context has been detached from its task. 1550 * context has been detached from its task.
1551 */ 1551 */
1552 static void perf_remove_from_context(struct perf_event *event, bool detach_group) 1552 static void perf_remove_from_context(struct perf_event *event, bool detach_group)
1553 { 1553 {
1554 struct perf_event_context *ctx = event->ctx; 1554 struct perf_event_context *ctx = event->ctx;
1555 struct task_struct *task = ctx->task; 1555 struct task_struct *task = ctx->task;
1556 struct remove_event re = { 1556 struct remove_event re = {
1557 .event = event, 1557 .event = event,
1558 .detach_group = detach_group, 1558 .detach_group = detach_group,
1559 }; 1559 };
1560 1560
1561 lockdep_assert_held(&ctx->mutex); 1561 lockdep_assert_held(&ctx->mutex);
1562 1562
1563 if (!task) { 1563 if (!task) {
1564 /* 1564 /*
1565 * Per cpu events are removed via an smp call. The removal can 1565 * Per cpu events are removed via an smp call. The removal can
1566 * fail if the CPU is currently offline, but in that case we 1566 * fail if the CPU is currently offline, but in that case we
1567 * already called __perf_remove_from_context from 1567 * already called __perf_remove_from_context from
1568 * perf_event_exit_cpu. 1568 * perf_event_exit_cpu.
1569 */ 1569 */
1570 cpu_function_call(event->cpu, __perf_remove_from_context, &re); 1570 cpu_function_call(event->cpu, __perf_remove_from_context, &re);
1571 return; 1571 return;
1572 } 1572 }
1573 1573
1574 retry: 1574 retry:
1575 if (!task_function_call(task, __perf_remove_from_context, &re)) 1575 if (!task_function_call(task, __perf_remove_from_context, &re))
1576 return; 1576 return;
1577 1577
1578 raw_spin_lock_irq(&ctx->lock); 1578 raw_spin_lock_irq(&ctx->lock);
1579 /* 1579 /*
1580 * If we failed to find a running task, but find the context active now 1580 * If we failed to find a running task, but find the context active now
1581 * that we've acquired the ctx->lock, retry. 1581 * that we've acquired the ctx->lock, retry.
1582 */ 1582 */
1583 if (ctx->is_active) { 1583 if (ctx->is_active) {
1584 raw_spin_unlock_irq(&ctx->lock); 1584 raw_spin_unlock_irq(&ctx->lock);
1585 /* 1585 /*
1586 * Reload the task pointer, it might have been changed by 1586 * Reload the task pointer, it might have been changed by
1587 * a concurrent perf_event_context_sched_out(). 1587 * a concurrent perf_event_context_sched_out().
1588 */ 1588 */
1589 task = ctx->task; 1589 task = ctx->task;
1590 goto retry; 1590 goto retry;
1591 } 1591 }
1592 1592
1593 /* 1593 /*
1594 * Since the task isn't running, its safe to remove the event, us 1594 * Since the task isn't running, its safe to remove the event, us
1595 * holding the ctx->lock ensures the task won't get scheduled in. 1595 * holding the ctx->lock ensures the task won't get scheduled in.
1596 */ 1596 */
1597 if (detach_group) 1597 if (detach_group)
1598 perf_group_detach(event); 1598 perf_group_detach(event);
1599 list_del_event(event, ctx); 1599 list_del_event(event, ctx);
1600 raw_spin_unlock_irq(&ctx->lock); 1600 raw_spin_unlock_irq(&ctx->lock);
1601 } 1601 }
1602 1602
1603 /* 1603 /*
1604 * Cross CPU call to disable a performance event 1604 * Cross CPU call to disable a performance event
1605 */ 1605 */
1606 int __perf_event_disable(void *info) 1606 int __perf_event_disable(void *info)
1607 { 1607 {
1608 struct perf_event *event = info; 1608 struct perf_event *event = info;
1609 struct perf_event_context *ctx = event->ctx; 1609 struct perf_event_context *ctx = event->ctx;
1610 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1610 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1611 1611
1612 /* 1612 /*
1613 * If this is a per-task event, need to check whether this 1613 * If this is a per-task event, need to check whether this
1614 * event's task is the current task on this cpu. 1614 * event's task is the current task on this cpu.
1615 * 1615 *
1616 * Can trigger due to concurrent perf_event_context_sched_out() 1616 * Can trigger due to concurrent perf_event_context_sched_out()
1617 * flipping contexts around. 1617 * flipping contexts around.
1618 */ 1618 */
1619 if (ctx->task && cpuctx->task_ctx != ctx) 1619 if (ctx->task && cpuctx->task_ctx != ctx)
1620 return -EINVAL; 1620 return -EINVAL;
1621 1621
1622 raw_spin_lock(&ctx->lock); 1622 raw_spin_lock(&ctx->lock);
1623 1623
1624 /* 1624 /*
1625 * If the event is on, turn it off. 1625 * If the event is on, turn it off.
1626 * If it is in error state, leave it in error state. 1626 * If it is in error state, leave it in error state.
1627 */ 1627 */
1628 if (event->state >= PERF_EVENT_STATE_INACTIVE) { 1628 if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1629 update_context_time(ctx); 1629 update_context_time(ctx);
1630 update_cgrp_time_from_event(event); 1630 update_cgrp_time_from_event(event);
1631 update_group_times(event); 1631 update_group_times(event);
1632 if (event == event->group_leader) 1632 if (event == event->group_leader)
1633 group_sched_out(event, cpuctx, ctx); 1633 group_sched_out(event, cpuctx, ctx);
1634 else 1634 else
1635 event_sched_out(event, cpuctx, ctx); 1635 event_sched_out(event, cpuctx, ctx);
1636 event->state = PERF_EVENT_STATE_OFF; 1636 event->state = PERF_EVENT_STATE_OFF;
1637 } 1637 }
1638 1638
1639 raw_spin_unlock(&ctx->lock); 1639 raw_spin_unlock(&ctx->lock);
1640 1640
1641 return 0; 1641 return 0;
1642 } 1642 }
1643 1643
1644 /* 1644 /*
1645 * Disable a event. 1645 * Disable a event.
1646 * 1646 *
1647 * If event->ctx is a cloned context, callers must make sure that 1647 * If event->ctx is a cloned context, callers must make sure that
1648 * every task struct that event->ctx->task could possibly point to 1648 * every task struct that event->ctx->task could possibly point to
1649 * remains valid. This condition is satisifed when called through 1649 * remains valid. This condition is satisifed when called through
1650 * perf_event_for_each_child or perf_event_for_each because they 1650 * perf_event_for_each_child or perf_event_for_each because they
1651 * hold the top-level event's child_mutex, so any descendant that 1651 * hold the top-level event's child_mutex, so any descendant that
1652 * goes to exit will block in sync_child_event. 1652 * goes to exit will block in sync_child_event.
1653 * When called from perf_pending_event it's OK because event->ctx 1653 * When called from perf_pending_event it's OK because event->ctx
1654 * is the current context on this CPU and preemption is disabled, 1654 * is the current context on this CPU and preemption is disabled,
1655 * hence we can't get into perf_event_task_sched_out for this context. 1655 * hence we can't get into perf_event_task_sched_out for this context.
1656 */ 1656 */
1657 void perf_event_disable(struct perf_event *event) 1657 void perf_event_disable(struct perf_event *event)
1658 { 1658 {
1659 struct perf_event_context *ctx = event->ctx; 1659 struct perf_event_context *ctx = event->ctx;
1660 struct task_struct *task = ctx->task; 1660 struct task_struct *task = ctx->task;
1661 1661
1662 if (!task) { 1662 if (!task) {
1663 /* 1663 /*
1664 * Disable the event on the cpu that it's on 1664 * Disable the event on the cpu that it's on
1665 */ 1665 */
1666 cpu_function_call(event->cpu, __perf_event_disable, event); 1666 cpu_function_call(event->cpu, __perf_event_disable, event);
1667 return; 1667 return;
1668 } 1668 }
1669 1669
1670 retry: 1670 retry:
1671 if (!task_function_call(task, __perf_event_disable, event)) 1671 if (!task_function_call(task, __perf_event_disable, event))
1672 return; 1672 return;
1673 1673
1674 raw_spin_lock_irq(&ctx->lock); 1674 raw_spin_lock_irq(&ctx->lock);
1675 /* 1675 /*
1676 * If the event is still active, we need to retry the cross-call. 1676 * If the event is still active, we need to retry the cross-call.
1677 */ 1677 */
1678 if (event->state == PERF_EVENT_STATE_ACTIVE) { 1678 if (event->state == PERF_EVENT_STATE_ACTIVE) {
1679 raw_spin_unlock_irq(&ctx->lock); 1679 raw_spin_unlock_irq(&ctx->lock);
1680 /* 1680 /*
1681 * Reload the task pointer, it might have been changed by 1681 * Reload the task pointer, it might have been changed by
1682 * a concurrent perf_event_context_sched_out(). 1682 * a concurrent perf_event_context_sched_out().
1683 */ 1683 */
1684 task = ctx->task; 1684 task = ctx->task;
1685 goto retry; 1685 goto retry;
1686 } 1686 }
1687 1687
1688 /* 1688 /*
1689 * Since we have the lock this context can't be scheduled 1689 * Since we have the lock this context can't be scheduled
1690 * in, so we can change the state safely. 1690 * in, so we can change the state safely.
1691 */ 1691 */
1692 if (event->state == PERF_EVENT_STATE_INACTIVE) { 1692 if (event->state == PERF_EVENT_STATE_INACTIVE) {
1693 update_group_times(event); 1693 update_group_times(event);
1694 event->state = PERF_EVENT_STATE_OFF; 1694 event->state = PERF_EVENT_STATE_OFF;
1695 } 1695 }
1696 raw_spin_unlock_irq(&ctx->lock); 1696 raw_spin_unlock_irq(&ctx->lock);
1697 } 1697 }
1698 EXPORT_SYMBOL_GPL(perf_event_disable); 1698 EXPORT_SYMBOL_GPL(perf_event_disable);
1699 1699
1700 static void perf_set_shadow_time(struct perf_event *event, 1700 static void perf_set_shadow_time(struct perf_event *event,
1701 struct perf_event_context *ctx, 1701 struct perf_event_context *ctx,
1702 u64 tstamp) 1702 u64 tstamp)
1703 { 1703 {
1704 /* 1704 /*
1705 * use the correct time source for the time snapshot 1705 * use the correct time source for the time snapshot
1706 * 1706 *
1707 * We could get by without this by leveraging the 1707 * We could get by without this by leveraging the
1708 * fact that to get to this function, the caller 1708 * fact that to get to this function, the caller
1709 * has most likely already called update_context_time() 1709 * has most likely already called update_context_time()
1710 * and update_cgrp_time_xx() and thus both timestamp 1710 * and update_cgrp_time_xx() and thus both timestamp
1711 * are identical (or very close). Given that tstamp is, 1711 * are identical (or very close). Given that tstamp is,
1712 * already adjusted for cgroup, we could say that: 1712 * already adjusted for cgroup, we could say that:
1713 * tstamp - ctx->timestamp 1713 * tstamp - ctx->timestamp
1714 * is equivalent to 1714 * is equivalent to
1715 * tstamp - cgrp->timestamp. 1715 * tstamp - cgrp->timestamp.
1716 * 1716 *
1717 * Then, in perf_output_read(), the calculation would 1717 * Then, in perf_output_read(), the calculation would
1718 * work with no changes because: 1718 * work with no changes because:
1719 * - event is guaranteed scheduled in 1719 * - event is guaranteed scheduled in
1720 * - no scheduled out in between 1720 * - no scheduled out in between
1721 * - thus the timestamp would be the same 1721 * - thus the timestamp would be the same
1722 * 1722 *
1723 * But this is a bit hairy. 1723 * But this is a bit hairy.
1724 * 1724 *
1725 * So instead, we have an explicit cgroup call to remain 1725 * So instead, we have an explicit cgroup call to remain
1726 * within the time time source all along. We believe it 1726 * within the time time source all along. We believe it
1727 * is cleaner and simpler to understand. 1727 * is cleaner and simpler to understand.
1728 */ 1728 */
1729 if (is_cgroup_event(event)) 1729 if (is_cgroup_event(event))
1730 perf_cgroup_set_shadow_time(event, tstamp); 1730 perf_cgroup_set_shadow_time(event, tstamp);
1731 else 1731 else
1732 event->shadow_ctx_time = tstamp - ctx->timestamp; 1732 event->shadow_ctx_time = tstamp - ctx->timestamp;
1733 } 1733 }
1734 1734
1735 #define MAX_INTERRUPTS (~0ULL) 1735 #define MAX_INTERRUPTS (~0ULL)
1736 1736
1737 static void perf_log_throttle(struct perf_event *event, int enable); 1737 static void perf_log_throttle(struct perf_event *event, int enable);
1738 1738
1739 static int 1739 static int
1740 event_sched_in(struct perf_event *event, 1740 event_sched_in(struct perf_event *event,
1741 struct perf_cpu_context *cpuctx, 1741 struct perf_cpu_context *cpuctx,
1742 struct perf_event_context *ctx) 1742 struct perf_event_context *ctx)
1743 { 1743 {
1744 u64 tstamp = perf_event_time(event); 1744 u64 tstamp = perf_event_time(event);
1745 int ret = 0; 1745 int ret = 0;
1746 1746
1747 lockdep_assert_held(&ctx->lock); 1747 lockdep_assert_held(&ctx->lock);
1748 1748
1749 if (event->state <= PERF_EVENT_STATE_OFF) 1749 if (event->state <= PERF_EVENT_STATE_OFF)
1750 return 0; 1750 return 0;
1751 1751
1752 event->state = PERF_EVENT_STATE_ACTIVE; 1752 event->state = PERF_EVENT_STATE_ACTIVE;
1753 event->oncpu = smp_processor_id(); 1753 event->oncpu = smp_processor_id();
1754 1754
1755 /* 1755 /*
1756 * Unthrottle events, since we scheduled we might have missed several 1756 * Unthrottle events, since we scheduled we might have missed several
1757 * ticks already, also for a heavily scheduling task there is little 1757 * ticks already, also for a heavily scheduling task there is little
1758 * guarantee it'll get a tick in a timely manner. 1758 * guarantee it'll get a tick in a timely manner.
1759 */ 1759 */
1760 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 1760 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1761 perf_log_throttle(event, 1); 1761 perf_log_throttle(event, 1);
1762 event->hw.interrupts = 0; 1762 event->hw.interrupts = 0;
1763 } 1763 }
1764 1764
1765 /* 1765 /*
1766 * The new state must be visible before we turn it on in the hardware: 1766 * The new state must be visible before we turn it on in the hardware:
1767 */ 1767 */
1768 smp_wmb(); 1768 smp_wmb();
1769 1769
1770 perf_pmu_disable(event->pmu); 1770 perf_pmu_disable(event->pmu);
1771 1771
1772 if (event->pmu->add(event, PERF_EF_START)) { 1772 if (event->pmu->add(event, PERF_EF_START)) {
1773 event->state = PERF_EVENT_STATE_INACTIVE; 1773 event->state = PERF_EVENT_STATE_INACTIVE;
1774 event->oncpu = -1; 1774 event->oncpu = -1;
1775 ret = -EAGAIN; 1775 ret = -EAGAIN;
1776 goto out; 1776 goto out;
1777 } 1777 }
1778 1778
1779 event->tstamp_running += tstamp - event->tstamp_stopped; 1779 event->tstamp_running += tstamp - event->tstamp_stopped;
1780 1780
1781 perf_set_shadow_time(event, ctx, tstamp); 1781 perf_set_shadow_time(event, ctx, tstamp);
1782 1782
1783 if (!is_software_event(event)) 1783 if (!is_software_event(event))
1784 cpuctx->active_oncpu++; 1784 cpuctx->active_oncpu++;
1785 ctx->nr_active++; 1785 ctx->nr_active++;
1786 if (event->attr.freq && event->attr.sample_freq) 1786 if (event->attr.freq && event->attr.sample_freq)
1787 ctx->nr_freq++; 1787 ctx->nr_freq++;
1788 1788
1789 if (event->attr.exclusive) 1789 if (event->attr.exclusive)
1790 cpuctx->exclusive = 1; 1790 cpuctx->exclusive = 1;
1791 1791
1792 if (is_orphaned_child(event)) 1792 if (is_orphaned_child(event))
1793 schedule_orphans_remove(ctx); 1793 schedule_orphans_remove(ctx);
1794 1794
1795 out: 1795 out:
1796 perf_pmu_enable(event->pmu); 1796 perf_pmu_enable(event->pmu);
1797 1797
1798 return ret; 1798 return ret;
1799 } 1799 }
1800 1800
1801 static int 1801 static int
1802 group_sched_in(struct perf_event *group_event, 1802 group_sched_in(struct perf_event *group_event,
1803 struct perf_cpu_context *cpuctx, 1803 struct perf_cpu_context *cpuctx,
1804 struct perf_event_context *ctx) 1804 struct perf_event_context *ctx)
1805 { 1805 {
1806 struct perf_event *event, *partial_group = NULL; 1806 struct perf_event *event, *partial_group = NULL;
1807 struct pmu *pmu = ctx->pmu; 1807 struct pmu *pmu = ctx->pmu;
1808 u64 now = ctx->time; 1808 u64 now = ctx->time;
1809 bool simulate = false; 1809 bool simulate = false;
1810 1810
1811 if (group_event->state == PERF_EVENT_STATE_OFF) 1811 if (group_event->state == PERF_EVENT_STATE_OFF)
1812 return 0; 1812 return 0;
1813 1813
1814 pmu->start_txn(pmu); 1814 pmu->start_txn(pmu);
1815 1815
1816 if (event_sched_in(group_event, cpuctx, ctx)) { 1816 if (event_sched_in(group_event, cpuctx, ctx)) {
1817 pmu->cancel_txn(pmu); 1817 pmu->cancel_txn(pmu);
1818 perf_cpu_hrtimer_restart(cpuctx); 1818 perf_cpu_hrtimer_restart(cpuctx);
1819 return -EAGAIN; 1819 return -EAGAIN;
1820 } 1820 }
1821 1821
1822 /* 1822 /*
1823 * Schedule in siblings as one group (if any): 1823 * Schedule in siblings as one group (if any):
1824 */ 1824 */
1825 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1825 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1826 if (event_sched_in(event, cpuctx, ctx)) { 1826 if (event_sched_in(event, cpuctx, ctx)) {
1827 partial_group = event; 1827 partial_group = event;
1828 goto group_error; 1828 goto group_error;
1829 } 1829 }
1830 } 1830 }
1831 1831
1832 if (!pmu->commit_txn(pmu)) 1832 if (!pmu->commit_txn(pmu))
1833 return 0; 1833 return 0;
1834 1834
1835 group_error: 1835 group_error:
1836 /* 1836 /*
1837 * Groups can be scheduled in as one unit only, so undo any 1837 * Groups can be scheduled in as one unit only, so undo any
1838 * partial group before returning: 1838 * partial group before returning:
1839 * The events up to the failed event are scheduled out normally, 1839 * The events up to the failed event are scheduled out normally,
1840 * tstamp_stopped will be updated. 1840 * tstamp_stopped will be updated.
1841 * 1841 *
1842 * The failed events and the remaining siblings need to have 1842 * The failed events and the remaining siblings need to have
1843 * their timings updated as if they had gone thru event_sched_in() 1843 * their timings updated as if they had gone thru event_sched_in()
1844 * and event_sched_out(). This is required to get consistent timings 1844 * and event_sched_out(). This is required to get consistent timings
1845 * across the group. This also takes care of the case where the group 1845 * across the group. This also takes care of the case where the group
1846 * could never be scheduled by ensuring tstamp_stopped is set to mark 1846 * could never be scheduled by ensuring tstamp_stopped is set to mark
1847 * the time the event was actually stopped, such that time delta 1847 * the time the event was actually stopped, such that time delta
1848 * calculation in update_event_times() is correct. 1848 * calculation in update_event_times() is correct.
1849 */ 1849 */
1850 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 1850 list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1851 if (event == partial_group) 1851 if (event == partial_group)
1852 simulate = true; 1852 simulate = true;
1853 1853
1854 if (simulate) { 1854 if (simulate) {
1855 event->tstamp_running += now - event->tstamp_stopped; 1855 event->tstamp_running += now - event->tstamp_stopped;
1856 event->tstamp_stopped = now; 1856 event->tstamp_stopped = now;
1857 } else { 1857 } else {
1858 event_sched_out(event, cpuctx, ctx); 1858 event_sched_out(event, cpuctx, ctx);
1859 } 1859 }
1860 } 1860 }
1861 event_sched_out(group_event, cpuctx, ctx); 1861 event_sched_out(group_event, cpuctx, ctx);
1862 1862
1863 pmu->cancel_txn(pmu); 1863 pmu->cancel_txn(pmu);
1864 1864
1865 perf_cpu_hrtimer_restart(cpuctx); 1865 perf_cpu_hrtimer_restart(cpuctx);
1866 1866
1867 return -EAGAIN; 1867 return -EAGAIN;
1868 } 1868 }
1869 1869
1870 /* 1870 /*
1871 * Work out whether we can put this event group on the CPU now. 1871 * Work out whether we can put this event group on the CPU now.
1872 */ 1872 */
1873 static int group_can_go_on(struct perf_event *event, 1873 static int group_can_go_on(struct perf_event *event,
1874 struct perf_cpu_context *cpuctx, 1874 struct perf_cpu_context *cpuctx,
1875 int can_add_hw) 1875 int can_add_hw)
1876 { 1876 {
1877 /* 1877 /*
1878 * Groups consisting entirely of software events can always go on. 1878 * Groups consisting entirely of software events can always go on.
1879 */ 1879 */
1880 if (event->group_flags & PERF_GROUP_SOFTWARE) 1880 if (event->group_flags & PERF_GROUP_SOFTWARE)
1881 return 1; 1881 return 1;
1882 /* 1882 /*
1883 * If an exclusive group is already on, no other hardware 1883 * If an exclusive group is already on, no other hardware
1884 * events can go on. 1884 * events can go on.
1885 */ 1885 */
1886 if (cpuctx->exclusive) 1886 if (cpuctx->exclusive)
1887 return 0; 1887 return 0;
1888 /* 1888 /*
1889 * If this group is exclusive and there are already 1889 * If this group is exclusive and there are already
1890 * events on the CPU, it can't go on. 1890 * events on the CPU, it can't go on.
1891 */ 1891 */
1892 if (event->attr.exclusive && cpuctx->active_oncpu) 1892 if (event->attr.exclusive && cpuctx->active_oncpu)
1893 return 0; 1893 return 0;
1894 /* 1894 /*
1895 * Otherwise, try to add it if all previous groups were able 1895 * Otherwise, try to add it if all previous groups were able
1896 * to go on. 1896 * to go on.
1897 */ 1897 */
1898 return can_add_hw; 1898 return can_add_hw;
1899 } 1899 }
1900 1900
1901 static void add_event_to_ctx(struct perf_event *event, 1901 static void add_event_to_ctx(struct perf_event *event,
1902 struct perf_event_context *ctx) 1902 struct perf_event_context *ctx)
1903 { 1903 {
1904 u64 tstamp = perf_event_time(event); 1904 u64 tstamp = perf_event_time(event);
1905 1905
1906 list_add_event(event, ctx); 1906 list_add_event(event, ctx);
1907 perf_group_attach(event); 1907 perf_group_attach(event);
1908 event->tstamp_enabled = tstamp; 1908 event->tstamp_enabled = tstamp;
1909 event->tstamp_running = tstamp; 1909 event->tstamp_running = tstamp;
1910 event->tstamp_stopped = tstamp; 1910 event->tstamp_stopped = tstamp;
1911 } 1911 }
1912 1912
1913 static void task_ctx_sched_out(struct perf_event_context *ctx); 1913 static void task_ctx_sched_out(struct perf_event_context *ctx);
1914 static void 1914 static void
1915 ctx_sched_in(struct perf_event_context *ctx, 1915 ctx_sched_in(struct perf_event_context *ctx,
1916 struct perf_cpu_context *cpuctx, 1916 struct perf_cpu_context *cpuctx,
1917 enum event_type_t event_type, 1917 enum event_type_t event_type,
1918 struct task_struct *task); 1918 struct task_struct *task);
1919 1919
1920 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 1920 static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1921 struct perf_event_context *ctx, 1921 struct perf_event_context *ctx,
1922 struct task_struct *task) 1922 struct task_struct *task)
1923 { 1923 {
1924 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 1924 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1925 if (ctx) 1925 if (ctx)
1926 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 1926 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1927 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 1927 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1928 if (ctx) 1928 if (ctx)
1929 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 1929 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1930 } 1930 }
1931 1931
1932 /* 1932 /*
1933 * Cross CPU call to install and enable a performance event 1933 * Cross CPU call to install and enable a performance event
1934 * 1934 *
1935 * Must be called with ctx->mutex held 1935 * Must be called with ctx->mutex held
1936 */ 1936 */
1937 static int __perf_install_in_context(void *info) 1937 static int __perf_install_in_context(void *info)
1938 { 1938 {
1939 struct perf_event *event = info; 1939 struct perf_event *event = info;
1940 struct perf_event_context *ctx = event->ctx; 1940 struct perf_event_context *ctx = event->ctx;
1941 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 1941 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1942 struct perf_event_context *task_ctx = cpuctx->task_ctx; 1942 struct perf_event_context *task_ctx = cpuctx->task_ctx;
1943 struct task_struct *task = current; 1943 struct task_struct *task = current;
1944 1944
1945 perf_ctx_lock(cpuctx, task_ctx); 1945 perf_ctx_lock(cpuctx, task_ctx);
1946 perf_pmu_disable(cpuctx->ctx.pmu); 1946 perf_pmu_disable(cpuctx->ctx.pmu);
1947 1947
1948 /* 1948 /*
1949 * If there was an active task_ctx schedule it out. 1949 * If there was an active task_ctx schedule it out.
1950 */ 1950 */
1951 if (task_ctx) 1951 if (task_ctx)
1952 task_ctx_sched_out(task_ctx); 1952 task_ctx_sched_out(task_ctx);
1953 1953
1954 /* 1954 /*
1955 * If the context we're installing events in is not the 1955 * If the context we're installing events in is not the
1956 * active task_ctx, flip them. 1956 * active task_ctx, flip them.
1957 */ 1957 */
1958 if (ctx->task && task_ctx != ctx) { 1958 if (ctx->task && task_ctx != ctx) {
1959 if (task_ctx) 1959 if (task_ctx)
1960 raw_spin_unlock(&task_ctx->lock); 1960 raw_spin_unlock(&task_ctx->lock);
1961 raw_spin_lock(&ctx->lock); 1961 raw_spin_lock(&ctx->lock);
1962 task_ctx = ctx; 1962 task_ctx = ctx;
1963 } 1963 }
1964 1964
1965 if (task_ctx) { 1965 if (task_ctx) {
1966 cpuctx->task_ctx = task_ctx; 1966 cpuctx->task_ctx = task_ctx;
1967 task = task_ctx->task; 1967 task = task_ctx->task;
1968 } 1968 }
1969 1969
1970 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 1970 cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1971 1971
1972 update_context_time(ctx); 1972 update_context_time(ctx);
1973 /* 1973 /*
1974 * update cgrp time only if current cgrp 1974 * update cgrp time only if current cgrp
1975 * matches event->cgrp. Must be done before 1975 * matches event->cgrp. Must be done before
1976 * calling add_event_to_ctx() 1976 * calling add_event_to_ctx()
1977 */ 1977 */
1978 update_cgrp_time_from_event(event); 1978 update_cgrp_time_from_event(event);
1979 1979
1980 add_event_to_ctx(event, ctx); 1980 add_event_to_ctx(event, ctx);
1981 1981
1982 /* 1982 /*
1983 * Schedule everything back in 1983 * Schedule everything back in
1984 */ 1984 */
1985 perf_event_sched_in(cpuctx, task_ctx, task); 1985 perf_event_sched_in(cpuctx, task_ctx, task);
1986 1986
1987 perf_pmu_enable(cpuctx->ctx.pmu); 1987 perf_pmu_enable(cpuctx->ctx.pmu);
1988 perf_ctx_unlock(cpuctx, task_ctx); 1988 perf_ctx_unlock(cpuctx, task_ctx);
1989 1989
1990 return 0; 1990 return 0;
1991 } 1991 }
1992 1992
1993 /* 1993 /*
1994 * Attach a performance event to a context 1994 * Attach a performance event to a context
1995 * 1995 *
1996 * First we add the event to the list with the hardware enable bit 1996 * First we add the event to the list with the hardware enable bit
1997 * in event->hw_config cleared. 1997 * in event->hw_config cleared.
1998 * 1998 *
1999 * If the event is attached to a task which is on a CPU we use a smp 1999 * If the event is attached to a task which is on a CPU we use a smp
2000 * call to enable it in the task context. The task might have been 2000 * call to enable it in the task context. The task might have been
2001 * scheduled away, but we check this in the smp call again. 2001 * scheduled away, but we check this in the smp call again.
2002 */ 2002 */
2003 static void 2003 static void
2004 perf_install_in_context(struct perf_event_context *ctx, 2004 perf_install_in_context(struct perf_event_context *ctx,
2005 struct perf_event *event, 2005 struct perf_event *event,
2006 int cpu) 2006 int cpu)
2007 { 2007 {
2008 struct task_struct *task = ctx->task; 2008 struct task_struct *task = ctx->task;
2009 2009
2010 lockdep_assert_held(&ctx->mutex); 2010 lockdep_assert_held(&ctx->mutex);
2011 2011
2012 event->ctx = ctx; 2012 event->ctx = ctx;
2013 if (event->cpu != -1) 2013 if (event->cpu != -1)
2014 event->cpu = cpu; 2014 event->cpu = cpu;
2015 2015
2016 if (!task) { 2016 if (!task) {
2017 /* 2017 /*
2018 * Per cpu events are installed via an smp call and 2018 * Per cpu events are installed via an smp call and
2019 * the install is always successful. 2019 * the install is always successful.
2020 */ 2020 */
2021 cpu_function_call(cpu, __perf_install_in_context, event); 2021 cpu_function_call(cpu, __perf_install_in_context, event);
2022 return; 2022 return;
2023 } 2023 }
2024 2024
2025 retry: 2025 retry:
2026 if (!task_function_call(task, __perf_install_in_context, event)) 2026 if (!task_function_call(task, __perf_install_in_context, event))
2027 return; 2027 return;
2028 2028
2029 raw_spin_lock_irq(&ctx->lock); 2029 raw_spin_lock_irq(&ctx->lock);
2030 /* 2030 /*
2031 * If we failed to find a running task, but find the context active now 2031 * If we failed to find a running task, but find the context active now
2032 * that we've acquired the ctx->lock, retry. 2032 * that we've acquired the ctx->lock, retry.
2033 */ 2033 */
2034 if (ctx->is_active) { 2034 if (ctx->is_active) {
2035 raw_spin_unlock_irq(&ctx->lock); 2035 raw_spin_unlock_irq(&ctx->lock);
2036 /* 2036 /*
2037 * Reload the task pointer, it might have been changed by 2037 * Reload the task pointer, it might have been changed by
2038 * a concurrent perf_event_context_sched_out(). 2038 * a concurrent perf_event_context_sched_out().
2039 */ 2039 */
2040 task = ctx->task; 2040 task = ctx->task;
2041 goto retry; 2041 goto retry;
2042 } 2042 }
2043 2043
2044 /* 2044 /*
2045 * Since the task isn't running, its safe to add the event, us holding 2045 * Since the task isn't running, its safe to add the event, us holding
2046 * the ctx->lock ensures the task won't get scheduled in. 2046 * the ctx->lock ensures the task won't get scheduled in.
2047 */ 2047 */
2048 add_event_to_ctx(event, ctx); 2048 add_event_to_ctx(event, ctx);
2049 raw_spin_unlock_irq(&ctx->lock); 2049 raw_spin_unlock_irq(&ctx->lock);
2050 } 2050 }
2051 2051
2052 /* 2052 /*
2053 * Put a event into inactive state and update time fields. 2053 * Put a event into inactive state and update time fields.
2054 * Enabling the leader of a group effectively enables all 2054 * Enabling the leader of a group effectively enables all
2055 * the group members that aren't explicitly disabled, so we 2055 * the group members that aren't explicitly disabled, so we
2056 * have to update their ->tstamp_enabled also. 2056 * have to update their ->tstamp_enabled also.
2057 * Note: this works for group members as well as group leaders 2057 * Note: this works for group members as well as group leaders
2058 * since the non-leader members' sibling_lists will be empty. 2058 * since the non-leader members' sibling_lists will be empty.
2059 */ 2059 */
2060 static void __perf_event_mark_enabled(struct perf_event *event) 2060 static void __perf_event_mark_enabled(struct perf_event *event)
2061 { 2061 {
2062 struct perf_event *sub; 2062 struct perf_event *sub;
2063 u64 tstamp = perf_event_time(event); 2063 u64 tstamp = perf_event_time(event);
2064 2064
2065 event->state = PERF_EVENT_STATE_INACTIVE; 2065 event->state = PERF_EVENT_STATE_INACTIVE;
2066 event->tstamp_enabled = tstamp - event->total_time_enabled; 2066 event->tstamp_enabled = tstamp - event->total_time_enabled;
2067 list_for_each_entry(sub, &event->sibling_list, group_entry) { 2067 list_for_each_entry(sub, &event->sibling_list, group_entry) {
2068 if (sub->state >= PERF_EVENT_STATE_INACTIVE) 2068 if (sub->state >= PERF_EVENT_STATE_INACTIVE)
2069 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 2069 sub->tstamp_enabled = tstamp - sub->total_time_enabled;
2070 } 2070 }
2071 } 2071 }
2072 2072
2073 /* 2073 /*
2074 * Cross CPU call to enable a performance event 2074 * Cross CPU call to enable a performance event
2075 */ 2075 */
2076 static int __perf_event_enable(void *info) 2076 static int __perf_event_enable(void *info)
2077 { 2077 {
2078 struct perf_event *event = info; 2078 struct perf_event *event = info;
2079 struct perf_event_context *ctx = event->ctx; 2079 struct perf_event_context *ctx = event->ctx;
2080 struct perf_event *leader = event->group_leader; 2080 struct perf_event *leader = event->group_leader;
2081 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2081 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2082 int err; 2082 int err;
2083 2083
2084 /* 2084 /*
2085 * There's a time window between 'ctx->is_active' check 2085 * There's a time window between 'ctx->is_active' check
2086 * in perf_event_enable function and this place having: 2086 * in perf_event_enable function and this place having:
2087 * - IRQs on 2087 * - IRQs on
2088 * - ctx->lock unlocked 2088 * - ctx->lock unlocked
2089 * 2089 *
2090 * where the task could be killed and 'ctx' deactivated 2090 * where the task could be killed and 'ctx' deactivated
2091 * by perf_event_exit_task. 2091 * by perf_event_exit_task.
2092 */ 2092 */
2093 if (!ctx->is_active) 2093 if (!ctx->is_active)
2094 return -EINVAL; 2094 return -EINVAL;
2095 2095
2096 raw_spin_lock(&ctx->lock); 2096 raw_spin_lock(&ctx->lock);
2097 update_context_time(ctx); 2097 update_context_time(ctx);
2098 2098
2099 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2099 if (event->state >= PERF_EVENT_STATE_INACTIVE)
2100 goto unlock; 2100 goto unlock;
2101 2101
2102 /* 2102 /*
2103 * set current task's cgroup time reference point 2103 * set current task's cgroup time reference point
2104 */ 2104 */
2105 perf_cgroup_set_timestamp(current, ctx); 2105 perf_cgroup_set_timestamp(current, ctx);
2106 2106
2107 __perf_event_mark_enabled(event); 2107 __perf_event_mark_enabled(event);
2108 2108
2109 if (!event_filter_match(event)) { 2109 if (!event_filter_match(event)) {
2110 if (is_cgroup_event(event)) 2110 if (is_cgroup_event(event))
2111 perf_cgroup_defer_enabled(event); 2111 perf_cgroup_defer_enabled(event);
2112 goto unlock; 2112 goto unlock;
2113 } 2113 }
2114 2114
2115 /* 2115 /*
2116 * If the event is in a group and isn't the group leader, 2116 * If the event is in a group and isn't the group leader,
2117 * then don't put it on unless the group is on. 2117 * then don't put it on unless the group is on.
2118 */ 2118 */
2119 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) 2119 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
2120 goto unlock; 2120 goto unlock;
2121 2121
2122 if (!group_can_go_on(event, cpuctx, 1)) { 2122 if (!group_can_go_on(event, cpuctx, 1)) {
2123 err = -EEXIST; 2123 err = -EEXIST;
2124 } else { 2124 } else {
2125 if (event == leader) 2125 if (event == leader)
2126 err = group_sched_in(event, cpuctx, ctx); 2126 err = group_sched_in(event, cpuctx, ctx);
2127 else 2127 else
2128 err = event_sched_in(event, cpuctx, ctx); 2128 err = event_sched_in(event, cpuctx, ctx);
2129 } 2129 }
2130 2130
2131 if (err) { 2131 if (err) {
2132 /* 2132 /*
2133 * If this event can't go on and it's part of a 2133 * If this event can't go on and it's part of a
2134 * group, then the whole group has to come off. 2134 * group, then the whole group has to come off.
2135 */ 2135 */
2136 if (leader != event) { 2136 if (leader != event) {
2137 group_sched_out(leader, cpuctx, ctx); 2137 group_sched_out(leader, cpuctx, ctx);
2138 perf_cpu_hrtimer_restart(cpuctx); 2138 perf_cpu_hrtimer_restart(cpuctx);
2139 } 2139 }
2140 if (leader->attr.pinned) { 2140 if (leader->attr.pinned) {
2141 update_group_times(leader); 2141 update_group_times(leader);
2142 leader->state = PERF_EVENT_STATE_ERROR; 2142 leader->state = PERF_EVENT_STATE_ERROR;
2143 } 2143 }
2144 } 2144 }
2145 2145
2146 unlock: 2146 unlock:
2147 raw_spin_unlock(&ctx->lock); 2147 raw_spin_unlock(&ctx->lock);
2148 2148
2149 return 0; 2149 return 0;
2150 } 2150 }
2151 2151
2152 /* 2152 /*
2153 * Enable a event. 2153 * Enable a event.
2154 * 2154 *
2155 * If event->ctx is a cloned context, callers must make sure that 2155 * If event->ctx is a cloned context, callers must make sure that
2156 * every task struct that event->ctx->task could possibly point to 2156 * every task struct that event->ctx->task could possibly point to
2157 * remains valid. This condition is satisfied when called through 2157 * remains valid. This condition is satisfied when called through
2158 * perf_event_for_each_child or perf_event_for_each as described 2158 * perf_event_for_each_child or perf_event_for_each as described
2159 * for perf_event_disable. 2159 * for perf_event_disable.
2160 */ 2160 */
2161 void perf_event_enable(struct perf_event *event) 2161 void perf_event_enable(struct perf_event *event)
2162 { 2162 {
2163 struct perf_event_context *ctx = event->ctx; 2163 struct perf_event_context *ctx = event->ctx;
2164 struct task_struct *task = ctx->task; 2164 struct task_struct *task = ctx->task;
2165 2165
2166 if (!task) { 2166 if (!task) {
2167 /* 2167 /*
2168 * Enable the event on the cpu that it's on 2168 * Enable the event on the cpu that it's on
2169 */ 2169 */
2170 cpu_function_call(event->cpu, __perf_event_enable, event); 2170 cpu_function_call(event->cpu, __perf_event_enable, event);
2171 return; 2171 return;
2172 } 2172 }
2173 2173
2174 raw_spin_lock_irq(&ctx->lock); 2174 raw_spin_lock_irq(&ctx->lock);
2175 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2175 if (event->state >= PERF_EVENT_STATE_INACTIVE)
2176 goto out; 2176 goto out;
2177 2177
2178 /* 2178 /*
2179 * If the event is in error state, clear that first. 2179 * If the event is in error state, clear that first.
2180 * That way, if we see the event in error state below, we 2180 * That way, if we see the event in error state below, we
2181 * know that it has gone back into error state, as distinct 2181 * know that it has gone back into error state, as distinct
2182 * from the task having been scheduled away before the 2182 * from the task having been scheduled away before the
2183 * cross-call arrived. 2183 * cross-call arrived.
2184 */ 2184 */
2185 if (event->state == PERF_EVENT_STATE_ERROR) 2185 if (event->state == PERF_EVENT_STATE_ERROR)
2186 event->state = PERF_EVENT_STATE_OFF; 2186 event->state = PERF_EVENT_STATE_OFF;
2187 2187
2188 retry: 2188 retry:
2189 if (!ctx->is_active) { 2189 if (!ctx->is_active) {
2190 __perf_event_mark_enabled(event); 2190 __perf_event_mark_enabled(event);
2191 goto out; 2191 goto out;
2192 } 2192 }
2193 2193
2194 raw_spin_unlock_irq(&ctx->lock); 2194 raw_spin_unlock_irq(&ctx->lock);
2195 2195
2196 if (!task_function_call(task, __perf_event_enable, event)) 2196 if (!task_function_call(task, __perf_event_enable, event))
2197 return; 2197 return;
2198 2198
2199 raw_spin_lock_irq(&ctx->lock); 2199 raw_spin_lock_irq(&ctx->lock);
2200 2200
2201 /* 2201 /*
2202 * If the context is active and the event is still off, 2202 * If the context is active and the event is still off,
2203 * we need to retry the cross-call. 2203 * we need to retry the cross-call.
2204 */ 2204 */
2205 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) { 2205 if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
2206 /* 2206 /*
2207 * task could have been flipped by a concurrent 2207 * task could have been flipped by a concurrent
2208 * perf_event_context_sched_out() 2208 * perf_event_context_sched_out()
2209 */ 2209 */
2210 task = ctx->task; 2210 task = ctx->task;
2211 goto retry; 2211 goto retry;
2212 } 2212 }
2213 2213
2214 out: 2214 out:
2215 raw_spin_unlock_irq(&ctx->lock); 2215 raw_spin_unlock_irq(&ctx->lock);
2216 } 2216 }
2217 EXPORT_SYMBOL_GPL(perf_event_enable); 2217 EXPORT_SYMBOL_GPL(perf_event_enable);
2218 2218
2219 int perf_event_refresh(struct perf_event *event, int refresh) 2219 int perf_event_refresh(struct perf_event *event, int refresh)
2220 { 2220 {
2221 /* 2221 /*
2222 * not supported on inherited events 2222 * not supported on inherited events
2223 */ 2223 */
2224 if (event->attr.inherit || !is_sampling_event(event)) 2224 if (event->attr.inherit || !is_sampling_event(event))
2225 return -EINVAL; 2225 return -EINVAL;
2226 2226
2227 atomic_add(refresh, &event->event_limit); 2227 atomic_add(refresh, &event->event_limit);
2228 perf_event_enable(event); 2228 perf_event_enable(event);
2229 2229
2230 return 0; 2230 return 0;
2231 } 2231 }
2232 EXPORT_SYMBOL_GPL(perf_event_refresh); 2232 EXPORT_SYMBOL_GPL(perf_event_refresh);
2233 2233
2234 static void ctx_sched_out(struct perf_event_context *ctx, 2234 static void ctx_sched_out(struct perf_event_context *ctx,
2235 struct perf_cpu_context *cpuctx, 2235 struct perf_cpu_context *cpuctx,
2236 enum event_type_t event_type) 2236 enum event_type_t event_type)
2237 { 2237 {
2238 struct perf_event *event; 2238 struct perf_event *event;
2239 int is_active = ctx->is_active; 2239 int is_active = ctx->is_active;
2240 2240
2241 ctx->is_active &= ~event_type; 2241 ctx->is_active &= ~event_type;
2242 if (likely(!ctx->nr_events)) 2242 if (likely(!ctx->nr_events))
2243 return; 2243 return;
2244 2244
2245 update_context_time(ctx); 2245 update_context_time(ctx);
2246 update_cgrp_time_from_cpuctx(cpuctx); 2246 update_cgrp_time_from_cpuctx(cpuctx);
2247 if (!ctx->nr_active) 2247 if (!ctx->nr_active)
2248 return; 2248 return;
2249 2249
2250 perf_pmu_disable(ctx->pmu); 2250 perf_pmu_disable(ctx->pmu);
2251 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) { 2251 if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2252 list_for_each_entry(event, &ctx->pinned_groups, group_entry) 2252 list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2253 group_sched_out(event, cpuctx, ctx); 2253 group_sched_out(event, cpuctx, ctx);
2254 } 2254 }
2255 2255
2256 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) { 2256 if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2257 list_for_each_entry(event, &ctx->flexible_groups, group_entry) 2257 list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2258 group_sched_out(event, cpuctx, ctx); 2258 group_sched_out(event, cpuctx, ctx);
2259 } 2259 }
2260 perf_pmu_enable(ctx->pmu); 2260 perf_pmu_enable(ctx->pmu);
2261 } 2261 }
2262 2262
2263 /* 2263 /*
2264 * Test whether two contexts are equivalent, i.e. whether they have both been 2264 * Test whether two contexts are equivalent, i.e. whether they have both been
2265 * cloned from the same version of the same context. 2265 * cloned from the same version of the same context.
2266 * 2266 *
2267 * Equivalence is measured using a generation number in the context that is 2267 * Equivalence is measured using a generation number in the context that is
2268 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2268 * incremented on each modification to it; see unclone_ctx(), list_add_event()
2269 * and list_del_event(). 2269 * and list_del_event().
2270 */ 2270 */
2271 static int context_equiv(struct perf_event_context *ctx1, 2271 static int context_equiv(struct perf_event_context *ctx1,
2272 struct perf_event_context *ctx2) 2272 struct perf_event_context *ctx2)
2273 { 2273 {
2274 lockdep_assert_held(&ctx1->lock); 2274 lockdep_assert_held(&ctx1->lock);
2275 lockdep_assert_held(&ctx2->lock); 2275 lockdep_assert_held(&ctx2->lock);
2276 2276
2277 /* Pinning disables the swap optimization */ 2277 /* Pinning disables the swap optimization */
2278 if (ctx1->pin_count || ctx2->pin_count) 2278 if (ctx1->pin_count || ctx2->pin_count)
2279 return 0; 2279 return 0;
2280 2280
2281 /* If ctx1 is the parent of ctx2 */ 2281 /* If ctx1 is the parent of ctx2 */
2282 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2282 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
2283 return 1; 2283 return 1;
2284 2284
2285 /* If ctx2 is the parent of ctx1 */ 2285 /* If ctx2 is the parent of ctx1 */
2286 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2286 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
2287 return 1; 2287 return 1;
2288 2288
2289 /* 2289 /*
2290 * If ctx1 and ctx2 have the same parent; we flatten the parent 2290 * If ctx1 and ctx2 have the same parent; we flatten the parent
2291 * hierarchy, see perf_event_init_context(). 2291 * hierarchy, see perf_event_init_context().
2292 */ 2292 */
2293 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2293 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
2294 ctx1->parent_gen == ctx2->parent_gen) 2294 ctx1->parent_gen == ctx2->parent_gen)
2295 return 1; 2295 return 1;
2296 2296
2297 /* Unmatched */ 2297 /* Unmatched */
2298 return 0; 2298 return 0;
2299 } 2299 }
2300 2300
2301 static void __perf_event_sync_stat(struct perf_event *event, 2301 static void __perf_event_sync_stat(struct perf_event *event,
2302 struct perf_event *next_event) 2302 struct perf_event *next_event)
2303 { 2303 {
2304 u64 value; 2304 u64 value;
2305 2305
2306 if (!event->attr.inherit_stat) 2306 if (!event->attr.inherit_stat)
2307 return; 2307 return;
2308 2308
2309 /* 2309 /*
2310 * Update the event value, we cannot use perf_event_read() 2310 * Update the event value, we cannot use perf_event_read()
2311 * because we're in the middle of a context switch and have IRQs 2311 * because we're in the middle of a context switch and have IRQs
2312 * disabled, which upsets smp_call_function_single(), however 2312 * disabled, which upsets smp_call_function_single(), however
2313 * we know the event must be on the current CPU, therefore we 2313 * we know the event must be on the current CPU, therefore we
2314 * don't need to use it. 2314 * don't need to use it.
2315 */ 2315 */
2316 switch (event->state) { 2316 switch (event->state) {
2317 case PERF_EVENT_STATE_ACTIVE: 2317 case PERF_EVENT_STATE_ACTIVE:
2318 event->pmu->read(event); 2318 event->pmu->read(event);
2319 /* fall-through */ 2319 /* fall-through */
2320 2320
2321 case PERF_EVENT_STATE_INACTIVE: 2321 case PERF_EVENT_STATE_INACTIVE:
2322 update_event_times(event); 2322 update_event_times(event);
2323 break; 2323 break;
2324 2324
2325 default: 2325 default:
2326 break; 2326 break;
2327 } 2327 }
2328 2328
2329 /* 2329 /*
2330 * In order to keep per-task stats reliable we need to flip the event 2330 * In order to keep per-task stats reliable we need to flip the event
2331 * values when we flip the contexts. 2331 * values when we flip the contexts.
2332 */ 2332 */
2333 value = local64_read(&next_event->count); 2333 value = local64_read(&next_event->count);
2334 value = local64_xchg(&event->count, value); 2334 value = local64_xchg(&event->count, value);
2335 local64_set(&next_event->count, value); 2335 local64_set(&next_event->count, value);
2336 2336
2337 swap(event->total_time_enabled, next_event->total_time_enabled); 2337 swap(event->total_time_enabled, next_event->total_time_enabled);
2338 swap(event->total_time_running, next_event->total_time_running); 2338 swap(event->total_time_running, next_event->total_time_running);
2339 2339
2340 /* 2340 /*
2341 * Since we swizzled the values, update the user visible data too. 2341 * Since we swizzled the values, update the user visible data too.
2342 */ 2342 */
2343 perf_event_update_userpage(event); 2343 perf_event_update_userpage(event);
2344 perf_event_update_userpage(next_event); 2344 perf_event_update_userpage(next_event);
2345 } 2345 }
2346 2346
2347 static void perf_event_sync_stat(struct perf_event_context *ctx, 2347 static void perf_event_sync_stat(struct perf_event_context *ctx,
2348 struct perf_event_context *next_ctx) 2348 struct perf_event_context *next_ctx)
2349 { 2349 {
2350 struct perf_event *event, *next_event; 2350 struct perf_event *event, *next_event;
2351 2351
2352 if (!ctx->nr_stat) 2352 if (!ctx->nr_stat)
2353 return; 2353 return;
2354 2354
2355 update_context_time(ctx); 2355 update_context_time(ctx);
2356 2356
2357 event = list_first_entry(&ctx->event_list, 2357 event = list_first_entry(&ctx->event_list,
2358 struct perf_event, event_entry); 2358 struct perf_event, event_entry);
2359 2359
2360 next_event = list_first_entry(&next_ctx->event_list, 2360 next_event = list_first_entry(&next_ctx->event_list,
2361 struct perf_event, event_entry); 2361 struct perf_event, event_entry);
2362 2362
2363 while (&event->event_entry != &ctx->event_list && 2363 while (&event->event_entry != &ctx->event_list &&
2364 &next_event->event_entry != &next_ctx->event_list) { 2364 &next_event->event_entry != &next_ctx->event_list) {
2365 2365
2366 __perf_event_sync_stat(event, next_event); 2366 __perf_event_sync_stat(event, next_event);
2367 2367
2368 event = list_next_entry(event, event_entry); 2368 event = list_next_entry(event, event_entry);
2369 next_event = list_next_entry(next_event, event_entry); 2369 next_event = list_next_entry(next_event, event_entry);
2370 } 2370 }
2371 } 2371 }
2372 2372
2373 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 2373 static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2374 struct task_struct *next) 2374 struct task_struct *next)
2375 { 2375 {
2376 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 2376 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2377 struct perf_event_context *next_ctx; 2377 struct perf_event_context *next_ctx;
2378 struct perf_event_context *parent, *next_parent; 2378 struct perf_event_context *parent, *next_parent;
2379 struct perf_cpu_context *cpuctx; 2379 struct perf_cpu_context *cpuctx;
2380 int do_switch = 1; 2380 int do_switch = 1;
2381 2381
2382 if (likely(!ctx)) 2382 if (likely(!ctx))
2383 return; 2383 return;
2384 2384
2385 cpuctx = __get_cpu_context(ctx); 2385 cpuctx = __get_cpu_context(ctx);
2386 if (!cpuctx->task_ctx) 2386 if (!cpuctx->task_ctx)
2387 return; 2387 return;
2388 2388
2389 rcu_read_lock(); 2389 rcu_read_lock();
2390 next_ctx = next->perf_event_ctxp[ctxn]; 2390 next_ctx = next->perf_event_ctxp[ctxn];
2391 if (!next_ctx) 2391 if (!next_ctx)
2392 goto unlock; 2392 goto unlock;
2393 2393
2394 parent = rcu_dereference(ctx->parent_ctx); 2394 parent = rcu_dereference(ctx->parent_ctx);
2395 next_parent = rcu_dereference(next_ctx->parent_ctx); 2395 next_parent = rcu_dereference(next_ctx->parent_ctx);
2396 2396
2397 /* If neither context have a parent context; they cannot be clones. */ 2397 /* If neither context have a parent context; they cannot be clones. */
2398 if (!parent && !next_parent) 2398 if (!parent && !next_parent)
2399 goto unlock; 2399 goto unlock;
2400 2400
2401 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 2401 if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
2402 /* 2402 /*
2403 * Looks like the two contexts are clones, so we might be 2403 * Looks like the two contexts are clones, so we might be
2404 * able to optimize the context switch. We lock both 2404 * able to optimize the context switch. We lock both
2405 * contexts and check that they are clones under the 2405 * contexts and check that they are clones under the
2406 * lock (including re-checking that neither has been 2406 * lock (including re-checking that neither has been
2407 * uncloned in the meantime). It doesn't matter which 2407 * uncloned in the meantime). It doesn't matter which
2408 * order we take the locks because no other cpu could 2408 * order we take the locks because no other cpu could
2409 * be trying to lock both of these tasks. 2409 * be trying to lock both of these tasks.
2410 */ 2410 */
2411 raw_spin_lock(&ctx->lock); 2411 raw_spin_lock(&ctx->lock);
2412 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 2412 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2413 if (context_equiv(ctx, next_ctx)) { 2413 if (context_equiv(ctx, next_ctx)) {
2414 /* 2414 /*
2415 * XXX do we need a memory barrier of sorts 2415 * XXX do we need a memory barrier of sorts
2416 * wrt to rcu_dereference() of perf_event_ctxp 2416 * wrt to rcu_dereference() of perf_event_ctxp
2417 */ 2417 */
2418 task->perf_event_ctxp[ctxn] = next_ctx; 2418 task->perf_event_ctxp[ctxn] = next_ctx;
2419 next->perf_event_ctxp[ctxn] = ctx; 2419 next->perf_event_ctxp[ctxn] = ctx;
2420 ctx->task = next; 2420 ctx->task = next;
2421 next_ctx->task = task; 2421 next_ctx->task = task;
2422 do_switch = 0; 2422 do_switch = 0;
2423 2423
2424 perf_event_sync_stat(ctx, next_ctx); 2424 perf_event_sync_stat(ctx, next_ctx);
2425 } 2425 }
2426 raw_spin_unlock(&next_ctx->lock); 2426 raw_spin_unlock(&next_ctx->lock);
2427 raw_spin_unlock(&ctx->lock); 2427 raw_spin_unlock(&ctx->lock);
2428 } 2428 }
2429 unlock: 2429 unlock:
2430 rcu_read_unlock(); 2430 rcu_read_unlock();
2431 2431
2432 if (do_switch) { 2432 if (do_switch) {
2433 raw_spin_lock(&ctx->lock); 2433 raw_spin_lock(&ctx->lock);
2434 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2434 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2435 cpuctx->task_ctx = NULL; 2435 cpuctx->task_ctx = NULL;
2436 raw_spin_unlock(&ctx->lock); 2436 raw_spin_unlock(&ctx->lock);
2437 } 2437 }
2438 } 2438 }
2439 2439
2440 #define for_each_task_context_nr(ctxn) \ 2440 #define for_each_task_context_nr(ctxn) \
2441 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 2441 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2442 2442
2443 /* 2443 /*
2444 * Called from scheduler to remove the events of the current task, 2444 * Called from scheduler to remove the events of the current task,
2445 * with interrupts disabled. 2445 * with interrupts disabled.
2446 * 2446 *
2447 * We stop each event and update the event value in event->count. 2447 * We stop each event and update the event value in event->count.
2448 * 2448 *
2449 * This does not protect us against NMI, but disable() 2449 * This does not protect us against NMI, but disable()
2450 * sets the disabled bit in the control field of event _before_ 2450 * sets the disabled bit in the control field of event _before_
2451 * accessing the event control register. If a NMI hits, then it will 2451 * accessing the event control register. If a NMI hits, then it will
2452 * not restart the event. 2452 * not restart the event.
2453 */ 2453 */
2454 void __perf_event_task_sched_out(struct task_struct *task, 2454 void __perf_event_task_sched_out(struct task_struct *task,
2455 struct task_struct *next) 2455 struct task_struct *next)
2456 { 2456 {
2457 int ctxn; 2457 int ctxn;
2458 2458
2459 for_each_task_context_nr(ctxn) 2459 for_each_task_context_nr(ctxn)
2460 perf_event_context_sched_out(task, ctxn, next); 2460 perf_event_context_sched_out(task, ctxn, next);
2461 2461
2462 /* 2462 /*
2463 * if cgroup events exist on this CPU, then we need 2463 * if cgroup events exist on this CPU, then we need
2464 * to check if we have to switch out PMU state. 2464 * to check if we have to switch out PMU state.
2465 * cgroup event are system-wide mode only 2465 * cgroup event are system-wide mode only
2466 */ 2466 */
2467 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2467 if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2468 perf_cgroup_sched_out(task, next); 2468 perf_cgroup_sched_out(task, next);
2469 } 2469 }
2470 2470
2471 static void task_ctx_sched_out(struct perf_event_context *ctx) 2471 static void task_ctx_sched_out(struct perf_event_context *ctx)
2472 { 2472 {
2473 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2473 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2474 2474
2475 if (!cpuctx->task_ctx) 2475 if (!cpuctx->task_ctx)
2476 return; 2476 return;
2477 2477
2478 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2478 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2479 return; 2479 return;
2480 2480
2481 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2481 ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2482 cpuctx->task_ctx = NULL; 2482 cpuctx->task_ctx = NULL;
2483 } 2483 }
2484 2484
2485 /* 2485 /*
2486 * Called with IRQs disabled 2486 * Called with IRQs disabled
2487 */ 2487 */
2488 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 2488 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2489 enum event_type_t event_type) 2489 enum event_type_t event_type)
2490 { 2490 {
2491 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 2491 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2492 } 2492 }
2493 2493
2494 static void 2494 static void
2495 ctx_pinned_sched_in(struct perf_event_context *ctx, 2495 ctx_pinned_sched_in(struct perf_event_context *ctx,
2496 struct perf_cpu_context *cpuctx) 2496 struct perf_cpu_context *cpuctx)
2497 { 2497 {
2498 struct perf_event *event; 2498 struct perf_event *event;
2499 2499
2500 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2500 list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2501 if (event->state <= PERF_EVENT_STATE_OFF) 2501 if (event->state <= PERF_EVENT_STATE_OFF)
2502 continue; 2502 continue;
2503 if (!event_filter_match(event)) 2503 if (!event_filter_match(event))
2504 continue; 2504 continue;
2505 2505
2506 /* may need to reset tstamp_enabled */ 2506 /* may need to reset tstamp_enabled */
2507 if (is_cgroup_event(event)) 2507 if (is_cgroup_event(event))
2508 perf_cgroup_mark_enabled(event, ctx); 2508 perf_cgroup_mark_enabled(event, ctx);
2509 2509
2510 if (group_can_go_on(event, cpuctx, 1)) 2510 if (group_can_go_on(event, cpuctx, 1))
2511 group_sched_in(event, cpuctx, ctx); 2511 group_sched_in(event, cpuctx, ctx);
2512 2512
2513 /* 2513 /*
2514 * If this pinned group hasn't been scheduled, 2514 * If this pinned group hasn't been scheduled,
2515 * put it in error state. 2515 * put it in error state.
2516 */ 2516 */
2517 if (event->state == PERF_EVENT_STATE_INACTIVE) { 2517 if (event->state == PERF_EVENT_STATE_INACTIVE) {
2518 update_group_times(event); 2518 update_group_times(event);
2519 event->state = PERF_EVENT_STATE_ERROR; 2519 event->state = PERF_EVENT_STATE_ERROR;
2520 } 2520 }
2521 } 2521 }
2522 } 2522 }
2523 2523
2524 static void 2524 static void
2525 ctx_flexible_sched_in(struct perf_event_context *ctx, 2525 ctx_flexible_sched_in(struct perf_event_context *ctx,
2526 struct perf_cpu_context *cpuctx) 2526 struct perf_cpu_context *cpuctx)
2527 { 2527 {
2528 struct perf_event *event; 2528 struct perf_event *event;
2529 int can_add_hw = 1; 2529 int can_add_hw = 1;
2530 2530
2531 list_for_each_entry(event, &ctx->flexible_groups, group_entry) { 2531 list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2532 /* Ignore events in OFF or ERROR state */ 2532 /* Ignore events in OFF or ERROR state */
2533 if (event->state <= PERF_EVENT_STATE_OFF) 2533 if (event->state <= PERF_EVENT_STATE_OFF)
2534 continue; 2534 continue;
2535 /* 2535 /*
2536 * Listen to the 'cpu' scheduling filter constraint 2536 * Listen to the 'cpu' scheduling filter constraint
2537 * of events: 2537 * of events:
2538 */ 2538 */
2539 if (!event_filter_match(event)) 2539 if (!event_filter_match(event))
2540 continue; 2540 continue;
2541 2541
2542 /* may need to reset tstamp_enabled */ 2542 /* may need to reset tstamp_enabled */
2543 if (is_cgroup_event(event)) 2543 if (is_cgroup_event(event))
2544 perf_cgroup_mark_enabled(event, ctx); 2544 perf_cgroup_mark_enabled(event, ctx);
2545 2545
2546 if (group_can_go_on(event, cpuctx, can_add_hw)) { 2546 if (group_can_go_on(event, cpuctx, can_add_hw)) {
2547 if (group_sched_in(event, cpuctx, ctx)) 2547 if (group_sched_in(event, cpuctx, ctx))
2548 can_add_hw = 0; 2548 can_add_hw = 0;
2549 } 2549 }
2550 } 2550 }
2551 } 2551 }
2552 2552
2553 static void 2553 static void
2554 ctx_sched_in(struct perf_event_context *ctx, 2554 ctx_sched_in(struct perf_event_context *ctx,
2555 struct perf_cpu_context *cpuctx, 2555 struct perf_cpu_context *cpuctx,
2556 enum event_type_t event_type, 2556 enum event_type_t event_type,
2557 struct task_struct *task) 2557 struct task_struct *task)
2558 { 2558 {
2559 u64 now; 2559 u64 now;
2560 int is_active = ctx->is_active; 2560 int is_active = ctx->is_active;
2561 2561
2562 ctx->is_active |= event_type; 2562 ctx->is_active |= event_type;
2563 if (likely(!ctx->nr_events)) 2563 if (likely(!ctx->nr_events))
2564 return; 2564 return;
2565 2565
2566 now = perf_clock(); 2566 now = perf_clock();
2567 ctx->timestamp = now; 2567 ctx->timestamp = now;
2568 perf_cgroup_set_timestamp(task, ctx); 2568 perf_cgroup_set_timestamp(task, ctx);
2569 /* 2569 /*
2570 * First go through the list and put on any pinned groups 2570 * First go through the list and put on any pinned groups
2571 * in order to give them the best chance of going on. 2571 * in order to give them the best chance of going on.
2572 */ 2572 */
2573 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) 2573 if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2574 ctx_pinned_sched_in(ctx, cpuctx); 2574 ctx_pinned_sched_in(ctx, cpuctx);
2575 2575
2576 /* Then walk through the lower prio flexible groups */ 2576 /* Then walk through the lower prio flexible groups */
2577 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) 2577 if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2578 ctx_flexible_sched_in(ctx, cpuctx); 2578 ctx_flexible_sched_in(ctx, cpuctx);
2579 } 2579 }
2580 2580
2581 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 2581 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2582 enum event_type_t event_type, 2582 enum event_type_t event_type,
2583 struct task_struct *task) 2583 struct task_struct *task)
2584 { 2584 {
2585 struct perf_event_context *ctx = &cpuctx->ctx; 2585 struct perf_event_context *ctx = &cpuctx->ctx;
2586 2586
2587 ctx_sched_in(ctx, cpuctx, event_type, task); 2587 ctx_sched_in(ctx, cpuctx, event_type, task);
2588 } 2588 }
2589 2589
2590 static void perf_event_context_sched_in(struct perf_event_context *ctx, 2590 static void perf_event_context_sched_in(struct perf_event_context *ctx,
2591 struct task_struct *task) 2591 struct task_struct *task)
2592 { 2592 {
2593 struct perf_cpu_context *cpuctx; 2593 struct perf_cpu_context *cpuctx;
2594 2594
2595 cpuctx = __get_cpu_context(ctx); 2595 cpuctx = __get_cpu_context(ctx);
2596 if (cpuctx->task_ctx == ctx) 2596 if (cpuctx->task_ctx == ctx)
2597 return; 2597 return;
2598 2598
2599 perf_ctx_lock(cpuctx, ctx); 2599 perf_ctx_lock(cpuctx, ctx);
2600 perf_pmu_disable(ctx->pmu); 2600 perf_pmu_disable(ctx->pmu);
2601 /* 2601 /*
2602 * We want to keep the following priority order: 2602 * We want to keep the following priority order:
2603 * cpu pinned (that don't need to move), task pinned, 2603 * cpu pinned (that don't need to move), task pinned,
2604 * cpu flexible, task flexible. 2604 * cpu flexible, task flexible.
2605 */ 2605 */
2606 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2606 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2607 2607
2608 if (ctx->nr_events) 2608 if (ctx->nr_events)
2609 cpuctx->task_ctx = ctx; 2609 cpuctx->task_ctx = ctx;
2610 2610
2611 perf_event_sched_in(cpuctx, cpuctx->task_ctx, task); 2611 perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2612 2612
2613 perf_pmu_enable(ctx->pmu); 2613 perf_pmu_enable(ctx->pmu);
2614 perf_ctx_unlock(cpuctx, ctx); 2614 perf_ctx_unlock(cpuctx, ctx);
2615 2615
2616 /* 2616 /*
2617 * Since these rotations are per-cpu, we need to ensure the 2617 * Since these rotations are per-cpu, we need to ensure the
2618 * cpu-context we got scheduled on is actually rotating. 2618 * cpu-context we got scheduled on is actually rotating.
2619 */ 2619 */
2620 perf_pmu_rotate_start(ctx->pmu); 2620 perf_pmu_rotate_start(ctx->pmu);
2621 } 2621 }
2622 2622
2623 /* 2623 /*
2624 * When sampling the branck stack in system-wide, it may be necessary 2624 * When sampling the branck stack in system-wide, it may be necessary
2625 * to flush the stack on context switch. This happens when the branch 2625 * to flush the stack on context switch. This happens when the branch
2626 * stack does not tag its entries with the pid of the current task. 2626 * stack does not tag its entries with the pid of the current task.
2627 * Otherwise it becomes impossible to associate a branch entry with a 2627 * Otherwise it becomes impossible to associate a branch entry with a
2628 * task. This ambiguity is more likely to appear when the branch stack 2628 * task. This ambiguity is more likely to appear when the branch stack
2629 * supports priv level filtering and the user sets it to monitor only 2629 * supports priv level filtering and the user sets it to monitor only
2630 * at the user level (which could be a useful measurement in system-wide 2630 * at the user level (which could be a useful measurement in system-wide
2631 * mode). In that case, the risk is high of having a branch stack with 2631 * mode). In that case, the risk is high of having a branch stack with
2632 * branch from multiple tasks. Flushing may mean dropping the existing 2632 * branch from multiple tasks. Flushing may mean dropping the existing
2633 * entries or stashing them somewhere in the PMU specific code layer. 2633 * entries or stashing them somewhere in the PMU specific code layer.
2634 * 2634 *
2635 * This function provides the context switch callback to the lower code 2635 * This function provides the context switch callback to the lower code
2636 * layer. It is invoked ONLY when there is at least one system-wide context 2636 * layer. It is invoked ONLY when there is at least one system-wide context
2637 * with at least one active event using taken branch sampling. 2637 * with at least one active event using taken branch sampling.
2638 */ 2638 */
2639 static void perf_branch_stack_sched_in(struct task_struct *prev, 2639 static void perf_branch_stack_sched_in(struct task_struct *prev,
2640 struct task_struct *task) 2640 struct task_struct *task)
2641 { 2641 {
2642 struct perf_cpu_context *cpuctx; 2642 struct perf_cpu_context *cpuctx;
2643 struct pmu *pmu; 2643 struct pmu *pmu;
2644 unsigned long flags; 2644 unsigned long flags;
2645 2645
2646 /* no need to flush branch stack if not changing task */ 2646 /* no need to flush branch stack if not changing task */
2647 if (prev == task) 2647 if (prev == task)
2648 return; 2648 return;
2649 2649
2650 local_irq_save(flags); 2650 local_irq_save(flags);
2651 2651
2652 rcu_read_lock(); 2652 rcu_read_lock();
2653 2653
2654 list_for_each_entry_rcu(pmu, &pmus, entry) { 2654 list_for_each_entry_rcu(pmu, &pmus, entry) {
2655 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2655 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2656 2656
2657 /* 2657 /*
2658 * check if the context has at least one 2658 * check if the context has at least one
2659 * event using PERF_SAMPLE_BRANCH_STACK 2659 * event using PERF_SAMPLE_BRANCH_STACK
2660 */ 2660 */
2661 if (cpuctx->ctx.nr_branch_stack > 0 2661 if (cpuctx->ctx.nr_branch_stack > 0
2662 && pmu->flush_branch_stack) { 2662 && pmu->flush_branch_stack) {
2663 2663
2664 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2664 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2665 2665
2666 perf_pmu_disable(pmu); 2666 perf_pmu_disable(pmu);
2667 2667
2668 pmu->flush_branch_stack(); 2668 pmu->flush_branch_stack();
2669 2669
2670 perf_pmu_enable(pmu); 2670 perf_pmu_enable(pmu);
2671 2671
2672 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2672 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2673 } 2673 }
2674 } 2674 }
2675 2675
2676 rcu_read_unlock(); 2676 rcu_read_unlock();
2677 2677
2678 local_irq_restore(flags); 2678 local_irq_restore(flags);
2679 } 2679 }
2680 2680
2681 /* 2681 /*
2682 * Called from scheduler to add the events of the current task 2682 * Called from scheduler to add the events of the current task
2683 * with interrupts disabled. 2683 * with interrupts disabled.
2684 * 2684 *
2685 * We restore the event value and then enable it. 2685 * We restore the event value and then enable it.
2686 * 2686 *
2687 * This does not protect us against NMI, but enable() 2687 * This does not protect us against NMI, but enable()
2688 * sets the enabled bit in the control field of event _before_ 2688 * sets the enabled bit in the control field of event _before_
2689 * accessing the event control register. If a NMI hits, then it will 2689 * accessing the event control register. If a NMI hits, then it will
2690 * keep the event running. 2690 * keep the event running.
2691 */ 2691 */
2692 void __perf_event_task_sched_in(struct task_struct *prev, 2692 void __perf_event_task_sched_in(struct task_struct *prev,
2693 struct task_struct *task) 2693 struct task_struct *task)
2694 { 2694 {
2695 struct perf_event_context *ctx; 2695 struct perf_event_context *ctx;
2696 int ctxn; 2696 int ctxn;
2697 2697
2698 for_each_task_context_nr(ctxn) { 2698 for_each_task_context_nr(ctxn) {
2699 ctx = task->perf_event_ctxp[ctxn]; 2699 ctx = task->perf_event_ctxp[ctxn];
2700 if (likely(!ctx)) 2700 if (likely(!ctx))
2701 continue; 2701 continue;
2702 2702
2703 perf_event_context_sched_in(ctx, task); 2703 perf_event_context_sched_in(ctx, task);
2704 } 2704 }
2705 /* 2705 /*
2706 * if cgroup events exist on this CPU, then we need 2706 * if cgroup events exist on this CPU, then we need
2707 * to check if we have to switch in PMU state. 2707 * to check if we have to switch in PMU state.
2708 * cgroup event are system-wide mode only 2708 * cgroup event are system-wide mode only
2709 */ 2709 */
2710 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2710 if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2711 perf_cgroup_sched_in(prev, task); 2711 perf_cgroup_sched_in(prev, task);
2712 2712
2713 /* check for system-wide branch_stack events */ 2713 /* check for system-wide branch_stack events */
2714 if (atomic_read(this_cpu_ptr(&perf_branch_stack_events))) 2714 if (atomic_read(this_cpu_ptr(&perf_branch_stack_events)))
2715 perf_branch_stack_sched_in(prev, task); 2715 perf_branch_stack_sched_in(prev, task);
2716 } 2716 }
2717 2717
2718 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 2718 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2719 { 2719 {
2720 u64 frequency = event->attr.sample_freq; 2720 u64 frequency = event->attr.sample_freq;
2721 u64 sec = NSEC_PER_SEC; 2721 u64 sec = NSEC_PER_SEC;
2722 u64 divisor, dividend; 2722 u64 divisor, dividend;
2723 2723
2724 int count_fls, nsec_fls, frequency_fls, sec_fls; 2724 int count_fls, nsec_fls, frequency_fls, sec_fls;
2725 2725
2726 count_fls = fls64(count); 2726 count_fls = fls64(count);
2727 nsec_fls = fls64(nsec); 2727 nsec_fls = fls64(nsec);
2728 frequency_fls = fls64(frequency); 2728 frequency_fls = fls64(frequency);
2729 sec_fls = 30; 2729 sec_fls = 30;
2730 2730
2731 /* 2731 /*
2732 * We got @count in @nsec, with a target of sample_freq HZ 2732 * We got @count in @nsec, with a target of sample_freq HZ
2733 * the target period becomes: 2733 * the target period becomes:
2734 * 2734 *
2735 * @count * 10^9 2735 * @count * 10^9
2736 * period = ------------------- 2736 * period = -------------------
2737 * @nsec * sample_freq 2737 * @nsec * sample_freq
2738 * 2738 *
2739 */ 2739 */
2740 2740
2741 /* 2741 /*
2742 * Reduce accuracy by one bit such that @a and @b converge 2742 * Reduce accuracy by one bit such that @a and @b converge
2743 * to a similar magnitude. 2743 * to a similar magnitude.
2744 */ 2744 */
2745 #define REDUCE_FLS(a, b) \ 2745 #define REDUCE_FLS(a, b) \
2746 do { \ 2746 do { \
2747 if (a##_fls > b##_fls) { \ 2747 if (a##_fls > b##_fls) { \
2748 a >>= 1; \ 2748 a >>= 1; \
2749 a##_fls--; \ 2749 a##_fls--; \
2750 } else { \ 2750 } else { \
2751 b >>= 1; \ 2751 b >>= 1; \
2752 b##_fls--; \ 2752 b##_fls--; \
2753 } \ 2753 } \
2754 } while (0) 2754 } while (0)
2755 2755
2756 /* 2756 /*
2757 * Reduce accuracy until either term fits in a u64, then proceed with 2757 * Reduce accuracy until either term fits in a u64, then proceed with
2758 * the other, so that finally we can do a u64/u64 division. 2758 * the other, so that finally we can do a u64/u64 division.
2759 */ 2759 */
2760 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 2760 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2761 REDUCE_FLS(nsec, frequency); 2761 REDUCE_FLS(nsec, frequency);
2762 REDUCE_FLS(sec, count); 2762 REDUCE_FLS(sec, count);
2763 } 2763 }
2764 2764
2765 if (count_fls + sec_fls > 64) { 2765 if (count_fls + sec_fls > 64) {
2766 divisor = nsec * frequency; 2766 divisor = nsec * frequency;
2767 2767
2768 while (count_fls + sec_fls > 64) { 2768 while (count_fls + sec_fls > 64) {
2769 REDUCE_FLS(count, sec); 2769 REDUCE_FLS(count, sec);
2770 divisor >>= 1; 2770 divisor >>= 1;
2771 } 2771 }
2772 2772
2773 dividend = count * sec; 2773 dividend = count * sec;
2774 } else { 2774 } else {
2775 dividend = count * sec; 2775 dividend = count * sec;
2776 2776
2777 while (nsec_fls + frequency_fls > 64) { 2777 while (nsec_fls + frequency_fls > 64) {
2778 REDUCE_FLS(nsec, frequency); 2778 REDUCE_FLS(nsec, frequency);
2779 dividend >>= 1; 2779 dividend >>= 1;
2780 } 2780 }
2781 2781
2782 divisor = nsec * frequency; 2782 divisor = nsec * frequency;
2783 } 2783 }
2784 2784
2785 if (!divisor) 2785 if (!divisor)
2786 return dividend; 2786 return dividend;
2787 2787
2788 return div64_u64(dividend, divisor); 2788 return div64_u64(dividend, divisor);
2789 } 2789 }
2790 2790
2791 static DEFINE_PER_CPU(int, perf_throttled_count); 2791 static DEFINE_PER_CPU(int, perf_throttled_count);
2792 static DEFINE_PER_CPU(u64, perf_throttled_seq); 2792 static DEFINE_PER_CPU(u64, perf_throttled_seq);
2793 2793
2794 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 2794 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2795 { 2795 {
2796 struct hw_perf_event *hwc = &event->hw; 2796 struct hw_perf_event *hwc = &event->hw;
2797 s64 period, sample_period; 2797 s64 period, sample_period;
2798 s64 delta; 2798 s64 delta;
2799 2799
2800 period = perf_calculate_period(event, nsec, count); 2800 period = perf_calculate_period(event, nsec, count);
2801 2801
2802 delta = (s64)(period - hwc->sample_period); 2802 delta = (s64)(period - hwc->sample_period);
2803 delta = (delta + 7) / 8; /* low pass filter */ 2803 delta = (delta + 7) / 8; /* low pass filter */
2804 2804
2805 sample_period = hwc->sample_period + delta; 2805 sample_period = hwc->sample_period + delta;
2806 2806
2807 if (!sample_period) 2807 if (!sample_period)
2808 sample_period = 1; 2808 sample_period = 1;
2809 2809
2810 hwc->sample_period = sample_period; 2810 hwc->sample_period = sample_period;
2811 2811
2812 if (local64_read(&hwc->period_left) > 8*sample_period) { 2812 if (local64_read(&hwc->period_left) > 8*sample_period) {
2813 if (disable) 2813 if (disable)
2814 event->pmu->stop(event, PERF_EF_UPDATE); 2814 event->pmu->stop(event, PERF_EF_UPDATE);
2815 2815
2816 local64_set(&hwc->period_left, 0); 2816 local64_set(&hwc->period_left, 0);
2817 2817
2818 if (disable) 2818 if (disable)
2819 event->pmu->start(event, PERF_EF_RELOAD); 2819 event->pmu->start(event, PERF_EF_RELOAD);
2820 } 2820 }
2821 } 2821 }
2822 2822
2823 /* 2823 /*
2824 * combine freq adjustment with unthrottling to avoid two passes over the 2824 * combine freq adjustment with unthrottling to avoid two passes over the
2825 * events. At the same time, make sure, having freq events does not change 2825 * events. At the same time, make sure, having freq events does not change
2826 * the rate of unthrottling as that would introduce bias. 2826 * the rate of unthrottling as that would introduce bias.
2827 */ 2827 */
2828 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 2828 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2829 int needs_unthr) 2829 int needs_unthr)
2830 { 2830 {
2831 struct perf_event *event; 2831 struct perf_event *event;
2832 struct hw_perf_event *hwc; 2832 struct hw_perf_event *hwc;
2833 u64 now, period = TICK_NSEC; 2833 u64 now, period = TICK_NSEC;
2834 s64 delta; 2834 s64 delta;
2835 2835
2836 /* 2836 /*
2837 * only need to iterate over all events iff: 2837 * only need to iterate over all events iff:
2838 * - context have events in frequency mode (needs freq adjust) 2838 * - context have events in frequency mode (needs freq adjust)
2839 * - there are events to unthrottle on this cpu 2839 * - there are events to unthrottle on this cpu
2840 */ 2840 */
2841 if (!(ctx->nr_freq || needs_unthr)) 2841 if (!(ctx->nr_freq || needs_unthr))
2842 return; 2842 return;
2843 2843
2844 raw_spin_lock(&ctx->lock); 2844 raw_spin_lock(&ctx->lock);
2845 perf_pmu_disable(ctx->pmu); 2845 perf_pmu_disable(ctx->pmu);
2846 2846
2847 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 2847 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2848 if (event->state != PERF_EVENT_STATE_ACTIVE) 2848 if (event->state != PERF_EVENT_STATE_ACTIVE)
2849 continue; 2849 continue;
2850 2850
2851 if (!event_filter_match(event)) 2851 if (!event_filter_match(event))
2852 continue; 2852 continue;
2853 2853
2854 perf_pmu_disable(event->pmu); 2854 perf_pmu_disable(event->pmu);
2855 2855
2856 hwc = &event->hw; 2856 hwc = &event->hw;
2857 2857
2858 if (hwc->interrupts == MAX_INTERRUPTS) { 2858 if (hwc->interrupts == MAX_INTERRUPTS) {
2859 hwc->interrupts = 0; 2859 hwc->interrupts = 0;
2860 perf_log_throttle(event, 1); 2860 perf_log_throttle(event, 1);
2861 event->pmu->start(event, 0); 2861 event->pmu->start(event, 0);
2862 } 2862 }
2863 2863
2864 if (!event->attr.freq || !event->attr.sample_freq) 2864 if (!event->attr.freq || !event->attr.sample_freq)
2865 goto next; 2865 goto next;
2866 2866
2867 /* 2867 /*
2868 * stop the event and update event->count 2868 * stop the event and update event->count
2869 */ 2869 */
2870 event->pmu->stop(event, PERF_EF_UPDATE); 2870 event->pmu->stop(event, PERF_EF_UPDATE);
2871 2871
2872 now = local64_read(&event->count); 2872 now = local64_read(&event->count);
2873 delta = now - hwc->freq_count_stamp; 2873 delta = now - hwc->freq_count_stamp;
2874 hwc->freq_count_stamp = now; 2874 hwc->freq_count_stamp = now;
2875 2875
2876 /* 2876 /*
2877 * restart the event 2877 * restart the event
2878 * reload only if value has changed 2878 * reload only if value has changed
2879 * we have stopped the event so tell that 2879 * we have stopped the event so tell that
2880 * to perf_adjust_period() to avoid stopping it 2880 * to perf_adjust_period() to avoid stopping it
2881 * twice. 2881 * twice.
2882 */ 2882 */
2883 if (delta > 0) 2883 if (delta > 0)
2884 perf_adjust_period(event, period, delta, false); 2884 perf_adjust_period(event, period, delta, false);
2885 2885
2886 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 2886 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2887 next: 2887 next:
2888 perf_pmu_enable(event->pmu); 2888 perf_pmu_enable(event->pmu);
2889 } 2889 }
2890 2890
2891 perf_pmu_enable(ctx->pmu); 2891 perf_pmu_enable(ctx->pmu);
2892 raw_spin_unlock(&ctx->lock); 2892 raw_spin_unlock(&ctx->lock);
2893 } 2893 }
2894 2894
2895 /* 2895 /*
2896 * Round-robin a context's events: 2896 * Round-robin a context's events:
2897 */ 2897 */
2898 static void rotate_ctx(struct perf_event_context *ctx) 2898 static void rotate_ctx(struct perf_event_context *ctx)
2899 { 2899 {
2900 /* 2900 /*
2901 * Rotate the first entry last of non-pinned groups. Rotation might be 2901 * Rotate the first entry last of non-pinned groups. Rotation might be
2902 * disabled by the inheritance code. 2902 * disabled by the inheritance code.
2903 */ 2903 */
2904 if (!ctx->rotate_disable) 2904 if (!ctx->rotate_disable)
2905 list_rotate_left(&ctx->flexible_groups); 2905 list_rotate_left(&ctx->flexible_groups);
2906 } 2906 }
2907 2907
2908 /* 2908 /*
2909 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized 2909 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2910 * because they're strictly cpu affine and rotate_start is called with IRQs 2910 * because they're strictly cpu affine and rotate_start is called with IRQs
2911 * disabled, while rotate_context is called from IRQ context. 2911 * disabled, while rotate_context is called from IRQ context.
2912 */ 2912 */
2913 static int perf_rotate_context(struct perf_cpu_context *cpuctx) 2913 static int perf_rotate_context(struct perf_cpu_context *cpuctx)
2914 { 2914 {
2915 struct perf_event_context *ctx = NULL; 2915 struct perf_event_context *ctx = NULL;
2916 int rotate = 0, remove = 1; 2916 int rotate = 0, remove = 1;
2917 2917
2918 if (cpuctx->ctx.nr_events) { 2918 if (cpuctx->ctx.nr_events) {
2919 remove = 0; 2919 remove = 0;
2920 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 2920 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2921 rotate = 1; 2921 rotate = 1;
2922 } 2922 }
2923 2923
2924 ctx = cpuctx->task_ctx; 2924 ctx = cpuctx->task_ctx;
2925 if (ctx && ctx->nr_events) { 2925 if (ctx && ctx->nr_events) {
2926 remove = 0; 2926 remove = 0;
2927 if (ctx->nr_events != ctx->nr_active) 2927 if (ctx->nr_events != ctx->nr_active)
2928 rotate = 1; 2928 rotate = 1;
2929 } 2929 }
2930 2930
2931 if (!rotate) 2931 if (!rotate)
2932 goto done; 2932 goto done;
2933 2933
2934 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2934 perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2935 perf_pmu_disable(cpuctx->ctx.pmu); 2935 perf_pmu_disable(cpuctx->ctx.pmu);
2936 2936
2937 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2937 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2938 if (ctx) 2938 if (ctx)
2939 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 2939 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2940 2940
2941 rotate_ctx(&cpuctx->ctx); 2941 rotate_ctx(&cpuctx->ctx);
2942 if (ctx) 2942 if (ctx)
2943 rotate_ctx(ctx); 2943 rotate_ctx(ctx);
2944 2944
2945 perf_event_sched_in(cpuctx, ctx, current); 2945 perf_event_sched_in(cpuctx, ctx, current);
2946 2946
2947 perf_pmu_enable(cpuctx->ctx.pmu); 2947 perf_pmu_enable(cpuctx->ctx.pmu);
2948 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2948 perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2949 done: 2949 done:
2950 if (remove) 2950 if (remove)
2951 list_del_init(&cpuctx->rotation_list); 2951 list_del_init(&cpuctx->rotation_list);
2952 2952
2953 return rotate; 2953 return rotate;
2954 } 2954 }
2955 2955
2956 #ifdef CONFIG_NO_HZ_FULL 2956 #ifdef CONFIG_NO_HZ_FULL
2957 bool perf_event_can_stop_tick(void) 2957 bool perf_event_can_stop_tick(void)
2958 { 2958 {
2959 if (atomic_read(&nr_freq_events) || 2959 if (atomic_read(&nr_freq_events) ||
2960 __this_cpu_read(perf_throttled_count)) 2960 __this_cpu_read(perf_throttled_count))
2961 return false; 2961 return false;
2962 else 2962 else
2963 return true; 2963 return true;
2964 } 2964 }
2965 #endif 2965 #endif
2966 2966
2967 void perf_event_task_tick(void) 2967 void perf_event_task_tick(void)
2968 { 2968 {
2969 struct list_head *head = this_cpu_ptr(&rotation_list); 2969 struct list_head *head = this_cpu_ptr(&rotation_list);
2970 struct perf_cpu_context *cpuctx, *tmp; 2970 struct perf_cpu_context *cpuctx, *tmp;
2971 struct perf_event_context *ctx; 2971 struct perf_event_context *ctx;
2972 int throttled; 2972 int throttled;
2973 2973
2974 WARN_ON(!irqs_disabled()); 2974 WARN_ON(!irqs_disabled());
2975 2975
2976 __this_cpu_inc(perf_throttled_seq); 2976 __this_cpu_inc(perf_throttled_seq);
2977 throttled = __this_cpu_xchg(perf_throttled_count, 0); 2977 throttled = __this_cpu_xchg(perf_throttled_count, 0);
2978 2978
2979 list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) { 2979 list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2980 ctx = &cpuctx->ctx; 2980 ctx = &cpuctx->ctx;
2981 perf_adjust_freq_unthr_context(ctx, throttled); 2981 perf_adjust_freq_unthr_context(ctx, throttled);
2982 2982
2983 ctx = cpuctx->task_ctx; 2983 ctx = cpuctx->task_ctx;
2984 if (ctx) 2984 if (ctx)
2985 perf_adjust_freq_unthr_context(ctx, throttled); 2985 perf_adjust_freq_unthr_context(ctx, throttled);
2986 } 2986 }
2987 } 2987 }
2988 2988
2989 static int event_enable_on_exec(struct perf_event *event, 2989 static int event_enable_on_exec(struct perf_event *event,
2990 struct perf_event_context *ctx) 2990 struct perf_event_context *ctx)
2991 { 2991 {
2992 if (!event->attr.enable_on_exec) 2992 if (!event->attr.enable_on_exec)
2993 return 0; 2993 return 0;
2994 2994
2995 event->attr.enable_on_exec = 0; 2995 event->attr.enable_on_exec = 0;
2996 if (event->state >= PERF_EVENT_STATE_INACTIVE) 2996 if (event->state >= PERF_EVENT_STATE_INACTIVE)
2997 return 0; 2997 return 0;
2998 2998
2999 __perf_event_mark_enabled(event); 2999 __perf_event_mark_enabled(event);
3000 3000
3001 return 1; 3001 return 1;
3002 } 3002 }
3003 3003
3004 /* 3004 /*
3005 * Enable all of a task's events that have been marked enable-on-exec. 3005 * Enable all of a task's events that have been marked enable-on-exec.
3006 * This expects task == current. 3006 * This expects task == current.
3007 */ 3007 */
3008 static void perf_event_enable_on_exec(struct perf_event_context *ctx) 3008 static void perf_event_enable_on_exec(struct perf_event_context *ctx)
3009 { 3009 {
3010 struct perf_event_context *clone_ctx = NULL; 3010 struct perf_event_context *clone_ctx = NULL;
3011 struct perf_event *event; 3011 struct perf_event *event;
3012 unsigned long flags; 3012 unsigned long flags;
3013 int enabled = 0; 3013 int enabled = 0;
3014 int ret; 3014 int ret;
3015 3015
3016 local_irq_save(flags); 3016 local_irq_save(flags);
3017 if (!ctx || !ctx->nr_events) 3017 if (!ctx || !ctx->nr_events)
3018 goto out; 3018 goto out;
3019 3019
3020 /* 3020 /*
3021 * We must ctxsw out cgroup events to avoid conflict 3021 * We must ctxsw out cgroup events to avoid conflict
3022 * when invoking perf_task_event_sched_in() later on 3022 * when invoking perf_task_event_sched_in() later on
3023 * in this function. Otherwise we end up trying to 3023 * in this function. Otherwise we end up trying to
3024 * ctxswin cgroup events which are already scheduled 3024 * ctxswin cgroup events which are already scheduled
3025 * in. 3025 * in.
3026 */ 3026 */
3027 perf_cgroup_sched_out(current, NULL); 3027 perf_cgroup_sched_out(current, NULL);
3028 3028
3029 raw_spin_lock(&ctx->lock); 3029 raw_spin_lock(&ctx->lock);
3030 task_ctx_sched_out(ctx); 3030 task_ctx_sched_out(ctx);
3031 3031
3032 list_for_each_entry(event, &ctx->event_list, event_entry) { 3032 list_for_each_entry(event, &ctx->event_list, event_entry) {
3033 ret = event_enable_on_exec(event, ctx); 3033 ret = event_enable_on_exec(event, ctx);
3034 if (ret) 3034 if (ret)
3035 enabled = 1; 3035 enabled = 1;
3036 } 3036 }
3037 3037
3038 /* 3038 /*
3039 * Unclone this context if we enabled any event. 3039 * Unclone this context if we enabled any event.
3040 */ 3040 */
3041 if (enabled) 3041 if (enabled)
3042 clone_ctx = unclone_ctx(ctx); 3042 clone_ctx = unclone_ctx(ctx);
3043 3043
3044 raw_spin_unlock(&ctx->lock); 3044 raw_spin_unlock(&ctx->lock);
3045 3045
3046 /* 3046 /*
3047 * Also calls ctxswin for cgroup events, if any: 3047 * Also calls ctxswin for cgroup events, if any:
3048 */ 3048 */
3049 perf_event_context_sched_in(ctx, ctx->task); 3049 perf_event_context_sched_in(ctx, ctx->task);
3050 out: 3050 out:
3051 local_irq_restore(flags); 3051 local_irq_restore(flags);
3052 3052
3053 if (clone_ctx) 3053 if (clone_ctx)
3054 put_ctx(clone_ctx); 3054 put_ctx(clone_ctx);
3055 } 3055 }
3056 3056
3057 void perf_event_exec(void) 3057 void perf_event_exec(void)
3058 { 3058 {
3059 struct perf_event_context *ctx; 3059 struct perf_event_context *ctx;
3060 int ctxn; 3060 int ctxn;
3061 3061
3062 rcu_read_lock(); 3062 rcu_read_lock();
3063 for_each_task_context_nr(ctxn) { 3063 for_each_task_context_nr(ctxn) {
3064 ctx = current->perf_event_ctxp[ctxn]; 3064 ctx = current->perf_event_ctxp[ctxn];
3065 if (!ctx) 3065 if (!ctx)
3066 continue; 3066 continue;
3067 3067
3068 perf_event_enable_on_exec(ctx); 3068 perf_event_enable_on_exec(ctx);
3069 } 3069 }
3070 rcu_read_unlock(); 3070 rcu_read_unlock();
3071 } 3071 }
3072 3072
3073 /* 3073 /*
3074 * Cross CPU call to read the hardware event 3074 * Cross CPU call to read the hardware event
3075 */ 3075 */
3076 static void __perf_event_read(void *info) 3076 static void __perf_event_read(void *info)
3077 { 3077 {
3078 struct perf_event *event = info; 3078 struct perf_event *event = info;
3079 struct perf_event_context *ctx = event->ctx; 3079 struct perf_event_context *ctx = event->ctx;
3080 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3080 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3081 3081
3082 /* 3082 /*
3083 * If this is a task context, we need to check whether it is 3083 * If this is a task context, we need to check whether it is
3084 * the current task context of this cpu. If not it has been 3084 * the current task context of this cpu. If not it has been
3085 * scheduled out before the smp call arrived. In that case 3085 * scheduled out before the smp call arrived. In that case
3086 * event->count would have been updated to a recent sample 3086 * event->count would have been updated to a recent sample
3087 * when the event was scheduled out. 3087 * when the event was scheduled out.
3088 */ 3088 */
3089 if (ctx->task && cpuctx->task_ctx != ctx) 3089 if (ctx->task && cpuctx->task_ctx != ctx)
3090 return; 3090 return;
3091 3091
3092 raw_spin_lock(&ctx->lock); 3092 raw_spin_lock(&ctx->lock);
3093 if (ctx->is_active) { 3093 if (ctx->is_active) {
3094 update_context_time(ctx); 3094 update_context_time(ctx);
3095 update_cgrp_time_from_event(event); 3095 update_cgrp_time_from_event(event);
3096 } 3096 }
3097 update_event_times(event); 3097 update_event_times(event);
3098 if (event->state == PERF_EVENT_STATE_ACTIVE) 3098 if (event->state == PERF_EVENT_STATE_ACTIVE)
3099 event->pmu->read(event); 3099 event->pmu->read(event);
3100 raw_spin_unlock(&ctx->lock); 3100 raw_spin_unlock(&ctx->lock);
3101 } 3101 }
3102 3102
3103 static inline u64 perf_event_count(struct perf_event *event) 3103 static inline u64 perf_event_count(struct perf_event *event)
3104 { 3104 {
3105 return local64_read(&event->count) + atomic64_read(&event->child_count); 3105 return local64_read(&event->count) + atomic64_read(&event->child_count);
3106 } 3106 }
3107 3107
3108 static u64 perf_event_read(struct perf_event *event) 3108 static u64 perf_event_read(struct perf_event *event)
3109 { 3109 {
3110 /* 3110 /*
3111 * If event is enabled and currently active on a CPU, update the 3111 * If event is enabled and currently active on a CPU, update the
3112 * value in the event structure: 3112 * value in the event structure:
3113 */ 3113 */
3114 if (event->state == PERF_EVENT_STATE_ACTIVE) { 3114 if (event->state == PERF_EVENT_STATE_ACTIVE) {
3115 smp_call_function_single(event->oncpu, 3115 smp_call_function_single(event->oncpu,
3116 __perf_event_read, event, 1); 3116 __perf_event_read, event, 1);
3117 } else if (event->state == PERF_EVENT_STATE_INACTIVE) { 3117 } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
3118 struct perf_event_context *ctx = event->ctx; 3118 struct perf_event_context *ctx = event->ctx;
3119 unsigned long flags; 3119 unsigned long flags;
3120 3120
3121 raw_spin_lock_irqsave(&ctx->lock, flags); 3121 raw_spin_lock_irqsave(&ctx->lock, flags);
3122 /* 3122 /*
3123 * may read while context is not active 3123 * may read while context is not active
3124 * (e.g., thread is blocked), in that case 3124 * (e.g., thread is blocked), in that case
3125 * we cannot update context time 3125 * we cannot update context time
3126 */ 3126 */
3127 if (ctx->is_active) { 3127 if (ctx->is_active) {
3128 update_context_time(ctx); 3128 update_context_time(ctx);
3129 update_cgrp_time_from_event(event); 3129 update_cgrp_time_from_event(event);
3130 } 3130 }
3131 update_event_times(event); 3131 update_event_times(event);
3132 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3132 raw_spin_unlock_irqrestore(&ctx->lock, flags);
3133 } 3133 }
3134 3134
3135 return perf_event_count(event); 3135 return perf_event_count(event);
3136 } 3136 }
3137 3137
3138 /* 3138 /*
3139 * Initialize the perf_event context in a task_struct: 3139 * Initialize the perf_event context in a task_struct:
3140 */ 3140 */
3141 static void __perf_event_init_context(struct perf_event_context *ctx) 3141 static void __perf_event_init_context(struct perf_event_context *ctx)
3142 { 3142 {
3143 raw_spin_lock_init(&ctx->lock); 3143 raw_spin_lock_init(&ctx->lock);
3144 mutex_init(&ctx->mutex); 3144 mutex_init(&ctx->mutex);
3145 INIT_LIST_HEAD(&ctx->pinned_groups); 3145 INIT_LIST_HEAD(&ctx->pinned_groups);
3146 INIT_LIST_HEAD(&ctx->flexible_groups); 3146 INIT_LIST_HEAD(&ctx->flexible_groups);
3147 INIT_LIST_HEAD(&ctx->event_list); 3147 INIT_LIST_HEAD(&ctx->event_list);
3148 atomic_set(&ctx->refcount, 1); 3148 atomic_set(&ctx->refcount, 1);
3149 INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work); 3149 INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work);
3150 } 3150 }
3151 3151
3152 static struct perf_event_context * 3152 static struct perf_event_context *
3153 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 3153 alloc_perf_context(struct pmu *pmu, struct task_struct *task)
3154 { 3154 {
3155 struct perf_event_context *ctx; 3155 struct perf_event_context *ctx;
3156 3156
3157 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 3157 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
3158 if (!ctx) 3158 if (!ctx)
3159 return NULL; 3159 return NULL;
3160 3160
3161 __perf_event_init_context(ctx); 3161 __perf_event_init_context(ctx);
3162 if (task) { 3162 if (task) {
3163 ctx->task = task; 3163 ctx->task = task;
3164 get_task_struct(task); 3164 get_task_struct(task);
3165 } 3165 }
3166 ctx->pmu = pmu; 3166 ctx->pmu = pmu;
3167 3167
3168 return ctx; 3168 return ctx;
3169 } 3169 }
3170 3170
3171 static struct task_struct * 3171 static struct task_struct *
3172 find_lively_task_by_vpid(pid_t vpid) 3172 find_lively_task_by_vpid(pid_t vpid)
3173 { 3173 {
3174 struct task_struct *task; 3174 struct task_struct *task;
3175 int err; 3175 int err;
3176 3176
3177 rcu_read_lock(); 3177 rcu_read_lock();
3178 if (!vpid) 3178 if (!vpid)
3179 task = current; 3179 task = current;
3180 else 3180 else
3181 task = find_task_by_vpid(vpid); 3181 task = find_task_by_vpid(vpid);
3182 if (task) 3182 if (task)
3183 get_task_struct(task); 3183 get_task_struct(task);
3184 rcu_read_unlock(); 3184 rcu_read_unlock();
3185 3185
3186 if (!task) 3186 if (!task)
3187 return ERR_PTR(-ESRCH); 3187 return ERR_PTR(-ESRCH);
3188 3188
3189 /* Reuse ptrace permission checks for now. */ 3189 /* Reuse ptrace permission checks for now. */
3190 err = -EACCES; 3190 err = -EACCES;
3191 if (!ptrace_may_access(task, PTRACE_MODE_READ)) 3191 if (!ptrace_may_access(task, PTRACE_MODE_READ))
3192 goto errout; 3192 goto errout;
3193 3193
3194 return task; 3194 return task;
3195 errout: 3195 errout:
3196 put_task_struct(task); 3196 put_task_struct(task);
3197 return ERR_PTR(err); 3197 return ERR_PTR(err);
3198 3198
3199 } 3199 }
3200 3200
3201 /* 3201 /*
3202 * Returns a matching context with refcount and pincount. 3202 * Returns a matching context with refcount and pincount.
3203 */ 3203 */
3204 static struct perf_event_context * 3204 static struct perf_event_context *
3205 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu) 3205 find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
3206 { 3206 {
3207 struct perf_event_context *ctx, *clone_ctx = NULL; 3207 struct perf_event_context *ctx, *clone_ctx = NULL;
3208 struct perf_cpu_context *cpuctx; 3208 struct perf_cpu_context *cpuctx;
3209 unsigned long flags; 3209 unsigned long flags;
3210 int ctxn, err; 3210 int ctxn, err;
3211 3211
3212 if (!task) { 3212 if (!task) {
3213 /* Must be root to operate on a CPU event: */ 3213 /* Must be root to operate on a CPU event: */
3214 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 3214 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3215 return ERR_PTR(-EACCES); 3215 return ERR_PTR(-EACCES);
3216 3216
3217 /* 3217 /*
3218 * We could be clever and allow to attach a event to an 3218 * We could be clever and allow to attach a event to an
3219 * offline CPU and activate it when the CPU comes up, but 3219 * offline CPU and activate it when the CPU comes up, but
3220 * that's for later. 3220 * that's for later.
3221 */ 3221 */
3222 if (!cpu_online(cpu)) 3222 if (!cpu_online(cpu))
3223 return ERR_PTR(-ENODEV); 3223 return ERR_PTR(-ENODEV);
3224 3224
3225 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 3225 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
3226 ctx = &cpuctx->ctx; 3226 ctx = &cpuctx->ctx;
3227 get_ctx(ctx); 3227 get_ctx(ctx);
3228 ++ctx->pin_count; 3228 ++ctx->pin_count;
3229 3229
3230 return ctx; 3230 return ctx;
3231 } 3231 }
3232 3232
3233 err = -EINVAL; 3233 err = -EINVAL;
3234 ctxn = pmu->task_ctx_nr; 3234 ctxn = pmu->task_ctx_nr;
3235 if (ctxn < 0) 3235 if (ctxn < 0)
3236 goto errout; 3236 goto errout;
3237 3237
3238 retry: 3238 retry:
3239 ctx = perf_lock_task_context(task, ctxn, &flags); 3239 ctx = perf_lock_task_context(task, ctxn, &flags);
3240 if (ctx) { 3240 if (ctx) {
3241 clone_ctx = unclone_ctx(ctx); 3241 clone_ctx = unclone_ctx(ctx);
3242 ++ctx->pin_count; 3242 ++ctx->pin_count;
3243 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3243 raw_spin_unlock_irqrestore(&ctx->lock, flags);
3244 3244
3245 if (clone_ctx) 3245 if (clone_ctx)
3246 put_ctx(clone_ctx); 3246 put_ctx(clone_ctx);
3247 } else { 3247 } else {
3248 ctx = alloc_perf_context(pmu, task); 3248 ctx = alloc_perf_context(pmu, task);
3249 err = -ENOMEM; 3249 err = -ENOMEM;
3250 if (!ctx) 3250 if (!ctx)
3251 goto errout; 3251 goto errout;
3252 3252
3253 err = 0; 3253 err = 0;
3254 mutex_lock(&task->perf_event_mutex); 3254 mutex_lock(&task->perf_event_mutex);
3255 /* 3255 /*
3256 * If it has already passed perf_event_exit_task(). 3256 * If it has already passed perf_event_exit_task().
3257 * we must see PF_EXITING, it takes this mutex too. 3257 * we must see PF_EXITING, it takes this mutex too.
3258 */ 3258 */
3259 if (task->flags & PF_EXITING) 3259 if (task->flags & PF_EXITING)
3260 err = -ESRCH; 3260 err = -ESRCH;
3261 else if (task->perf_event_ctxp[ctxn]) 3261 else if (task->perf_event_ctxp[ctxn])
3262 err = -EAGAIN; 3262 err = -EAGAIN;
3263 else { 3263 else {
3264 get_ctx(ctx); 3264 get_ctx(ctx);
3265 ++ctx->pin_count; 3265 ++ctx->pin_count;
3266 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 3266 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3267 } 3267 }
3268 mutex_unlock(&task->perf_event_mutex); 3268 mutex_unlock(&task->perf_event_mutex);
3269 3269
3270 if (unlikely(err)) { 3270 if (unlikely(err)) {
3271 put_ctx(ctx); 3271 put_ctx(ctx);
3272 3272
3273 if (err == -EAGAIN) 3273 if (err == -EAGAIN)
3274 goto retry; 3274 goto retry;
3275 goto errout; 3275 goto errout;
3276 } 3276 }
3277 } 3277 }
3278 3278
3279 return ctx; 3279 return ctx;
3280 3280
3281 errout: 3281 errout:
3282 return ERR_PTR(err); 3282 return ERR_PTR(err);
3283 } 3283 }
3284 3284
3285 static void perf_event_free_filter(struct perf_event *event); 3285 static void perf_event_free_filter(struct perf_event *event);
3286 3286
3287 static void free_event_rcu(struct rcu_head *head) 3287 static void free_event_rcu(struct rcu_head *head)
3288 { 3288 {
3289 struct perf_event *event; 3289 struct perf_event *event;
3290 3290
3291 event = container_of(head, struct perf_event, rcu_head); 3291 event = container_of(head, struct perf_event, rcu_head);
3292 if (event->ns) 3292 if (event->ns)
3293 put_pid_ns(event->ns); 3293 put_pid_ns(event->ns);
3294 perf_event_free_filter(event); 3294 perf_event_free_filter(event);
3295 kfree(event); 3295 kfree(event);
3296 } 3296 }
3297 3297
3298 static void ring_buffer_put(struct ring_buffer *rb); 3298 static void ring_buffer_put(struct ring_buffer *rb);
3299 static void ring_buffer_attach(struct perf_event *event, 3299 static void ring_buffer_attach(struct perf_event *event,
3300 struct ring_buffer *rb); 3300 struct ring_buffer *rb);
3301 3301
3302 static void unaccount_event_cpu(struct perf_event *event, int cpu) 3302 static void unaccount_event_cpu(struct perf_event *event, int cpu)
3303 { 3303 {
3304 if (event->parent) 3304 if (event->parent)
3305 return; 3305 return;
3306 3306
3307 if (has_branch_stack(event)) { 3307 if (has_branch_stack(event)) {
3308 if (!(event->attach_state & PERF_ATTACH_TASK)) 3308 if (!(event->attach_state & PERF_ATTACH_TASK))
3309 atomic_dec(&per_cpu(perf_branch_stack_events, cpu)); 3309 atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
3310 } 3310 }
3311 if (is_cgroup_event(event)) 3311 if (is_cgroup_event(event))
3312 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 3312 atomic_dec(&per_cpu(perf_cgroup_events, cpu));
3313 } 3313 }
3314 3314
3315 static void unaccount_event(struct perf_event *event) 3315 static void unaccount_event(struct perf_event *event)
3316 { 3316 {
3317 if (event->parent) 3317 if (event->parent)
3318 return; 3318 return;
3319 3319
3320 if (event->attach_state & PERF_ATTACH_TASK) 3320 if (event->attach_state & PERF_ATTACH_TASK)
3321 static_key_slow_dec_deferred(&perf_sched_events); 3321 static_key_slow_dec_deferred(&perf_sched_events);
3322 if (event->attr.mmap || event->attr.mmap_data) 3322 if (event->attr.mmap || event->attr.mmap_data)
3323 atomic_dec(&nr_mmap_events); 3323 atomic_dec(&nr_mmap_events);
3324 if (event->attr.comm) 3324 if (event->attr.comm)
3325 atomic_dec(&nr_comm_events); 3325 atomic_dec(&nr_comm_events);
3326 if (event->attr.task) 3326 if (event->attr.task)
3327 atomic_dec(&nr_task_events); 3327 atomic_dec(&nr_task_events);
3328 if (event->attr.freq) 3328 if (event->attr.freq)
3329 atomic_dec(&nr_freq_events); 3329 atomic_dec(&nr_freq_events);
3330 if (is_cgroup_event(event)) 3330 if (is_cgroup_event(event))
3331 static_key_slow_dec_deferred(&perf_sched_events); 3331 static_key_slow_dec_deferred(&perf_sched_events);
3332 if (has_branch_stack(event)) 3332 if (has_branch_stack(event))
3333 static_key_slow_dec_deferred(&perf_sched_events); 3333 static_key_slow_dec_deferred(&perf_sched_events);
3334 3334
3335 unaccount_event_cpu(event, event->cpu); 3335 unaccount_event_cpu(event, event->cpu);
3336 } 3336 }
3337 3337
3338 static void __free_event(struct perf_event *event) 3338 static void __free_event(struct perf_event *event)
3339 { 3339 {
3340 if (!event->parent) { 3340 if (!event->parent) {
3341 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 3341 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3342 put_callchain_buffers(); 3342 put_callchain_buffers();
3343 } 3343 }
3344 3344
3345 if (event->destroy) 3345 if (event->destroy)
3346 event->destroy(event); 3346 event->destroy(event);
3347 3347
3348 if (event->ctx) 3348 if (event->ctx)
3349 put_ctx(event->ctx); 3349 put_ctx(event->ctx);
3350 3350
3351 if (event->pmu) 3351 if (event->pmu)
3352 module_put(event->pmu->module); 3352 module_put(event->pmu->module);
3353 3353
3354 call_rcu(&event->rcu_head, free_event_rcu); 3354 call_rcu(&event->rcu_head, free_event_rcu);
3355 } 3355 }
3356 3356
3357 static void _free_event(struct perf_event *event) 3357 static void _free_event(struct perf_event *event)
3358 { 3358 {
3359 irq_work_sync(&event->pending); 3359 irq_work_sync(&event->pending);
3360 3360
3361 unaccount_event(event); 3361 unaccount_event(event);
3362 3362
3363 if (event->rb) { 3363 if (event->rb) {
3364 /* 3364 /*
3365 * Can happen when we close an event with re-directed output. 3365 * Can happen when we close an event with re-directed output.
3366 * 3366 *
3367 * Since we have a 0 refcount, perf_mmap_close() will skip 3367 * Since we have a 0 refcount, perf_mmap_close() will skip
3368 * over us; possibly making our ring_buffer_put() the last. 3368 * over us; possibly making our ring_buffer_put() the last.
3369 */ 3369 */
3370 mutex_lock(&event->mmap_mutex); 3370 mutex_lock(&event->mmap_mutex);
3371 ring_buffer_attach(event, NULL); 3371 ring_buffer_attach(event, NULL);
3372 mutex_unlock(&event->mmap_mutex); 3372 mutex_unlock(&event->mmap_mutex);
3373 } 3373 }
3374 3374
3375 if (is_cgroup_event(event)) 3375 if (is_cgroup_event(event))
3376 perf_detach_cgroup(event); 3376 perf_detach_cgroup(event);
3377 3377
3378 __free_event(event); 3378 __free_event(event);
3379 } 3379 }
3380 3380
3381 /* 3381 /*
3382 * Used to free events which have a known refcount of 1, such as in error paths 3382 * Used to free events which have a known refcount of 1, such as in error paths
3383 * where the event isn't exposed yet and inherited events. 3383 * where the event isn't exposed yet and inherited events.
3384 */ 3384 */
3385 static void free_event(struct perf_event *event) 3385 static void free_event(struct perf_event *event)
3386 { 3386 {
3387 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 3387 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
3388 "unexpected event refcount: %ld; ptr=%p\n", 3388 "unexpected event refcount: %ld; ptr=%p\n",
3389 atomic_long_read(&event->refcount), event)) { 3389 atomic_long_read(&event->refcount), event)) {
3390 /* leak to avoid use-after-free */ 3390 /* leak to avoid use-after-free */
3391 return; 3391 return;
3392 } 3392 }
3393 3393
3394 _free_event(event); 3394 _free_event(event);
3395 } 3395 }
3396 3396
3397 /* 3397 /*
3398 * Remove user event from the owner task. 3398 * Remove user event from the owner task.
3399 */ 3399 */
3400 static void perf_remove_from_owner(struct perf_event *event) 3400 static void perf_remove_from_owner(struct perf_event *event)
3401 { 3401 {
3402 struct task_struct *owner; 3402 struct task_struct *owner;
3403 3403
3404 rcu_read_lock(); 3404 rcu_read_lock();
3405 owner = ACCESS_ONCE(event->owner); 3405 owner = ACCESS_ONCE(event->owner);
3406 /* 3406 /*
3407 * Matches the smp_wmb() in perf_event_exit_task(). If we observe 3407 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
3408 * !owner it means the list deletion is complete and we can indeed 3408 * !owner it means the list deletion is complete and we can indeed
3409 * free this event, otherwise we need to serialize on 3409 * free this event, otherwise we need to serialize on
3410 * owner->perf_event_mutex. 3410 * owner->perf_event_mutex.
3411 */ 3411 */
3412 smp_read_barrier_depends(); 3412 smp_read_barrier_depends();
3413 if (owner) { 3413 if (owner) {
3414 /* 3414 /*
3415 * Since delayed_put_task_struct() also drops the last 3415 * Since delayed_put_task_struct() also drops the last
3416 * task reference we can safely take a new reference 3416 * task reference we can safely take a new reference
3417 * while holding the rcu_read_lock(). 3417 * while holding the rcu_read_lock().
3418 */ 3418 */
3419 get_task_struct(owner); 3419 get_task_struct(owner);
3420 } 3420 }
3421 rcu_read_unlock(); 3421 rcu_read_unlock();
3422 3422
3423 if (owner) { 3423 if (owner) {
3424 mutex_lock(&owner->perf_event_mutex); 3424 mutex_lock(&owner->perf_event_mutex);
3425 /* 3425 /*
3426 * We have to re-check the event->owner field, if it is cleared 3426 * We have to re-check the event->owner field, if it is cleared
3427 * we raced with perf_event_exit_task(), acquiring the mutex 3427 * we raced with perf_event_exit_task(), acquiring the mutex
3428 * ensured they're done, and we can proceed with freeing the 3428 * ensured they're done, and we can proceed with freeing the
3429 * event. 3429 * event.
3430 */ 3430 */
3431 if (event->owner) 3431 if (event->owner)
3432 list_del_init(&event->owner_entry); 3432 list_del_init(&event->owner_entry);
3433 mutex_unlock(&owner->perf_event_mutex); 3433 mutex_unlock(&owner->perf_event_mutex);
3434 put_task_struct(owner); 3434 put_task_struct(owner);
3435 } 3435 }
3436 } 3436 }
3437 3437
3438 /* 3438 /*
3439 * Called when the last reference to the file is gone. 3439 * Called when the last reference to the file is gone.
3440 */ 3440 */
3441 static void put_event(struct perf_event *event) 3441 static void put_event(struct perf_event *event)
3442 { 3442 {
3443 struct perf_event_context *ctx = event->ctx; 3443 struct perf_event_context *ctx = event->ctx;
3444 3444
3445 if (!atomic_long_dec_and_test(&event->refcount)) 3445 if (!atomic_long_dec_and_test(&event->refcount))
3446 return; 3446 return;
3447 3447
3448 if (!is_kernel_event(event)) 3448 if (!is_kernel_event(event))
3449 perf_remove_from_owner(event); 3449 perf_remove_from_owner(event);
3450 3450
3451 WARN_ON_ONCE(ctx->parent_ctx); 3451 WARN_ON_ONCE(ctx->parent_ctx);
3452 /* 3452 /*
3453 * There are two ways this annotation is useful: 3453 * There are two ways this annotation is useful:
3454 * 3454 *
3455 * 1) there is a lock recursion from perf_event_exit_task 3455 * 1) there is a lock recursion from perf_event_exit_task
3456 * see the comment there. 3456 * see the comment there.
3457 * 3457 *
3458 * 2) there is a lock-inversion with mmap_sem through 3458 * 2) there is a lock-inversion with mmap_sem through
3459 * perf_event_read_group(), which takes faults while 3459 * perf_event_read_group(), which takes faults while
3460 * holding ctx->mutex, however this is called after 3460 * holding ctx->mutex, however this is called after
3461 * the last filedesc died, so there is no possibility 3461 * the last filedesc died, so there is no possibility
3462 * to trigger the AB-BA case. 3462 * to trigger the AB-BA case.
3463 */ 3463 */
3464 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING); 3464 mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
3465 perf_remove_from_context(event, true); 3465 perf_remove_from_context(event, true);
3466 mutex_unlock(&ctx->mutex); 3466 mutex_unlock(&ctx->mutex);
3467 3467
3468 _free_event(event); 3468 _free_event(event);
3469 } 3469 }
3470 3470
3471 int perf_event_release_kernel(struct perf_event *event) 3471 int perf_event_release_kernel(struct perf_event *event)
3472 { 3472 {
3473 put_event(event); 3473 put_event(event);
3474 return 0; 3474 return 0;
3475 } 3475 }
3476 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 3476 EXPORT_SYMBOL_GPL(perf_event_release_kernel);
3477 3477
3478 static int perf_release(struct inode *inode, struct file *file) 3478 static int perf_release(struct inode *inode, struct file *file)
3479 { 3479 {
3480 put_event(file->private_data); 3480 put_event(file->private_data);
3481 return 0; 3481 return 0;
3482 } 3482 }
3483 3483
3484 /* 3484 /*
3485 * Remove all orphanes events from the context. 3485 * Remove all orphanes events from the context.
3486 */ 3486 */
3487 static void orphans_remove_work(struct work_struct *work) 3487 static void orphans_remove_work(struct work_struct *work)
3488 { 3488 {
3489 struct perf_event_context *ctx; 3489 struct perf_event_context *ctx;
3490 struct perf_event *event, *tmp; 3490 struct perf_event *event, *tmp;
3491 3491
3492 ctx = container_of(work, struct perf_event_context, 3492 ctx = container_of(work, struct perf_event_context,
3493 orphans_remove.work); 3493 orphans_remove.work);
3494 3494
3495 mutex_lock(&ctx->mutex); 3495 mutex_lock(&ctx->mutex);
3496 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) { 3496 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
3497 struct perf_event *parent_event = event->parent; 3497 struct perf_event *parent_event = event->parent;
3498 3498
3499 if (!is_orphaned_child(event)) 3499 if (!is_orphaned_child(event))
3500 continue; 3500 continue;
3501 3501
3502 perf_remove_from_context(event, true); 3502 perf_remove_from_context(event, true);
3503 3503
3504 mutex_lock(&parent_event->child_mutex); 3504 mutex_lock(&parent_event->child_mutex);
3505 list_del_init(&event->child_list); 3505 list_del_init(&event->child_list);
3506 mutex_unlock(&parent_event->child_mutex); 3506 mutex_unlock(&parent_event->child_mutex);
3507 3507
3508 free_event(event); 3508 free_event(event);
3509 put_event(parent_event); 3509 put_event(parent_event);
3510 } 3510 }
3511 3511
3512 raw_spin_lock_irq(&ctx->lock); 3512 raw_spin_lock_irq(&ctx->lock);
3513 ctx->orphans_remove_sched = false; 3513 ctx->orphans_remove_sched = false;
3514 raw_spin_unlock_irq(&ctx->lock); 3514 raw_spin_unlock_irq(&ctx->lock);
3515 mutex_unlock(&ctx->mutex); 3515 mutex_unlock(&ctx->mutex);
3516 3516
3517 put_ctx(ctx); 3517 put_ctx(ctx);
3518 } 3518 }
3519 3519
3520 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 3520 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3521 { 3521 {
3522 struct perf_event *child; 3522 struct perf_event *child;
3523 u64 total = 0; 3523 u64 total = 0;
3524 3524
3525 *enabled = 0; 3525 *enabled = 0;
3526 *running = 0; 3526 *running = 0;
3527 3527
3528 mutex_lock(&event->child_mutex); 3528 mutex_lock(&event->child_mutex);
3529 total += perf_event_read(event); 3529 total += perf_event_read(event);
3530 *enabled += event->total_time_enabled + 3530 *enabled += event->total_time_enabled +
3531 atomic64_read(&event->child_total_time_enabled); 3531 atomic64_read(&event->child_total_time_enabled);
3532 *running += event->total_time_running + 3532 *running += event->total_time_running +
3533 atomic64_read(&event->child_total_time_running); 3533 atomic64_read(&event->child_total_time_running);
3534 3534
3535 list_for_each_entry(child, &event->child_list, child_list) { 3535 list_for_each_entry(child, &event->child_list, child_list) {
3536 total += perf_event_read(child); 3536 total += perf_event_read(child);
3537 *enabled += child->total_time_enabled; 3537 *enabled += child->total_time_enabled;
3538 *running += child->total_time_running; 3538 *running += child->total_time_running;
3539 } 3539 }
3540 mutex_unlock(&event->child_mutex); 3540 mutex_unlock(&event->child_mutex);
3541 3541
3542 return total; 3542 return total;
3543 } 3543 }
3544 EXPORT_SYMBOL_GPL(perf_event_read_value); 3544 EXPORT_SYMBOL_GPL(perf_event_read_value);
3545 3545
3546 static int perf_event_read_group(struct perf_event *event, 3546 static int perf_event_read_group(struct perf_event *event,
3547 u64 read_format, char __user *buf) 3547 u64 read_format, char __user *buf)
3548 { 3548 {
3549 struct perf_event *leader = event->group_leader, *sub; 3549 struct perf_event *leader = event->group_leader, *sub;
3550 int n = 0, size = 0, ret = -EFAULT; 3550 int n = 0, size = 0, ret = -EFAULT;
3551 struct perf_event_context *ctx = leader->ctx; 3551 struct perf_event_context *ctx = leader->ctx;
3552 u64 values[5]; 3552 u64 values[5];
3553 u64 count, enabled, running; 3553 u64 count, enabled, running;
3554 3554
3555 mutex_lock(&ctx->mutex); 3555 mutex_lock(&ctx->mutex);
3556 count = perf_event_read_value(leader, &enabled, &running); 3556 count = perf_event_read_value(leader, &enabled, &running);
3557 3557
3558 values[n++] = 1 + leader->nr_siblings; 3558 values[n++] = 1 + leader->nr_siblings;
3559 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3559 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3560 values[n++] = enabled; 3560 values[n++] = enabled;
3561 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3561 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3562 values[n++] = running; 3562 values[n++] = running;
3563 values[n++] = count; 3563 values[n++] = count;
3564 if (read_format & PERF_FORMAT_ID) 3564 if (read_format & PERF_FORMAT_ID)
3565 values[n++] = primary_event_id(leader); 3565 values[n++] = primary_event_id(leader);
3566 3566
3567 size = n * sizeof(u64); 3567 size = n * sizeof(u64);
3568 3568
3569 if (copy_to_user(buf, values, size)) 3569 if (copy_to_user(buf, values, size))
3570 goto unlock; 3570 goto unlock;
3571 3571
3572 ret = size; 3572 ret = size;
3573 3573
3574 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 3574 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3575 n = 0; 3575 n = 0;
3576 3576
3577 values[n++] = perf_event_read_value(sub, &enabled, &running); 3577 values[n++] = perf_event_read_value(sub, &enabled, &running);
3578 if (read_format & PERF_FORMAT_ID) 3578 if (read_format & PERF_FORMAT_ID)
3579 values[n++] = primary_event_id(sub); 3579 values[n++] = primary_event_id(sub);
3580 3580
3581 size = n * sizeof(u64); 3581 size = n * sizeof(u64);
3582 3582
3583 if (copy_to_user(buf + ret, values, size)) { 3583 if (copy_to_user(buf + ret, values, size)) {
3584 ret = -EFAULT; 3584 ret = -EFAULT;
3585 goto unlock; 3585 goto unlock;
3586 } 3586 }
3587 3587
3588 ret += size; 3588 ret += size;
3589 } 3589 }
3590 unlock: 3590 unlock:
3591 mutex_unlock(&ctx->mutex); 3591 mutex_unlock(&ctx->mutex);
3592 3592
3593 return ret; 3593 return ret;
3594 } 3594 }
3595 3595
3596 static int perf_event_read_one(struct perf_event *event, 3596 static int perf_event_read_one(struct perf_event *event,
3597 u64 read_format, char __user *buf) 3597 u64 read_format, char __user *buf)
3598 { 3598 {
3599 u64 enabled, running; 3599 u64 enabled, running;
3600 u64 values[4]; 3600 u64 values[4];
3601 int n = 0; 3601 int n = 0;
3602 3602
3603 values[n++] = perf_event_read_value(event, &enabled, &running); 3603 values[n++] = perf_event_read_value(event, &enabled, &running);
3604 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 3604 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3605 values[n++] = enabled; 3605 values[n++] = enabled;
3606 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 3606 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3607 values[n++] = running; 3607 values[n++] = running;
3608 if (read_format & PERF_FORMAT_ID) 3608 if (read_format & PERF_FORMAT_ID)
3609 values[n++] = primary_event_id(event); 3609 values[n++] = primary_event_id(event);
3610 3610
3611 if (copy_to_user(buf, values, n * sizeof(u64))) 3611 if (copy_to_user(buf, values, n * sizeof(u64)))
3612 return -EFAULT; 3612 return -EFAULT;
3613 3613
3614 return n * sizeof(u64); 3614 return n * sizeof(u64);
3615 } 3615 }
3616 3616
3617 static bool is_event_hup(struct perf_event *event) 3617 static bool is_event_hup(struct perf_event *event)
3618 { 3618 {
3619 bool no_children; 3619 bool no_children;
3620 3620
3621 if (event->state != PERF_EVENT_STATE_EXIT) 3621 if (event->state != PERF_EVENT_STATE_EXIT)
3622 return false; 3622 return false;
3623 3623
3624 mutex_lock(&event->child_mutex); 3624 mutex_lock(&event->child_mutex);
3625 no_children = list_empty(&event->child_list); 3625 no_children = list_empty(&event->child_list);
3626 mutex_unlock(&event->child_mutex); 3626 mutex_unlock(&event->child_mutex);
3627 return no_children; 3627 return no_children;
3628 } 3628 }
3629 3629
3630 /* 3630 /*
3631 * Read the performance event - simple non blocking version for now 3631 * Read the performance event - simple non blocking version for now
3632 */ 3632 */
3633 static ssize_t 3633 static ssize_t
3634 perf_read_hw(struct perf_event *event, char __user *buf, size_t count) 3634 perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3635 { 3635 {
3636 u64 read_format = event->attr.read_format; 3636 u64 read_format = event->attr.read_format;
3637 int ret; 3637 int ret;
3638 3638
3639 /* 3639 /*
3640 * Return end-of-file for a read on a event that is in 3640 * Return end-of-file for a read on a event that is in
3641 * error state (i.e. because it was pinned but it couldn't be 3641 * error state (i.e. because it was pinned but it couldn't be
3642 * scheduled on to the CPU at some point). 3642 * scheduled on to the CPU at some point).
3643 */ 3643 */
3644 if (event->state == PERF_EVENT_STATE_ERROR) 3644 if (event->state == PERF_EVENT_STATE_ERROR)
3645 return 0; 3645 return 0;
3646 3646
3647 if (count < event->read_size) 3647 if (count < event->read_size)
3648 return -ENOSPC; 3648 return -ENOSPC;
3649 3649
3650 WARN_ON_ONCE(event->ctx->parent_ctx); 3650 WARN_ON_ONCE(event->ctx->parent_ctx);
3651 if (read_format & PERF_FORMAT_GROUP) 3651 if (read_format & PERF_FORMAT_GROUP)
3652 ret = perf_event_read_group(event, read_format, buf); 3652 ret = perf_event_read_group(event, read_format, buf);
3653 else 3653 else
3654 ret = perf_event_read_one(event, read_format, buf); 3654 ret = perf_event_read_one(event, read_format, buf);
3655 3655
3656 return ret; 3656 return ret;
3657 } 3657 }
3658 3658
3659 static ssize_t 3659 static ssize_t
3660 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 3660 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3661 { 3661 {
3662 struct perf_event *event = file->private_data; 3662 struct perf_event *event = file->private_data;
3663 3663
3664 return perf_read_hw(event, buf, count); 3664 return perf_read_hw(event, buf, count);
3665 } 3665 }
3666 3666
3667 static unsigned int perf_poll(struct file *file, poll_table *wait) 3667 static unsigned int perf_poll(struct file *file, poll_table *wait)
3668 { 3668 {
3669 struct perf_event *event = file->private_data; 3669 struct perf_event *event = file->private_data;
3670 struct ring_buffer *rb; 3670 struct ring_buffer *rb;
3671 unsigned int events = POLLHUP; 3671 unsigned int events = POLLHUP;
3672 3672
3673 poll_wait(file, &event->waitq, wait); 3673 poll_wait(file, &event->waitq, wait);
3674 3674
3675 if (is_event_hup(event)) 3675 if (is_event_hup(event))
3676 return events; 3676 return events;
3677 3677
3678 /* 3678 /*
3679 * Pin the event->rb by taking event->mmap_mutex; otherwise 3679 * Pin the event->rb by taking event->mmap_mutex; otherwise
3680 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 3680 * perf_event_set_output() can swizzle our rb and make us miss wakeups.
3681 */ 3681 */
3682 mutex_lock(&event->mmap_mutex); 3682 mutex_lock(&event->mmap_mutex);
3683 rb = event->rb; 3683 rb = event->rb;
3684 if (rb) 3684 if (rb)
3685 events = atomic_xchg(&rb->poll, 0); 3685 events = atomic_xchg(&rb->poll, 0);
3686 mutex_unlock(&event->mmap_mutex); 3686 mutex_unlock(&event->mmap_mutex);
3687 return events; 3687 return events;
3688 } 3688 }
3689 3689
3690 static void perf_event_reset(struct perf_event *event) 3690 static void perf_event_reset(struct perf_event *event)
3691 { 3691 {
3692 (void)perf_event_read(event); 3692 (void)perf_event_read(event);
3693 local64_set(&event->count, 0); 3693 local64_set(&event->count, 0);
3694 perf_event_update_userpage(event); 3694 perf_event_update_userpage(event);
3695 } 3695 }
3696 3696
3697 /* 3697 /*
3698 * Holding the top-level event's child_mutex means that any 3698 * Holding the top-level event's child_mutex means that any
3699 * descendant process that has inherited this event will block 3699 * descendant process that has inherited this event will block
3700 * in sync_child_event if it goes to exit, thus satisfying the 3700 * in sync_child_event if it goes to exit, thus satisfying the
3701 * task existence requirements of perf_event_enable/disable. 3701 * task existence requirements of perf_event_enable/disable.
3702 */ 3702 */
3703 static void perf_event_for_each_child(struct perf_event *event, 3703 static void perf_event_for_each_child(struct perf_event *event,
3704 void (*func)(struct perf_event *)) 3704 void (*func)(struct perf_event *))
3705 { 3705 {
3706 struct perf_event *child; 3706 struct perf_event *child;
3707 3707
3708 WARN_ON_ONCE(event->ctx->parent_ctx); 3708 WARN_ON_ONCE(event->ctx->parent_ctx);
3709 mutex_lock(&event->child_mutex); 3709 mutex_lock(&event->child_mutex);
3710 func(event); 3710 func(event);
3711 list_for_each_entry(child, &event->child_list, child_list) 3711 list_for_each_entry(child, &event->child_list, child_list)
3712 func(child); 3712 func(child);
3713 mutex_unlock(&event->child_mutex); 3713 mutex_unlock(&event->child_mutex);
3714 } 3714 }
3715 3715
3716 static void perf_event_for_each(struct perf_event *event, 3716 static void perf_event_for_each(struct perf_event *event,
3717 void (*func)(struct perf_event *)) 3717 void (*func)(struct perf_event *))
3718 { 3718 {
3719 struct perf_event_context *ctx = event->ctx; 3719 struct perf_event_context *ctx = event->ctx;
3720 struct perf_event *sibling; 3720 struct perf_event *sibling;
3721 3721
3722 WARN_ON_ONCE(ctx->parent_ctx); 3722 WARN_ON_ONCE(ctx->parent_ctx);
3723 mutex_lock(&ctx->mutex); 3723 mutex_lock(&ctx->mutex);
3724 event = event->group_leader; 3724 event = event->group_leader;
3725 3725
3726 perf_event_for_each_child(event, func); 3726 perf_event_for_each_child(event, func);
3727 list_for_each_entry(sibling, &event->sibling_list, group_entry) 3727 list_for_each_entry(sibling, &event->sibling_list, group_entry)
3728 perf_event_for_each_child(sibling, func); 3728 perf_event_for_each_child(sibling, func);
3729 mutex_unlock(&ctx->mutex); 3729 mutex_unlock(&ctx->mutex);
3730 } 3730 }
3731 3731
3732 static int perf_event_period(struct perf_event *event, u64 __user *arg) 3732 static int perf_event_period(struct perf_event *event, u64 __user *arg)
3733 { 3733 {
3734 struct perf_event_context *ctx = event->ctx; 3734 struct perf_event_context *ctx = event->ctx;
3735 int ret = 0, active; 3735 int ret = 0, active;
3736 u64 value; 3736 u64 value;
3737 3737
3738 if (!is_sampling_event(event)) 3738 if (!is_sampling_event(event))
3739 return -EINVAL; 3739 return -EINVAL;
3740 3740
3741 if (copy_from_user(&value, arg, sizeof(value))) 3741 if (copy_from_user(&value, arg, sizeof(value)))
3742 return -EFAULT; 3742 return -EFAULT;
3743 3743
3744 if (!value) 3744 if (!value)
3745 return -EINVAL; 3745 return -EINVAL;
3746 3746
3747 raw_spin_lock_irq(&ctx->lock); 3747 raw_spin_lock_irq(&ctx->lock);
3748 if (event->attr.freq) { 3748 if (event->attr.freq) {
3749 if (value > sysctl_perf_event_sample_rate) { 3749 if (value > sysctl_perf_event_sample_rate) {
3750 ret = -EINVAL; 3750 ret = -EINVAL;
3751 goto unlock; 3751 goto unlock;
3752 } 3752 }
3753 3753
3754 event->attr.sample_freq = value; 3754 event->attr.sample_freq = value;
3755 } else { 3755 } else {
3756 event->attr.sample_period = value; 3756 event->attr.sample_period = value;
3757 event->hw.sample_period = value; 3757 event->hw.sample_period = value;
3758 } 3758 }
3759 3759
3760 active = (event->state == PERF_EVENT_STATE_ACTIVE); 3760 active = (event->state == PERF_EVENT_STATE_ACTIVE);
3761 if (active) { 3761 if (active) {
3762 perf_pmu_disable(ctx->pmu); 3762 perf_pmu_disable(ctx->pmu);
3763 event->pmu->stop(event, PERF_EF_UPDATE); 3763 event->pmu->stop(event, PERF_EF_UPDATE);
3764 } 3764 }
3765 3765
3766 local64_set(&event->hw.period_left, 0); 3766 local64_set(&event->hw.period_left, 0);
3767 3767
3768 if (active) { 3768 if (active) {
3769 event->pmu->start(event, PERF_EF_RELOAD); 3769 event->pmu->start(event, PERF_EF_RELOAD);
3770 perf_pmu_enable(ctx->pmu); 3770 perf_pmu_enable(ctx->pmu);
3771 } 3771 }
3772 3772
3773 unlock: 3773 unlock:
3774 raw_spin_unlock_irq(&ctx->lock); 3774 raw_spin_unlock_irq(&ctx->lock);
3775 3775
3776 return ret; 3776 return ret;
3777 } 3777 }
3778 3778
3779 static const struct file_operations perf_fops; 3779 static const struct file_operations perf_fops;
3780 3780
3781 static inline int perf_fget_light(int fd, struct fd *p) 3781 static inline int perf_fget_light(int fd, struct fd *p)
3782 { 3782 {
3783 struct fd f = fdget(fd); 3783 struct fd f = fdget(fd);
3784 if (!f.file) 3784 if (!f.file)
3785 return -EBADF; 3785 return -EBADF;
3786 3786
3787 if (f.file->f_op != &perf_fops) { 3787 if (f.file->f_op != &perf_fops) {
3788 fdput(f); 3788 fdput(f);
3789 return -EBADF; 3789 return -EBADF;
3790 } 3790 }
3791 *p = f; 3791 *p = f;
3792 return 0; 3792 return 0;
3793 } 3793 }
3794 3794
3795 static int perf_event_set_output(struct perf_event *event, 3795 static int perf_event_set_output(struct perf_event *event,
3796 struct perf_event *output_event); 3796 struct perf_event *output_event);
3797 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 3797 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3798 3798
3799 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 3799 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3800 { 3800 {
3801 struct perf_event *event = file->private_data; 3801 struct perf_event *event = file->private_data;
3802 void (*func)(struct perf_event *); 3802 void (*func)(struct perf_event *);
3803 u32 flags = arg; 3803 u32 flags = arg;
3804 3804
3805 switch (cmd) { 3805 switch (cmd) {
3806 case PERF_EVENT_IOC_ENABLE: 3806 case PERF_EVENT_IOC_ENABLE:
3807 func = perf_event_enable; 3807 func = perf_event_enable;
3808 break; 3808 break;
3809 case PERF_EVENT_IOC_DISABLE: 3809 case PERF_EVENT_IOC_DISABLE:
3810 func = perf_event_disable; 3810 func = perf_event_disable;
3811 break; 3811 break;
3812 case PERF_EVENT_IOC_RESET: 3812 case PERF_EVENT_IOC_RESET:
3813 func = perf_event_reset; 3813 func = perf_event_reset;
3814 break; 3814 break;
3815 3815
3816 case PERF_EVENT_IOC_REFRESH: 3816 case PERF_EVENT_IOC_REFRESH:
3817 return perf_event_refresh(event, arg); 3817 return perf_event_refresh(event, arg);
3818 3818
3819 case PERF_EVENT_IOC_PERIOD: 3819 case PERF_EVENT_IOC_PERIOD:
3820 return perf_event_period(event, (u64 __user *)arg); 3820 return perf_event_period(event, (u64 __user *)arg);
3821 3821
3822 case PERF_EVENT_IOC_ID: 3822 case PERF_EVENT_IOC_ID:
3823 { 3823 {
3824 u64 id = primary_event_id(event); 3824 u64 id = primary_event_id(event);
3825 3825
3826 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 3826 if (copy_to_user((void __user *)arg, &id, sizeof(id)))
3827 return -EFAULT; 3827 return -EFAULT;
3828 return 0; 3828 return 0;
3829 } 3829 }
3830 3830
3831 case PERF_EVENT_IOC_SET_OUTPUT: 3831 case PERF_EVENT_IOC_SET_OUTPUT:
3832 { 3832 {
3833 int ret; 3833 int ret;
3834 if (arg != -1) { 3834 if (arg != -1) {
3835 struct perf_event *output_event; 3835 struct perf_event *output_event;
3836 struct fd output; 3836 struct fd output;
3837 ret = perf_fget_light(arg, &output); 3837 ret = perf_fget_light(arg, &output);
3838 if (ret) 3838 if (ret)
3839 return ret; 3839 return ret;
3840 output_event = output.file->private_data; 3840 output_event = output.file->private_data;
3841 ret = perf_event_set_output(event, output_event); 3841 ret = perf_event_set_output(event, output_event);
3842 fdput(output); 3842 fdput(output);
3843 } else { 3843 } else {
3844 ret = perf_event_set_output(event, NULL); 3844 ret = perf_event_set_output(event, NULL);
3845 } 3845 }
3846 return ret; 3846 return ret;
3847 } 3847 }
3848 3848
3849 case PERF_EVENT_IOC_SET_FILTER: 3849 case PERF_EVENT_IOC_SET_FILTER:
3850 return perf_event_set_filter(event, (void __user *)arg); 3850 return perf_event_set_filter(event, (void __user *)arg);
3851 3851
3852 default: 3852 default:
3853 return -ENOTTY; 3853 return -ENOTTY;
3854 } 3854 }
3855 3855
3856 if (flags & PERF_IOC_FLAG_GROUP) 3856 if (flags & PERF_IOC_FLAG_GROUP)
3857 perf_event_for_each(event, func); 3857 perf_event_for_each(event, func);
3858 else 3858 else
3859 perf_event_for_each_child(event, func); 3859 perf_event_for_each_child(event, func);
3860 3860
3861 return 0; 3861 return 0;
3862 } 3862 }
3863 3863
3864 #ifdef CONFIG_COMPAT 3864 #ifdef CONFIG_COMPAT
3865 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 3865 static long perf_compat_ioctl(struct file *file, unsigned int cmd,
3866 unsigned long arg) 3866 unsigned long arg)
3867 { 3867 {
3868 switch (_IOC_NR(cmd)) { 3868 switch (_IOC_NR(cmd)) {
3869 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 3869 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
3870 case _IOC_NR(PERF_EVENT_IOC_ID): 3870 case _IOC_NR(PERF_EVENT_IOC_ID):
3871 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 3871 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
3872 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 3872 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
3873 cmd &= ~IOCSIZE_MASK; 3873 cmd &= ~IOCSIZE_MASK;
3874 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 3874 cmd |= sizeof(void *) << IOCSIZE_SHIFT;
3875 } 3875 }
3876 break; 3876 break;
3877 } 3877 }
3878 return perf_ioctl(file, cmd, arg); 3878 return perf_ioctl(file, cmd, arg);
3879 } 3879 }
3880 #else 3880 #else
3881 # define perf_compat_ioctl NULL 3881 # define perf_compat_ioctl NULL
3882 #endif 3882 #endif
3883 3883
3884 int perf_event_task_enable(void) 3884 int perf_event_task_enable(void)
3885 { 3885 {
3886 struct perf_event *event; 3886 struct perf_event *event;
3887 3887
3888 mutex_lock(&current->perf_event_mutex); 3888 mutex_lock(&current->perf_event_mutex);
3889 list_for_each_entry(event, &current->perf_event_list, owner_entry) 3889 list_for_each_entry(event, &current->perf_event_list, owner_entry)
3890 perf_event_for_each_child(event, perf_event_enable); 3890 perf_event_for_each_child(event, perf_event_enable);
3891 mutex_unlock(&current->perf_event_mutex); 3891 mutex_unlock(&current->perf_event_mutex);
3892 3892
3893 return 0; 3893 return 0;
3894 } 3894 }
3895 3895
3896 int perf_event_task_disable(void) 3896 int perf_event_task_disable(void)
3897 { 3897 {
3898 struct perf_event *event; 3898 struct perf_event *event;
3899 3899
3900 mutex_lock(&current->perf_event_mutex); 3900 mutex_lock(&current->perf_event_mutex);
3901 list_for_each_entry(event, &current->perf_event_list, owner_entry) 3901 list_for_each_entry(event, &current->perf_event_list, owner_entry)
3902 perf_event_for_each_child(event, perf_event_disable); 3902 perf_event_for_each_child(event, perf_event_disable);
3903 mutex_unlock(&current->perf_event_mutex); 3903 mutex_unlock(&current->perf_event_mutex);
3904 3904
3905 return 0; 3905 return 0;
3906 } 3906 }
3907 3907
3908 static int perf_event_index(struct perf_event *event) 3908 static int perf_event_index(struct perf_event *event)
3909 { 3909 {
3910 if (event->hw.state & PERF_HES_STOPPED) 3910 if (event->hw.state & PERF_HES_STOPPED)
3911 return 0; 3911 return 0;
3912 3912
3913 if (event->state != PERF_EVENT_STATE_ACTIVE) 3913 if (event->state != PERF_EVENT_STATE_ACTIVE)
3914 return 0; 3914 return 0;
3915 3915
3916 return event->pmu->event_idx(event); 3916 return event->pmu->event_idx(event);
3917 } 3917 }
3918 3918
3919 static void calc_timer_values(struct perf_event *event, 3919 static void calc_timer_values(struct perf_event *event,
3920 u64 *now, 3920 u64 *now,
3921 u64 *enabled, 3921 u64 *enabled,
3922 u64 *running) 3922 u64 *running)
3923 { 3923 {
3924 u64 ctx_time; 3924 u64 ctx_time;
3925 3925
3926 *now = perf_clock(); 3926 *now = perf_clock();
3927 ctx_time = event->shadow_ctx_time + *now; 3927 ctx_time = event->shadow_ctx_time + *now;
3928 *enabled = ctx_time - event->tstamp_enabled; 3928 *enabled = ctx_time - event->tstamp_enabled;
3929 *running = ctx_time - event->tstamp_running; 3929 *running = ctx_time - event->tstamp_running;
3930 } 3930 }
3931 3931
3932 static void perf_event_init_userpage(struct perf_event *event) 3932 static void perf_event_init_userpage(struct perf_event *event)
3933 { 3933 {
3934 struct perf_event_mmap_page *userpg; 3934 struct perf_event_mmap_page *userpg;
3935 struct ring_buffer *rb; 3935 struct ring_buffer *rb;
3936 3936
3937 rcu_read_lock(); 3937 rcu_read_lock();
3938 rb = rcu_dereference(event->rb); 3938 rb = rcu_dereference(event->rb);
3939 if (!rb) 3939 if (!rb)
3940 goto unlock; 3940 goto unlock;
3941 3941
3942 userpg = rb->user_page; 3942 userpg = rb->user_page;
3943 3943
3944 /* Allow new userspace to detect that bit 0 is deprecated */ 3944 /* Allow new userspace to detect that bit 0 is deprecated */
3945 userpg->cap_bit0_is_deprecated = 1; 3945 userpg->cap_bit0_is_deprecated = 1;
3946 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 3946 userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
3947 3947
3948 unlock: 3948 unlock:
3949 rcu_read_unlock(); 3949 rcu_read_unlock();
3950 } 3950 }
3951 3951
3952 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now) 3952 void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3953 { 3953 {
3954 } 3954 }
3955 3955
3956 /* 3956 /*
3957 * Callers need to ensure there can be no nesting of this function, otherwise 3957 * Callers need to ensure there can be no nesting of this function, otherwise
3958 * the seqlock logic goes bad. We can not serialize this because the arch 3958 * the seqlock logic goes bad. We can not serialize this because the arch
3959 * code calls this from NMI context. 3959 * code calls this from NMI context.
3960 */ 3960 */
3961 void perf_event_update_userpage(struct perf_event *event) 3961 void perf_event_update_userpage(struct perf_event *event)
3962 { 3962 {
3963 struct perf_event_mmap_page *userpg; 3963 struct perf_event_mmap_page *userpg;
3964 struct ring_buffer *rb; 3964 struct ring_buffer *rb;
3965 u64 enabled, running, now; 3965 u64 enabled, running, now;
3966 3966
3967 rcu_read_lock(); 3967 rcu_read_lock();
3968 rb = rcu_dereference(event->rb); 3968 rb = rcu_dereference(event->rb);
3969 if (!rb) 3969 if (!rb)
3970 goto unlock; 3970 goto unlock;
3971 3971
3972 /* 3972 /*
3973 * compute total_time_enabled, total_time_running 3973 * compute total_time_enabled, total_time_running
3974 * based on snapshot values taken when the event 3974 * based on snapshot values taken when the event
3975 * was last scheduled in. 3975 * was last scheduled in.
3976 * 3976 *
3977 * we cannot simply called update_context_time() 3977 * we cannot simply called update_context_time()
3978 * because of locking issue as we can be called in 3978 * because of locking issue as we can be called in
3979 * NMI context 3979 * NMI context
3980 */ 3980 */
3981 calc_timer_values(event, &now, &enabled, &running); 3981 calc_timer_values(event, &now, &enabled, &running);
3982 3982
3983 userpg = rb->user_page; 3983 userpg = rb->user_page;
3984 /* 3984 /*
3985 * Disable preemption so as to not let the corresponding user-space 3985 * Disable preemption so as to not let the corresponding user-space
3986 * spin too long if we get preempted. 3986 * spin too long if we get preempted.
3987 */ 3987 */
3988 preempt_disable(); 3988 preempt_disable();
3989 ++userpg->lock; 3989 ++userpg->lock;
3990 barrier(); 3990 barrier();
3991 userpg->index = perf_event_index(event); 3991 userpg->index = perf_event_index(event);
3992 userpg->offset = perf_event_count(event); 3992 userpg->offset = perf_event_count(event);
3993 if (userpg->index) 3993 if (userpg->index)
3994 userpg->offset -= local64_read(&event->hw.prev_count); 3994 userpg->offset -= local64_read(&event->hw.prev_count);
3995 3995
3996 userpg->time_enabled = enabled + 3996 userpg->time_enabled = enabled +
3997 atomic64_read(&event->child_total_time_enabled); 3997 atomic64_read(&event->child_total_time_enabled);
3998 3998
3999 userpg->time_running = running + 3999 userpg->time_running = running +
4000 atomic64_read(&event->child_total_time_running); 4000 atomic64_read(&event->child_total_time_running);
4001 4001
4002 arch_perf_update_userpage(userpg, now); 4002 arch_perf_update_userpage(userpg, now);
4003 4003
4004 barrier(); 4004 barrier();
4005 ++userpg->lock; 4005 ++userpg->lock;
4006 preempt_enable(); 4006 preempt_enable();
4007 unlock: 4007 unlock:
4008 rcu_read_unlock(); 4008 rcu_read_unlock();
4009 } 4009 }
4010 4010
4011 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 4011 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
4012 { 4012 {
4013 struct perf_event *event = vma->vm_file->private_data; 4013 struct perf_event *event = vma->vm_file->private_data;
4014 struct ring_buffer *rb; 4014 struct ring_buffer *rb;
4015 int ret = VM_FAULT_SIGBUS; 4015 int ret = VM_FAULT_SIGBUS;
4016 4016
4017 if (vmf->flags & FAULT_FLAG_MKWRITE) { 4017 if (vmf->flags & FAULT_FLAG_MKWRITE) {
4018 if (vmf->pgoff == 0) 4018 if (vmf->pgoff == 0)
4019 ret = 0; 4019 ret = 0;
4020 return ret; 4020 return ret;
4021 } 4021 }
4022 4022
4023 rcu_read_lock(); 4023 rcu_read_lock();
4024 rb = rcu_dereference(event->rb); 4024 rb = rcu_dereference(event->rb);
4025 if (!rb) 4025 if (!rb)
4026 goto unlock; 4026 goto unlock;
4027 4027
4028 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 4028 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
4029 goto unlock; 4029 goto unlock;
4030 4030
4031 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 4031 vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
4032 if (!vmf->page) 4032 if (!vmf->page)
4033 goto unlock; 4033 goto unlock;
4034 4034
4035 get_page(vmf->page); 4035 get_page(vmf->page);
4036 vmf->page->mapping = vma->vm_file->f_mapping; 4036 vmf->page->mapping = vma->vm_file->f_mapping;
4037 vmf->page->index = vmf->pgoff; 4037 vmf->page->index = vmf->pgoff;
4038 4038
4039 ret = 0; 4039 ret = 0;
4040 unlock: 4040 unlock:
4041 rcu_read_unlock(); 4041 rcu_read_unlock();
4042 4042
4043 return ret; 4043 return ret;
4044 } 4044 }
4045 4045
4046 static void ring_buffer_attach(struct perf_event *event, 4046 static void ring_buffer_attach(struct perf_event *event,
4047 struct ring_buffer *rb) 4047 struct ring_buffer *rb)
4048 { 4048 {
4049 struct ring_buffer *old_rb = NULL; 4049 struct ring_buffer *old_rb = NULL;
4050 unsigned long flags; 4050 unsigned long flags;
4051 4051
4052 if (event->rb) { 4052 if (event->rb) {
4053 /* 4053 /*
4054 * Should be impossible, we set this when removing 4054 * Should be impossible, we set this when removing
4055 * event->rb_entry and wait/clear when adding event->rb_entry. 4055 * event->rb_entry and wait/clear when adding event->rb_entry.
4056 */ 4056 */
4057 WARN_ON_ONCE(event->rcu_pending); 4057 WARN_ON_ONCE(event->rcu_pending);
4058 4058
4059 old_rb = event->rb; 4059 old_rb = event->rb;
4060 event->rcu_batches = get_state_synchronize_rcu(); 4060 event->rcu_batches = get_state_synchronize_rcu();
4061 event->rcu_pending = 1; 4061 event->rcu_pending = 1;
4062 4062
4063 spin_lock_irqsave(&old_rb->event_lock, flags); 4063 spin_lock_irqsave(&old_rb->event_lock, flags);
4064 list_del_rcu(&event->rb_entry); 4064 list_del_rcu(&event->rb_entry);
4065 spin_unlock_irqrestore(&old_rb->event_lock, flags); 4065 spin_unlock_irqrestore(&old_rb->event_lock, flags);
4066 } 4066 }
4067 4067
4068 if (event->rcu_pending && rb) { 4068 if (event->rcu_pending && rb) {
4069 cond_synchronize_rcu(event->rcu_batches); 4069 cond_synchronize_rcu(event->rcu_batches);
4070 event->rcu_pending = 0; 4070 event->rcu_pending = 0;
4071 } 4071 }
4072 4072
4073 if (rb) { 4073 if (rb) {
4074 spin_lock_irqsave(&rb->event_lock, flags); 4074 spin_lock_irqsave(&rb->event_lock, flags);
4075 list_add_rcu(&event->rb_entry, &rb->event_list); 4075 list_add_rcu(&event->rb_entry, &rb->event_list);
4076 spin_unlock_irqrestore(&rb->event_lock, flags); 4076 spin_unlock_irqrestore(&rb->event_lock, flags);
4077 } 4077 }
4078 4078
4079 rcu_assign_pointer(event->rb, rb); 4079 rcu_assign_pointer(event->rb, rb);
4080 4080
4081 if (old_rb) { 4081 if (old_rb) {
4082 ring_buffer_put(old_rb); 4082 ring_buffer_put(old_rb);
4083 /* 4083 /*
4084 * Since we detached before setting the new rb, so that we 4084 * Since we detached before setting the new rb, so that we
4085 * could attach the new rb, we could have missed a wakeup. 4085 * could attach the new rb, we could have missed a wakeup.
4086 * Provide it now. 4086 * Provide it now.
4087 */ 4087 */
4088 wake_up_all(&event->waitq); 4088 wake_up_all(&event->waitq);
4089 } 4089 }
4090 } 4090 }
4091 4091
4092 static void ring_buffer_wakeup(struct perf_event *event) 4092 static void ring_buffer_wakeup(struct perf_event *event)
4093 { 4093 {
4094 struct ring_buffer *rb; 4094 struct ring_buffer *rb;
4095 4095
4096 rcu_read_lock(); 4096 rcu_read_lock();
4097 rb = rcu_dereference(event->rb); 4097 rb = rcu_dereference(event->rb);
4098 if (rb) { 4098 if (rb) {
4099 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 4099 list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
4100 wake_up_all(&event->waitq); 4100 wake_up_all(&event->waitq);
4101 } 4101 }
4102 rcu_read_unlock(); 4102 rcu_read_unlock();
4103 } 4103 }
4104 4104
4105 static void rb_free_rcu(struct rcu_head *rcu_head) 4105 static void rb_free_rcu(struct rcu_head *rcu_head)
4106 { 4106 {
4107 struct ring_buffer *rb; 4107 struct ring_buffer *rb;
4108 4108
4109 rb = container_of(rcu_head, struct ring_buffer, rcu_head); 4109 rb = container_of(rcu_head, struct ring_buffer, rcu_head);
4110 rb_free(rb); 4110 rb_free(rb);
4111 } 4111 }
4112 4112
4113 static struct ring_buffer *ring_buffer_get(struct perf_event *event) 4113 static struct ring_buffer *ring_buffer_get(struct perf_event *event)
4114 { 4114 {
4115 struct ring_buffer *rb; 4115 struct ring_buffer *rb;
4116 4116
4117 rcu_read_lock(); 4117 rcu_read_lock();
4118 rb = rcu_dereference(event->rb); 4118 rb = rcu_dereference(event->rb);
4119 if (rb) { 4119 if (rb) {
4120 if (!atomic_inc_not_zero(&rb->refcount)) 4120 if (!atomic_inc_not_zero(&rb->refcount))
4121 rb = NULL; 4121 rb = NULL;
4122 } 4122 }
4123 rcu_read_unlock(); 4123 rcu_read_unlock();
4124 4124
4125 return rb; 4125 return rb;
4126 } 4126 }
4127 4127
4128 static void ring_buffer_put(struct ring_buffer *rb) 4128 static void ring_buffer_put(struct ring_buffer *rb)
4129 { 4129 {
4130 if (!atomic_dec_and_test(&rb->refcount)) 4130 if (!atomic_dec_and_test(&rb->refcount))
4131 return; 4131 return;
4132 4132
4133 WARN_ON_ONCE(!list_empty(&rb->event_list)); 4133 WARN_ON_ONCE(!list_empty(&rb->event_list));
4134 4134
4135 call_rcu(&rb->rcu_head, rb_free_rcu); 4135 call_rcu(&rb->rcu_head, rb_free_rcu);
4136 } 4136 }
4137 4137
4138 static void perf_mmap_open(struct vm_area_struct *vma) 4138 static void perf_mmap_open(struct vm_area_struct *vma)
4139 { 4139 {
4140 struct perf_event *event = vma->vm_file->private_data; 4140 struct perf_event *event = vma->vm_file->private_data;
4141 4141
4142 atomic_inc(&event->mmap_count); 4142 atomic_inc(&event->mmap_count);
4143 atomic_inc(&event->rb->mmap_count); 4143 atomic_inc(&event->rb->mmap_count);
4144 } 4144 }
4145 4145
4146 /* 4146 /*
4147 * A buffer can be mmap()ed multiple times; either directly through the same 4147 * A buffer can be mmap()ed multiple times; either directly through the same
4148 * event, or through other events by use of perf_event_set_output(). 4148 * event, or through other events by use of perf_event_set_output().
4149 * 4149 *
4150 * In order to undo the VM accounting done by perf_mmap() we need to destroy 4150 * In order to undo the VM accounting done by perf_mmap() we need to destroy
4151 * the buffer here, where we still have a VM context. This means we need 4151 * the buffer here, where we still have a VM context. This means we need
4152 * to detach all events redirecting to us. 4152 * to detach all events redirecting to us.
4153 */ 4153 */
4154 static void perf_mmap_close(struct vm_area_struct *vma) 4154 static void perf_mmap_close(struct vm_area_struct *vma)
4155 { 4155 {
4156 struct perf_event *event = vma->vm_file->private_data; 4156 struct perf_event *event = vma->vm_file->private_data;
4157 4157
4158 struct ring_buffer *rb = ring_buffer_get(event); 4158 struct ring_buffer *rb = ring_buffer_get(event);
4159 struct user_struct *mmap_user = rb->mmap_user; 4159 struct user_struct *mmap_user = rb->mmap_user;
4160 int mmap_locked = rb->mmap_locked; 4160 int mmap_locked = rb->mmap_locked;
4161 unsigned long size = perf_data_size(rb); 4161 unsigned long size = perf_data_size(rb);
4162 4162
4163 atomic_dec(&rb->mmap_count); 4163 atomic_dec(&rb->mmap_count);
4164 4164
4165 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 4165 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
4166 goto out_put; 4166 goto out_put;
4167 4167
4168 ring_buffer_attach(event, NULL); 4168 ring_buffer_attach(event, NULL);
4169 mutex_unlock(&event->mmap_mutex); 4169 mutex_unlock(&event->mmap_mutex);
4170 4170
4171 /* If there's still other mmap()s of this buffer, we're done. */ 4171 /* If there's still other mmap()s of this buffer, we're done. */
4172 if (atomic_read(&rb->mmap_count)) 4172 if (atomic_read(&rb->mmap_count))
4173 goto out_put; 4173 goto out_put;
4174 4174
4175 /* 4175 /*
4176 * No other mmap()s, detach from all other events that might redirect 4176 * No other mmap()s, detach from all other events that might redirect
4177 * into the now unreachable buffer. Somewhat complicated by the 4177 * into the now unreachable buffer. Somewhat complicated by the
4178 * fact that rb::event_lock otherwise nests inside mmap_mutex. 4178 * fact that rb::event_lock otherwise nests inside mmap_mutex.
4179 */ 4179 */
4180 again: 4180 again:
4181 rcu_read_lock(); 4181 rcu_read_lock();
4182 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 4182 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
4183 if (!atomic_long_inc_not_zero(&event->refcount)) { 4183 if (!atomic_long_inc_not_zero(&event->refcount)) {
4184 /* 4184 /*
4185 * This event is en-route to free_event() which will 4185 * This event is en-route to free_event() which will
4186 * detach it and remove it from the list. 4186 * detach it and remove it from the list.
4187 */ 4187 */
4188 continue; 4188 continue;
4189 } 4189 }
4190 rcu_read_unlock(); 4190 rcu_read_unlock();
4191 4191
4192 mutex_lock(&event->mmap_mutex); 4192 mutex_lock(&event->mmap_mutex);
4193 /* 4193 /*
4194 * Check we didn't race with perf_event_set_output() which can 4194 * Check we didn't race with perf_event_set_output() which can
4195 * swizzle the rb from under us while we were waiting to 4195 * swizzle the rb from under us while we were waiting to
4196 * acquire mmap_mutex. 4196 * acquire mmap_mutex.
4197 * 4197 *
4198 * If we find a different rb; ignore this event, a next 4198 * If we find a different rb; ignore this event, a next
4199 * iteration will no longer find it on the list. We have to 4199 * iteration will no longer find it on the list. We have to
4200 * still restart the iteration to make sure we're not now 4200 * still restart the iteration to make sure we're not now
4201 * iterating the wrong list. 4201 * iterating the wrong list.
4202 */ 4202 */
4203 if (event->rb == rb) 4203 if (event->rb == rb)
4204 ring_buffer_attach(event, NULL); 4204 ring_buffer_attach(event, NULL);
4205 4205
4206 mutex_unlock(&event->mmap_mutex); 4206 mutex_unlock(&event->mmap_mutex);
4207 put_event(event); 4207 put_event(event);
4208 4208
4209 /* 4209 /*
4210 * Restart the iteration; either we're on the wrong list or 4210 * Restart the iteration; either we're on the wrong list or
4211 * destroyed its integrity by doing a deletion. 4211 * destroyed its integrity by doing a deletion.
4212 */ 4212 */
4213 goto again; 4213 goto again;
4214 } 4214 }
4215 rcu_read_unlock(); 4215 rcu_read_unlock();
4216 4216
4217 /* 4217 /*
4218 * It could be there's still a few 0-ref events on the list; they'll 4218 * It could be there's still a few 0-ref events on the list; they'll
4219 * get cleaned up by free_event() -- they'll also still have their 4219 * get cleaned up by free_event() -- they'll also still have their
4220 * ref on the rb and will free it whenever they are done with it. 4220 * ref on the rb and will free it whenever they are done with it.
4221 * 4221 *
4222 * Aside from that, this buffer is 'fully' detached and unmapped, 4222 * Aside from that, this buffer is 'fully' detached and unmapped,
4223 * undo the VM accounting. 4223 * undo the VM accounting.
4224 */ 4224 */
4225 4225
4226 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 4226 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
4227 vma->vm_mm->pinned_vm -= mmap_locked; 4227 vma->vm_mm->pinned_vm -= mmap_locked;
4228 free_uid(mmap_user); 4228 free_uid(mmap_user);
4229 4229
4230 out_put: 4230 out_put:
4231 ring_buffer_put(rb); /* could be last */ 4231 ring_buffer_put(rb); /* could be last */
4232 } 4232 }
4233 4233
4234 static const struct vm_operations_struct perf_mmap_vmops = { 4234 static const struct vm_operations_struct perf_mmap_vmops = {
4235 .open = perf_mmap_open, 4235 .open = perf_mmap_open,
4236 .close = perf_mmap_close, 4236 .close = perf_mmap_close,
4237 .fault = perf_mmap_fault, 4237 .fault = perf_mmap_fault,
4238 .page_mkwrite = perf_mmap_fault, 4238 .page_mkwrite = perf_mmap_fault,
4239 }; 4239 };
4240 4240
4241 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 4241 static int perf_mmap(struct file *file, struct vm_area_struct *vma)
4242 { 4242 {
4243 struct perf_event *event = file->private_data; 4243 struct perf_event *event = file->private_data;
4244 unsigned long user_locked, user_lock_limit; 4244 unsigned long user_locked, user_lock_limit;
4245 struct user_struct *user = current_user(); 4245 struct user_struct *user = current_user();
4246 unsigned long locked, lock_limit; 4246 unsigned long locked, lock_limit;
4247 struct ring_buffer *rb; 4247 struct ring_buffer *rb;
4248 unsigned long vma_size; 4248 unsigned long vma_size;
4249 unsigned long nr_pages; 4249 unsigned long nr_pages;
4250 long user_extra, extra; 4250 long user_extra, extra;
4251 int ret = 0, flags = 0; 4251 int ret = 0, flags = 0;
4252 4252
4253 /* 4253 /*
4254 * Don't allow mmap() of inherited per-task counters. This would 4254 * Don't allow mmap() of inherited per-task counters. This would
4255 * create a performance issue due to all children writing to the 4255 * create a performance issue due to all children writing to the
4256 * same rb. 4256 * same rb.
4257 */ 4257 */
4258 if (event->cpu == -1 && event->attr.inherit) 4258 if (event->cpu == -1 && event->attr.inherit)
4259 return -EINVAL; 4259 return -EINVAL;
4260 4260
4261 if (!(vma->vm_flags & VM_SHARED)) 4261 if (!(vma->vm_flags & VM_SHARED))
4262 return -EINVAL; 4262 return -EINVAL;
4263 4263
4264 vma_size = vma->vm_end - vma->vm_start; 4264 vma_size = vma->vm_end - vma->vm_start;
4265 nr_pages = (vma_size / PAGE_SIZE) - 1; 4265 nr_pages = (vma_size / PAGE_SIZE) - 1;
4266 4266
4267 /* 4267 /*
4268 * If we have rb pages ensure they're a power-of-two number, so we 4268 * If we have rb pages ensure they're a power-of-two number, so we
4269 * can do bitmasks instead of modulo. 4269 * can do bitmasks instead of modulo.
4270 */ 4270 */
4271 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 4271 if (nr_pages != 0 && !is_power_of_2(nr_pages))
4272 return -EINVAL; 4272 return -EINVAL;
4273 4273
4274 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 4274 if (vma_size != PAGE_SIZE * (1 + nr_pages))
4275 return -EINVAL; 4275 return -EINVAL;
4276 4276
4277 if (vma->vm_pgoff != 0) 4277 if (vma->vm_pgoff != 0)
4278 return -EINVAL; 4278 return -EINVAL;
4279 4279
4280 WARN_ON_ONCE(event->ctx->parent_ctx); 4280 WARN_ON_ONCE(event->ctx->parent_ctx);
4281 again: 4281 again:
4282 mutex_lock(&event->mmap_mutex); 4282 mutex_lock(&event->mmap_mutex);
4283 if (event->rb) { 4283 if (event->rb) {
4284 if (event->rb->nr_pages != nr_pages) { 4284 if (event->rb->nr_pages != nr_pages) {
4285 ret = -EINVAL; 4285 ret = -EINVAL;
4286 goto unlock; 4286 goto unlock;
4287 } 4287 }
4288 4288
4289 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 4289 if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
4290 /* 4290 /*
4291 * Raced against perf_mmap_close() through 4291 * Raced against perf_mmap_close() through
4292 * perf_event_set_output(). Try again, hope for better 4292 * perf_event_set_output(). Try again, hope for better
4293 * luck. 4293 * luck.
4294 */ 4294 */
4295 mutex_unlock(&event->mmap_mutex); 4295 mutex_unlock(&event->mmap_mutex);
4296 goto again; 4296 goto again;
4297 } 4297 }
4298 4298
4299 goto unlock; 4299 goto unlock;
4300 } 4300 }
4301 4301
4302 user_extra = nr_pages + 1; 4302 user_extra = nr_pages + 1;
4303 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 4303 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
4304 4304
4305 /* 4305 /*
4306 * Increase the limit linearly with more CPUs: 4306 * Increase the limit linearly with more CPUs:
4307 */ 4307 */
4308 user_lock_limit *= num_online_cpus(); 4308 user_lock_limit *= num_online_cpus();
4309 4309
4310 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 4310 user_locked = atomic_long_read(&user->locked_vm) + user_extra;
4311 4311
4312 extra = 0; 4312 extra = 0;
4313 if (user_locked > user_lock_limit) 4313 if (user_locked > user_lock_limit)
4314 extra = user_locked - user_lock_limit; 4314 extra = user_locked - user_lock_limit;
4315 4315
4316 lock_limit = rlimit(RLIMIT_MEMLOCK); 4316 lock_limit = rlimit(RLIMIT_MEMLOCK);
4317 lock_limit >>= PAGE_SHIFT; 4317 lock_limit >>= PAGE_SHIFT;
4318 locked = vma->vm_mm->pinned_vm + extra; 4318 locked = vma->vm_mm->pinned_vm + extra;
4319 4319
4320 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 4320 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
4321 !capable(CAP_IPC_LOCK)) { 4321 !capable(CAP_IPC_LOCK)) {
4322 ret = -EPERM; 4322 ret = -EPERM;
4323 goto unlock; 4323 goto unlock;
4324 } 4324 }
4325 4325
4326 WARN_ON(event->rb); 4326 WARN_ON(event->rb);
4327 4327
4328 if (vma->vm_flags & VM_WRITE) 4328 if (vma->vm_flags & VM_WRITE)
4329 flags |= RING_BUFFER_WRITABLE; 4329 flags |= RING_BUFFER_WRITABLE;
4330 4330
4331 rb = rb_alloc(nr_pages, 4331 rb = rb_alloc(nr_pages,
4332 event->attr.watermark ? event->attr.wakeup_watermark : 0, 4332 event->attr.watermark ? event->attr.wakeup_watermark : 0,
4333 event->cpu, flags); 4333 event->cpu, flags);
4334 4334
4335 if (!rb) { 4335 if (!rb) {
4336 ret = -ENOMEM; 4336 ret = -ENOMEM;
4337 goto unlock; 4337 goto unlock;
4338 } 4338 }
4339 4339
4340 atomic_set(&rb->mmap_count, 1); 4340 atomic_set(&rb->mmap_count, 1);
4341 rb->mmap_locked = extra; 4341 rb->mmap_locked = extra;
4342 rb->mmap_user = get_current_user(); 4342 rb->mmap_user = get_current_user();
4343 4343
4344 atomic_long_add(user_extra, &user->locked_vm); 4344 atomic_long_add(user_extra, &user->locked_vm);
4345 vma->vm_mm->pinned_vm += extra; 4345 vma->vm_mm->pinned_vm += extra;
4346 4346
4347 ring_buffer_attach(event, rb); 4347 ring_buffer_attach(event, rb);
4348 4348
4349 perf_event_init_userpage(event); 4349 perf_event_init_userpage(event);
4350 perf_event_update_userpage(event); 4350 perf_event_update_userpage(event);
4351 4351
4352 unlock: 4352 unlock:
4353 if (!ret) 4353 if (!ret)
4354 atomic_inc(&event->mmap_count); 4354 atomic_inc(&event->mmap_count);
4355 mutex_unlock(&event->mmap_mutex); 4355 mutex_unlock(&event->mmap_mutex);
4356 4356
4357 /* 4357 /*
4358 * Since pinned accounting is per vm we cannot allow fork() to copy our 4358 * Since pinned accounting is per vm we cannot allow fork() to copy our
4359 * vma. 4359 * vma.
4360 */ 4360 */
4361 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 4361 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
4362 vma->vm_ops = &perf_mmap_vmops; 4362 vma->vm_ops = &perf_mmap_vmops;
4363 4363
4364 return ret; 4364 return ret;
4365 } 4365 }
4366 4366
4367 static int perf_fasync(int fd, struct file *filp, int on) 4367 static int perf_fasync(int fd, struct file *filp, int on)
4368 { 4368 {
4369 struct inode *inode = file_inode(filp); 4369 struct inode *inode = file_inode(filp);
4370 struct perf_event *event = filp->private_data; 4370 struct perf_event *event = filp->private_data;
4371 int retval; 4371 int retval;
4372 4372
4373 mutex_lock(&inode->i_mutex); 4373 mutex_lock(&inode->i_mutex);
4374 retval = fasync_helper(fd, filp, on, &event->fasync); 4374 retval = fasync_helper(fd, filp, on, &event->fasync);
4375 mutex_unlock(&inode->i_mutex); 4375 mutex_unlock(&inode->i_mutex);
4376 4376
4377 if (retval < 0) 4377 if (retval < 0)
4378 return retval; 4378 return retval;
4379 4379
4380 return 0; 4380 return 0;
4381 } 4381 }
4382 4382
4383 static const struct file_operations perf_fops = { 4383 static const struct file_operations perf_fops = {
4384 .llseek = no_llseek, 4384 .llseek = no_llseek,
4385 .release = perf_release, 4385 .release = perf_release,
4386 .read = perf_read, 4386 .read = perf_read,
4387 .poll = perf_poll, 4387 .poll = perf_poll,
4388 .unlocked_ioctl = perf_ioctl, 4388 .unlocked_ioctl = perf_ioctl,
4389 .compat_ioctl = perf_compat_ioctl, 4389 .compat_ioctl = perf_compat_ioctl,
4390 .mmap = perf_mmap, 4390 .mmap = perf_mmap,
4391 .fasync = perf_fasync, 4391 .fasync = perf_fasync,
4392 }; 4392 };
4393 4393
4394 /* 4394 /*
4395 * Perf event wakeup 4395 * Perf event wakeup
4396 * 4396 *
4397 * If there's data, ensure we set the poll() state and publish everything 4397 * If there's data, ensure we set the poll() state and publish everything
4398 * to user-space before waking everybody up. 4398 * to user-space before waking everybody up.
4399 */ 4399 */
4400 4400
4401 void perf_event_wakeup(struct perf_event *event) 4401 void perf_event_wakeup(struct perf_event *event)
4402 { 4402 {
4403 ring_buffer_wakeup(event); 4403 ring_buffer_wakeup(event);
4404 4404
4405 if (event->pending_kill) { 4405 if (event->pending_kill) {
4406 kill_fasync(&event->fasync, SIGIO, event->pending_kill); 4406 kill_fasync(&event->fasync, SIGIO, event->pending_kill);
4407 event->pending_kill = 0; 4407 event->pending_kill = 0;
4408 } 4408 }
4409 } 4409 }
4410 4410
4411 static void perf_pending_event(struct irq_work *entry) 4411 static void perf_pending_event(struct irq_work *entry)
4412 { 4412 {
4413 struct perf_event *event = container_of(entry, 4413 struct perf_event *event = container_of(entry,
4414 struct perf_event, pending); 4414 struct perf_event, pending);
4415 4415
4416 if (event->pending_disable) { 4416 if (event->pending_disable) {
4417 event->pending_disable = 0; 4417 event->pending_disable = 0;
4418 __perf_event_disable(event); 4418 __perf_event_disable(event);
4419 } 4419 }
4420 4420
4421 if (event->pending_wakeup) { 4421 if (event->pending_wakeup) {
4422 event->pending_wakeup = 0; 4422 event->pending_wakeup = 0;
4423 perf_event_wakeup(event); 4423 perf_event_wakeup(event);
4424 } 4424 }
4425 } 4425 }
4426 4426
4427 /* 4427 /*
4428 * We assume there is only KVM supporting the callbacks. 4428 * We assume there is only KVM supporting the callbacks.
4429 * Later on, we might change it to a list if there is 4429 * Later on, we might change it to a list if there is
4430 * another virtualization implementation supporting the callbacks. 4430 * another virtualization implementation supporting the callbacks.
4431 */ 4431 */
4432 struct perf_guest_info_callbacks *perf_guest_cbs; 4432 struct perf_guest_info_callbacks *perf_guest_cbs;
4433 4433
4434 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4434 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4435 { 4435 {
4436 perf_guest_cbs = cbs; 4436 perf_guest_cbs = cbs;
4437 return 0; 4437 return 0;
4438 } 4438 }
4439 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 4439 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
4440 4440
4441 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 4441 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4442 { 4442 {
4443 perf_guest_cbs = NULL; 4443 perf_guest_cbs = NULL;
4444 return 0; 4444 return 0;
4445 } 4445 }
4446 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 4446 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
4447 4447
4448 static void 4448 static void
4449 perf_output_sample_regs(struct perf_output_handle *handle, 4449 perf_output_sample_regs(struct perf_output_handle *handle,
4450 struct pt_regs *regs, u64 mask) 4450 struct pt_regs *regs, u64 mask)
4451 { 4451 {
4452 int bit; 4452 int bit;
4453 4453
4454 for_each_set_bit(bit, (const unsigned long *) &mask, 4454 for_each_set_bit(bit, (const unsigned long *) &mask,
4455 sizeof(mask) * BITS_PER_BYTE) { 4455 sizeof(mask) * BITS_PER_BYTE) {
4456 u64 val; 4456 u64 val;
4457 4457
4458 val = perf_reg_value(regs, bit); 4458 val = perf_reg_value(regs, bit);
4459 perf_output_put(handle, val); 4459 perf_output_put(handle, val);
4460 } 4460 }
4461 } 4461 }
4462 4462
4463 static void perf_sample_regs_user(struct perf_regs *regs_user, 4463 static void perf_sample_regs_user(struct perf_regs *regs_user,
4464 struct pt_regs *regs) 4464 struct pt_regs *regs,
4465 struct pt_regs *regs_user_copy)
4465 { 4466 {
4466 if (!user_mode(regs)) { 4467 if (user_mode(regs)) {
4467 if (current->mm) 4468 regs_user->abi = perf_reg_abi(current);
4468 regs = task_pt_regs(current);
4469 else
4470 regs = NULL;
4471 }
4472
4473 if (regs) {
4474 regs_user->abi = perf_reg_abi(current);
4475 regs_user->regs = regs; 4469 regs_user->regs = regs;
4470 } else if (current->mm) {
4471 perf_get_regs_user(regs_user, regs, regs_user_copy);
4476 } else { 4472 } else {
4477 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 4473 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
4478 regs_user->regs = NULL; 4474 regs_user->regs = NULL;
4479 } 4475 }
4480 } 4476 }
4481 4477
4482 static void perf_sample_regs_intr(struct perf_regs *regs_intr, 4478 static void perf_sample_regs_intr(struct perf_regs *regs_intr,
4483 struct pt_regs *regs) 4479 struct pt_regs *regs)
4484 { 4480 {
4485 regs_intr->regs = regs; 4481 regs_intr->regs = regs;
4486 regs_intr->abi = perf_reg_abi(current); 4482 regs_intr->abi = perf_reg_abi(current);
4487 } 4483 }
4488 4484
4489 4485
4490 /* 4486 /*
4491 * Get remaining task size from user stack pointer. 4487 * Get remaining task size from user stack pointer.
4492 * 4488 *
4493 * It'd be better to take stack vma map and limit this more 4489 * It'd be better to take stack vma map and limit this more
4494 * precisly, but there's no way to get it safely under interrupt, 4490 * precisly, but there's no way to get it safely under interrupt,
4495 * so using TASK_SIZE as limit. 4491 * so using TASK_SIZE as limit.
4496 */ 4492 */
4497 static u64 perf_ustack_task_size(struct pt_regs *regs) 4493 static u64 perf_ustack_task_size(struct pt_regs *regs)
4498 { 4494 {
4499 unsigned long addr = perf_user_stack_pointer(regs); 4495 unsigned long addr = perf_user_stack_pointer(regs);
4500 4496
4501 if (!addr || addr >= TASK_SIZE) 4497 if (!addr || addr >= TASK_SIZE)
4502 return 0; 4498 return 0;
4503 4499
4504 return TASK_SIZE - addr; 4500 return TASK_SIZE - addr;
4505 } 4501 }
4506 4502
4507 static u16 4503 static u16
4508 perf_sample_ustack_size(u16 stack_size, u16 header_size, 4504 perf_sample_ustack_size(u16 stack_size, u16 header_size,
4509 struct pt_regs *regs) 4505 struct pt_regs *regs)
4510 { 4506 {
4511 u64 task_size; 4507 u64 task_size;
4512 4508
4513 /* No regs, no stack pointer, no dump. */ 4509 /* No regs, no stack pointer, no dump. */
4514 if (!regs) 4510 if (!regs)
4515 return 0; 4511 return 0;
4516 4512
4517 /* 4513 /*
4518 * Check if we fit in with the requested stack size into the: 4514 * Check if we fit in with the requested stack size into the:
4519 * - TASK_SIZE 4515 * - TASK_SIZE
4520 * If we don't, we limit the size to the TASK_SIZE. 4516 * If we don't, we limit the size to the TASK_SIZE.
4521 * 4517 *
4522 * - remaining sample size 4518 * - remaining sample size
4523 * If we don't, we customize the stack size to 4519 * If we don't, we customize the stack size to
4524 * fit in to the remaining sample size. 4520 * fit in to the remaining sample size.
4525 */ 4521 */
4526 4522
4527 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 4523 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
4528 stack_size = min(stack_size, (u16) task_size); 4524 stack_size = min(stack_size, (u16) task_size);
4529 4525
4530 /* Current header size plus static size and dynamic size. */ 4526 /* Current header size plus static size and dynamic size. */
4531 header_size += 2 * sizeof(u64); 4527 header_size += 2 * sizeof(u64);
4532 4528
4533 /* Do we fit in with the current stack dump size? */ 4529 /* Do we fit in with the current stack dump size? */
4534 if ((u16) (header_size + stack_size) < header_size) { 4530 if ((u16) (header_size + stack_size) < header_size) {
4535 /* 4531 /*
4536 * If we overflow the maximum size for the sample, 4532 * If we overflow the maximum size for the sample,
4537 * we customize the stack dump size to fit in. 4533 * we customize the stack dump size to fit in.
4538 */ 4534 */
4539 stack_size = USHRT_MAX - header_size - sizeof(u64); 4535 stack_size = USHRT_MAX - header_size - sizeof(u64);
4540 stack_size = round_up(stack_size, sizeof(u64)); 4536 stack_size = round_up(stack_size, sizeof(u64));
4541 } 4537 }
4542 4538
4543 return stack_size; 4539 return stack_size;
4544 } 4540 }
4545 4541
4546 static void 4542 static void
4547 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 4543 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
4548 struct pt_regs *regs) 4544 struct pt_regs *regs)
4549 { 4545 {
4550 /* Case of a kernel thread, nothing to dump */ 4546 /* Case of a kernel thread, nothing to dump */
4551 if (!regs) { 4547 if (!regs) {
4552 u64 size = 0; 4548 u64 size = 0;
4553 perf_output_put(handle, size); 4549 perf_output_put(handle, size);
4554 } else { 4550 } else {
4555 unsigned long sp; 4551 unsigned long sp;
4556 unsigned int rem; 4552 unsigned int rem;
4557 u64 dyn_size; 4553 u64 dyn_size;
4558 4554
4559 /* 4555 /*
4560 * We dump: 4556 * We dump:
4561 * static size 4557 * static size
4562 * - the size requested by user or the best one we can fit 4558 * - the size requested by user or the best one we can fit
4563 * in to the sample max size 4559 * in to the sample max size
4564 * data 4560 * data
4565 * - user stack dump data 4561 * - user stack dump data
4566 * dynamic size 4562 * dynamic size
4567 * - the actual dumped size 4563 * - the actual dumped size
4568 */ 4564 */
4569 4565
4570 /* Static size. */ 4566 /* Static size. */
4571 perf_output_put(handle, dump_size); 4567 perf_output_put(handle, dump_size);
4572 4568
4573 /* Data. */ 4569 /* Data. */
4574 sp = perf_user_stack_pointer(regs); 4570 sp = perf_user_stack_pointer(regs);
4575 rem = __output_copy_user(handle, (void *) sp, dump_size); 4571 rem = __output_copy_user(handle, (void *) sp, dump_size);
4576 dyn_size = dump_size - rem; 4572 dyn_size = dump_size - rem;
4577 4573
4578 perf_output_skip(handle, rem); 4574 perf_output_skip(handle, rem);
4579 4575
4580 /* Dynamic size. */ 4576 /* Dynamic size. */
4581 perf_output_put(handle, dyn_size); 4577 perf_output_put(handle, dyn_size);
4582 } 4578 }
4583 } 4579 }
4584 4580
4585 static void __perf_event_header__init_id(struct perf_event_header *header, 4581 static void __perf_event_header__init_id(struct perf_event_header *header,
4586 struct perf_sample_data *data, 4582 struct perf_sample_data *data,
4587 struct perf_event *event) 4583 struct perf_event *event)
4588 { 4584 {
4589 u64 sample_type = event->attr.sample_type; 4585 u64 sample_type = event->attr.sample_type;
4590 4586
4591 data->type = sample_type; 4587 data->type = sample_type;
4592 header->size += event->id_header_size; 4588 header->size += event->id_header_size;
4593 4589
4594 if (sample_type & PERF_SAMPLE_TID) { 4590 if (sample_type & PERF_SAMPLE_TID) {
4595 /* namespace issues */ 4591 /* namespace issues */
4596 data->tid_entry.pid = perf_event_pid(event, current); 4592 data->tid_entry.pid = perf_event_pid(event, current);
4597 data->tid_entry.tid = perf_event_tid(event, current); 4593 data->tid_entry.tid = perf_event_tid(event, current);
4598 } 4594 }
4599 4595
4600 if (sample_type & PERF_SAMPLE_TIME) 4596 if (sample_type & PERF_SAMPLE_TIME)
4601 data->time = perf_clock(); 4597 data->time = perf_clock();
4602 4598
4603 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 4599 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
4604 data->id = primary_event_id(event); 4600 data->id = primary_event_id(event);
4605 4601
4606 if (sample_type & PERF_SAMPLE_STREAM_ID) 4602 if (sample_type & PERF_SAMPLE_STREAM_ID)
4607 data->stream_id = event->id; 4603 data->stream_id = event->id;
4608 4604
4609 if (sample_type & PERF_SAMPLE_CPU) { 4605 if (sample_type & PERF_SAMPLE_CPU) {
4610 data->cpu_entry.cpu = raw_smp_processor_id(); 4606 data->cpu_entry.cpu = raw_smp_processor_id();
4611 data->cpu_entry.reserved = 0; 4607 data->cpu_entry.reserved = 0;
4612 } 4608 }
4613 } 4609 }
4614 4610
4615 void perf_event_header__init_id(struct perf_event_header *header, 4611 void perf_event_header__init_id(struct perf_event_header *header,
4616 struct perf_sample_data *data, 4612 struct perf_sample_data *data,
4617 struct perf_event *event) 4613 struct perf_event *event)
4618 { 4614 {
4619 if (event->attr.sample_id_all) 4615 if (event->attr.sample_id_all)
4620 __perf_event_header__init_id(header, data, event); 4616 __perf_event_header__init_id(header, data, event);
4621 } 4617 }
4622 4618
4623 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 4619 static void __perf_event__output_id_sample(struct perf_output_handle *handle,
4624 struct perf_sample_data *data) 4620 struct perf_sample_data *data)
4625 { 4621 {
4626 u64 sample_type = data->type; 4622 u64 sample_type = data->type;
4627 4623
4628 if (sample_type & PERF_SAMPLE_TID) 4624 if (sample_type & PERF_SAMPLE_TID)
4629 perf_output_put(handle, data->tid_entry); 4625 perf_output_put(handle, data->tid_entry);
4630 4626
4631 if (sample_type & PERF_SAMPLE_TIME) 4627 if (sample_type & PERF_SAMPLE_TIME)
4632 perf_output_put(handle, data->time); 4628 perf_output_put(handle, data->time);
4633 4629
4634 if (sample_type & PERF_SAMPLE_ID) 4630 if (sample_type & PERF_SAMPLE_ID)
4635 perf_output_put(handle, data->id); 4631 perf_output_put(handle, data->id);
4636 4632
4637 if (sample_type & PERF_SAMPLE_STREAM_ID) 4633 if (sample_type & PERF_SAMPLE_STREAM_ID)
4638 perf_output_put(handle, data->stream_id); 4634 perf_output_put(handle, data->stream_id);
4639 4635
4640 if (sample_type & PERF_SAMPLE_CPU) 4636 if (sample_type & PERF_SAMPLE_CPU)
4641 perf_output_put(handle, data->cpu_entry); 4637 perf_output_put(handle, data->cpu_entry);
4642 4638
4643 if (sample_type & PERF_SAMPLE_IDENTIFIER) 4639 if (sample_type & PERF_SAMPLE_IDENTIFIER)
4644 perf_output_put(handle, data->id); 4640 perf_output_put(handle, data->id);
4645 } 4641 }
4646 4642
4647 void perf_event__output_id_sample(struct perf_event *event, 4643 void perf_event__output_id_sample(struct perf_event *event,
4648 struct perf_output_handle *handle, 4644 struct perf_output_handle *handle,
4649 struct perf_sample_data *sample) 4645 struct perf_sample_data *sample)
4650 { 4646 {
4651 if (event->attr.sample_id_all) 4647 if (event->attr.sample_id_all)
4652 __perf_event__output_id_sample(handle, sample); 4648 __perf_event__output_id_sample(handle, sample);
4653 } 4649 }
4654 4650
4655 static void perf_output_read_one(struct perf_output_handle *handle, 4651 static void perf_output_read_one(struct perf_output_handle *handle,
4656 struct perf_event *event, 4652 struct perf_event *event,
4657 u64 enabled, u64 running) 4653 u64 enabled, u64 running)
4658 { 4654 {
4659 u64 read_format = event->attr.read_format; 4655 u64 read_format = event->attr.read_format;
4660 u64 values[4]; 4656 u64 values[4];
4661 int n = 0; 4657 int n = 0;
4662 4658
4663 values[n++] = perf_event_count(event); 4659 values[n++] = perf_event_count(event);
4664 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 4660 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
4665 values[n++] = enabled + 4661 values[n++] = enabled +
4666 atomic64_read(&event->child_total_time_enabled); 4662 atomic64_read(&event->child_total_time_enabled);
4667 } 4663 }
4668 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 4664 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
4669 values[n++] = running + 4665 values[n++] = running +
4670 atomic64_read(&event->child_total_time_running); 4666 atomic64_read(&event->child_total_time_running);
4671 } 4667 }
4672 if (read_format & PERF_FORMAT_ID) 4668 if (read_format & PERF_FORMAT_ID)
4673 values[n++] = primary_event_id(event); 4669 values[n++] = primary_event_id(event);
4674 4670
4675 __output_copy(handle, values, n * sizeof(u64)); 4671 __output_copy(handle, values, n * sizeof(u64));
4676 } 4672 }
4677 4673
4678 /* 4674 /*
4679 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult. 4675 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
4680 */ 4676 */
4681 static void perf_output_read_group(struct perf_output_handle *handle, 4677 static void perf_output_read_group(struct perf_output_handle *handle,
4682 struct perf_event *event, 4678 struct perf_event *event,
4683 u64 enabled, u64 running) 4679 u64 enabled, u64 running)
4684 { 4680 {
4685 struct perf_event *leader = event->group_leader, *sub; 4681 struct perf_event *leader = event->group_leader, *sub;
4686 u64 read_format = event->attr.read_format; 4682 u64 read_format = event->attr.read_format;
4687 u64 values[5]; 4683 u64 values[5];
4688 int n = 0; 4684 int n = 0;
4689 4685
4690 values[n++] = 1 + leader->nr_siblings; 4686 values[n++] = 1 + leader->nr_siblings;
4691 4687
4692 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 4688 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
4693 values[n++] = enabled; 4689 values[n++] = enabled;
4694 4690
4695 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 4691 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
4696 values[n++] = running; 4692 values[n++] = running;
4697 4693
4698 if (leader != event) 4694 if (leader != event)
4699 leader->pmu->read(leader); 4695 leader->pmu->read(leader);
4700 4696
4701 values[n++] = perf_event_count(leader); 4697 values[n++] = perf_event_count(leader);
4702 if (read_format & PERF_FORMAT_ID) 4698 if (read_format & PERF_FORMAT_ID)
4703 values[n++] = primary_event_id(leader); 4699 values[n++] = primary_event_id(leader);
4704 4700
4705 __output_copy(handle, values, n * sizeof(u64)); 4701 __output_copy(handle, values, n * sizeof(u64));
4706 4702
4707 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 4703 list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4708 n = 0; 4704 n = 0;
4709 4705
4710 if ((sub != event) && 4706 if ((sub != event) &&
4711 (sub->state == PERF_EVENT_STATE_ACTIVE)) 4707 (sub->state == PERF_EVENT_STATE_ACTIVE))
4712 sub->pmu->read(sub); 4708 sub->pmu->read(sub);
4713 4709
4714 values[n++] = perf_event_count(sub); 4710 values[n++] = perf_event_count(sub);
4715 if (read_format & PERF_FORMAT_ID) 4711 if (read_format & PERF_FORMAT_ID)
4716 values[n++] = primary_event_id(sub); 4712 values[n++] = primary_event_id(sub);
4717 4713
4718 __output_copy(handle, values, n * sizeof(u64)); 4714 __output_copy(handle, values, n * sizeof(u64));
4719 } 4715 }
4720 } 4716 }
4721 4717
4722 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 4718 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4723 PERF_FORMAT_TOTAL_TIME_RUNNING) 4719 PERF_FORMAT_TOTAL_TIME_RUNNING)
4724 4720
4725 static void perf_output_read(struct perf_output_handle *handle, 4721 static void perf_output_read(struct perf_output_handle *handle,
4726 struct perf_event *event) 4722 struct perf_event *event)
4727 { 4723 {
4728 u64 enabled = 0, running = 0, now; 4724 u64 enabled = 0, running = 0, now;
4729 u64 read_format = event->attr.read_format; 4725 u64 read_format = event->attr.read_format;
4730 4726
4731 /* 4727 /*
4732 * compute total_time_enabled, total_time_running 4728 * compute total_time_enabled, total_time_running
4733 * based on snapshot values taken when the event 4729 * based on snapshot values taken when the event
4734 * was last scheduled in. 4730 * was last scheduled in.
4735 * 4731 *
4736 * we cannot simply called update_context_time() 4732 * we cannot simply called update_context_time()
4737 * because of locking issue as we are called in 4733 * because of locking issue as we are called in
4738 * NMI context 4734 * NMI context
4739 */ 4735 */
4740 if (read_format & PERF_FORMAT_TOTAL_TIMES) 4736 if (read_format & PERF_FORMAT_TOTAL_TIMES)
4741 calc_timer_values(event, &now, &enabled, &running); 4737 calc_timer_values(event, &now, &enabled, &running);
4742 4738
4743 if (event->attr.read_format & PERF_FORMAT_GROUP) 4739 if (event->attr.read_format & PERF_FORMAT_GROUP)
4744 perf_output_read_group(handle, event, enabled, running); 4740 perf_output_read_group(handle, event, enabled, running);
4745 else 4741 else
4746 perf_output_read_one(handle, event, enabled, running); 4742 perf_output_read_one(handle, event, enabled, running);
4747 } 4743 }
4748 4744
4749 void perf_output_sample(struct perf_output_handle *handle, 4745 void perf_output_sample(struct perf_output_handle *handle,
4750 struct perf_event_header *header, 4746 struct perf_event_header *header,
4751 struct perf_sample_data *data, 4747 struct perf_sample_data *data,
4752 struct perf_event *event) 4748 struct perf_event *event)
4753 { 4749 {
4754 u64 sample_type = data->type; 4750 u64 sample_type = data->type;
4755 4751
4756 perf_output_put(handle, *header); 4752 perf_output_put(handle, *header);
4757 4753
4758 if (sample_type & PERF_SAMPLE_IDENTIFIER) 4754 if (sample_type & PERF_SAMPLE_IDENTIFIER)
4759 perf_output_put(handle, data->id); 4755 perf_output_put(handle, data->id);
4760 4756
4761 if (sample_type & PERF_SAMPLE_IP) 4757 if (sample_type & PERF_SAMPLE_IP)
4762 perf_output_put(handle, data->ip); 4758 perf_output_put(handle, data->ip);
4763 4759
4764 if (sample_type & PERF_SAMPLE_TID) 4760 if (sample_type & PERF_SAMPLE_TID)
4765 perf_output_put(handle, data->tid_entry); 4761 perf_output_put(handle, data->tid_entry);
4766 4762
4767 if (sample_type & PERF_SAMPLE_TIME) 4763 if (sample_type & PERF_SAMPLE_TIME)
4768 perf_output_put(handle, data->time); 4764 perf_output_put(handle, data->time);
4769 4765
4770 if (sample_type & PERF_SAMPLE_ADDR) 4766 if (sample_type & PERF_SAMPLE_ADDR)
4771 perf_output_put(handle, data->addr); 4767 perf_output_put(handle, data->addr);
4772 4768
4773 if (sample_type & PERF_SAMPLE_ID) 4769 if (sample_type & PERF_SAMPLE_ID)
4774 perf_output_put(handle, data->id); 4770 perf_output_put(handle, data->id);
4775 4771
4776 if (sample_type & PERF_SAMPLE_STREAM_ID) 4772 if (sample_type & PERF_SAMPLE_STREAM_ID)
4777 perf_output_put(handle, data->stream_id); 4773 perf_output_put(handle, data->stream_id);
4778 4774
4779 if (sample_type & PERF_SAMPLE_CPU) 4775 if (sample_type & PERF_SAMPLE_CPU)
4780 perf_output_put(handle, data->cpu_entry); 4776 perf_output_put(handle, data->cpu_entry);
4781 4777
4782 if (sample_type & PERF_SAMPLE_PERIOD) 4778 if (sample_type & PERF_SAMPLE_PERIOD)
4783 perf_output_put(handle, data->period); 4779 perf_output_put(handle, data->period);
4784 4780
4785 if (sample_type & PERF_SAMPLE_READ) 4781 if (sample_type & PERF_SAMPLE_READ)
4786 perf_output_read(handle, event); 4782 perf_output_read(handle, event);
4787 4783
4788 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 4784 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4789 if (data->callchain) { 4785 if (data->callchain) {
4790 int size = 1; 4786 int size = 1;
4791 4787
4792 if (data->callchain) 4788 if (data->callchain)
4793 size += data->callchain->nr; 4789 size += data->callchain->nr;
4794 4790
4795 size *= sizeof(u64); 4791 size *= sizeof(u64);
4796 4792
4797 __output_copy(handle, data->callchain, size); 4793 __output_copy(handle, data->callchain, size);
4798 } else { 4794 } else {
4799 u64 nr = 0; 4795 u64 nr = 0;
4800 perf_output_put(handle, nr); 4796 perf_output_put(handle, nr);
4801 } 4797 }
4802 } 4798 }
4803 4799
4804 if (sample_type & PERF_SAMPLE_RAW) { 4800 if (sample_type & PERF_SAMPLE_RAW) {
4805 if (data->raw) { 4801 if (data->raw) {
4806 perf_output_put(handle, data->raw->size); 4802 perf_output_put(handle, data->raw->size);
4807 __output_copy(handle, data->raw->data, 4803 __output_copy(handle, data->raw->data,
4808 data->raw->size); 4804 data->raw->size);
4809 } else { 4805 } else {
4810 struct { 4806 struct {
4811 u32 size; 4807 u32 size;
4812 u32 data; 4808 u32 data;
4813 } raw = { 4809 } raw = {
4814 .size = sizeof(u32), 4810 .size = sizeof(u32),
4815 .data = 0, 4811 .data = 0,
4816 }; 4812 };
4817 perf_output_put(handle, raw); 4813 perf_output_put(handle, raw);
4818 } 4814 }
4819 } 4815 }
4820 4816
4821 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 4817 if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4822 if (data->br_stack) { 4818 if (data->br_stack) {
4823 size_t size; 4819 size_t size;
4824 4820
4825 size = data->br_stack->nr 4821 size = data->br_stack->nr
4826 * sizeof(struct perf_branch_entry); 4822 * sizeof(struct perf_branch_entry);
4827 4823
4828 perf_output_put(handle, data->br_stack->nr); 4824 perf_output_put(handle, data->br_stack->nr);
4829 perf_output_copy(handle, data->br_stack->entries, size); 4825 perf_output_copy(handle, data->br_stack->entries, size);
4830 } else { 4826 } else {
4831 /* 4827 /*
4832 * we always store at least the value of nr 4828 * we always store at least the value of nr
4833 */ 4829 */
4834 u64 nr = 0; 4830 u64 nr = 0;
4835 perf_output_put(handle, nr); 4831 perf_output_put(handle, nr);
4836 } 4832 }
4837 } 4833 }
4838 4834
4839 if (sample_type & PERF_SAMPLE_REGS_USER) { 4835 if (sample_type & PERF_SAMPLE_REGS_USER) {
4840 u64 abi = data->regs_user.abi; 4836 u64 abi = data->regs_user.abi;
4841 4837
4842 /* 4838 /*
4843 * If there are no regs to dump, notice it through 4839 * If there are no regs to dump, notice it through
4844 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 4840 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
4845 */ 4841 */
4846 perf_output_put(handle, abi); 4842 perf_output_put(handle, abi);
4847 4843
4848 if (abi) { 4844 if (abi) {
4849 u64 mask = event->attr.sample_regs_user; 4845 u64 mask = event->attr.sample_regs_user;
4850 perf_output_sample_regs(handle, 4846 perf_output_sample_regs(handle,
4851 data->regs_user.regs, 4847 data->regs_user.regs,
4852 mask); 4848 mask);
4853 } 4849 }
4854 } 4850 }
4855 4851
4856 if (sample_type & PERF_SAMPLE_STACK_USER) { 4852 if (sample_type & PERF_SAMPLE_STACK_USER) {
4857 perf_output_sample_ustack(handle, 4853 perf_output_sample_ustack(handle,
4858 data->stack_user_size, 4854 data->stack_user_size,
4859 data->regs_user.regs); 4855 data->regs_user.regs);
4860 } 4856 }
4861 4857
4862 if (sample_type & PERF_SAMPLE_WEIGHT) 4858 if (sample_type & PERF_SAMPLE_WEIGHT)
4863 perf_output_put(handle, data->weight); 4859 perf_output_put(handle, data->weight);
4864 4860
4865 if (sample_type & PERF_SAMPLE_DATA_SRC) 4861 if (sample_type & PERF_SAMPLE_DATA_SRC)
4866 perf_output_put(handle, data->data_src.val); 4862 perf_output_put(handle, data->data_src.val);
4867 4863
4868 if (sample_type & PERF_SAMPLE_TRANSACTION) 4864 if (sample_type & PERF_SAMPLE_TRANSACTION)
4869 perf_output_put(handle, data->txn); 4865 perf_output_put(handle, data->txn);
4870 4866
4871 if (sample_type & PERF_SAMPLE_REGS_INTR) { 4867 if (sample_type & PERF_SAMPLE_REGS_INTR) {
4872 u64 abi = data->regs_intr.abi; 4868 u64 abi = data->regs_intr.abi;
4873 /* 4869 /*
4874 * If there are no regs to dump, notice it through 4870 * If there are no regs to dump, notice it through
4875 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 4871 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
4876 */ 4872 */
4877 perf_output_put(handle, abi); 4873 perf_output_put(handle, abi);
4878 4874
4879 if (abi) { 4875 if (abi) {
4880 u64 mask = event->attr.sample_regs_intr; 4876 u64 mask = event->attr.sample_regs_intr;
4881 4877
4882 perf_output_sample_regs(handle, 4878 perf_output_sample_regs(handle,
4883 data->regs_intr.regs, 4879 data->regs_intr.regs,
4884 mask); 4880 mask);
4885 } 4881 }
4886 } 4882 }
4887 4883
4888 if (!event->attr.watermark) { 4884 if (!event->attr.watermark) {
4889 int wakeup_events = event->attr.wakeup_events; 4885 int wakeup_events = event->attr.wakeup_events;
4890 4886
4891 if (wakeup_events) { 4887 if (wakeup_events) {
4892 struct ring_buffer *rb = handle->rb; 4888 struct ring_buffer *rb = handle->rb;
4893 int events = local_inc_return(&rb->events); 4889 int events = local_inc_return(&rb->events);
4894 4890
4895 if (events >= wakeup_events) { 4891 if (events >= wakeup_events) {
4896 local_sub(wakeup_events, &rb->events); 4892 local_sub(wakeup_events, &rb->events);
4897 local_inc(&rb->wakeup); 4893 local_inc(&rb->wakeup);
4898 } 4894 }
4899 } 4895 }
4900 } 4896 }
4901 } 4897 }
4902 4898
4903 void perf_prepare_sample(struct perf_event_header *header, 4899 void perf_prepare_sample(struct perf_event_header *header,
4904 struct perf_sample_data *data, 4900 struct perf_sample_data *data,
4905 struct perf_event *event, 4901 struct perf_event *event,
4906 struct pt_regs *regs) 4902 struct pt_regs *regs)
4907 { 4903 {
4908 u64 sample_type = event->attr.sample_type; 4904 u64 sample_type = event->attr.sample_type;
4909 4905
4910 header->type = PERF_RECORD_SAMPLE; 4906 header->type = PERF_RECORD_SAMPLE;
4911 header->size = sizeof(*header) + event->header_size; 4907 header->size = sizeof(*header) + event->header_size;
4912 4908
4913 header->misc = 0; 4909 header->misc = 0;
4914 header->misc |= perf_misc_flags(regs); 4910 header->misc |= perf_misc_flags(regs);
4915 4911
4916 __perf_event_header__init_id(header, data, event); 4912 __perf_event_header__init_id(header, data, event);
4917 4913
4918 if (sample_type & PERF_SAMPLE_IP) 4914 if (sample_type & PERF_SAMPLE_IP)
4919 data->ip = perf_instruction_pointer(regs); 4915 data->ip = perf_instruction_pointer(regs);
4920 4916
4921 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 4917 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4922 int size = 1; 4918 int size = 1;
4923 4919
4924 data->callchain = perf_callchain(event, regs); 4920 data->callchain = perf_callchain(event, regs);
4925 4921
4926 if (data->callchain) 4922 if (data->callchain)
4927 size += data->callchain->nr; 4923 size += data->callchain->nr;
4928 4924
4929 header->size += size * sizeof(u64); 4925 header->size += size * sizeof(u64);
4930 } 4926 }
4931 4927
4932 if (sample_type & PERF_SAMPLE_RAW) { 4928 if (sample_type & PERF_SAMPLE_RAW) {
4933 int size = sizeof(u32); 4929 int size = sizeof(u32);
4934 4930
4935 if (data->raw) 4931 if (data->raw)
4936 size += data->raw->size; 4932 size += data->raw->size;
4937 else 4933 else
4938 size += sizeof(u32); 4934 size += sizeof(u32);
4939 4935
4940 WARN_ON_ONCE(size & (sizeof(u64)-1)); 4936 WARN_ON_ONCE(size & (sizeof(u64)-1));
4941 header->size += size; 4937 header->size += size;
4942 } 4938 }
4943 4939
4944 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 4940 if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4945 int size = sizeof(u64); /* nr */ 4941 int size = sizeof(u64); /* nr */
4946 if (data->br_stack) { 4942 if (data->br_stack) {
4947 size += data->br_stack->nr 4943 size += data->br_stack->nr
4948 * sizeof(struct perf_branch_entry); 4944 * sizeof(struct perf_branch_entry);
4949 } 4945 }
4950 header->size += size; 4946 header->size += size;
4951 } 4947 }
4952 4948
4953 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) 4949 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER))
4954 perf_sample_regs_user(&data->regs_user, regs); 4950 perf_sample_regs_user(&data->regs_user, regs,
4951 &data->regs_user_copy);
4955 4952
4956 if (sample_type & PERF_SAMPLE_REGS_USER) { 4953 if (sample_type & PERF_SAMPLE_REGS_USER) {
4957 /* regs dump ABI info */ 4954 /* regs dump ABI info */
4958 int size = sizeof(u64); 4955 int size = sizeof(u64);
4959 4956
4960 if (data->regs_user.regs) { 4957 if (data->regs_user.regs) {
4961 u64 mask = event->attr.sample_regs_user; 4958 u64 mask = event->attr.sample_regs_user;
4962 size += hweight64(mask) * sizeof(u64); 4959 size += hweight64(mask) * sizeof(u64);
4963 } 4960 }
4964 4961
4965 header->size += size; 4962 header->size += size;
4966 } 4963 }
4967 4964
4968 if (sample_type & PERF_SAMPLE_STACK_USER) { 4965 if (sample_type & PERF_SAMPLE_STACK_USER) {
4969 /* 4966 /*
4970 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 4967 * Either we need PERF_SAMPLE_STACK_USER bit to be allways
4971 * processed as the last one or have additional check added 4968 * processed as the last one or have additional check added
4972 * in case new sample type is added, because we could eat 4969 * in case new sample type is added, because we could eat
4973 * up the rest of the sample size. 4970 * up the rest of the sample size.
4974 */ 4971 */
4975 u16 stack_size = event->attr.sample_stack_user; 4972 u16 stack_size = event->attr.sample_stack_user;
4976 u16 size = sizeof(u64); 4973 u16 size = sizeof(u64);
4977 4974
4978 stack_size = perf_sample_ustack_size(stack_size, header->size, 4975 stack_size = perf_sample_ustack_size(stack_size, header->size,
4979 data->regs_user.regs); 4976 data->regs_user.regs);
4980 4977
4981 /* 4978 /*
4982 * If there is something to dump, add space for the dump 4979 * If there is something to dump, add space for the dump
4983 * itself and for the field that tells the dynamic size, 4980 * itself and for the field that tells the dynamic size,
4984 * which is how many have been actually dumped. 4981 * which is how many have been actually dumped.
4985 */ 4982 */
4986 if (stack_size) 4983 if (stack_size)
4987 size += sizeof(u64) + stack_size; 4984 size += sizeof(u64) + stack_size;
4988 4985
4989 data->stack_user_size = stack_size; 4986 data->stack_user_size = stack_size;
4990 header->size += size; 4987 header->size += size;
4991 } 4988 }
4992 4989
4993 if (sample_type & PERF_SAMPLE_REGS_INTR) { 4990 if (sample_type & PERF_SAMPLE_REGS_INTR) {
4994 /* regs dump ABI info */ 4991 /* regs dump ABI info */
4995 int size = sizeof(u64); 4992 int size = sizeof(u64);
4996 4993
4997 perf_sample_regs_intr(&data->regs_intr, regs); 4994 perf_sample_regs_intr(&data->regs_intr, regs);
4998 4995
4999 if (data->regs_intr.regs) { 4996 if (data->regs_intr.regs) {
5000 u64 mask = event->attr.sample_regs_intr; 4997 u64 mask = event->attr.sample_regs_intr;
5001 4998
5002 size += hweight64(mask) * sizeof(u64); 4999 size += hweight64(mask) * sizeof(u64);
5003 } 5000 }
5004 5001
5005 header->size += size; 5002 header->size += size;
5006 } 5003 }
5007 } 5004 }
5008 5005
5009 static void perf_event_output(struct perf_event *event, 5006 static void perf_event_output(struct perf_event *event,
5010 struct perf_sample_data *data, 5007 struct perf_sample_data *data,
5011 struct pt_regs *regs) 5008 struct pt_regs *regs)
5012 { 5009 {
5013 struct perf_output_handle handle; 5010 struct perf_output_handle handle;
5014 struct perf_event_header header; 5011 struct perf_event_header header;
5015 5012
5016 /* protect the callchain buffers */ 5013 /* protect the callchain buffers */
5017 rcu_read_lock(); 5014 rcu_read_lock();
5018 5015
5019 perf_prepare_sample(&header, data, event, regs); 5016 perf_prepare_sample(&header, data, event, regs);
5020 5017
5021 if (perf_output_begin(&handle, event, header.size)) 5018 if (perf_output_begin(&handle, event, header.size))
5022 goto exit; 5019 goto exit;
5023 5020
5024 perf_output_sample(&handle, &header, data, event); 5021 perf_output_sample(&handle, &header, data, event);
5025 5022
5026 perf_output_end(&handle); 5023 perf_output_end(&handle);
5027 5024
5028 exit: 5025 exit:
5029 rcu_read_unlock(); 5026 rcu_read_unlock();
5030 } 5027 }
5031 5028
5032 /* 5029 /*
5033 * read event_id 5030 * read event_id
5034 */ 5031 */
5035 5032
5036 struct perf_read_event { 5033 struct perf_read_event {
5037 struct perf_event_header header; 5034 struct perf_event_header header;
5038 5035
5039 u32 pid; 5036 u32 pid;
5040 u32 tid; 5037 u32 tid;
5041 }; 5038 };
5042 5039
5043 static void 5040 static void
5044 perf_event_read_event(struct perf_event *event, 5041 perf_event_read_event(struct perf_event *event,
5045 struct task_struct *task) 5042 struct task_struct *task)
5046 { 5043 {
5047 struct perf_output_handle handle; 5044 struct perf_output_handle handle;
5048 struct perf_sample_data sample; 5045 struct perf_sample_data sample;
5049 struct perf_read_event read_event = { 5046 struct perf_read_event read_event = {
5050 .header = { 5047 .header = {
5051 .type = PERF_RECORD_READ, 5048 .type = PERF_RECORD_READ,
5052 .misc = 0, 5049 .misc = 0,
5053 .size = sizeof(read_event) + event->read_size, 5050 .size = sizeof(read_event) + event->read_size,
5054 }, 5051 },
5055 .pid = perf_event_pid(event, task), 5052 .pid = perf_event_pid(event, task),
5056 .tid = perf_event_tid(event, task), 5053 .tid = perf_event_tid(event, task),
5057 }; 5054 };
5058 int ret; 5055 int ret;
5059 5056
5060 perf_event_header__init_id(&read_event.header, &sample, event); 5057 perf_event_header__init_id(&read_event.header, &sample, event);
5061 ret = perf_output_begin(&handle, event, read_event.header.size); 5058 ret = perf_output_begin(&handle, event, read_event.header.size);
5062 if (ret) 5059 if (ret)
5063 return; 5060 return;
5064 5061
5065 perf_output_put(&handle, read_event); 5062 perf_output_put(&handle, read_event);
5066 perf_output_read(&handle, event); 5063 perf_output_read(&handle, event);
5067 perf_event__output_id_sample(event, &handle, &sample); 5064 perf_event__output_id_sample(event, &handle, &sample);
5068 5065
5069 perf_output_end(&handle); 5066 perf_output_end(&handle);
5070 } 5067 }
5071 5068
5072 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data); 5069 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
5073 5070
5074 static void 5071 static void
5075 perf_event_aux_ctx(struct perf_event_context *ctx, 5072 perf_event_aux_ctx(struct perf_event_context *ctx,
5076 perf_event_aux_output_cb output, 5073 perf_event_aux_output_cb output,
5077 void *data) 5074 void *data)
5078 { 5075 {
5079 struct perf_event *event; 5076 struct perf_event *event;
5080 5077
5081 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 5078 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5082 if (event->state < PERF_EVENT_STATE_INACTIVE) 5079 if (event->state < PERF_EVENT_STATE_INACTIVE)
5083 continue; 5080 continue;
5084 if (!event_filter_match(event)) 5081 if (!event_filter_match(event))
5085 continue; 5082 continue;
5086 output(event, data); 5083 output(event, data);
5087 } 5084 }
5088 } 5085 }
5089 5086
5090 static void 5087 static void
5091 perf_event_aux(perf_event_aux_output_cb output, void *data, 5088 perf_event_aux(perf_event_aux_output_cb output, void *data,
5092 struct perf_event_context *task_ctx) 5089 struct perf_event_context *task_ctx)
5093 { 5090 {
5094 struct perf_cpu_context *cpuctx; 5091 struct perf_cpu_context *cpuctx;
5095 struct perf_event_context *ctx; 5092 struct perf_event_context *ctx;
5096 struct pmu *pmu; 5093 struct pmu *pmu;
5097 int ctxn; 5094 int ctxn;
5098 5095
5099 rcu_read_lock(); 5096 rcu_read_lock();
5100 list_for_each_entry_rcu(pmu, &pmus, entry) { 5097 list_for_each_entry_rcu(pmu, &pmus, entry) {
5101 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); 5098 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
5102 if (cpuctx->unique_pmu != pmu) 5099 if (cpuctx->unique_pmu != pmu)
5103 goto next; 5100 goto next;
5104 perf_event_aux_ctx(&cpuctx->ctx, output, data); 5101 perf_event_aux_ctx(&cpuctx->ctx, output, data);
5105 if (task_ctx) 5102 if (task_ctx)
5106 goto next; 5103 goto next;
5107 ctxn = pmu->task_ctx_nr; 5104 ctxn = pmu->task_ctx_nr;
5108 if (ctxn < 0) 5105 if (ctxn < 0)
5109 goto next; 5106 goto next;
5110 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 5107 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
5111 if (ctx) 5108 if (ctx)
5112 perf_event_aux_ctx(ctx, output, data); 5109 perf_event_aux_ctx(ctx, output, data);
5113 next: 5110 next:
5114 put_cpu_ptr(pmu->pmu_cpu_context); 5111 put_cpu_ptr(pmu->pmu_cpu_context);
5115 } 5112 }
5116 5113
5117 if (task_ctx) { 5114 if (task_ctx) {
5118 preempt_disable(); 5115 preempt_disable();
5119 perf_event_aux_ctx(task_ctx, output, data); 5116 perf_event_aux_ctx(task_ctx, output, data);
5120 preempt_enable(); 5117 preempt_enable();
5121 } 5118 }
5122 rcu_read_unlock(); 5119 rcu_read_unlock();
5123 } 5120 }
5124 5121
5125 /* 5122 /*
5126 * task tracking -- fork/exit 5123 * task tracking -- fork/exit
5127 * 5124 *
5128 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 5125 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
5129 */ 5126 */
5130 5127
5131 struct perf_task_event { 5128 struct perf_task_event {
5132 struct task_struct *task; 5129 struct task_struct *task;
5133 struct perf_event_context *task_ctx; 5130 struct perf_event_context *task_ctx;
5134 5131
5135 struct { 5132 struct {
5136 struct perf_event_header header; 5133 struct perf_event_header header;
5137 5134
5138 u32 pid; 5135 u32 pid;
5139 u32 ppid; 5136 u32 ppid;
5140 u32 tid; 5137 u32 tid;
5141 u32 ptid; 5138 u32 ptid;
5142 u64 time; 5139 u64 time;
5143 } event_id; 5140 } event_id;
5144 }; 5141 };
5145 5142
5146 static int perf_event_task_match(struct perf_event *event) 5143 static int perf_event_task_match(struct perf_event *event)
5147 { 5144 {
5148 return event->attr.comm || event->attr.mmap || 5145 return event->attr.comm || event->attr.mmap ||
5149 event->attr.mmap2 || event->attr.mmap_data || 5146 event->attr.mmap2 || event->attr.mmap_data ||
5150 event->attr.task; 5147 event->attr.task;
5151 } 5148 }
5152 5149
5153 static void perf_event_task_output(struct perf_event *event, 5150 static void perf_event_task_output(struct perf_event *event,
5154 void *data) 5151 void *data)
5155 { 5152 {
5156 struct perf_task_event *task_event = data; 5153 struct perf_task_event *task_event = data;
5157 struct perf_output_handle handle; 5154 struct perf_output_handle handle;
5158 struct perf_sample_data sample; 5155 struct perf_sample_data sample;
5159 struct task_struct *task = task_event->task; 5156 struct task_struct *task = task_event->task;
5160 int ret, size = task_event->event_id.header.size; 5157 int ret, size = task_event->event_id.header.size;
5161 5158
5162 if (!perf_event_task_match(event)) 5159 if (!perf_event_task_match(event))
5163 return; 5160 return;
5164 5161
5165 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 5162 perf_event_header__init_id(&task_event->event_id.header, &sample, event);
5166 5163
5167 ret = perf_output_begin(&handle, event, 5164 ret = perf_output_begin(&handle, event,
5168 task_event->event_id.header.size); 5165 task_event->event_id.header.size);
5169 if (ret) 5166 if (ret)
5170 goto out; 5167 goto out;
5171 5168
5172 task_event->event_id.pid = perf_event_pid(event, task); 5169 task_event->event_id.pid = perf_event_pid(event, task);
5173 task_event->event_id.ppid = perf_event_pid(event, current); 5170 task_event->event_id.ppid = perf_event_pid(event, current);
5174 5171
5175 task_event->event_id.tid = perf_event_tid(event, task); 5172 task_event->event_id.tid = perf_event_tid(event, task);
5176 task_event->event_id.ptid = perf_event_tid(event, current); 5173 task_event->event_id.ptid = perf_event_tid(event, current);
5177 5174
5178 perf_output_put(&handle, task_event->event_id); 5175 perf_output_put(&handle, task_event->event_id);
5179 5176
5180 perf_event__output_id_sample(event, &handle, &sample); 5177 perf_event__output_id_sample(event, &handle, &sample);
5181 5178
5182 perf_output_end(&handle); 5179 perf_output_end(&handle);
5183 out: 5180 out:
5184 task_event->event_id.header.size = size; 5181 task_event->event_id.header.size = size;
5185 } 5182 }
5186 5183
5187 static void perf_event_task(struct task_struct *task, 5184 static void perf_event_task(struct task_struct *task,
5188 struct perf_event_context *task_ctx, 5185 struct perf_event_context *task_ctx,
5189 int new) 5186 int new)
5190 { 5187 {
5191 struct perf_task_event task_event; 5188 struct perf_task_event task_event;
5192 5189
5193 if (!atomic_read(&nr_comm_events) && 5190 if (!atomic_read(&nr_comm_events) &&
5194 !atomic_read(&nr_mmap_events) && 5191 !atomic_read(&nr_mmap_events) &&
5195 !atomic_read(&nr_task_events)) 5192 !atomic_read(&nr_task_events))
5196 return; 5193 return;
5197 5194
5198 task_event = (struct perf_task_event){ 5195 task_event = (struct perf_task_event){
5199 .task = task, 5196 .task = task,
5200 .task_ctx = task_ctx, 5197 .task_ctx = task_ctx,
5201 .event_id = { 5198 .event_id = {
5202 .header = { 5199 .header = {
5203 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 5200 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
5204 .misc = 0, 5201 .misc = 0,
5205 .size = sizeof(task_event.event_id), 5202 .size = sizeof(task_event.event_id),
5206 }, 5203 },
5207 /* .pid */ 5204 /* .pid */
5208 /* .ppid */ 5205 /* .ppid */
5209 /* .tid */ 5206 /* .tid */
5210 /* .ptid */ 5207 /* .ptid */
5211 .time = perf_clock(), 5208 .time = perf_clock(),
5212 }, 5209 },
5213 }; 5210 };
5214 5211
5215 perf_event_aux(perf_event_task_output, 5212 perf_event_aux(perf_event_task_output,
5216 &task_event, 5213 &task_event,
5217 task_ctx); 5214 task_ctx);
5218 } 5215 }
5219 5216
5220 void perf_event_fork(struct task_struct *task) 5217 void perf_event_fork(struct task_struct *task)
5221 { 5218 {
5222 perf_event_task(task, NULL, 1); 5219 perf_event_task(task, NULL, 1);
5223 } 5220 }
5224 5221
5225 /* 5222 /*
5226 * comm tracking 5223 * comm tracking
5227 */ 5224 */
5228 5225
5229 struct perf_comm_event { 5226 struct perf_comm_event {
5230 struct task_struct *task; 5227 struct task_struct *task;
5231 char *comm; 5228 char *comm;
5232 int comm_size; 5229 int comm_size;
5233 5230
5234 struct { 5231 struct {
5235 struct perf_event_header header; 5232 struct perf_event_header header;
5236 5233
5237 u32 pid; 5234 u32 pid;
5238 u32 tid; 5235 u32 tid;
5239 } event_id; 5236 } event_id;
5240 }; 5237 };
5241 5238
5242 static int perf_event_comm_match(struct perf_event *event) 5239 static int perf_event_comm_match(struct perf_event *event)
5243 { 5240 {
5244 return event->attr.comm; 5241 return event->attr.comm;
5245 } 5242 }
5246 5243
5247 static void perf_event_comm_output(struct perf_event *event, 5244 static void perf_event_comm_output(struct perf_event *event,
5248 void *data) 5245 void *data)
5249 { 5246 {
5250 struct perf_comm_event *comm_event = data; 5247 struct perf_comm_event *comm_event = data;
5251 struct perf_output_handle handle; 5248 struct perf_output_handle handle;
5252 struct perf_sample_data sample; 5249 struct perf_sample_data sample;
5253 int size = comm_event->event_id.header.size; 5250 int size = comm_event->event_id.header.size;
5254 int ret; 5251 int ret;
5255 5252
5256 if (!perf_event_comm_match(event)) 5253 if (!perf_event_comm_match(event))
5257 return; 5254 return;
5258 5255
5259 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 5256 perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
5260 ret = perf_output_begin(&handle, event, 5257 ret = perf_output_begin(&handle, event,
5261 comm_event->event_id.header.size); 5258 comm_event->event_id.header.size);
5262 5259
5263 if (ret) 5260 if (ret)
5264 goto out; 5261 goto out;
5265 5262
5266 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 5263 comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
5267 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 5264 comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
5268 5265
5269 perf_output_put(&handle, comm_event->event_id); 5266 perf_output_put(&handle, comm_event->event_id);
5270 __output_copy(&handle, comm_event->comm, 5267 __output_copy(&handle, comm_event->comm,
5271 comm_event->comm_size); 5268 comm_event->comm_size);
5272 5269
5273 perf_event__output_id_sample(event, &handle, &sample); 5270 perf_event__output_id_sample(event, &handle, &sample);
5274 5271
5275 perf_output_end(&handle); 5272 perf_output_end(&handle);
5276 out: 5273 out:
5277 comm_event->event_id.header.size = size; 5274 comm_event->event_id.header.size = size;
5278 } 5275 }
5279 5276
5280 static void perf_event_comm_event(struct perf_comm_event *comm_event) 5277 static void perf_event_comm_event(struct perf_comm_event *comm_event)
5281 { 5278 {
5282 char comm[TASK_COMM_LEN]; 5279 char comm[TASK_COMM_LEN];
5283 unsigned int size; 5280 unsigned int size;
5284 5281
5285 memset(comm, 0, sizeof(comm)); 5282 memset(comm, 0, sizeof(comm));
5286 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 5283 strlcpy(comm, comm_event->task->comm, sizeof(comm));
5287 size = ALIGN(strlen(comm)+1, sizeof(u64)); 5284 size = ALIGN(strlen(comm)+1, sizeof(u64));
5288 5285
5289 comm_event->comm = comm; 5286 comm_event->comm = comm;
5290 comm_event->comm_size = size; 5287 comm_event->comm_size = size;
5291 5288
5292 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 5289 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
5293 5290
5294 perf_event_aux(perf_event_comm_output, 5291 perf_event_aux(perf_event_comm_output,
5295 comm_event, 5292 comm_event,
5296 NULL); 5293 NULL);
5297 } 5294 }
5298 5295
5299 void perf_event_comm(struct task_struct *task, bool exec) 5296 void perf_event_comm(struct task_struct *task, bool exec)
5300 { 5297 {
5301 struct perf_comm_event comm_event; 5298 struct perf_comm_event comm_event;
5302 5299
5303 if (!atomic_read(&nr_comm_events)) 5300 if (!atomic_read(&nr_comm_events))
5304 return; 5301 return;
5305 5302
5306 comm_event = (struct perf_comm_event){ 5303 comm_event = (struct perf_comm_event){
5307 .task = task, 5304 .task = task,
5308 /* .comm */ 5305 /* .comm */
5309 /* .comm_size */ 5306 /* .comm_size */
5310 .event_id = { 5307 .event_id = {
5311 .header = { 5308 .header = {
5312 .type = PERF_RECORD_COMM, 5309 .type = PERF_RECORD_COMM,
5313 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 5310 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
5314 /* .size */ 5311 /* .size */
5315 }, 5312 },
5316 /* .pid */ 5313 /* .pid */
5317 /* .tid */ 5314 /* .tid */
5318 }, 5315 },
5319 }; 5316 };
5320 5317
5321 perf_event_comm_event(&comm_event); 5318 perf_event_comm_event(&comm_event);
5322 } 5319 }
5323 5320
5324 /* 5321 /*
5325 * mmap tracking 5322 * mmap tracking
5326 */ 5323 */
5327 5324
5328 struct perf_mmap_event { 5325 struct perf_mmap_event {
5329 struct vm_area_struct *vma; 5326 struct vm_area_struct *vma;
5330 5327
5331 const char *file_name; 5328 const char *file_name;
5332 int file_size; 5329 int file_size;
5333 int maj, min; 5330 int maj, min;
5334 u64 ino; 5331 u64 ino;
5335 u64 ino_generation; 5332 u64 ino_generation;
5336 u32 prot, flags; 5333 u32 prot, flags;
5337 5334
5338 struct { 5335 struct {
5339 struct perf_event_header header; 5336 struct perf_event_header header;
5340 5337
5341 u32 pid; 5338 u32 pid;
5342 u32 tid; 5339 u32 tid;
5343 u64 start; 5340 u64 start;
5344 u64 len; 5341 u64 len;
5345 u64 pgoff; 5342 u64 pgoff;
5346 } event_id; 5343 } event_id;
5347 }; 5344 };
5348 5345
5349 static int perf_event_mmap_match(struct perf_event *event, 5346 static int perf_event_mmap_match(struct perf_event *event,
5350 void *data) 5347 void *data)
5351 { 5348 {
5352 struct perf_mmap_event *mmap_event = data; 5349 struct perf_mmap_event *mmap_event = data;
5353 struct vm_area_struct *vma = mmap_event->vma; 5350 struct vm_area_struct *vma = mmap_event->vma;
5354 int executable = vma->vm_flags & VM_EXEC; 5351 int executable = vma->vm_flags & VM_EXEC;
5355 5352
5356 return (!executable && event->attr.mmap_data) || 5353 return (!executable && event->attr.mmap_data) ||
5357 (executable && (event->attr.mmap || event->attr.mmap2)); 5354 (executable && (event->attr.mmap || event->attr.mmap2));
5358 } 5355 }
5359 5356
5360 static void perf_event_mmap_output(struct perf_event *event, 5357 static void perf_event_mmap_output(struct perf_event *event,
5361 void *data) 5358 void *data)
5362 { 5359 {
5363 struct perf_mmap_event *mmap_event = data; 5360 struct perf_mmap_event *mmap_event = data;
5364 struct perf_output_handle handle; 5361 struct perf_output_handle handle;
5365 struct perf_sample_data sample; 5362 struct perf_sample_data sample;
5366 int size = mmap_event->event_id.header.size; 5363 int size = mmap_event->event_id.header.size;
5367 int ret; 5364 int ret;
5368 5365
5369 if (!perf_event_mmap_match(event, data)) 5366 if (!perf_event_mmap_match(event, data))
5370 return; 5367 return;
5371 5368
5372 if (event->attr.mmap2) { 5369 if (event->attr.mmap2) {
5373 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 5370 mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
5374 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 5371 mmap_event->event_id.header.size += sizeof(mmap_event->maj);
5375 mmap_event->event_id.header.size += sizeof(mmap_event->min); 5372 mmap_event->event_id.header.size += sizeof(mmap_event->min);
5376 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 5373 mmap_event->event_id.header.size += sizeof(mmap_event->ino);
5377 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 5374 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
5378 mmap_event->event_id.header.size += sizeof(mmap_event->prot); 5375 mmap_event->event_id.header.size += sizeof(mmap_event->prot);
5379 mmap_event->event_id.header.size += sizeof(mmap_event->flags); 5376 mmap_event->event_id.header.size += sizeof(mmap_event->flags);
5380 } 5377 }
5381 5378
5382 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 5379 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
5383 ret = perf_output_begin(&handle, event, 5380 ret = perf_output_begin(&handle, event,
5384 mmap_event->event_id.header.size); 5381 mmap_event->event_id.header.size);
5385 if (ret) 5382 if (ret)
5386 goto out; 5383 goto out;
5387 5384
5388 mmap_event->event_id.pid = perf_event_pid(event, current); 5385 mmap_event->event_id.pid = perf_event_pid(event, current);
5389 mmap_event->event_id.tid = perf_event_tid(event, current); 5386 mmap_event->event_id.tid = perf_event_tid(event, current);
5390 5387
5391 perf_output_put(&handle, mmap_event->event_id); 5388 perf_output_put(&handle, mmap_event->event_id);
5392 5389
5393 if (event->attr.mmap2) { 5390 if (event->attr.mmap2) {
5394 perf_output_put(&handle, mmap_event->maj); 5391 perf_output_put(&handle, mmap_event->maj);
5395 perf_output_put(&handle, mmap_event->min); 5392 perf_output_put(&handle, mmap_event->min);
5396 perf_output_put(&handle, mmap_event->ino); 5393 perf_output_put(&handle, mmap_event->ino);
5397 perf_output_put(&handle, mmap_event->ino_generation); 5394 perf_output_put(&handle, mmap_event->ino_generation);
5398 perf_output_put(&handle, mmap_event->prot); 5395 perf_output_put(&handle, mmap_event->prot);
5399 perf_output_put(&handle, mmap_event->flags); 5396 perf_output_put(&handle, mmap_event->flags);
5400 } 5397 }
5401 5398
5402 __output_copy(&handle, mmap_event->file_name, 5399 __output_copy(&handle, mmap_event->file_name,
5403 mmap_event->file_size); 5400 mmap_event->file_size);
5404 5401
5405 perf_event__output_id_sample(event, &handle, &sample); 5402 perf_event__output_id_sample(event, &handle, &sample);
5406 5403
5407 perf_output_end(&handle); 5404 perf_output_end(&handle);
5408 out: 5405 out:
5409 mmap_event->event_id.header.size = size; 5406 mmap_event->event_id.header.size = size;
5410 } 5407 }
5411 5408
5412 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 5409 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
5413 { 5410 {
5414 struct vm_area_struct *vma = mmap_event->vma; 5411 struct vm_area_struct *vma = mmap_event->vma;
5415 struct file *file = vma->vm_file; 5412 struct file *file = vma->vm_file;
5416 int maj = 0, min = 0; 5413 int maj = 0, min = 0;
5417 u64 ino = 0, gen = 0; 5414 u64 ino = 0, gen = 0;
5418 u32 prot = 0, flags = 0; 5415 u32 prot = 0, flags = 0;
5419 unsigned int size; 5416 unsigned int size;
5420 char tmp[16]; 5417 char tmp[16];
5421 char *buf = NULL; 5418 char *buf = NULL;
5422 char *name; 5419 char *name;
5423 5420
5424 if (file) { 5421 if (file) {
5425 struct inode *inode; 5422 struct inode *inode;
5426 dev_t dev; 5423 dev_t dev;
5427 5424
5428 buf = kmalloc(PATH_MAX, GFP_KERNEL); 5425 buf = kmalloc(PATH_MAX, GFP_KERNEL);
5429 if (!buf) { 5426 if (!buf) {
5430 name = "//enomem"; 5427 name = "//enomem";
5431 goto cpy_name; 5428 goto cpy_name;
5432 } 5429 }
5433 /* 5430 /*
5434 * d_path() works from the end of the rb backwards, so we 5431 * d_path() works from the end of the rb backwards, so we
5435 * need to add enough zero bytes after the string to handle 5432 * need to add enough zero bytes after the string to handle
5436 * the 64bit alignment we do later. 5433 * the 64bit alignment we do later.
5437 */ 5434 */
5438 name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64)); 5435 name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64));
5439 if (IS_ERR(name)) { 5436 if (IS_ERR(name)) {
5440 name = "//toolong"; 5437 name = "//toolong";
5441 goto cpy_name; 5438 goto cpy_name;
5442 } 5439 }
5443 inode = file_inode(vma->vm_file); 5440 inode = file_inode(vma->vm_file);
5444 dev = inode->i_sb->s_dev; 5441 dev = inode->i_sb->s_dev;
5445 ino = inode->i_ino; 5442 ino = inode->i_ino;
5446 gen = inode->i_generation; 5443 gen = inode->i_generation;
5447 maj = MAJOR(dev); 5444 maj = MAJOR(dev);
5448 min = MINOR(dev); 5445 min = MINOR(dev);
5449 5446
5450 if (vma->vm_flags & VM_READ) 5447 if (vma->vm_flags & VM_READ)
5451 prot |= PROT_READ; 5448 prot |= PROT_READ;
5452 if (vma->vm_flags & VM_WRITE) 5449 if (vma->vm_flags & VM_WRITE)
5453 prot |= PROT_WRITE; 5450 prot |= PROT_WRITE;
5454 if (vma->vm_flags & VM_EXEC) 5451 if (vma->vm_flags & VM_EXEC)
5455 prot |= PROT_EXEC; 5452 prot |= PROT_EXEC;
5456 5453
5457 if (vma->vm_flags & VM_MAYSHARE) 5454 if (vma->vm_flags & VM_MAYSHARE)
5458 flags = MAP_SHARED; 5455 flags = MAP_SHARED;
5459 else 5456 else
5460 flags = MAP_PRIVATE; 5457 flags = MAP_PRIVATE;
5461 5458
5462 if (vma->vm_flags & VM_DENYWRITE) 5459 if (vma->vm_flags & VM_DENYWRITE)
5463 flags |= MAP_DENYWRITE; 5460 flags |= MAP_DENYWRITE;
5464 if (vma->vm_flags & VM_MAYEXEC) 5461 if (vma->vm_flags & VM_MAYEXEC)
5465 flags |= MAP_EXECUTABLE; 5462 flags |= MAP_EXECUTABLE;
5466 if (vma->vm_flags & VM_LOCKED) 5463 if (vma->vm_flags & VM_LOCKED)
5467 flags |= MAP_LOCKED; 5464 flags |= MAP_LOCKED;
5468 if (vma->vm_flags & VM_HUGETLB) 5465 if (vma->vm_flags & VM_HUGETLB)
5469 flags |= MAP_HUGETLB; 5466 flags |= MAP_HUGETLB;
5470 5467
5471 goto got_name; 5468 goto got_name;
5472 } else { 5469 } else {
5473 if (vma->vm_ops && vma->vm_ops->name) { 5470 if (vma->vm_ops && vma->vm_ops->name) {
5474 name = (char *) vma->vm_ops->name(vma); 5471 name = (char *) vma->vm_ops->name(vma);
5475 if (name) 5472 if (name)
5476 goto cpy_name; 5473 goto cpy_name;
5477 } 5474 }
5478 5475
5479 name = (char *)arch_vma_name(vma); 5476 name = (char *)arch_vma_name(vma);
5480 if (name) 5477 if (name)
5481 goto cpy_name; 5478 goto cpy_name;
5482 5479
5483 if (vma->vm_start <= vma->vm_mm->start_brk && 5480 if (vma->vm_start <= vma->vm_mm->start_brk &&
5484 vma->vm_end >= vma->vm_mm->brk) { 5481 vma->vm_end >= vma->vm_mm->brk) {
5485 name = "[heap]"; 5482 name = "[heap]";
5486 goto cpy_name; 5483 goto cpy_name;
5487 } 5484 }
5488 if (vma->vm_start <= vma->vm_mm->start_stack && 5485 if (vma->vm_start <= vma->vm_mm->start_stack &&
5489 vma->vm_end >= vma->vm_mm->start_stack) { 5486 vma->vm_end >= vma->vm_mm->start_stack) {
5490 name = "[stack]"; 5487 name = "[stack]";
5491 goto cpy_name; 5488 goto cpy_name;
5492 } 5489 }
5493 5490
5494 name = "//anon"; 5491 name = "//anon";
5495 goto cpy_name; 5492 goto cpy_name;
5496 } 5493 }
5497 5494
5498 cpy_name: 5495 cpy_name:
5499 strlcpy(tmp, name, sizeof(tmp)); 5496 strlcpy(tmp, name, sizeof(tmp));
5500 name = tmp; 5497 name = tmp;
5501 got_name: 5498 got_name:
5502 /* 5499 /*
5503 * Since our buffer works in 8 byte units we need to align our string 5500 * Since our buffer works in 8 byte units we need to align our string
5504 * size to a multiple of 8. However, we must guarantee the tail end is 5501 * size to a multiple of 8. However, we must guarantee the tail end is
5505 * zero'd out to avoid leaking random bits to userspace. 5502 * zero'd out to avoid leaking random bits to userspace.
5506 */ 5503 */
5507 size = strlen(name)+1; 5504 size = strlen(name)+1;
5508 while (!IS_ALIGNED(size, sizeof(u64))) 5505 while (!IS_ALIGNED(size, sizeof(u64)))
5509 name[size++] = '\0'; 5506 name[size++] = '\0';
5510 5507
5511 mmap_event->file_name = name; 5508 mmap_event->file_name = name;
5512 mmap_event->file_size = size; 5509 mmap_event->file_size = size;
5513 mmap_event->maj = maj; 5510 mmap_event->maj = maj;
5514 mmap_event->min = min; 5511 mmap_event->min = min;
5515 mmap_event->ino = ino; 5512 mmap_event->ino = ino;
5516 mmap_event->ino_generation = gen; 5513 mmap_event->ino_generation = gen;
5517 mmap_event->prot = prot; 5514 mmap_event->prot = prot;
5518 mmap_event->flags = flags; 5515 mmap_event->flags = flags;
5519 5516
5520 if (!(vma->vm_flags & VM_EXEC)) 5517 if (!(vma->vm_flags & VM_EXEC))
5521 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 5518 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
5522 5519
5523 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 5520 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
5524 5521
5525 perf_event_aux(perf_event_mmap_output, 5522 perf_event_aux(perf_event_mmap_output,
5526 mmap_event, 5523 mmap_event,
5527 NULL); 5524 NULL);
5528 5525
5529 kfree(buf); 5526 kfree(buf);
5530 } 5527 }
5531 5528
5532 void perf_event_mmap(struct vm_area_struct *vma) 5529 void perf_event_mmap(struct vm_area_struct *vma)
5533 { 5530 {
5534 struct perf_mmap_event mmap_event; 5531 struct perf_mmap_event mmap_event;
5535 5532
5536 if (!atomic_read(&nr_mmap_events)) 5533 if (!atomic_read(&nr_mmap_events))
5537 return; 5534 return;
5538 5535
5539 mmap_event = (struct perf_mmap_event){ 5536 mmap_event = (struct perf_mmap_event){
5540 .vma = vma, 5537 .vma = vma,
5541 /* .file_name */ 5538 /* .file_name */
5542 /* .file_size */ 5539 /* .file_size */
5543 .event_id = { 5540 .event_id = {
5544 .header = { 5541 .header = {
5545 .type = PERF_RECORD_MMAP, 5542 .type = PERF_RECORD_MMAP,
5546 .misc = PERF_RECORD_MISC_USER, 5543 .misc = PERF_RECORD_MISC_USER,
5547 /* .size */ 5544 /* .size */
5548 }, 5545 },
5549 /* .pid */ 5546 /* .pid */
5550 /* .tid */ 5547 /* .tid */
5551 .start = vma->vm_start, 5548 .start = vma->vm_start,
5552 .len = vma->vm_end - vma->vm_start, 5549 .len = vma->vm_end - vma->vm_start,
5553 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 5550 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT,
5554 }, 5551 },
5555 /* .maj (attr_mmap2 only) */ 5552 /* .maj (attr_mmap2 only) */
5556 /* .min (attr_mmap2 only) */ 5553 /* .min (attr_mmap2 only) */
5557 /* .ino (attr_mmap2 only) */ 5554 /* .ino (attr_mmap2 only) */
5558 /* .ino_generation (attr_mmap2 only) */ 5555 /* .ino_generation (attr_mmap2 only) */
5559 /* .prot (attr_mmap2 only) */ 5556 /* .prot (attr_mmap2 only) */
5560 /* .flags (attr_mmap2 only) */ 5557 /* .flags (attr_mmap2 only) */
5561 }; 5558 };
5562 5559
5563 perf_event_mmap_event(&mmap_event); 5560 perf_event_mmap_event(&mmap_event);
5564 } 5561 }
5565 5562
5566 /* 5563 /*
5567 * IRQ throttle logging 5564 * IRQ throttle logging
5568 */ 5565 */
5569 5566
5570 static void perf_log_throttle(struct perf_event *event, int enable) 5567 static void perf_log_throttle(struct perf_event *event, int enable)
5571 { 5568 {
5572 struct perf_output_handle handle; 5569 struct perf_output_handle handle;
5573 struct perf_sample_data sample; 5570 struct perf_sample_data sample;
5574 int ret; 5571 int ret;
5575 5572
5576 struct { 5573 struct {
5577 struct perf_event_header header; 5574 struct perf_event_header header;
5578 u64 time; 5575 u64 time;
5579 u64 id; 5576 u64 id;
5580 u64 stream_id; 5577 u64 stream_id;
5581 } throttle_event = { 5578 } throttle_event = {
5582 .header = { 5579 .header = {
5583 .type = PERF_RECORD_THROTTLE, 5580 .type = PERF_RECORD_THROTTLE,
5584 .misc = 0, 5581 .misc = 0,
5585 .size = sizeof(throttle_event), 5582 .size = sizeof(throttle_event),
5586 }, 5583 },
5587 .time = perf_clock(), 5584 .time = perf_clock(),
5588 .id = primary_event_id(event), 5585 .id = primary_event_id(event),
5589 .stream_id = event->id, 5586 .stream_id = event->id,
5590 }; 5587 };
5591 5588
5592 if (enable) 5589 if (enable)
5593 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 5590 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
5594 5591
5595 perf_event_header__init_id(&throttle_event.header, &sample, event); 5592 perf_event_header__init_id(&throttle_event.header, &sample, event);
5596 5593
5597 ret = perf_output_begin(&handle, event, 5594 ret = perf_output_begin(&handle, event,
5598 throttle_event.header.size); 5595 throttle_event.header.size);
5599 if (ret) 5596 if (ret)
5600 return; 5597 return;
5601 5598
5602 perf_output_put(&handle, throttle_event); 5599 perf_output_put(&handle, throttle_event);
5603 perf_event__output_id_sample(event, &handle, &sample); 5600 perf_event__output_id_sample(event, &handle, &sample);
5604 perf_output_end(&handle); 5601 perf_output_end(&handle);
5605 } 5602 }
5606 5603
5607 /* 5604 /*
5608 * Generic event overflow handling, sampling. 5605 * Generic event overflow handling, sampling.
5609 */ 5606 */
5610 5607
5611 static int __perf_event_overflow(struct perf_event *event, 5608 static int __perf_event_overflow(struct perf_event *event,
5612 int throttle, struct perf_sample_data *data, 5609 int throttle, struct perf_sample_data *data,
5613 struct pt_regs *regs) 5610 struct pt_regs *regs)
5614 { 5611 {
5615 int events = atomic_read(&event->event_limit); 5612 int events = atomic_read(&event->event_limit);
5616 struct hw_perf_event *hwc = &event->hw; 5613 struct hw_perf_event *hwc = &event->hw;
5617 u64 seq; 5614 u64 seq;
5618 int ret = 0; 5615 int ret = 0;
5619 5616
5620 /* 5617 /*
5621 * Non-sampling counters might still use the PMI to fold short 5618 * Non-sampling counters might still use the PMI to fold short
5622 * hardware counters, ignore those. 5619 * hardware counters, ignore those.
5623 */ 5620 */
5624 if (unlikely(!is_sampling_event(event))) 5621 if (unlikely(!is_sampling_event(event)))
5625 return 0; 5622 return 0;
5626 5623
5627 seq = __this_cpu_read(perf_throttled_seq); 5624 seq = __this_cpu_read(perf_throttled_seq);
5628 if (seq != hwc->interrupts_seq) { 5625 if (seq != hwc->interrupts_seq) {
5629 hwc->interrupts_seq = seq; 5626 hwc->interrupts_seq = seq;
5630 hwc->interrupts = 1; 5627 hwc->interrupts = 1;
5631 } else { 5628 } else {
5632 hwc->interrupts++; 5629 hwc->interrupts++;
5633 if (unlikely(throttle 5630 if (unlikely(throttle
5634 && hwc->interrupts >= max_samples_per_tick)) { 5631 && hwc->interrupts >= max_samples_per_tick)) {
5635 __this_cpu_inc(perf_throttled_count); 5632 __this_cpu_inc(perf_throttled_count);
5636 hwc->interrupts = MAX_INTERRUPTS; 5633 hwc->interrupts = MAX_INTERRUPTS;
5637 perf_log_throttle(event, 0); 5634 perf_log_throttle(event, 0);
5638 tick_nohz_full_kick(); 5635 tick_nohz_full_kick();
5639 ret = 1; 5636 ret = 1;
5640 } 5637 }
5641 } 5638 }
5642 5639
5643 if (event->attr.freq) { 5640 if (event->attr.freq) {
5644 u64 now = perf_clock(); 5641 u64 now = perf_clock();
5645 s64 delta = now - hwc->freq_time_stamp; 5642 s64 delta = now - hwc->freq_time_stamp;
5646 5643
5647 hwc->freq_time_stamp = now; 5644 hwc->freq_time_stamp = now;
5648 5645
5649 if (delta > 0 && delta < 2*TICK_NSEC) 5646 if (delta > 0 && delta < 2*TICK_NSEC)
5650 perf_adjust_period(event, delta, hwc->last_period, true); 5647 perf_adjust_period(event, delta, hwc->last_period, true);
5651 } 5648 }
5652 5649
5653 /* 5650 /*
5654 * XXX event_limit might not quite work as expected on inherited 5651 * XXX event_limit might not quite work as expected on inherited
5655 * events 5652 * events
5656 */ 5653 */
5657 5654
5658 event->pending_kill = POLL_IN; 5655 event->pending_kill = POLL_IN;
5659 if (events && atomic_dec_and_test(&event->event_limit)) { 5656 if (events && atomic_dec_and_test(&event->event_limit)) {
5660 ret = 1; 5657 ret = 1;
5661 event->pending_kill = POLL_HUP; 5658 event->pending_kill = POLL_HUP;
5662 event->pending_disable = 1; 5659 event->pending_disable = 1;
5663 irq_work_queue(&event->pending); 5660 irq_work_queue(&event->pending);
5664 } 5661 }
5665 5662
5666 if (event->overflow_handler) 5663 if (event->overflow_handler)
5667 event->overflow_handler(event, data, regs); 5664 event->overflow_handler(event, data, regs);
5668 else 5665 else
5669 perf_event_output(event, data, regs); 5666 perf_event_output(event, data, regs);
5670 5667
5671 if (event->fasync && event->pending_kill) { 5668 if (event->fasync && event->pending_kill) {
5672 event->pending_wakeup = 1; 5669 event->pending_wakeup = 1;
5673 irq_work_queue(&event->pending); 5670 irq_work_queue(&event->pending);
5674 } 5671 }
5675 5672
5676 return ret; 5673 return ret;
5677 } 5674 }
5678 5675
5679 int perf_event_overflow(struct perf_event *event, 5676 int perf_event_overflow(struct perf_event *event,
5680 struct perf_sample_data *data, 5677 struct perf_sample_data *data,
5681 struct pt_regs *regs) 5678 struct pt_regs *regs)
5682 { 5679 {
5683 return __perf_event_overflow(event, 1, data, regs); 5680 return __perf_event_overflow(event, 1, data, regs);
5684 } 5681 }
5685 5682
5686 /* 5683 /*
5687 * Generic software event infrastructure 5684 * Generic software event infrastructure
5688 */ 5685 */
5689 5686
5690 struct swevent_htable { 5687 struct swevent_htable {
5691 struct swevent_hlist *swevent_hlist; 5688 struct swevent_hlist *swevent_hlist;
5692 struct mutex hlist_mutex; 5689 struct mutex hlist_mutex;
5693 int hlist_refcount; 5690 int hlist_refcount;
5694 5691
5695 /* Recursion avoidance in each contexts */ 5692 /* Recursion avoidance in each contexts */
5696 int recursion[PERF_NR_CONTEXTS]; 5693 int recursion[PERF_NR_CONTEXTS];
5697 5694
5698 /* Keeps track of cpu being initialized/exited */ 5695 /* Keeps track of cpu being initialized/exited */
5699 bool online; 5696 bool online;
5700 }; 5697 };
5701 5698
5702 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 5699 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
5703 5700
5704 /* 5701 /*
5705 * We directly increment event->count and keep a second value in 5702 * We directly increment event->count and keep a second value in
5706 * event->hw.period_left to count intervals. This period event 5703 * event->hw.period_left to count intervals. This period event
5707 * is kept in the range [-sample_period, 0] so that we can use the 5704 * is kept in the range [-sample_period, 0] so that we can use the
5708 * sign as trigger. 5705 * sign as trigger.
5709 */ 5706 */
5710 5707
5711 u64 perf_swevent_set_period(struct perf_event *event) 5708 u64 perf_swevent_set_period(struct perf_event *event)
5712 { 5709 {
5713 struct hw_perf_event *hwc = &event->hw; 5710 struct hw_perf_event *hwc = &event->hw;
5714 u64 period = hwc->last_period; 5711 u64 period = hwc->last_period;
5715 u64 nr, offset; 5712 u64 nr, offset;
5716 s64 old, val; 5713 s64 old, val;
5717 5714
5718 hwc->last_period = hwc->sample_period; 5715 hwc->last_period = hwc->sample_period;
5719 5716
5720 again: 5717 again:
5721 old = val = local64_read(&hwc->period_left); 5718 old = val = local64_read(&hwc->period_left);
5722 if (val < 0) 5719 if (val < 0)
5723 return 0; 5720 return 0;
5724 5721
5725 nr = div64_u64(period + val, period); 5722 nr = div64_u64(period + val, period);
5726 offset = nr * period; 5723 offset = nr * period;
5727 val -= offset; 5724 val -= offset;
5728 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 5725 if (local64_cmpxchg(&hwc->period_left, old, val) != old)
5729 goto again; 5726 goto again;
5730 5727
5731 return nr; 5728 return nr;
5732 } 5729 }
5733 5730
5734 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 5731 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
5735 struct perf_sample_data *data, 5732 struct perf_sample_data *data,
5736 struct pt_regs *regs) 5733 struct pt_regs *regs)
5737 { 5734 {
5738 struct hw_perf_event *hwc = &event->hw; 5735 struct hw_perf_event *hwc = &event->hw;
5739 int throttle = 0; 5736 int throttle = 0;
5740 5737
5741 if (!overflow) 5738 if (!overflow)
5742 overflow = perf_swevent_set_period(event); 5739 overflow = perf_swevent_set_period(event);
5743 5740
5744 if (hwc->interrupts == MAX_INTERRUPTS) 5741 if (hwc->interrupts == MAX_INTERRUPTS)
5745 return; 5742 return;
5746 5743
5747 for (; overflow; overflow--) { 5744 for (; overflow; overflow--) {
5748 if (__perf_event_overflow(event, throttle, 5745 if (__perf_event_overflow(event, throttle,
5749 data, regs)) { 5746 data, regs)) {
5750 /* 5747 /*
5751 * We inhibit the overflow from happening when 5748 * We inhibit the overflow from happening when
5752 * hwc->interrupts == MAX_INTERRUPTS. 5749 * hwc->interrupts == MAX_INTERRUPTS.
5753 */ 5750 */
5754 break; 5751 break;
5755 } 5752 }
5756 throttle = 1; 5753 throttle = 1;
5757 } 5754 }
5758 } 5755 }
5759 5756
5760 static void perf_swevent_event(struct perf_event *event, u64 nr, 5757 static void perf_swevent_event(struct perf_event *event, u64 nr,
5761 struct perf_sample_data *data, 5758 struct perf_sample_data *data,
5762 struct pt_regs *regs) 5759 struct pt_regs *regs)
5763 { 5760 {
5764 struct hw_perf_event *hwc = &event->hw; 5761 struct hw_perf_event *hwc = &event->hw;
5765 5762
5766 local64_add(nr, &event->count); 5763 local64_add(nr, &event->count);
5767 5764
5768 if (!regs) 5765 if (!regs)
5769 return; 5766 return;
5770 5767
5771 if (!is_sampling_event(event)) 5768 if (!is_sampling_event(event))
5772 return; 5769 return;
5773 5770
5774 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 5771 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
5775 data->period = nr; 5772 data->period = nr;
5776 return perf_swevent_overflow(event, 1, data, regs); 5773 return perf_swevent_overflow(event, 1, data, regs);
5777 } else 5774 } else
5778 data->period = event->hw.last_period; 5775 data->period = event->hw.last_period;
5779 5776
5780 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 5777 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
5781 return perf_swevent_overflow(event, 1, data, regs); 5778 return perf_swevent_overflow(event, 1, data, regs);
5782 5779
5783 if (local64_add_negative(nr, &hwc->period_left)) 5780 if (local64_add_negative(nr, &hwc->period_left))
5784 return; 5781 return;
5785 5782
5786 perf_swevent_overflow(event, 0, data, regs); 5783 perf_swevent_overflow(event, 0, data, regs);
5787 } 5784 }
5788 5785
5789 static int perf_exclude_event(struct perf_event *event, 5786 static int perf_exclude_event(struct perf_event *event,
5790 struct pt_regs *regs) 5787 struct pt_regs *regs)
5791 { 5788 {
5792 if (event->hw.state & PERF_HES_STOPPED) 5789 if (event->hw.state & PERF_HES_STOPPED)
5793 return 1; 5790 return 1;
5794 5791
5795 if (regs) { 5792 if (regs) {
5796 if (event->attr.exclude_user && user_mode(regs)) 5793 if (event->attr.exclude_user && user_mode(regs))
5797 return 1; 5794 return 1;
5798 5795
5799 if (event->attr.exclude_kernel && !user_mode(regs)) 5796 if (event->attr.exclude_kernel && !user_mode(regs))
5800 return 1; 5797 return 1;
5801 } 5798 }
5802 5799
5803 return 0; 5800 return 0;
5804 } 5801 }
5805 5802
5806 static int perf_swevent_match(struct perf_event *event, 5803 static int perf_swevent_match(struct perf_event *event,
5807 enum perf_type_id type, 5804 enum perf_type_id type,
5808 u32 event_id, 5805 u32 event_id,
5809 struct perf_sample_data *data, 5806 struct perf_sample_data *data,
5810 struct pt_regs *regs) 5807 struct pt_regs *regs)
5811 { 5808 {
5812 if (event->attr.type != type) 5809 if (event->attr.type != type)
5813 return 0; 5810 return 0;
5814 5811
5815 if (event->attr.config != event_id) 5812 if (event->attr.config != event_id)
5816 return 0; 5813 return 0;
5817 5814
5818 if (perf_exclude_event(event, regs)) 5815 if (perf_exclude_event(event, regs))
5819 return 0; 5816 return 0;
5820 5817
5821 return 1; 5818 return 1;
5822 } 5819 }
5823 5820
5824 static inline u64 swevent_hash(u64 type, u32 event_id) 5821 static inline u64 swevent_hash(u64 type, u32 event_id)
5825 { 5822 {
5826 u64 val = event_id | (type << 32); 5823 u64 val = event_id | (type << 32);
5827 5824
5828 return hash_64(val, SWEVENT_HLIST_BITS); 5825 return hash_64(val, SWEVENT_HLIST_BITS);
5829 } 5826 }
5830 5827
5831 static inline struct hlist_head * 5828 static inline struct hlist_head *
5832 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 5829 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
5833 { 5830 {
5834 u64 hash = swevent_hash(type, event_id); 5831 u64 hash = swevent_hash(type, event_id);
5835 5832
5836 return &hlist->heads[hash]; 5833 return &hlist->heads[hash];
5837 } 5834 }
5838 5835
5839 /* For the read side: events when they trigger */ 5836 /* For the read side: events when they trigger */
5840 static inline struct hlist_head * 5837 static inline struct hlist_head *
5841 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 5838 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
5842 { 5839 {
5843 struct swevent_hlist *hlist; 5840 struct swevent_hlist *hlist;
5844 5841
5845 hlist = rcu_dereference(swhash->swevent_hlist); 5842 hlist = rcu_dereference(swhash->swevent_hlist);
5846 if (!hlist) 5843 if (!hlist)
5847 return NULL; 5844 return NULL;
5848 5845
5849 return __find_swevent_head(hlist, type, event_id); 5846 return __find_swevent_head(hlist, type, event_id);
5850 } 5847 }
5851 5848
5852 /* For the event head insertion and removal in the hlist */ 5849 /* For the event head insertion and removal in the hlist */
5853 static inline struct hlist_head * 5850 static inline struct hlist_head *
5854 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 5851 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5855 { 5852 {
5856 struct swevent_hlist *hlist; 5853 struct swevent_hlist *hlist;
5857 u32 event_id = event->attr.config; 5854 u32 event_id = event->attr.config;
5858 u64 type = event->attr.type; 5855 u64 type = event->attr.type;
5859 5856
5860 /* 5857 /*
5861 * Event scheduling is always serialized against hlist allocation 5858 * Event scheduling is always serialized against hlist allocation
5862 * and release. Which makes the protected version suitable here. 5859 * and release. Which makes the protected version suitable here.
5863 * The context lock guarantees that. 5860 * The context lock guarantees that.
5864 */ 5861 */
5865 hlist = rcu_dereference_protected(swhash->swevent_hlist, 5862 hlist = rcu_dereference_protected(swhash->swevent_hlist,
5866 lockdep_is_held(&event->ctx->lock)); 5863 lockdep_is_held(&event->ctx->lock));
5867 if (!hlist) 5864 if (!hlist)
5868 return NULL; 5865 return NULL;
5869 5866
5870 return __find_swevent_head(hlist, type, event_id); 5867 return __find_swevent_head(hlist, type, event_id);
5871 } 5868 }
5872 5869
5873 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 5870 static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5874 u64 nr, 5871 u64 nr,
5875 struct perf_sample_data *data, 5872 struct perf_sample_data *data,
5876 struct pt_regs *regs) 5873 struct pt_regs *regs)
5877 { 5874 {
5878 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 5875 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5879 struct perf_event *event; 5876 struct perf_event *event;
5880 struct hlist_head *head; 5877 struct hlist_head *head;
5881 5878
5882 rcu_read_lock(); 5879 rcu_read_lock();
5883 head = find_swevent_head_rcu(swhash, type, event_id); 5880 head = find_swevent_head_rcu(swhash, type, event_id);
5884 if (!head) 5881 if (!head)
5885 goto end; 5882 goto end;
5886 5883
5887 hlist_for_each_entry_rcu(event, head, hlist_entry) { 5884 hlist_for_each_entry_rcu(event, head, hlist_entry) {
5888 if (perf_swevent_match(event, type, event_id, data, regs)) 5885 if (perf_swevent_match(event, type, event_id, data, regs))
5889 perf_swevent_event(event, nr, data, regs); 5886 perf_swevent_event(event, nr, data, regs);
5890 } 5887 }
5891 end: 5888 end:
5892 rcu_read_unlock(); 5889 rcu_read_unlock();
5893 } 5890 }
5894 5891
5895 int perf_swevent_get_recursion_context(void) 5892 int perf_swevent_get_recursion_context(void)
5896 { 5893 {
5897 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 5894 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5898 5895
5899 return get_recursion_context(swhash->recursion); 5896 return get_recursion_context(swhash->recursion);
5900 } 5897 }
5901 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 5898 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
5902 5899
5903 inline void perf_swevent_put_recursion_context(int rctx) 5900 inline void perf_swevent_put_recursion_context(int rctx)
5904 { 5901 {
5905 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 5902 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5906 5903
5907 put_recursion_context(swhash->recursion, rctx); 5904 put_recursion_context(swhash->recursion, rctx);
5908 } 5905 }
5909 5906
5910 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 5907 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
5911 { 5908 {
5912 struct perf_sample_data data; 5909 struct perf_sample_data data;
5913 int rctx; 5910 int rctx;
5914 5911
5915 preempt_disable_notrace(); 5912 preempt_disable_notrace();
5916 rctx = perf_swevent_get_recursion_context(); 5913 rctx = perf_swevent_get_recursion_context();
5917 if (rctx < 0) 5914 if (rctx < 0)
5918 return; 5915 return;
5919 5916
5920 perf_sample_data_init(&data, addr, 0); 5917 perf_sample_data_init(&data, addr, 0);
5921 5918
5922 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 5919 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
5923 5920
5924 perf_swevent_put_recursion_context(rctx); 5921 perf_swevent_put_recursion_context(rctx);
5925 preempt_enable_notrace(); 5922 preempt_enable_notrace();
5926 } 5923 }
5927 5924
5928 static void perf_swevent_read(struct perf_event *event) 5925 static void perf_swevent_read(struct perf_event *event)
5929 { 5926 {
5930 } 5927 }
5931 5928
5932 static int perf_swevent_add(struct perf_event *event, int flags) 5929 static int perf_swevent_add(struct perf_event *event, int flags)
5933 { 5930 {
5934 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 5931 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5935 struct hw_perf_event *hwc = &event->hw; 5932 struct hw_perf_event *hwc = &event->hw;
5936 struct hlist_head *head; 5933 struct hlist_head *head;
5937 5934
5938 if (is_sampling_event(event)) { 5935 if (is_sampling_event(event)) {
5939 hwc->last_period = hwc->sample_period; 5936 hwc->last_period = hwc->sample_period;
5940 perf_swevent_set_period(event); 5937 perf_swevent_set_period(event);
5941 } 5938 }
5942 5939
5943 hwc->state = !(flags & PERF_EF_START); 5940 hwc->state = !(flags & PERF_EF_START);
5944 5941
5945 head = find_swevent_head(swhash, event); 5942 head = find_swevent_head(swhash, event);
5946 if (!head) { 5943 if (!head) {
5947 /* 5944 /*
5948 * We can race with cpu hotplug code. Do not 5945 * We can race with cpu hotplug code. Do not
5949 * WARN if the cpu just got unplugged. 5946 * WARN if the cpu just got unplugged.
5950 */ 5947 */
5951 WARN_ON_ONCE(swhash->online); 5948 WARN_ON_ONCE(swhash->online);
5952 return -EINVAL; 5949 return -EINVAL;
5953 } 5950 }
5954 5951
5955 hlist_add_head_rcu(&event->hlist_entry, head); 5952 hlist_add_head_rcu(&event->hlist_entry, head);
5956 5953
5957 return 0; 5954 return 0;
5958 } 5955 }
5959 5956
5960 static void perf_swevent_del(struct perf_event *event, int flags) 5957 static void perf_swevent_del(struct perf_event *event, int flags)
5961 { 5958 {
5962 hlist_del_rcu(&event->hlist_entry); 5959 hlist_del_rcu(&event->hlist_entry);
5963 } 5960 }
5964 5961
5965 static void perf_swevent_start(struct perf_event *event, int flags) 5962 static void perf_swevent_start(struct perf_event *event, int flags)
5966 { 5963 {
5967 event->hw.state = 0; 5964 event->hw.state = 0;
5968 } 5965 }
5969 5966
5970 static void perf_swevent_stop(struct perf_event *event, int flags) 5967 static void perf_swevent_stop(struct perf_event *event, int flags)
5971 { 5968 {
5972 event->hw.state = PERF_HES_STOPPED; 5969 event->hw.state = PERF_HES_STOPPED;
5973 } 5970 }
5974 5971
5975 /* Deref the hlist from the update side */ 5972 /* Deref the hlist from the update side */
5976 static inline struct swevent_hlist * 5973 static inline struct swevent_hlist *
5977 swevent_hlist_deref(struct swevent_htable *swhash) 5974 swevent_hlist_deref(struct swevent_htable *swhash)
5978 { 5975 {
5979 return rcu_dereference_protected(swhash->swevent_hlist, 5976 return rcu_dereference_protected(swhash->swevent_hlist,
5980 lockdep_is_held(&swhash->hlist_mutex)); 5977 lockdep_is_held(&swhash->hlist_mutex));
5981 } 5978 }
5982 5979
5983 static void swevent_hlist_release(struct swevent_htable *swhash) 5980 static void swevent_hlist_release(struct swevent_htable *swhash)
5984 { 5981 {
5985 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 5982 struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5986 5983
5987 if (!hlist) 5984 if (!hlist)
5988 return; 5985 return;
5989 5986
5990 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 5987 RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
5991 kfree_rcu(hlist, rcu_head); 5988 kfree_rcu(hlist, rcu_head);
5992 } 5989 }
5993 5990
5994 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu) 5991 static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
5995 { 5992 {
5996 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 5993 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
5997 5994
5998 mutex_lock(&swhash->hlist_mutex); 5995 mutex_lock(&swhash->hlist_mutex);
5999 5996
6000 if (!--swhash->hlist_refcount) 5997 if (!--swhash->hlist_refcount)
6001 swevent_hlist_release(swhash); 5998 swevent_hlist_release(swhash);
6002 5999
6003 mutex_unlock(&swhash->hlist_mutex); 6000 mutex_unlock(&swhash->hlist_mutex);
6004 } 6001 }
6005 6002
6006 static void swevent_hlist_put(struct perf_event *event) 6003 static void swevent_hlist_put(struct perf_event *event)
6007 { 6004 {
6008 int cpu; 6005 int cpu;
6009 6006
6010 for_each_possible_cpu(cpu) 6007 for_each_possible_cpu(cpu)
6011 swevent_hlist_put_cpu(event, cpu); 6008 swevent_hlist_put_cpu(event, cpu);
6012 } 6009 }
6013 6010
6014 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu) 6011 static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
6015 { 6012 {
6016 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 6013 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6017 int err = 0; 6014 int err = 0;
6018 6015
6019 mutex_lock(&swhash->hlist_mutex); 6016 mutex_lock(&swhash->hlist_mutex);
6020 6017
6021 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { 6018 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
6022 struct swevent_hlist *hlist; 6019 struct swevent_hlist *hlist;
6023 6020
6024 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 6021 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
6025 if (!hlist) { 6022 if (!hlist) {
6026 err = -ENOMEM; 6023 err = -ENOMEM;
6027 goto exit; 6024 goto exit;
6028 } 6025 }
6029 rcu_assign_pointer(swhash->swevent_hlist, hlist); 6026 rcu_assign_pointer(swhash->swevent_hlist, hlist);
6030 } 6027 }
6031 swhash->hlist_refcount++; 6028 swhash->hlist_refcount++;
6032 exit: 6029 exit:
6033 mutex_unlock(&swhash->hlist_mutex); 6030 mutex_unlock(&swhash->hlist_mutex);
6034 6031
6035 return err; 6032 return err;
6036 } 6033 }
6037 6034
6038 static int swevent_hlist_get(struct perf_event *event) 6035 static int swevent_hlist_get(struct perf_event *event)
6039 { 6036 {
6040 int err; 6037 int err;
6041 int cpu, failed_cpu; 6038 int cpu, failed_cpu;
6042 6039
6043 get_online_cpus(); 6040 get_online_cpus();
6044 for_each_possible_cpu(cpu) { 6041 for_each_possible_cpu(cpu) {
6045 err = swevent_hlist_get_cpu(event, cpu); 6042 err = swevent_hlist_get_cpu(event, cpu);
6046 if (err) { 6043 if (err) {
6047 failed_cpu = cpu; 6044 failed_cpu = cpu;
6048 goto fail; 6045 goto fail;
6049 } 6046 }
6050 } 6047 }
6051 put_online_cpus(); 6048 put_online_cpus();
6052 6049
6053 return 0; 6050 return 0;
6054 fail: 6051 fail:
6055 for_each_possible_cpu(cpu) { 6052 for_each_possible_cpu(cpu) {
6056 if (cpu == failed_cpu) 6053 if (cpu == failed_cpu)
6057 break; 6054 break;
6058 swevent_hlist_put_cpu(event, cpu); 6055 swevent_hlist_put_cpu(event, cpu);
6059 } 6056 }
6060 6057
6061 put_online_cpus(); 6058 put_online_cpus();
6062 return err; 6059 return err;
6063 } 6060 }
6064 6061
6065 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 6062 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
6066 6063
6067 static void sw_perf_event_destroy(struct perf_event *event) 6064 static void sw_perf_event_destroy(struct perf_event *event)
6068 { 6065 {
6069 u64 event_id = event->attr.config; 6066 u64 event_id = event->attr.config;
6070 6067
6071 WARN_ON(event->parent); 6068 WARN_ON(event->parent);
6072 6069
6073 static_key_slow_dec(&perf_swevent_enabled[event_id]); 6070 static_key_slow_dec(&perf_swevent_enabled[event_id]);
6074 swevent_hlist_put(event); 6071 swevent_hlist_put(event);
6075 } 6072 }
6076 6073
6077 static int perf_swevent_init(struct perf_event *event) 6074 static int perf_swevent_init(struct perf_event *event)
6078 { 6075 {
6079 u64 event_id = event->attr.config; 6076 u64 event_id = event->attr.config;
6080 6077
6081 if (event->attr.type != PERF_TYPE_SOFTWARE) 6078 if (event->attr.type != PERF_TYPE_SOFTWARE)
6082 return -ENOENT; 6079 return -ENOENT;
6083 6080
6084 /* 6081 /*
6085 * no branch sampling for software events 6082 * no branch sampling for software events
6086 */ 6083 */
6087 if (has_branch_stack(event)) 6084 if (has_branch_stack(event))
6088 return -EOPNOTSUPP; 6085 return -EOPNOTSUPP;
6089 6086
6090 switch (event_id) { 6087 switch (event_id) {
6091 case PERF_COUNT_SW_CPU_CLOCK: 6088 case PERF_COUNT_SW_CPU_CLOCK:
6092 case PERF_COUNT_SW_TASK_CLOCK: 6089 case PERF_COUNT_SW_TASK_CLOCK:
6093 return -ENOENT; 6090 return -ENOENT;
6094 6091
6095 default: 6092 default:
6096 break; 6093 break;
6097 } 6094 }
6098 6095
6099 if (event_id >= PERF_COUNT_SW_MAX) 6096 if (event_id >= PERF_COUNT_SW_MAX)
6100 return -ENOENT; 6097 return -ENOENT;
6101 6098
6102 if (!event->parent) { 6099 if (!event->parent) {
6103 int err; 6100 int err;
6104 6101
6105 err = swevent_hlist_get(event); 6102 err = swevent_hlist_get(event);
6106 if (err) 6103 if (err)
6107 return err; 6104 return err;
6108 6105
6109 static_key_slow_inc(&perf_swevent_enabled[event_id]); 6106 static_key_slow_inc(&perf_swevent_enabled[event_id]);
6110 event->destroy = sw_perf_event_destroy; 6107 event->destroy = sw_perf_event_destroy;
6111 } 6108 }
6112 6109
6113 return 0; 6110 return 0;
6114 } 6111 }
6115 6112
6116 static struct pmu perf_swevent = { 6113 static struct pmu perf_swevent = {
6117 .task_ctx_nr = perf_sw_context, 6114 .task_ctx_nr = perf_sw_context,
6118 6115
6119 .event_init = perf_swevent_init, 6116 .event_init = perf_swevent_init,
6120 .add = perf_swevent_add, 6117 .add = perf_swevent_add,
6121 .del = perf_swevent_del, 6118 .del = perf_swevent_del,
6122 .start = perf_swevent_start, 6119 .start = perf_swevent_start,
6123 .stop = perf_swevent_stop, 6120 .stop = perf_swevent_stop,
6124 .read = perf_swevent_read, 6121 .read = perf_swevent_read,
6125 }; 6122 };
6126 6123
6127 #ifdef CONFIG_EVENT_TRACING 6124 #ifdef CONFIG_EVENT_TRACING
6128 6125
6129 static int perf_tp_filter_match(struct perf_event *event, 6126 static int perf_tp_filter_match(struct perf_event *event,
6130 struct perf_sample_data *data) 6127 struct perf_sample_data *data)
6131 { 6128 {
6132 void *record = data->raw->data; 6129 void *record = data->raw->data;
6133 6130
6134 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 6131 if (likely(!event->filter) || filter_match_preds(event->filter, record))
6135 return 1; 6132 return 1;
6136 return 0; 6133 return 0;
6137 } 6134 }
6138 6135
6139 static int perf_tp_event_match(struct perf_event *event, 6136 static int perf_tp_event_match(struct perf_event *event,
6140 struct perf_sample_data *data, 6137 struct perf_sample_data *data,
6141 struct pt_regs *regs) 6138 struct pt_regs *regs)
6142 { 6139 {
6143 if (event->hw.state & PERF_HES_STOPPED) 6140 if (event->hw.state & PERF_HES_STOPPED)
6144 return 0; 6141 return 0;
6145 /* 6142 /*
6146 * All tracepoints are from kernel-space. 6143 * All tracepoints are from kernel-space.
6147 */ 6144 */
6148 if (event->attr.exclude_kernel) 6145 if (event->attr.exclude_kernel)
6149 return 0; 6146 return 0;
6150 6147
6151 if (!perf_tp_filter_match(event, data)) 6148 if (!perf_tp_filter_match(event, data))
6152 return 0; 6149 return 0;
6153 6150
6154 return 1; 6151 return 1;
6155 } 6152 }
6156 6153
6157 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, 6154 void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
6158 struct pt_regs *regs, struct hlist_head *head, int rctx, 6155 struct pt_regs *regs, struct hlist_head *head, int rctx,
6159 struct task_struct *task) 6156 struct task_struct *task)
6160 { 6157 {
6161 struct perf_sample_data data; 6158 struct perf_sample_data data;
6162 struct perf_event *event; 6159 struct perf_event *event;
6163 6160
6164 struct perf_raw_record raw = { 6161 struct perf_raw_record raw = {
6165 .size = entry_size, 6162 .size = entry_size,
6166 .data = record, 6163 .data = record,
6167 }; 6164 };
6168 6165
6169 perf_sample_data_init(&data, addr, 0); 6166 perf_sample_data_init(&data, addr, 0);
6170 data.raw = &raw; 6167 data.raw = &raw;
6171 6168
6172 hlist_for_each_entry_rcu(event, head, hlist_entry) { 6169 hlist_for_each_entry_rcu(event, head, hlist_entry) {
6173 if (perf_tp_event_match(event, &data, regs)) 6170 if (perf_tp_event_match(event, &data, regs))
6174 perf_swevent_event(event, count, &data, regs); 6171 perf_swevent_event(event, count, &data, regs);
6175 } 6172 }
6176 6173
6177 /* 6174 /*
6178 * If we got specified a target task, also iterate its context and 6175 * If we got specified a target task, also iterate its context and
6179 * deliver this event there too. 6176 * deliver this event there too.
6180 */ 6177 */
6181 if (task && task != current) { 6178 if (task && task != current) {
6182 struct perf_event_context *ctx; 6179 struct perf_event_context *ctx;
6183 struct trace_entry *entry = record; 6180 struct trace_entry *entry = record;
6184 6181
6185 rcu_read_lock(); 6182 rcu_read_lock();
6186 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 6183 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
6187 if (!ctx) 6184 if (!ctx)
6188 goto unlock; 6185 goto unlock;
6189 6186
6190 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 6187 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
6191 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6188 if (event->attr.type != PERF_TYPE_TRACEPOINT)
6192 continue; 6189 continue;
6193 if (event->attr.config != entry->type) 6190 if (event->attr.config != entry->type)
6194 continue; 6191 continue;
6195 if (perf_tp_event_match(event, &data, regs)) 6192 if (perf_tp_event_match(event, &data, regs))
6196 perf_swevent_event(event, count, &data, regs); 6193 perf_swevent_event(event, count, &data, regs);
6197 } 6194 }
6198 unlock: 6195 unlock:
6199 rcu_read_unlock(); 6196 rcu_read_unlock();
6200 } 6197 }
6201 6198
6202 perf_swevent_put_recursion_context(rctx); 6199 perf_swevent_put_recursion_context(rctx);
6203 } 6200 }
6204 EXPORT_SYMBOL_GPL(perf_tp_event); 6201 EXPORT_SYMBOL_GPL(perf_tp_event);
6205 6202
6206 static void tp_perf_event_destroy(struct perf_event *event) 6203 static void tp_perf_event_destroy(struct perf_event *event)
6207 { 6204 {
6208 perf_trace_destroy(event); 6205 perf_trace_destroy(event);
6209 } 6206 }
6210 6207
6211 static int perf_tp_event_init(struct perf_event *event) 6208 static int perf_tp_event_init(struct perf_event *event)
6212 { 6209 {
6213 int err; 6210 int err;
6214 6211
6215 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6212 if (event->attr.type != PERF_TYPE_TRACEPOINT)
6216 return -ENOENT; 6213 return -ENOENT;
6217 6214
6218 /* 6215 /*
6219 * no branch sampling for tracepoint events 6216 * no branch sampling for tracepoint events
6220 */ 6217 */
6221 if (has_branch_stack(event)) 6218 if (has_branch_stack(event))
6222 return -EOPNOTSUPP; 6219 return -EOPNOTSUPP;
6223 6220
6224 err = perf_trace_init(event); 6221 err = perf_trace_init(event);
6225 if (err) 6222 if (err)
6226 return err; 6223 return err;
6227 6224
6228 event->destroy = tp_perf_event_destroy; 6225 event->destroy = tp_perf_event_destroy;
6229 6226
6230 return 0; 6227 return 0;
6231 } 6228 }
6232 6229
6233 static struct pmu perf_tracepoint = { 6230 static struct pmu perf_tracepoint = {
6234 .task_ctx_nr = perf_sw_context, 6231 .task_ctx_nr = perf_sw_context,
6235 6232
6236 .event_init = perf_tp_event_init, 6233 .event_init = perf_tp_event_init,
6237 .add = perf_trace_add, 6234 .add = perf_trace_add,
6238 .del = perf_trace_del, 6235 .del = perf_trace_del,
6239 .start = perf_swevent_start, 6236 .start = perf_swevent_start,
6240 .stop = perf_swevent_stop, 6237 .stop = perf_swevent_stop,
6241 .read = perf_swevent_read, 6238 .read = perf_swevent_read,
6242 }; 6239 };
6243 6240
6244 static inline void perf_tp_register(void) 6241 static inline void perf_tp_register(void)
6245 { 6242 {
6246 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 6243 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
6247 } 6244 }
6248 6245
6249 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 6246 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6250 { 6247 {
6251 char *filter_str; 6248 char *filter_str;
6252 int ret; 6249 int ret;
6253 6250
6254 if (event->attr.type != PERF_TYPE_TRACEPOINT) 6251 if (event->attr.type != PERF_TYPE_TRACEPOINT)
6255 return -EINVAL; 6252 return -EINVAL;
6256 6253
6257 filter_str = strndup_user(arg, PAGE_SIZE); 6254 filter_str = strndup_user(arg, PAGE_SIZE);
6258 if (IS_ERR(filter_str)) 6255 if (IS_ERR(filter_str))
6259 return PTR_ERR(filter_str); 6256 return PTR_ERR(filter_str);
6260 6257
6261 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); 6258 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
6262 6259
6263 kfree(filter_str); 6260 kfree(filter_str);
6264 return ret; 6261 return ret;
6265 } 6262 }
6266 6263
6267 static void perf_event_free_filter(struct perf_event *event) 6264 static void perf_event_free_filter(struct perf_event *event)
6268 { 6265 {
6269 ftrace_profile_free_filter(event); 6266 ftrace_profile_free_filter(event);
6270 } 6267 }
6271 6268
6272 #else 6269 #else
6273 6270
6274 static inline void perf_tp_register(void) 6271 static inline void perf_tp_register(void)
6275 { 6272 {
6276 } 6273 }
6277 6274
6278 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 6275 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6279 { 6276 {
6280 return -ENOENT; 6277 return -ENOENT;
6281 } 6278 }
6282 6279
6283 static void perf_event_free_filter(struct perf_event *event) 6280 static void perf_event_free_filter(struct perf_event *event)
6284 { 6281 {
6285 } 6282 }
6286 6283
6287 #endif /* CONFIG_EVENT_TRACING */ 6284 #endif /* CONFIG_EVENT_TRACING */
6288 6285
6289 #ifdef CONFIG_HAVE_HW_BREAKPOINT 6286 #ifdef CONFIG_HAVE_HW_BREAKPOINT
6290 void perf_bp_event(struct perf_event *bp, void *data) 6287 void perf_bp_event(struct perf_event *bp, void *data)
6291 { 6288 {
6292 struct perf_sample_data sample; 6289 struct perf_sample_data sample;
6293 struct pt_regs *regs = data; 6290 struct pt_regs *regs = data;
6294 6291
6295 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 6292 perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
6296 6293
6297 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 6294 if (!bp->hw.state && !perf_exclude_event(bp, regs))
6298 perf_swevent_event(bp, 1, &sample, regs); 6295 perf_swevent_event(bp, 1, &sample, regs);
6299 } 6296 }
6300 #endif 6297 #endif
6301 6298
6302 /* 6299 /*
6303 * hrtimer based swevent callback 6300 * hrtimer based swevent callback
6304 */ 6301 */
6305 6302
6306 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 6303 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
6307 { 6304 {
6308 enum hrtimer_restart ret = HRTIMER_RESTART; 6305 enum hrtimer_restart ret = HRTIMER_RESTART;
6309 struct perf_sample_data data; 6306 struct perf_sample_data data;
6310 struct pt_regs *regs; 6307 struct pt_regs *regs;
6311 struct perf_event *event; 6308 struct perf_event *event;
6312 u64 period; 6309 u64 period;
6313 6310
6314 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 6311 event = container_of(hrtimer, struct perf_event, hw.hrtimer);
6315 6312
6316 if (event->state != PERF_EVENT_STATE_ACTIVE) 6313 if (event->state != PERF_EVENT_STATE_ACTIVE)
6317 return HRTIMER_NORESTART; 6314 return HRTIMER_NORESTART;
6318 6315
6319 event->pmu->read(event); 6316 event->pmu->read(event);
6320 6317
6321 perf_sample_data_init(&data, 0, event->hw.last_period); 6318 perf_sample_data_init(&data, 0, event->hw.last_period);
6322 regs = get_irq_regs(); 6319 regs = get_irq_regs();
6323 6320
6324 if (regs && !perf_exclude_event(event, regs)) { 6321 if (regs && !perf_exclude_event(event, regs)) {
6325 if (!(event->attr.exclude_idle && is_idle_task(current))) 6322 if (!(event->attr.exclude_idle && is_idle_task(current)))
6326 if (__perf_event_overflow(event, 1, &data, regs)) 6323 if (__perf_event_overflow(event, 1, &data, regs))
6327 ret = HRTIMER_NORESTART; 6324 ret = HRTIMER_NORESTART;
6328 } 6325 }
6329 6326
6330 period = max_t(u64, 10000, event->hw.sample_period); 6327 period = max_t(u64, 10000, event->hw.sample_period);
6331 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 6328 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
6332 6329
6333 return ret; 6330 return ret;
6334 } 6331 }
6335 6332
6336 static void perf_swevent_start_hrtimer(struct perf_event *event) 6333 static void perf_swevent_start_hrtimer(struct perf_event *event)
6337 { 6334 {
6338 struct hw_perf_event *hwc = &event->hw; 6335 struct hw_perf_event *hwc = &event->hw;
6339 s64 period; 6336 s64 period;
6340 6337
6341 if (!is_sampling_event(event)) 6338 if (!is_sampling_event(event))
6342 return; 6339 return;
6343 6340
6344 period = local64_read(&hwc->period_left); 6341 period = local64_read(&hwc->period_left);
6345 if (period) { 6342 if (period) {
6346 if (period < 0) 6343 if (period < 0)
6347 period = 10000; 6344 period = 10000;
6348 6345
6349 local64_set(&hwc->period_left, 0); 6346 local64_set(&hwc->period_left, 0);
6350 } else { 6347 } else {
6351 period = max_t(u64, 10000, hwc->sample_period); 6348 period = max_t(u64, 10000, hwc->sample_period);
6352 } 6349 }
6353 __hrtimer_start_range_ns(&hwc->hrtimer, 6350 __hrtimer_start_range_ns(&hwc->hrtimer,
6354 ns_to_ktime(period), 0, 6351 ns_to_ktime(period), 0,
6355 HRTIMER_MODE_REL_PINNED, 0); 6352 HRTIMER_MODE_REL_PINNED, 0);
6356 } 6353 }
6357 6354
6358 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 6355 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
6359 { 6356 {
6360 struct hw_perf_event *hwc = &event->hw; 6357 struct hw_perf_event *hwc = &event->hw;
6361 6358
6362 if (is_sampling_event(event)) { 6359 if (is_sampling_event(event)) {
6363 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 6360 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
6364 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 6361 local64_set(&hwc->period_left, ktime_to_ns(remaining));
6365 6362
6366 hrtimer_cancel(&hwc->hrtimer); 6363 hrtimer_cancel(&hwc->hrtimer);
6367 } 6364 }
6368 } 6365 }
6369 6366
6370 static void perf_swevent_init_hrtimer(struct perf_event *event) 6367 static void perf_swevent_init_hrtimer(struct perf_event *event)
6371 { 6368 {
6372 struct hw_perf_event *hwc = &event->hw; 6369 struct hw_perf_event *hwc = &event->hw;
6373 6370
6374 if (!is_sampling_event(event)) 6371 if (!is_sampling_event(event))
6375 return; 6372 return;
6376 6373
6377 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 6374 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6378 hwc->hrtimer.function = perf_swevent_hrtimer; 6375 hwc->hrtimer.function = perf_swevent_hrtimer;
6379 6376
6380 /* 6377 /*
6381 * Since hrtimers have a fixed rate, we can do a static freq->period 6378 * Since hrtimers have a fixed rate, we can do a static freq->period
6382 * mapping and avoid the whole period adjust feedback stuff. 6379 * mapping and avoid the whole period adjust feedback stuff.
6383 */ 6380 */
6384 if (event->attr.freq) { 6381 if (event->attr.freq) {
6385 long freq = event->attr.sample_freq; 6382 long freq = event->attr.sample_freq;
6386 6383
6387 event->attr.sample_period = NSEC_PER_SEC / freq; 6384 event->attr.sample_period = NSEC_PER_SEC / freq;
6388 hwc->sample_period = event->attr.sample_period; 6385 hwc->sample_period = event->attr.sample_period;
6389 local64_set(&hwc->period_left, hwc->sample_period); 6386 local64_set(&hwc->period_left, hwc->sample_period);
6390 hwc->last_period = hwc->sample_period; 6387 hwc->last_period = hwc->sample_period;
6391 event->attr.freq = 0; 6388 event->attr.freq = 0;
6392 } 6389 }
6393 } 6390 }
6394 6391
6395 /* 6392 /*
6396 * Software event: cpu wall time clock 6393 * Software event: cpu wall time clock
6397 */ 6394 */
6398 6395
6399 static void cpu_clock_event_update(struct perf_event *event) 6396 static void cpu_clock_event_update(struct perf_event *event)
6400 { 6397 {
6401 s64 prev; 6398 s64 prev;
6402 u64 now; 6399 u64 now;
6403 6400
6404 now = local_clock(); 6401 now = local_clock();
6405 prev = local64_xchg(&event->hw.prev_count, now); 6402 prev = local64_xchg(&event->hw.prev_count, now);
6406 local64_add(now - prev, &event->count); 6403 local64_add(now - prev, &event->count);
6407 } 6404 }
6408 6405
6409 static void cpu_clock_event_start(struct perf_event *event, int flags) 6406 static void cpu_clock_event_start(struct perf_event *event, int flags)
6410 { 6407 {
6411 local64_set(&event->hw.prev_count, local_clock()); 6408 local64_set(&event->hw.prev_count, local_clock());
6412 perf_swevent_start_hrtimer(event); 6409 perf_swevent_start_hrtimer(event);
6413 } 6410 }
6414 6411
6415 static void cpu_clock_event_stop(struct perf_event *event, int flags) 6412 static void cpu_clock_event_stop(struct perf_event *event, int flags)
6416 { 6413 {
6417 perf_swevent_cancel_hrtimer(event); 6414 perf_swevent_cancel_hrtimer(event);
6418 cpu_clock_event_update(event); 6415 cpu_clock_event_update(event);
6419 } 6416 }
6420 6417
6421 static int cpu_clock_event_add(struct perf_event *event, int flags) 6418 static int cpu_clock_event_add(struct perf_event *event, int flags)
6422 { 6419 {
6423 if (flags & PERF_EF_START) 6420 if (flags & PERF_EF_START)
6424 cpu_clock_event_start(event, flags); 6421 cpu_clock_event_start(event, flags);
6425 6422
6426 return 0; 6423 return 0;
6427 } 6424 }
6428 6425
6429 static void cpu_clock_event_del(struct perf_event *event, int flags) 6426 static void cpu_clock_event_del(struct perf_event *event, int flags)
6430 { 6427 {
6431 cpu_clock_event_stop(event, flags); 6428 cpu_clock_event_stop(event, flags);
6432 } 6429 }
6433 6430
6434 static void cpu_clock_event_read(struct perf_event *event) 6431 static void cpu_clock_event_read(struct perf_event *event)
6435 { 6432 {
6436 cpu_clock_event_update(event); 6433 cpu_clock_event_update(event);
6437 } 6434 }
6438 6435
6439 static int cpu_clock_event_init(struct perf_event *event) 6436 static int cpu_clock_event_init(struct perf_event *event)
6440 { 6437 {
6441 if (event->attr.type != PERF_TYPE_SOFTWARE) 6438 if (event->attr.type != PERF_TYPE_SOFTWARE)
6442 return -ENOENT; 6439 return -ENOENT;
6443 6440
6444 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 6441 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
6445 return -ENOENT; 6442 return -ENOENT;
6446 6443
6447 /* 6444 /*
6448 * no branch sampling for software events 6445 * no branch sampling for software events
6449 */ 6446 */
6450 if (has_branch_stack(event)) 6447 if (has_branch_stack(event))
6451 return -EOPNOTSUPP; 6448 return -EOPNOTSUPP;
6452 6449
6453 perf_swevent_init_hrtimer(event); 6450 perf_swevent_init_hrtimer(event);
6454 6451
6455 return 0; 6452 return 0;
6456 } 6453 }
6457 6454
6458 static struct pmu perf_cpu_clock = { 6455 static struct pmu perf_cpu_clock = {
6459 .task_ctx_nr = perf_sw_context, 6456 .task_ctx_nr = perf_sw_context,
6460 6457
6461 .event_init = cpu_clock_event_init, 6458 .event_init = cpu_clock_event_init,
6462 .add = cpu_clock_event_add, 6459 .add = cpu_clock_event_add,
6463 .del = cpu_clock_event_del, 6460 .del = cpu_clock_event_del,
6464 .start = cpu_clock_event_start, 6461 .start = cpu_clock_event_start,
6465 .stop = cpu_clock_event_stop, 6462 .stop = cpu_clock_event_stop,
6466 .read = cpu_clock_event_read, 6463 .read = cpu_clock_event_read,
6467 }; 6464 };
6468 6465
6469 /* 6466 /*
6470 * Software event: task time clock 6467 * Software event: task time clock
6471 */ 6468 */
6472 6469
6473 static void task_clock_event_update(struct perf_event *event, u64 now) 6470 static void task_clock_event_update(struct perf_event *event, u64 now)
6474 { 6471 {
6475 u64 prev; 6472 u64 prev;
6476 s64 delta; 6473 s64 delta;
6477 6474
6478 prev = local64_xchg(&event->hw.prev_count, now); 6475 prev = local64_xchg(&event->hw.prev_count, now);
6479 delta = now - prev; 6476 delta = now - prev;
6480 local64_add(delta, &event->count); 6477 local64_add(delta, &event->count);
6481 } 6478 }
6482 6479
6483 static void task_clock_event_start(struct perf_event *event, int flags) 6480 static void task_clock_event_start(struct perf_event *event, int flags)
6484 { 6481 {
6485 local64_set(&event->hw.prev_count, event->ctx->time); 6482 local64_set(&event->hw.prev_count, event->ctx->time);
6486 perf_swevent_start_hrtimer(event); 6483 perf_swevent_start_hrtimer(event);
6487 } 6484 }
6488 6485
6489 static void task_clock_event_stop(struct perf_event *event, int flags) 6486 static void task_clock_event_stop(struct perf_event *event, int flags)
6490 { 6487 {
6491 perf_swevent_cancel_hrtimer(event); 6488 perf_swevent_cancel_hrtimer(event);
6492 task_clock_event_update(event, event->ctx->time); 6489 task_clock_event_update(event, event->ctx->time);
6493 } 6490 }
6494 6491
6495 static int task_clock_event_add(struct perf_event *event, int flags) 6492 static int task_clock_event_add(struct perf_event *event, int flags)
6496 { 6493 {
6497 if (flags & PERF_EF_START) 6494 if (flags & PERF_EF_START)
6498 task_clock_event_start(event, flags); 6495 task_clock_event_start(event, flags);
6499 6496
6500 return 0; 6497 return 0;
6501 } 6498 }
6502 6499
6503 static void task_clock_event_del(struct perf_event *event, int flags) 6500 static void task_clock_event_del(struct perf_event *event, int flags)
6504 { 6501 {
6505 task_clock_event_stop(event, PERF_EF_UPDATE); 6502 task_clock_event_stop(event, PERF_EF_UPDATE);
6506 } 6503 }
6507 6504
6508 static void task_clock_event_read(struct perf_event *event) 6505 static void task_clock_event_read(struct perf_event *event)
6509 { 6506 {
6510 u64 now = perf_clock(); 6507 u64 now = perf_clock();
6511 u64 delta = now - event->ctx->timestamp; 6508 u64 delta = now - event->ctx->timestamp;
6512 u64 time = event->ctx->time + delta; 6509 u64 time = event->ctx->time + delta;
6513 6510
6514 task_clock_event_update(event, time); 6511 task_clock_event_update(event, time);
6515 } 6512 }
6516 6513
6517 static int task_clock_event_init(struct perf_event *event) 6514 static int task_clock_event_init(struct perf_event *event)
6518 { 6515 {
6519 if (event->attr.type != PERF_TYPE_SOFTWARE) 6516 if (event->attr.type != PERF_TYPE_SOFTWARE)
6520 return -ENOENT; 6517 return -ENOENT;
6521 6518
6522 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 6519 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
6523 return -ENOENT; 6520 return -ENOENT;
6524 6521
6525 /* 6522 /*
6526 * no branch sampling for software events 6523 * no branch sampling for software events
6527 */ 6524 */
6528 if (has_branch_stack(event)) 6525 if (has_branch_stack(event))
6529 return -EOPNOTSUPP; 6526 return -EOPNOTSUPP;
6530 6527
6531 perf_swevent_init_hrtimer(event); 6528 perf_swevent_init_hrtimer(event);
6532 6529
6533 return 0; 6530 return 0;
6534 } 6531 }
6535 6532
6536 static struct pmu perf_task_clock = { 6533 static struct pmu perf_task_clock = {
6537 .task_ctx_nr = perf_sw_context, 6534 .task_ctx_nr = perf_sw_context,
6538 6535
6539 .event_init = task_clock_event_init, 6536 .event_init = task_clock_event_init,
6540 .add = task_clock_event_add, 6537 .add = task_clock_event_add,
6541 .del = task_clock_event_del, 6538 .del = task_clock_event_del,
6542 .start = task_clock_event_start, 6539 .start = task_clock_event_start,
6543 .stop = task_clock_event_stop, 6540 .stop = task_clock_event_stop,
6544 .read = task_clock_event_read, 6541 .read = task_clock_event_read,
6545 }; 6542 };
6546 6543
6547 static void perf_pmu_nop_void(struct pmu *pmu) 6544 static void perf_pmu_nop_void(struct pmu *pmu)
6548 { 6545 {
6549 } 6546 }
6550 6547
6551 static int perf_pmu_nop_int(struct pmu *pmu) 6548 static int perf_pmu_nop_int(struct pmu *pmu)
6552 { 6549 {
6553 return 0; 6550 return 0;
6554 } 6551 }
6555 6552
6556 static void perf_pmu_start_txn(struct pmu *pmu) 6553 static void perf_pmu_start_txn(struct pmu *pmu)
6557 { 6554 {
6558 perf_pmu_disable(pmu); 6555 perf_pmu_disable(pmu);
6559 } 6556 }
6560 6557
6561 static int perf_pmu_commit_txn(struct pmu *pmu) 6558 static int perf_pmu_commit_txn(struct pmu *pmu)
6562 { 6559 {
6563 perf_pmu_enable(pmu); 6560 perf_pmu_enable(pmu);
6564 return 0; 6561 return 0;
6565 } 6562 }
6566 6563
6567 static void perf_pmu_cancel_txn(struct pmu *pmu) 6564 static void perf_pmu_cancel_txn(struct pmu *pmu)
6568 { 6565 {
6569 perf_pmu_enable(pmu); 6566 perf_pmu_enable(pmu);
6570 } 6567 }
6571 6568
6572 static int perf_event_idx_default(struct perf_event *event) 6569 static int perf_event_idx_default(struct perf_event *event)
6573 { 6570 {
6574 return 0; 6571 return 0;
6575 } 6572 }
6576 6573
6577 /* 6574 /*
6578 * Ensures all contexts with the same task_ctx_nr have the same 6575 * Ensures all contexts with the same task_ctx_nr have the same
6579 * pmu_cpu_context too. 6576 * pmu_cpu_context too.
6580 */ 6577 */
6581 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 6578 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn)
6582 { 6579 {
6583 struct pmu *pmu; 6580 struct pmu *pmu;
6584 6581
6585 if (ctxn < 0) 6582 if (ctxn < 0)
6586 return NULL; 6583 return NULL;
6587 6584
6588 list_for_each_entry(pmu, &pmus, entry) { 6585 list_for_each_entry(pmu, &pmus, entry) {
6589 if (pmu->task_ctx_nr == ctxn) 6586 if (pmu->task_ctx_nr == ctxn)
6590 return pmu->pmu_cpu_context; 6587 return pmu->pmu_cpu_context;
6591 } 6588 }
6592 6589
6593 return NULL; 6590 return NULL;
6594 } 6591 }
6595 6592
6596 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu) 6593 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
6597 { 6594 {
6598 int cpu; 6595 int cpu;
6599 6596
6600 for_each_possible_cpu(cpu) { 6597 for_each_possible_cpu(cpu) {
6601 struct perf_cpu_context *cpuctx; 6598 struct perf_cpu_context *cpuctx;
6602 6599
6603 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6600 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6604 6601
6605 if (cpuctx->unique_pmu == old_pmu) 6602 if (cpuctx->unique_pmu == old_pmu)
6606 cpuctx->unique_pmu = pmu; 6603 cpuctx->unique_pmu = pmu;
6607 } 6604 }
6608 } 6605 }
6609 6606
6610 static void free_pmu_context(struct pmu *pmu) 6607 static void free_pmu_context(struct pmu *pmu)
6611 { 6608 {
6612 struct pmu *i; 6609 struct pmu *i;
6613 6610
6614 mutex_lock(&pmus_lock); 6611 mutex_lock(&pmus_lock);
6615 /* 6612 /*
6616 * Like a real lame refcount. 6613 * Like a real lame refcount.
6617 */ 6614 */
6618 list_for_each_entry(i, &pmus, entry) { 6615 list_for_each_entry(i, &pmus, entry) {
6619 if (i->pmu_cpu_context == pmu->pmu_cpu_context) { 6616 if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
6620 update_pmu_context(i, pmu); 6617 update_pmu_context(i, pmu);
6621 goto out; 6618 goto out;
6622 } 6619 }
6623 } 6620 }
6624 6621
6625 free_percpu(pmu->pmu_cpu_context); 6622 free_percpu(pmu->pmu_cpu_context);
6626 out: 6623 out:
6627 mutex_unlock(&pmus_lock); 6624 mutex_unlock(&pmus_lock);
6628 } 6625 }
6629 static struct idr pmu_idr; 6626 static struct idr pmu_idr;
6630 6627
6631 static ssize_t 6628 static ssize_t
6632 type_show(struct device *dev, struct device_attribute *attr, char *page) 6629 type_show(struct device *dev, struct device_attribute *attr, char *page)
6633 { 6630 {
6634 struct pmu *pmu = dev_get_drvdata(dev); 6631 struct pmu *pmu = dev_get_drvdata(dev);
6635 6632
6636 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 6633 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
6637 } 6634 }
6638 static DEVICE_ATTR_RO(type); 6635 static DEVICE_ATTR_RO(type);
6639 6636
6640 static ssize_t 6637 static ssize_t
6641 perf_event_mux_interval_ms_show(struct device *dev, 6638 perf_event_mux_interval_ms_show(struct device *dev,
6642 struct device_attribute *attr, 6639 struct device_attribute *attr,
6643 char *page) 6640 char *page)
6644 { 6641 {
6645 struct pmu *pmu = dev_get_drvdata(dev); 6642 struct pmu *pmu = dev_get_drvdata(dev);
6646 6643
6647 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 6644 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
6648 } 6645 }
6649 6646
6650 static ssize_t 6647 static ssize_t
6651 perf_event_mux_interval_ms_store(struct device *dev, 6648 perf_event_mux_interval_ms_store(struct device *dev,
6652 struct device_attribute *attr, 6649 struct device_attribute *attr,
6653 const char *buf, size_t count) 6650 const char *buf, size_t count)
6654 { 6651 {
6655 struct pmu *pmu = dev_get_drvdata(dev); 6652 struct pmu *pmu = dev_get_drvdata(dev);
6656 int timer, cpu, ret; 6653 int timer, cpu, ret;
6657 6654
6658 ret = kstrtoint(buf, 0, &timer); 6655 ret = kstrtoint(buf, 0, &timer);
6659 if (ret) 6656 if (ret)
6660 return ret; 6657 return ret;
6661 6658
6662 if (timer < 1) 6659 if (timer < 1)
6663 return -EINVAL; 6660 return -EINVAL;
6664 6661
6665 /* same value, noting to do */ 6662 /* same value, noting to do */
6666 if (timer == pmu->hrtimer_interval_ms) 6663 if (timer == pmu->hrtimer_interval_ms)
6667 return count; 6664 return count;
6668 6665
6669 pmu->hrtimer_interval_ms = timer; 6666 pmu->hrtimer_interval_ms = timer;
6670 6667
6671 /* update all cpuctx for this PMU */ 6668 /* update all cpuctx for this PMU */
6672 for_each_possible_cpu(cpu) { 6669 for_each_possible_cpu(cpu) {
6673 struct perf_cpu_context *cpuctx; 6670 struct perf_cpu_context *cpuctx;
6674 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6671 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6675 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 6672 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
6676 6673
6677 if (hrtimer_active(&cpuctx->hrtimer)) 6674 if (hrtimer_active(&cpuctx->hrtimer))
6678 hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval); 6675 hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
6679 } 6676 }
6680 6677
6681 return count; 6678 return count;
6682 } 6679 }
6683 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 6680 static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
6684 6681
6685 static struct attribute *pmu_dev_attrs[] = { 6682 static struct attribute *pmu_dev_attrs[] = {
6686 &dev_attr_type.attr, 6683 &dev_attr_type.attr,
6687 &dev_attr_perf_event_mux_interval_ms.attr, 6684 &dev_attr_perf_event_mux_interval_ms.attr,
6688 NULL, 6685 NULL,
6689 }; 6686 };
6690 ATTRIBUTE_GROUPS(pmu_dev); 6687 ATTRIBUTE_GROUPS(pmu_dev);
6691 6688
6692 static int pmu_bus_running; 6689 static int pmu_bus_running;
6693 static struct bus_type pmu_bus = { 6690 static struct bus_type pmu_bus = {
6694 .name = "event_source", 6691 .name = "event_source",
6695 .dev_groups = pmu_dev_groups, 6692 .dev_groups = pmu_dev_groups,
6696 }; 6693 };
6697 6694
6698 static void pmu_dev_release(struct device *dev) 6695 static void pmu_dev_release(struct device *dev)
6699 { 6696 {
6700 kfree(dev); 6697 kfree(dev);
6701 } 6698 }
6702 6699
6703 static int pmu_dev_alloc(struct pmu *pmu) 6700 static int pmu_dev_alloc(struct pmu *pmu)
6704 { 6701 {
6705 int ret = -ENOMEM; 6702 int ret = -ENOMEM;
6706 6703
6707 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 6704 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
6708 if (!pmu->dev) 6705 if (!pmu->dev)
6709 goto out; 6706 goto out;
6710 6707
6711 pmu->dev->groups = pmu->attr_groups; 6708 pmu->dev->groups = pmu->attr_groups;
6712 device_initialize(pmu->dev); 6709 device_initialize(pmu->dev);
6713 ret = dev_set_name(pmu->dev, "%s", pmu->name); 6710 ret = dev_set_name(pmu->dev, "%s", pmu->name);
6714 if (ret) 6711 if (ret)
6715 goto free_dev; 6712 goto free_dev;
6716 6713
6717 dev_set_drvdata(pmu->dev, pmu); 6714 dev_set_drvdata(pmu->dev, pmu);
6718 pmu->dev->bus = &pmu_bus; 6715 pmu->dev->bus = &pmu_bus;
6719 pmu->dev->release = pmu_dev_release; 6716 pmu->dev->release = pmu_dev_release;
6720 ret = device_add(pmu->dev); 6717 ret = device_add(pmu->dev);
6721 if (ret) 6718 if (ret)
6722 goto free_dev; 6719 goto free_dev;
6723 6720
6724 out: 6721 out:
6725 return ret; 6722 return ret;
6726 6723
6727 free_dev: 6724 free_dev:
6728 put_device(pmu->dev); 6725 put_device(pmu->dev);
6729 goto out; 6726 goto out;
6730 } 6727 }
6731 6728
6732 static struct lock_class_key cpuctx_mutex; 6729 static struct lock_class_key cpuctx_mutex;
6733 static struct lock_class_key cpuctx_lock; 6730 static struct lock_class_key cpuctx_lock;
6734 6731
6735 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 6732 int perf_pmu_register(struct pmu *pmu, const char *name, int type)
6736 { 6733 {
6737 int cpu, ret; 6734 int cpu, ret;
6738 6735
6739 mutex_lock(&pmus_lock); 6736 mutex_lock(&pmus_lock);
6740 ret = -ENOMEM; 6737 ret = -ENOMEM;
6741 pmu->pmu_disable_count = alloc_percpu(int); 6738 pmu->pmu_disable_count = alloc_percpu(int);
6742 if (!pmu->pmu_disable_count) 6739 if (!pmu->pmu_disable_count)
6743 goto unlock; 6740 goto unlock;
6744 6741
6745 pmu->type = -1; 6742 pmu->type = -1;
6746 if (!name) 6743 if (!name)
6747 goto skip_type; 6744 goto skip_type;
6748 pmu->name = name; 6745 pmu->name = name;
6749 6746
6750 if (type < 0) { 6747 if (type < 0) {
6751 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 6748 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
6752 if (type < 0) { 6749 if (type < 0) {
6753 ret = type; 6750 ret = type;
6754 goto free_pdc; 6751 goto free_pdc;
6755 } 6752 }
6756 } 6753 }
6757 pmu->type = type; 6754 pmu->type = type;
6758 6755
6759 if (pmu_bus_running) { 6756 if (pmu_bus_running) {
6760 ret = pmu_dev_alloc(pmu); 6757 ret = pmu_dev_alloc(pmu);
6761 if (ret) 6758 if (ret)
6762 goto free_idr; 6759 goto free_idr;
6763 } 6760 }
6764 6761
6765 skip_type: 6762 skip_type:
6766 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 6763 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
6767 if (pmu->pmu_cpu_context) 6764 if (pmu->pmu_cpu_context)
6768 goto got_cpu_context; 6765 goto got_cpu_context;
6769 6766
6770 ret = -ENOMEM; 6767 ret = -ENOMEM;
6771 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 6768 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
6772 if (!pmu->pmu_cpu_context) 6769 if (!pmu->pmu_cpu_context)
6773 goto free_dev; 6770 goto free_dev;
6774 6771
6775 for_each_possible_cpu(cpu) { 6772 for_each_possible_cpu(cpu) {
6776 struct perf_cpu_context *cpuctx; 6773 struct perf_cpu_context *cpuctx;
6777 6774
6778 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 6775 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6779 __perf_event_init_context(&cpuctx->ctx); 6776 __perf_event_init_context(&cpuctx->ctx);
6780 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 6777 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
6781 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 6778 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
6782 cpuctx->ctx.type = cpu_context; 6779 cpuctx->ctx.type = cpu_context;
6783 cpuctx->ctx.pmu = pmu; 6780 cpuctx->ctx.pmu = pmu;
6784 6781
6785 __perf_cpu_hrtimer_init(cpuctx, cpu); 6782 __perf_cpu_hrtimer_init(cpuctx, cpu);
6786 6783
6787 INIT_LIST_HEAD(&cpuctx->rotation_list); 6784 INIT_LIST_HEAD(&cpuctx->rotation_list);
6788 cpuctx->unique_pmu = pmu; 6785 cpuctx->unique_pmu = pmu;
6789 } 6786 }
6790 6787
6791 got_cpu_context: 6788 got_cpu_context:
6792 if (!pmu->start_txn) { 6789 if (!pmu->start_txn) {
6793 if (pmu->pmu_enable) { 6790 if (pmu->pmu_enable) {
6794 /* 6791 /*
6795 * If we have pmu_enable/pmu_disable calls, install 6792 * If we have pmu_enable/pmu_disable calls, install
6796 * transaction stubs that use that to try and batch 6793 * transaction stubs that use that to try and batch
6797 * hardware accesses. 6794 * hardware accesses.
6798 */ 6795 */
6799 pmu->start_txn = perf_pmu_start_txn; 6796 pmu->start_txn = perf_pmu_start_txn;
6800 pmu->commit_txn = perf_pmu_commit_txn; 6797 pmu->commit_txn = perf_pmu_commit_txn;
6801 pmu->cancel_txn = perf_pmu_cancel_txn; 6798 pmu->cancel_txn = perf_pmu_cancel_txn;
6802 } else { 6799 } else {
6803 pmu->start_txn = perf_pmu_nop_void; 6800 pmu->start_txn = perf_pmu_nop_void;
6804 pmu->commit_txn = perf_pmu_nop_int; 6801 pmu->commit_txn = perf_pmu_nop_int;
6805 pmu->cancel_txn = perf_pmu_nop_void; 6802 pmu->cancel_txn = perf_pmu_nop_void;
6806 } 6803 }
6807 } 6804 }
6808 6805
6809 if (!pmu->pmu_enable) { 6806 if (!pmu->pmu_enable) {
6810 pmu->pmu_enable = perf_pmu_nop_void; 6807 pmu->pmu_enable = perf_pmu_nop_void;
6811 pmu->pmu_disable = perf_pmu_nop_void; 6808 pmu->pmu_disable = perf_pmu_nop_void;
6812 } 6809 }
6813 6810
6814 if (!pmu->event_idx) 6811 if (!pmu->event_idx)
6815 pmu->event_idx = perf_event_idx_default; 6812 pmu->event_idx = perf_event_idx_default;
6816 6813
6817 list_add_rcu(&pmu->entry, &pmus); 6814 list_add_rcu(&pmu->entry, &pmus);
6818 ret = 0; 6815 ret = 0;
6819 unlock: 6816 unlock:
6820 mutex_unlock(&pmus_lock); 6817 mutex_unlock(&pmus_lock);
6821 6818
6822 return ret; 6819 return ret;
6823 6820
6824 free_dev: 6821 free_dev:
6825 device_del(pmu->dev); 6822 device_del(pmu->dev);
6826 put_device(pmu->dev); 6823 put_device(pmu->dev);
6827 6824
6828 free_idr: 6825 free_idr:
6829 if (pmu->type >= PERF_TYPE_MAX) 6826 if (pmu->type >= PERF_TYPE_MAX)
6830 idr_remove(&pmu_idr, pmu->type); 6827 idr_remove(&pmu_idr, pmu->type);
6831 6828
6832 free_pdc: 6829 free_pdc:
6833 free_percpu(pmu->pmu_disable_count); 6830 free_percpu(pmu->pmu_disable_count);
6834 goto unlock; 6831 goto unlock;
6835 } 6832 }
6836 EXPORT_SYMBOL_GPL(perf_pmu_register); 6833 EXPORT_SYMBOL_GPL(perf_pmu_register);
6837 6834
6838 void perf_pmu_unregister(struct pmu *pmu) 6835 void perf_pmu_unregister(struct pmu *pmu)
6839 { 6836 {
6840 mutex_lock(&pmus_lock); 6837 mutex_lock(&pmus_lock);
6841 list_del_rcu(&pmu->entry); 6838 list_del_rcu(&pmu->entry);
6842 mutex_unlock(&pmus_lock); 6839 mutex_unlock(&pmus_lock);
6843 6840
6844 /* 6841 /*
6845 * We dereference the pmu list under both SRCU and regular RCU, so 6842 * We dereference the pmu list under both SRCU and regular RCU, so
6846 * synchronize against both of those. 6843 * synchronize against both of those.
6847 */ 6844 */
6848 synchronize_srcu(&pmus_srcu); 6845 synchronize_srcu(&pmus_srcu);
6849 synchronize_rcu(); 6846 synchronize_rcu();
6850 6847
6851 free_percpu(pmu->pmu_disable_count); 6848 free_percpu(pmu->pmu_disable_count);
6852 if (pmu->type >= PERF_TYPE_MAX) 6849 if (pmu->type >= PERF_TYPE_MAX)
6853 idr_remove(&pmu_idr, pmu->type); 6850 idr_remove(&pmu_idr, pmu->type);
6854 device_del(pmu->dev); 6851 device_del(pmu->dev);
6855 put_device(pmu->dev); 6852 put_device(pmu->dev);
6856 free_pmu_context(pmu); 6853 free_pmu_context(pmu);
6857 } 6854 }
6858 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 6855 EXPORT_SYMBOL_GPL(perf_pmu_unregister);
6859 6856
6860 struct pmu *perf_init_event(struct perf_event *event) 6857 struct pmu *perf_init_event(struct perf_event *event)
6861 { 6858 {
6862 struct pmu *pmu = NULL; 6859 struct pmu *pmu = NULL;
6863 int idx; 6860 int idx;
6864 int ret; 6861 int ret;
6865 6862
6866 idx = srcu_read_lock(&pmus_srcu); 6863 idx = srcu_read_lock(&pmus_srcu);
6867 6864
6868 rcu_read_lock(); 6865 rcu_read_lock();
6869 pmu = idr_find(&pmu_idr, event->attr.type); 6866 pmu = idr_find(&pmu_idr, event->attr.type);
6870 rcu_read_unlock(); 6867 rcu_read_unlock();
6871 if (pmu) { 6868 if (pmu) {
6872 if (!try_module_get(pmu->module)) { 6869 if (!try_module_get(pmu->module)) {
6873 pmu = ERR_PTR(-ENODEV); 6870 pmu = ERR_PTR(-ENODEV);
6874 goto unlock; 6871 goto unlock;
6875 } 6872 }
6876 event->pmu = pmu; 6873 event->pmu = pmu;
6877 ret = pmu->event_init(event); 6874 ret = pmu->event_init(event);
6878 if (ret) 6875 if (ret)
6879 pmu = ERR_PTR(ret); 6876 pmu = ERR_PTR(ret);
6880 goto unlock; 6877 goto unlock;
6881 } 6878 }
6882 6879
6883 list_for_each_entry_rcu(pmu, &pmus, entry) { 6880 list_for_each_entry_rcu(pmu, &pmus, entry) {
6884 if (!try_module_get(pmu->module)) { 6881 if (!try_module_get(pmu->module)) {
6885 pmu = ERR_PTR(-ENODEV); 6882 pmu = ERR_PTR(-ENODEV);
6886 goto unlock; 6883 goto unlock;
6887 } 6884 }
6888 event->pmu = pmu; 6885 event->pmu = pmu;
6889 ret = pmu->event_init(event); 6886 ret = pmu->event_init(event);
6890 if (!ret) 6887 if (!ret)
6891 goto unlock; 6888 goto unlock;
6892 6889
6893 if (ret != -ENOENT) { 6890 if (ret != -ENOENT) {
6894 pmu = ERR_PTR(ret); 6891 pmu = ERR_PTR(ret);
6895 goto unlock; 6892 goto unlock;
6896 } 6893 }
6897 } 6894 }
6898 pmu = ERR_PTR(-ENOENT); 6895 pmu = ERR_PTR(-ENOENT);
6899 unlock: 6896 unlock:
6900 srcu_read_unlock(&pmus_srcu, idx); 6897 srcu_read_unlock(&pmus_srcu, idx);
6901 6898
6902 return pmu; 6899 return pmu;
6903 } 6900 }
6904 6901
6905 static void account_event_cpu(struct perf_event *event, int cpu) 6902 static void account_event_cpu(struct perf_event *event, int cpu)
6906 { 6903 {
6907 if (event->parent) 6904 if (event->parent)
6908 return; 6905 return;
6909 6906
6910 if (has_branch_stack(event)) { 6907 if (has_branch_stack(event)) {
6911 if (!(event->attach_state & PERF_ATTACH_TASK)) 6908 if (!(event->attach_state & PERF_ATTACH_TASK))
6912 atomic_inc(&per_cpu(perf_branch_stack_events, cpu)); 6909 atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
6913 } 6910 }
6914 if (is_cgroup_event(event)) 6911 if (is_cgroup_event(event))
6915 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 6912 atomic_inc(&per_cpu(perf_cgroup_events, cpu));
6916 } 6913 }
6917 6914
6918 static void account_event(struct perf_event *event) 6915 static void account_event(struct perf_event *event)
6919 { 6916 {
6920 if (event->parent) 6917 if (event->parent)
6921 return; 6918 return;
6922 6919
6923 if (event->attach_state & PERF_ATTACH_TASK) 6920 if (event->attach_state & PERF_ATTACH_TASK)
6924 static_key_slow_inc(&perf_sched_events.key); 6921 static_key_slow_inc(&perf_sched_events.key);
6925 if (event->attr.mmap || event->attr.mmap_data) 6922 if (event->attr.mmap || event->attr.mmap_data)
6926 atomic_inc(&nr_mmap_events); 6923 atomic_inc(&nr_mmap_events);
6927 if (event->attr.comm) 6924 if (event->attr.comm)
6928 atomic_inc(&nr_comm_events); 6925 atomic_inc(&nr_comm_events);
6929 if (event->attr.task) 6926 if (event->attr.task)
6930 atomic_inc(&nr_task_events); 6927 atomic_inc(&nr_task_events);
6931 if (event->attr.freq) { 6928 if (event->attr.freq) {
6932 if (atomic_inc_return(&nr_freq_events) == 1) 6929 if (atomic_inc_return(&nr_freq_events) == 1)
6933 tick_nohz_full_kick_all(); 6930 tick_nohz_full_kick_all();
6934 } 6931 }
6935 if (has_branch_stack(event)) 6932 if (has_branch_stack(event))
6936 static_key_slow_inc(&perf_sched_events.key); 6933 static_key_slow_inc(&perf_sched_events.key);
6937 if (is_cgroup_event(event)) 6934 if (is_cgroup_event(event))
6938 static_key_slow_inc(&perf_sched_events.key); 6935 static_key_slow_inc(&perf_sched_events.key);
6939 6936
6940 account_event_cpu(event, event->cpu); 6937 account_event_cpu(event, event->cpu);
6941 } 6938 }
6942 6939
6943 /* 6940 /*
6944 * Allocate and initialize a event structure 6941 * Allocate and initialize a event structure
6945 */ 6942 */
6946 static struct perf_event * 6943 static struct perf_event *
6947 perf_event_alloc(struct perf_event_attr *attr, int cpu, 6944 perf_event_alloc(struct perf_event_attr *attr, int cpu,
6948 struct task_struct *task, 6945 struct task_struct *task,
6949 struct perf_event *group_leader, 6946 struct perf_event *group_leader,
6950 struct perf_event *parent_event, 6947 struct perf_event *parent_event,
6951 perf_overflow_handler_t overflow_handler, 6948 perf_overflow_handler_t overflow_handler,
6952 void *context) 6949 void *context)
6953 { 6950 {
6954 struct pmu *pmu; 6951 struct pmu *pmu;
6955 struct perf_event *event; 6952 struct perf_event *event;
6956 struct hw_perf_event *hwc; 6953 struct hw_perf_event *hwc;
6957 long err = -EINVAL; 6954 long err = -EINVAL;
6958 6955
6959 if ((unsigned)cpu >= nr_cpu_ids) { 6956 if ((unsigned)cpu >= nr_cpu_ids) {
6960 if (!task || cpu != -1) 6957 if (!task || cpu != -1)
6961 return ERR_PTR(-EINVAL); 6958 return ERR_PTR(-EINVAL);
6962 } 6959 }
6963 6960
6964 event = kzalloc(sizeof(*event), GFP_KERNEL); 6961 event = kzalloc(sizeof(*event), GFP_KERNEL);
6965 if (!event) 6962 if (!event)
6966 return ERR_PTR(-ENOMEM); 6963 return ERR_PTR(-ENOMEM);
6967 6964
6968 /* 6965 /*
6969 * Single events are their own group leaders, with an 6966 * Single events are their own group leaders, with an
6970 * empty sibling list: 6967 * empty sibling list:
6971 */ 6968 */
6972 if (!group_leader) 6969 if (!group_leader)
6973 group_leader = event; 6970 group_leader = event;
6974 6971
6975 mutex_init(&event->child_mutex); 6972 mutex_init(&event->child_mutex);
6976 INIT_LIST_HEAD(&event->child_list); 6973 INIT_LIST_HEAD(&event->child_list);
6977 6974
6978 INIT_LIST_HEAD(&event->group_entry); 6975 INIT_LIST_HEAD(&event->group_entry);
6979 INIT_LIST_HEAD(&event->event_entry); 6976 INIT_LIST_HEAD(&event->event_entry);
6980 INIT_LIST_HEAD(&event->sibling_list); 6977 INIT_LIST_HEAD(&event->sibling_list);
6981 INIT_LIST_HEAD(&event->rb_entry); 6978 INIT_LIST_HEAD(&event->rb_entry);
6982 INIT_LIST_HEAD(&event->active_entry); 6979 INIT_LIST_HEAD(&event->active_entry);
6983 INIT_HLIST_NODE(&event->hlist_entry); 6980 INIT_HLIST_NODE(&event->hlist_entry);
6984 6981
6985 6982
6986 init_waitqueue_head(&event->waitq); 6983 init_waitqueue_head(&event->waitq);
6987 init_irq_work(&event->pending, perf_pending_event); 6984 init_irq_work(&event->pending, perf_pending_event);
6988 6985
6989 mutex_init(&event->mmap_mutex); 6986 mutex_init(&event->mmap_mutex);
6990 6987
6991 atomic_long_set(&event->refcount, 1); 6988 atomic_long_set(&event->refcount, 1);
6992 event->cpu = cpu; 6989 event->cpu = cpu;
6993 event->attr = *attr; 6990 event->attr = *attr;
6994 event->group_leader = group_leader; 6991 event->group_leader = group_leader;
6995 event->pmu = NULL; 6992 event->pmu = NULL;
6996 event->oncpu = -1; 6993 event->oncpu = -1;
6997 6994
6998 event->parent = parent_event; 6995 event->parent = parent_event;
6999 6996
7000 event->ns = get_pid_ns(task_active_pid_ns(current)); 6997 event->ns = get_pid_ns(task_active_pid_ns(current));
7001 event->id = atomic64_inc_return(&perf_event_id); 6998 event->id = atomic64_inc_return(&perf_event_id);
7002 6999
7003 event->state = PERF_EVENT_STATE_INACTIVE; 7000 event->state = PERF_EVENT_STATE_INACTIVE;
7004 7001
7005 if (task) { 7002 if (task) {
7006 event->attach_state = PERF_ATTACH_TASK; 7003 event->attach_state = PERF_ATTACH_TASK;
7007 7004
7008 if (attr->type == PERF_TYPE_TRACEPOINT) 7005 if (attr->type == PERF_TYPE_TRACEPOINT)
7009 event->hw.tp_target = task; 7006 event->hw.tp_target = task;
7010 #ifdef CONFIG_HAVE_HW_BREAKPOINT 7007 #ifdef CONFIG_HAVE_HW_BREAKPOINT
7011 /* 7008 /*
7012 * hw_breakpoint is a bit difficult here.. 7009 * hw_breakpoint is a bit difficult here..
7013 */ 7010 */
7014 else if (attr->type == PERF_TYPE_BREAKPOINT) 7011 else if (attr->type == PERF_TYPE_BREAKPOINT)
7015 event->hw.bp_target = task; 7012 event->hw.bp_target = task;
7016 #endif 7013 #endif
7017 } 7014 }
7018 7015
7019 if (!overflow_handler && parent_event) { 7016 if (!overflow_handler && parent_event) {
7020 overflow_handler = parent_event->overflow_handler; 7017 overflow_handler = parent_event->overflow_handler;
7021 context = parent_event->overflow_handler_context; 7018 context = parent_event->overflow_handler_context;
7022 } 7019 }
7023 7020
7024 event->overflow_handler = overflow_handler; 7021 event->overflow_handler = overflow_handler;
7025 event->overflow_handler_context = context; 7022 event->overflow_handler_context = context;
7026 7023
7027 perf_event__state_init(event); 7024 perf_event__state_init(event);
7028 7025
7029 pmu = NULL; 7026 pmu = NULL;
7030 7027
7031 hwc = &event->hw; 7028 hwc = &event->hw;
7032 hwc->sample_period = attr->sample_period; 7029 hwc->sample_period = attr->sample_period;
7033 if (attr->freq && attr->sample_freq) 7030 if (attr->freq && attr->sample_freq)
7034 hwc->sample_period = 1; 7031 hwc->sample_period = 1;
7035 hwc->last_period = hwc->sample_period; 7032 hwc->last_period = hwc->sample_period;
7036 7033
7037 local64_set(&hwc->period_left, hwc->sample_period); 7034 local64_set(&hwc->period_left, hwc->sample_period);
7038 7035
7039 /* 7036 /*
7040 * we currently do not support PERF_FORMAT_GROUP on inherited events 7037 * we currently do not support PERF_FORMAT_GROUP on inherited events
7041 */ 7038 */
7042 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP)) 7039 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
7043 goto err_ns; 7040 goto err_ns;
7044 7041
7045 pmu = perf_init_event(event); 7042 pmu = perf_init_event(event);
7046 if (!pmu) 7043 if (!pmu)
7047 goto err_ns; 7044 goto err_ns;
7048 else if (IS_ERR(pmu)) { 7045 else if (IS_ERR(pmu)) {
7049 err = PTR_ERR(pmu); 7046 err = PTR_ERR(pmu);
7050 goto err_ns; 7047 goto err_ns;
7051 } 7048 }
7052 7049
7053 if (!event->parent) { 7050 if (!event->parent) {
7054 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 7051 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
7055 err = get_callchain_buffers(); 7052 err = get_callchain_buffers();
7056 if (err) 7053 if (err)
7057 goto err_pmu; 7054 goto err_pmu;
7058 } 7055 }
7059 } 7056 }
7060 7057
7061 return event; 7058 return event;
7062 7059
7063 err_pmu: 7060 err_pmu:
7064 if (event->destroy) 7061 if (event->destroy)
7065 event->destroy(event); 7062 event->destroy(event);
7066 module_put(pmu->module); 7063 module_put(pmu->module);
7067 err_ns: 7064 err_ns:
7068 if (event->ns) 7065 if (event->ns)
7069 put_pid_ns(event->ns); 7066 put_pid_ns(event->ns);
7070 kfree(event); 7067 kfree(event);
7071 7068
7072 return ERR_PTR(err); 7069 return ERR_PTR(err);
7073 } 7070 }
7074 7071
7075 static int perf_copy_attr(struct perf_event_attr __user *uattr, 7072 static int perf_copy_attr(struct perf_event_attr __user *uattr,
7076 struct perf_event_attr *attr) 7073 struct perf_event_attr *attr)
7077 { 7074 {
7078 u32 size; 7075 u32 size;
7079 int ret; 7076 int ret;
7080 7077
7081 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) 7078 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
7082 return -EFAULT; 7079 return -EFAULT;
7083 7080
7084 /* 7081 /*
7085 * zero the full structure, so that a short copy will be nice. 7082 * zero the full structure, so that a short copy will be nice.
7086 */ 7083 */
7087 memset(attr, 0, sizeof(*attr)); 7084 memset(attr, 0, sizeof(*attr));
7088 7085
7089 ret = get_user(size, &uattr->size); 7086 ret = get_user(size, &uattr->size);
7090 if (ret) 7087 if (ret)
7091 return ret; 7088 return ret;
7092 7089
7093 if (size > PAGE_SIZE) /* silly large */ 7090 if (size > PAGE_SIZE) /* silly large */
7094 goto err_size; 7091 goto err_size;
7095 7092
7096 if (!size) /* abi compat */ 7093 if (!size) /* abi compat */
7097 size = PERF_ATTR_SIZE_VER0; 7094 size = PERF_ATTR_SIZE_VER0;
7098 7095
7099 if (size < PERF_ATTR_SIZE_VER0) 7096 if (size < PERF_ATTR_SIZE_VER0)
7100 goto err_size; 7097 goto err_size;
7101 7098
7102 /* 7099 /*
7103 * If we're handed a bigger struct than we know of, 7100 * If we're handed a bigger struct than we know of,
7104 * ensure all the unknown bits are 0 - i.e. new 7101 * ensure all the unknown bits are 0 - i.e. new
7105 * user-space does not rely on any kernel feature 7102 * user-space does not rely on any kernel feature
7106 * extensions we dont know about yet. 7103 * extensions we dont know about yet.
7107 */ 7104 */
7108 if (size > sizeof(*attr)) { 7105 if (size > sizeof(*attr)) {
7109 unsigned char __user *addr; 7106 unsigned char __user *addr;
7110 unsigned char __user *end; 7107 unsigned char __user *end;
7111 unsigned char val; 7108 unsigned char val;
7112 7109
7113 addr = (void __user *)uattr + sizeof(*attr); 7110 addr = (void __user *)uattr + sizeof(*attr);
7114 end = (void __user *)uattr + size; 7111 end = (void __user *)uattr + size;
7115 7112
7116 for (; addr < end; addr++) { 7113 for (; addr < end; addr++) {
7117 ret = get_user(val, addr); 7114 ret = get_user(val, addr);
7118 if (ret) 7115 if (ret)
7119 return ret; 7116 return ret;
7120 if (val) 7117 if (val)
7121 goto err_size; 7118 goto err_size;
7122 } 7119 }
7123 size = sizeof(*attr); 7120 size = sizeof(*attr);
7124 } 7121 }
7125 7122
7126 ret = copy_from_user(attr, uattr, size); 7123 ret = copy_from_user(attr, uattr, size);
7127 if (ret) 7124 if (ret)
7128 return -EFAULT; 7125 return -EFAULT;
7129 7126
7130 if (attr->__reserved_1) 7127 if (attr->__reserved_1)
7131 return -EINVAL; 7128 return -EINVAL;
7132 7129
7133 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 7130 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
7134 return -EINVAL; 7131 return -EINVAL;
7135 7132
7136 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 7133 if (attr->read_format & ~(PERF_FORMAT_MAX-1))
7137 return -EINVAL; 7134 return -EINVAL;
7138 7135
7139 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 7136 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
7140 u64 mask = attr->branch_sample_type; 7137 u64 mask = attr->branch_sample_type;
7141 7138
7142 /* only using defined bits */ 7139 /* only using defined bits */
7143 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 7140 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
7144 return -EINVAL; 7141 return -EINVAL;
7145 7142
7146 /* at least one branch bit must be set */ 7143 /* at least one branch bit must be set */
7147 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 7144 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
7148 return -EINVAL; 7145 return -EINVAL;
7149 7146
7150 /* propagate priv level, when not set for branch */ 7147 /* propagate priv level, when not set for branch */
7151 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 7148 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
7152 7149
7153 /* exclude_kernel checked on syscall entry */ 7150 /* exclude_kernel checked on syscall entry */
7154 if (!attr->exclude_kernel) 7151 if (!attr->exclude_kernel)
7155 mask |= PERF_SAMPLE_BRANCH_KERNEL; 7152 mask |= PERF_SAMPLE_BRANCH_KERNEL;
7156 7153
7157 if (!attr->exclude_user) 7154 if (!attr->exclude_user)
7158 mask |= PERF_SAMPLE_BRANCH_USER; 7155 mask |= PERF_SAMPLE_BRANCH_USER;
7159 7156
7160 if (!attr->exclude_hv) 7157 if (!attr->exclude_hv)
7161 mask |= PERF_SAMPLE_BRANCH_HV; 7158 mask |= PERF_SAMPLE_BRANCH_HV;
7162 /* 7159 /*
7163 * adjust user setting (for HW filter setup) 7160 * adjust user setting (for HW filter setup)
7164 */ 7161 */
7165 attr->branch_sample_type = mask; 7162 attr->branch_sample_type = mask;
7166 } 7163 }
7167 /* privileged levels capture (kernel, hv): check permissions */ 7164 /* privileged levels capture (kernel, hv): check permissions */
7168 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 7165 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
7169 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 7166 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7170 return -EACCES; 7167 return -EACCES;
7171 } 7168 }
7172 7169
7173 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 7170 if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
7174 ret = perf_reg_validate(attr->sample_regs_user); 7171 ret = perf_reg_validate(attr->sample_regs_user);
7175 if (ret) 7172 if (ret)
7176 return ret; 7173 return ret;
7177 } 7174 }
7178 7175
7179 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 7176 if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
7180 if (!arch_perf_have_user_stack_dump()) 7177 if (!arch_perf_have_user_stack_dump())
7181 return -ENOSYS; 7178 return -ENOSYS;
7182 7179
7183 /* 7180 /*
7184 * We have __u32 type for the size, but so far 7181 * We have __u32 type for the size, but so far
7185 * we can only use __u16 as maximum due to the 7182 * we can only use __u16 as maximum due to the
7186 * __u16 sample size limit. 7183 * __u16 sample size limit.
7187 */ 7184 */
7188 if (attr->sample_stack_user >= USHRT_MAX) 7185 if (attr->sample_stack_user >= USHRT_MAX)
7189 ret = -EINVAL; 7186 ret = -EINVAL;
7190 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 7187 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
7191 ret = -EINVAL; 7188 ret = -EINVAL;
7192 } 7189 }
7193 7190
7194 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) 7191 if (attr->sample_type & PERF_SAMPLE_REGS_INTR)
7195 ret = perf_reg_validate(attr->sample_regs_intr); 7192 ret = perf_reg_validate(attr->sample_regs_intr);
7196 out: 7193 out:
7197 return ret; 7194 return ret;
7198 7195
7199 err_size: 7196 err_size:
7200 put_user(sizeof(*attr), &uattr->size); 7197 put_user(sizeof(*attr), &uattr->size);
7201 ret = -E2BIG; 7198 ret = -E2BIG;
7202 goto out; 7199 goto out;
7203 } 7200 }
7204 7201
7205 static int 7202 static int
7206 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 7203 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
7207 { 7204 {
7208 struct ring_buffer *rb = NULL; 7205 struct ring_buffer *rb = NULL;
7209 int ret = -EINVAL; 7206 int ret = -EINVAL;
7210 7207
7211 if (!output_event) 7208 if (!output_event)
7212 goto set; 7209 goto set;
7213 7210
7214 /* don't allow circular references */ 7211 /* don't allow circular references */
7215 if (event == output_event) 7212 if (event == output_event)
7216 goto out; 7213 goto out;
7217 7214
7218 /* 7215 /*
7219 * Don't allow cross-cpu buffers 7216 * Don't allow cross-cpu buffers
7220 */ 7217 */
7221 if (output_event->cpu != event->cpu) 7218 if (output_event->cpu != event->cpu)
7222 goto out; 7219 goto out;
7223 7220
7224 /* 7221 /*
7225 * If its not a per-cpu rb, it must be the same task. 7222 * If its not a per-cpu rb, it must be the same task.
7226 */ 7223 */
7227 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 7224 if (output_event->cpu == -1 && output_event->ctx != event->ctx)
7228 goto out; 7225 goto out;
7229 7226
7230 set: 7227 set:
7231 mutex_lock(&event->mmap_mutex); 7228 mutex_lock(&event->mmap_mutex);
7232 /* Can't redirect output if we've got an active mmap() */ 7229 /* Can't redirect output if we've got an active mmap() */
7233 if (atomic_read(&event->mmap_count)) 7230 if (atomic_read(&event->mmap_count))
7234 goto unlock; 7231 goto unlock;
7235 7232
7236 if (output_event) { 7233 if (output_event) {
7237 /* get the rb we want to redirect to */ 7234 /* get the rb we want to redirect to */
7238 rb = ring_buffer_get(output_event); 7235 rb = ring_buffer_get(output_event);
7239 if (!rb) 7236 if (!rb)
7240 goto unlock; 7237 goto unlock;
7241 } 7238 }
7242 7239
7243 ring_buffer_attach(event, rb); 7240 ring_buffer_attach(event, rb);
7244 7241
7245 ret = 0; 7242 ret = 0;
7246 unlock: 7243 unlock:
7247 mutex_unlock(&event->mmap_mutex); 7244 mutex_unlock(&event->mmap_mutex);
7248 7245
7249 out: 7246 out:
7250 return ret; 7247 return ret;
7251 } 7248 }
7252 7249
7253 /** 7250 /**
7254 * sys_perf_event_open - open a performance event, associate it to a task/cpu 7251 * sys_perf_event_open - open a performance event, associate it to a task/cpu
7255 * 7252 *
7256 * @attr_uptr: event_id type attributes for monitoring/sampling 7253 * @attr_uptr: event_id type attributes for monitoring/sampling
7257 * @pid: target pid 7254 * @pid: target pid
7258 * @cpu: target cpu 7255 * @cpu: target cpu
7259 * @group_fd: group leader event fd 7256 * @group_fd: group leader event fd
7260 */ 7257 */
7261 SYSCALL_DEFINE5(perf_event_open, 7258 SYSCALL_DEFINE5(perf_event_open,
7262 struct perf_event_attr __user *, attr_uptr, 7259 struct perf_event_attr __user *, attr_uptr,
7263 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 7260 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
7264 { 7261 {
7265 struct perf_event *group_leader = NULL, *output_event = NULL; 7262 struct perf_event *group_leader = NULL, *output_event = NULL;
7266 struct perf_event *event, *sibling; 7263 struct perf_event *event, *sibling;
7267 struct perf_event_attr attr; 7264 struct perf_event_attr attr;
7268 struct perf_event_context *ctx; 7265 struct perf_event_context *ctx;
7269 struct file *event_file = NULL; 7266 struct file *event_file = NULL;
7270 struct fd group = {NULL, 0}; 7267 struct fd group = {NULL, 0};
7271 struct task_struct *task = NULL; 7268 struct task_struct *task = NULL;
7272 struct pmu *pmu; 7269 struct pmu *pmu;
7273 int event_fd; 7270 int event_fd;
7274 int move_group = 0; 7271 int move_group = 0;
7275 int err; 7272 int err;
7276 int f_flags = O_RDWR; 7273 int f_flags = O_RDWR;
7277 7274
7278 /* for future expandability... */ 7275 /* for future expandability... */
7279 if (flags & ~PERF_FLAG_ALL) 7276 if (flags & ~PERF_FLAG_ALL)
7280 return -EINVAL; 7277 return -EINVAL;
7281 7278
7282 err = perf_copy_attr(attr_uptr, &attr); 7279 err = perf_copy_attr(attr_uptr, &attr);
7283 if (err) 7280 if (err)
7284 return err; 7281 return err;
7285 7282
7286 if (!attr.exclude_kernel) { 7283 if (!attr.exclude_kernel) {
7287 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 7284 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7288 return -EACCES; 7285 return -EACCES;
7289 } 7286 }
7290 7287
7291 if (attr.freq) { 7288 if (attr.freq) {
7292 if (attr.sample_freq > sysctl_perf_event_sample_rate) 7289 if (attr.sample_freq > sysctl_perf_event_sample_rate)
7293 return -EINVAL; 7290 return -EINVAL;
7294 } else { 7291 } else {
7295 if (attr.sample_period & (1ULL << 63)) 7292 if (attr.sample_period & (1ULL << 63))
7296 return -EINVAL; 7293 return -EINVAL;
7297 } 7294 }
7298 7295
7299 /* 7296 /*
7300 * In cgroup mode, the pid argument is used to pass the fd 7297 * In cgroup mode, the pid argument is used to pass the fd
7301 * opened to the cgroup directory in cgroupfs. The cpu argument 7298 * opened to the cgroup directory in cgroupfs. The cpu argument
7302 * designates the cpu on which to monitor threads from that 7299 * designates the cpu on which to monitor threads from that
7303 * cgroup. 7300 * cgroup.
7304 */ 7301 */
7305 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 7302 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
7306 return -EINVAL; 7303 return -EINVAL;
7307 7304
7308 if (flags & PERF_FLAG_FD_CLOEXEC) 7305 if (flags & PERF_FLAG_FD_CLOEXEC)
7309 f_flags |= O_CLOEXEC; 7306 f_flags |= O_CLOEXEC;
7310 7307
7311 event_fd = get_unused_fd_flags(f_flags); 7308 event_fd = get_unused_fd_flags(f_flags);
7312 if (event_fd < 0) 7309 if (event_fd < 0)
7313 return event_fd; 7310 return event_fd;
7314 7311
7315 if (group_fd != -1) { 7312 if (group_fd != -1) {
7316 err = perf_fget_light(group_fd, &group); 7313 err = perf_fget_light(group_fd, &group);
7317 if (err) 7314 if (err)
7318 goto err_fd; 7315 goto err_fd;
7319 group_leader = group.file->private_data; 7316 group_leader = group.file->private_data;
7320 if (flags & PERF_FLAG_FD_OUTPUT) 7317 if (flags & PERF_FLAG_FD_OUTPUT)
7321 output_event = group_leader; 7318 output_event = group_leader;
7322 if (flags & PERF_FLAG_FD_NO_GROUP) 7319 if (flags & PERF_FLAG_FD_NO_GROUP)
7323 group_leader = NULL; 7320 group_leader = NULL;
7324 } 7321 }
7325 7322
7326 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 7323 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
7327 task = find_lively_task_by_vpid(pid); 7324 task = find_lively_task_by_vpid(pid);
7328 if (IS_ERR(task)) { 7325 if (IS_ERR(task)) {
7329 err = PTR_ERR(task); 7326 err = PTR_ERR(task);
7330 goto err_group_fd; 7327 goto err_group_fd;
7331 } 7328 }
7332 } 7329 }
7333 7330
7334 if (task && group_leader && 7331 if (task && group_leader &&
7335 group_leader->attr.inherit != attr.inherit) { 7332 group_leader->attr.inherit != attr.inherit) {
7336 err = -EINVAL; 7333 err = -EINVAL;
7337 goto err_task; 7334 goto err_task;
7338 } 7335 }
7339 7336
7340 get_online_cpus(); 7337 get_online_cpus();
7341 7338
7342 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 7339 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
7343 NULL, NULL); 7340 NULL, NULL);
7344 if (IS_ERR(event)) { 7341 if (IS_ERR(event)) {
7345 err = PTR_ERR(event); 7342 err = PTR_ERR(event);
7346 goto err_cpus; 7343 goto err_cpus;
7347 } 7344 }
7348 7345
7349 if (flags & PERF_FLAG_PID_CGROUP) { 7346 if (flags & PERF_FLAG_PID_CGROUP) {
7350 err = perf_cgroup_connect(pid, event, &attr, group_leader); 7347 err = perf_cgroup_connect(pid, event, &attr, group_leader);
7351 if (err) { 7348 if (err) {
7352 __free_event(event); 7349 __free_event(event);
7353 goto err_cpus; 7350 goto err_cpus;
7354 } 7351 }
7355 } 7352 }
7356 7353
7357 if (is_sampling_event(event)) { 7354 if (is_sampling_event(event)) {
7358 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 7355 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
7359 err = -ENOTSUPP; 7356 err = -ENOTSUPP;
7360 goto err_alloc; 7357 goto err_alloc;
7361 } 7358 }
7362 } 7359 }
7363 7360
7364 account_event(event); 7361 account_event(event);
7365 7362
7366 /* 7363 /*
7367 * Special case software events and allow them to be part of 7364 * Special case software events and allow them to be part of
7368 * any hardware group. 7365 * any hardware group.
7369 */ 7366 */
7370 pmu = event->pmu; 7367 pmu = event->pmu;
7371 7368
7372 if (group_leader && 7369 if (group_leader &&
7373 (is_software_event(event) != is_software_event(group_leader))) { 7370 (is_software_event(event) != is_software_event(group_leader))) {
7374 if (is_software_event(event)) { 7371 if (is_software_event(event)) {
7375 /* 7372 /*
7376 * If event and group_leader are not both a software 7373 * If event and group_leader are not both a software
7377 * event, and event is, then group leader is not. 7374 * event, and event is, then group leader is not.
7378 * 7375 *
7379 * Allow the addition of software events to !software 7376 * Allow the addition of software events to !software
7380 * groups, this is safe because software events never 7377 * groups, this is safe because software events never
7381 * fail to schedule. 7378 * fail to schedule.
7382 */ 7379 */
7383 pmu = group_leader->pmu; 7380 pmu = group_leader->pmu;
7384 } else if (is_software_event(group_leader) && 7381 } else if (is_software_event(group_leader) &&
7385 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) { 7382 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
7386 /* 7383 /*
7387 * In case the group is a pure software group, and we 7384 * In case the group is a pure software group, and we
7388 * try to add a hardware event, move the whole group to 7385 * try to add a hardware event, move the whole group to
7389 * the hardware context. 7386 * the hardware context.
7390 */ 7387 */
7391 move_group = 1; 7388 move_group = 1;
7392 } 7389 }
7393 } 7390 }
7394 7391
7395 /* 7392 /*
7396 * Get the target context (task or percpu): 7393 * Get the target context (task or percpu):
7397 */ 7394 */
7398 ctx = find_get_context(pmu, task, event->cpu); 7395 ctx = find_get_context(pmu, task, event->cpu);
7399 if (IS_ERR(ctx)) { 7396 if (IS_ERR(ctx)) {
7400 err = PTR_ERR(ctx); 7397 err = PTR_ERR(ctx);
7401 goto err_alloc; 7398 goto err_alloc;
7402 } 7399 }
7403 7400
7404 if (task) { 7401 if (task) {
7405 put_task_struct(task); 7402 put_task_struct(task);
7406 task = NULL; 7403 task = NULL;
7407 } 7404 }
7408 7405
7409 /* 7406 /*
7410 * Look up the group leader (we will attach this event to it): 7407 * Look up the group leader (we will attach this event to it):
7411 */ 7408 */
7412 if (group_leader) { 7409 if (group_leader) {
7413 err = -EINVAL; 7410 err = -EINVAL;
7414 7411
7415 /* 7412 /*
7416 * Do not allow a recursive hierarchy (this new sibling 7413 * Do not allow a recursive hierarchy (this new sibling
7417 * becoming part of another group-sibling): 7414 * becoming part of another group-sibling):
7418 */ 7415 */
7419 if (group_leader->group_leader != group_leader) 7416 if (group_leader->group_leader != group_leader)
7420 goto err_context; 7417 goto err_context;
7421 /* 7418 /*
7422 * Do not allow to attach to a group in a different 7419 * Do not allow to attach to a group in a different
7423 * task or CPU context: 7420 * task or CPU context:
7424 */ 7421 */
7425 if (move_group) { 7422 if (move_group) {
7426 if (group_leader->ctx->type != ctx->type) 7423 if (group_leader->ctx->type != ctx->type)
7427 goto err_context; 7424 goto err_context;
7428 } else { 7425 } else {
7429 if (group_leader->ctx != ctx) 7426 if (group_leader->ctx != ctx)
7430 goto err_context; 7427 goto err_context;
7431 } 7428 }
7432 7429
7433 /* 7430 /*
7434 * Only a group leader can be exclusive or pinned 7431 * Only a group leader can be exclusive or pinned
7435 */ 7432 */
7436 if (attr.exclusive || attr.pinned) 7433 if (attr.exclusive || attr.pinned)
7437 goto err_context; 7434 goto err_context;
7438 } 7435 }
7439 7436
7440 if (output_event) { 7437 if (output_event) {
7441 err = perf_event_set_output(event, output_event); 7438 err = perf_event_set_output(event, output_event);
7442 if (err) 7439 if (err)
7443 goto err_context; 7440 goto err_context;
7444 } 7441 }
7445 7442
7446 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 7443 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,
7447 f_flags); 7444 f_flags);
7448 if (IS_ERR(event_file)) { 7445 if (IS_ERR(event_file)) {
7449 err = PTR_ERR(event_file); 7446 err = PTR_ERR(event_file);
7450 goto err_context; 7447 goto err_context;
7451 } 7448 }
7452 7449
7453 if (move_group) { 7450 if (move_group) {
7454 struct perf_event_context *gctx = group_leader->ctx; 7451 struct perf_event_context *gctx = group_leader->ctx;
7455 7452
7456 mutex_lock(&gctx->mutex); 7453 mutex_lock(&gctx->mutex);
7457 perf_remove_from_context(group_leader, false); 7454 perf_remove_from_context(group_leader, false);
7458 7455
7459 /* 7456 /*
7460 * Removing from the context ends up with disabled 7457 * Removing from the context ends up with disabled
7461 * event. What we want here is event in the initial 7458 * event. What we want here is event in the initial
7462 * startup state, ready to be add into new context. 7459 * startup state, ready to be add into new context.
7463 */ 7460 */
7464 perf_event__state_init(group_leader); 7461 perf_event__state_init(group_leader);
7465 list_for_each_entry(sibling, &group_leader->sibling_list, 7462 list_for_each_entry(sibling, &group_leader->sibling_list,
7466 group_entry) { 7463 group_entry) {
7467 perf_remove_from_context(sibling, false); 7464 perf_remove_from_context(sibling, false);
7468 perf_event__state_init(sibling); 7465 perf_event__state_init(sibling);
7469 put_ctx(gctx); 7466 put_ctx(gctx);
7470 } 7467 }
7471 mutex_unlock(&gctx->mutex); 7468 mutex_unlock(&gctx->mutex);
7472 put_ctx(gctx); 7469 put_ctx(gctx);
7473 } 7470 }
7474 7471
7475 WARN_ON_ONCE(ctx->parent_ctx); 7472 WARN_ON_ONCE(ctx->parent_ctx);
7476 mutex_lock(&ctx->mutex); 7473 mutex_lock(&ctx->mutex);
7477 7474
7478 if (move_group) { 7475 if (move_group) {
7479 synchronize_rcu(); 7476 synchronize_rcu();
7480 perf_install_in_context(ctx, group_leader, group_leader->cpu); 7477 perf_install_in_context(ctx, group_leader, group_leader->cpu);
7481 get_ctx(ctx); 7478 get_ctx(ctx);
7482 list_for_each_entry(sibling, &group_leader->sibling_list, 7479 list_for_each_entry(sibling, &group_leader->sibling_list,
7483 group_entry) { 7480 group_entry) {
7484 perf_install_in_context(ctx, sibling, sibling->cpu); 7481 perf_install_in_context(ctx, sibling, sibling->cpu);
7485 get_ctx(ctx); 7482 get_ctx(ctx);
7486 } 7483 }
7487 } 7484 }
7488 7485
7489 perf_install_in_context(ctx, event, event->cpu); 7486 perf_install_in_context(ctx, event, event->cpu);
7490 perf_unpin_context(ctx); 7487 perf_unpin_context(ctx);
7491 mutex_unlock(&ctx->mutex); 7488 mutex_unlock(&ctx->mutex);
7492 7489
7493 put_online_cpus(); 7490 put_online_cpus();
7494 7491
7495 event->owner = current; 7492 event->owner = current;
7496 7493
7497 mutex_lock(&current->perf_event_mutex); 7494 mutex_lock(&current->perf_event_mutex);
7498 list_add_tail(&event->owner_entry, &current->perf_event_list); 7495 list_add_tail(&event->owner_entry, &current->perf_event_list);
7499 mutex_unlock(&current->perf_event_mutex); 7496 mutex_unlock(&current->perf_event_mutex);
7500 7497
7501 /* 7498 /*
7502 * Precalculate sample_data sizes 7499 * Precalculate sample_data sizes
7503 */ 7500 */
7504 perf_event__header_size(event); 7501 perf_event__header_size(event);
7505 perf_event__id_header_size(event); 7502 perf_event__id_header_size(event);
7506 7503
7507 /* 7504 /*
7508 * Drop the reference on the group_event after placing the 7505 * Drop the reference on the group_event after placing the
7509 * new event on the sibling_list. This ensures destruction 7506 * new event on the sibling_list. This ensures destruction
7510 * of the group leader will find the pointer to itself in 7507 * of the group leader will find the pointer to itself in
7511 * perf_group_detach(). 7508 * perf_group_detach().
7512 */ 7509 */
7513 fdput(group); 7510 fdput(group);
7514 fd_install(event_fd, event_file); 7511 fd_install(event_fd, event_file);
7515 return event_fd; 7512 return event_fd;
7516 7513
7517 err_context: 7514 err_context:
7518 perf_unpin_context(ctx); 7515 perf_unpin_context(ctx);
7519 put_ctx(ctx); 7516 put_ctx(ctx);
7520 err_alloc: 7517 err_alloc:
7521 free_event(event); 7518 free_event(event);
7522 err_cpus: 7519 err_cpus:
7523 put_online_cpus(); 7520 put_online_cpus();
7524 err_task: 7521 err_task:
7525 if (task) 7522 if (task)
7526 put_task_struct(task); 7523 put_task_struct(task);
7527 err_group_fd: 7524 err_group_fd:
7528 fdput(group); 7525 fdput(group);
7529 err_fd: 7526 err_fd:
7530 put_unused_fd(event_fd); 7527 put_unused_fd(event_fd);
7531 return err; 7528 return err;
7532 } 7529 }
7533 7530
7534 /** 7531 /**
7535 * perf_event_create_kernel_counter 7532 * perf_event_create_kernel_counter
7536 * 7533 *
7537 * @attr: attributes of the counter to create 7534 * @attr: attributes of the counter to create
7538 * @cpu: cpu in which the counter is bound 7535 * @cpu: cpu in which the counter is bound
7539 * @task: task to profile (NULL for percpu) 7536 * @task: task to profile (NULL for percpu)
7540 */ 7537 */
7541 struct perf_event * 7538 struct perf_event *
7542 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 7539 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
7543 struct task_struct *task, 7540 struct task_struct *task,
7544 perf_overflow_handler_t overflow_handler, 7541 perf_overflow_handler_t overflow_handler,
7545 void *context) 7542 void *context)
7546 { 7543 {
7547 struct perf_event_context *ctx; 7544 struct perf_event_context *ctx;
7548 struct perf_event *event; 7545 struct perf_event *event;
7549 int err; 7546 int err;
7550 7547
7551 /* 7548 /*
7552 * Get the target context (task or percpu): 7549 * Get the target context (task or percpu):
7553 */ 7550 */
7554 7551
7555 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 7552 event = perf_event_alloc(attr, cpu, task, NULL, NULL,
7556 overflow_handler, context); 7553 overflow_handler, context);
7557 if (IS_ERR(event)) { 7554 if (IS_ERR(event)) {
7558 err = PTR_ERR(event); 7555 err = PTR_ERR(event);
7559 goto err; 7556 goto err;
7560 } 7557 }
7561 7558
7562 /* Mark owner so we could distinguish it from user events. */ 7559 /* Mark owner so we could distinguish it from user events. */
7563 event->owner = EVENT_OWNER_KERNEL; 7560 event->owner = EVENT_OWNER_KERNEL;
7564 7561
7565 account_event(event); 7562 account_event(event);
7566 7563
7567 ctx = find_get_context(event->pmu, task, cpu); 7564 ctx = find_get_context(event->pmu, task, cpu);
7568 if (IS_ERR(ctx)) { 7565 if (IS_ERR(ctx)) {
7569 err = PTR_ERR(ctx); 7566 err = PTR_ERR(ctx);
7570 goto err_free; 7567 goto err_free;
7571 } 7568 }
7572 7569
7573 WARN_ON_ONCE(ctx->parent_ctx); 7570 WARN_ON_ONCE(ctx->parent_ctx);
7574 mutex_lock(&ctx->mutex); 7571 mutex_lock(&ctx->mutex);
7575 perf_install_in_context(ctx, event, cpu); 7572 perf_install_in_context(ctx, event, cpu);
7576 perf_unpin_context(ctx); 7573 perf_unpin_context(ctx);
7577 mutex_unlock(&ctx->mutex); 7574 mutex_unlock(&ctx->mutex);
7578 7575
7579 return event; 7576 return event;
7580 7577
7581 err_free: 7578 err_free:
7582 free_event(event); 7579 free_event(event);
7583 err: 7580 err:
7584 return ERR_PTR(err); 7581 return ERR_PTR(err);
7585 } 7582 }
7586 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 7583 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
7587 7584
7588 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 7585 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
7589 { 7586 {
7590 struct perf_event_context *src_ctx; 7587 struct perf_event_context *src_ctx;
7591 struct perf_event_context *dst_ctx; 7588 struct perf_event_context *dst_ctx;
7592 struct perf_event *event, *tmp; 7589 struct perf_event *event, *tmp;
7593 LIST_HEAD(events); 7590 LIST_HEAD(events);
7594 7591
7595 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 7592 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
7596 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 7593 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
7597 7594
7598 mutex_lock(&src_ctx->mutex); 7595 mutex_lock(&src_ctx->mutex);
7599 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 7596 list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
7600 event_entry) { 7597 event_entry) {
7601 perf_remove_from_context(event, false); 7598 perf_remove_from_context(event, false);
7602 unaccount_event_cpu(event, src_cpu); 7599 unaccount_event_cpu(event, src_cpu);
7603 put_ctx(src_ctx); 7600 put_ctx(src_ctx);
7604 list_add(&event->migrate_entry, &events); 7601 list_add(&event->migrate_entry, &events);
7605 } 7602 }
7606 mutex_unlock(&src_ctx->mutex); 7603 mutex_unlock(&src_ctx->mutex);
7607 7604
7608 synchronize_rcu(); 7605 synchronize_rcu();
7609 7606
7610 mutex_lock(&dst_ctx->mutex); 7607 mutex_lock(&dst_ctx->mutex);
7611 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 7608 list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
7612 list_del(&event->migrate_entry); 7609 list_del(&event->migrate_entry);
7613 if (event->state >= PERF_EVENT_STATE_OFF) 7610 if (event->state >= PERF_EVENT_STATE_OFF)
7614 event->state = PERF_EVENT_STATE_INACTIVE; 7611 event->state = PERF_EVENT_STATE_INACTIVE;
7615 account_event_cpu(event, dst_cpu); 7612 account_event_cpu(event, dst_cpu);
7616 perf_install_in_context(dst_ctx, event, dst_cpu); 7613 perf_install_in_context(dst_ctx, event, dst_cpu);
7617 get_ctx(dst_ctx); 7614 get_ctx(dst_ctx);
7618 } 7615 }
7619 mutex_unlock(&dst_ctx->mutex); 7616 mutex_unlock(&dst_ctx->mutex);
7620 } 7617 }
7621 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 7618 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
7622 7619
7623 static void sync_child_event(struct perf_event *child_event, 7620 static void sync_child_event(struct perf_event *child_event,
7624 struct task_struct *child) 7621 struct task_struct *child)
7625 { 7622 {
7626 struct perf_event *parent_event = child_event->parent; 7623 struct perf_event *parent_event = child_event->parent;
7627 u64 child_val; 7624 u64 child_val;
7628 7625
7629 if (child_event->attr.inherit_stat) 7626 if (child_event->attr.inherit_stat)
7630 perf_event_read_event(child_event, child); 7627 perf_event_read_event(child_event, child);
7631 7628
7632 child_val = perf_event_count(child_event); 7629 child_val = perf_event_count(child_event);
7633 7630
7634 /* 7631 /*
7635 * Add back the child's count to the parent's count: 7632 * Add back the child's count to the parent's count:
7636 */ 7633 */
7637 atomic64_add(child_val, &parent_event->child_count); 7634 atomic64_add(child_val, &parent_event->child_count);
7638 atomic64_add(child_event->total_time_enabled, 7635 atomic64_add(child_event->total_time_enabled,
7639 &parent_event->child_total_time_enabled); 7636 &parent_event->child_total_time_enabled);
7640 atomic64_add(child_event->total_time_running, 7637 atomic64_add(child_event->total_time_running,
7641 &parent_event->child_total_time_running); 7638 &parent_event->child_total_time_running);
7642 7639
7643 /* 7640 /*
7644 * Remove this event from the parent's list 7641 * Remove this event from the parent's list
7645 */ 7642 */
7646 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 7643 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7647 mutex_lock(&parent_event->child_mutex); 7644 mutex_lock(&parent_event->child_mutex);
7648 list_del_init(&child_event->child_list); 7645 list_del_init(&child_event->child_list);
7649 mutex_unlock(&parent_event->child_mutex); 7646 mutex_unlock(&parent_event->child_mutex);
7650 7647
7651 /* 7648 /*
7652 * Make sure user/parent get notified, that we just 7649 * Make sure user/parent get notified, that we just
7653 * lost one event. 7650 * lost one event.
7654 */ 7651 */
7655 perf_event_wakeup(parent_event); 7652 perf_event_wakeup(parent_event);
7656 7653
7657 /* 7654 /*
7658 * Release the parent event, if this was the last 7655 * Release the parent event, if this was the last
7659 * reference to it. 7656 * reference to it.
7660 */ 7657 */
7661 put_event(parent_event); 7658 put_event(parent_event);
7662 } 7659 }
7663 7660
7664 static void 7661 static void
7665 __perf_event_exit_task(struct perf_event *child_event, 7662 __perf_event_exit_task(struct perf_event *child_event,
7666 struct perf_event_context *child_ctx, 7663 struct perf_event_context *child_ctx,
7667 struct task_struct *child) 7664 struct task_struct *child)
7668 { 7665 {
7669 /* 7666 /*
7670 * Do not destroy the 'original' grouping; because of the context 7667 * Do not destroy the 'original' grouping; because of the context
7671 * switch optimization the original events could've ended up in a 7668 * switch optimization the original events could've ended up in a
7672 * random child task. 7669 * random child task.
7673 * 7670 *
7674 * If we were to destroy the original group, all group related 7671 * If we were to destroy the original group, all group related
7675 * operations would cease to function properly after this random 7672 * operations would cease to function properly after this random
7676 * child dies. 7673 * child dies.
7677 * 7674 *
7678 * Do destroy all inherited groups, we don't care about those 7675 * Do destroy all inherited groups, we don't care about those
7679 * and being thorough is better. 7676 * and being thorough is better.
7680 */ 7677 */
7681 perf_remove_from_context(child_event, !!child_event->parent); 7678 perf_remove_from_context(child_event, !!child_event->parent);
7682 7679
7683 /* 7680 /*
7684 * It can happen that the parent exits first, and has events 7681 * It can happen that the parent exits first, and has events
7685 * that are still around due to the child reference. These 7682 * that are still around due to the child reference. These
7686 * events need to be zapped. 7683 * events need to be zapped.
7687 */ 7684 */
7688 if (child_event->parent) { 7685 if (child_event->parent) {
7689 sync_child_event(child_event, child); 7686 sync_child_event(child_event, child);
7690 free_event(child_event); 7687 free_event(child_event);
7691 } else { 7688 } else {
7692 child_event->state = PERF_EVENT_STATE_EXIT; 7689 child_event->state = PERF_EVENT_STATE_EXIT;
7693 perf_event_wakeup(child_event); 7690 perf_event_wakeup(child_event);
7694 } 7691 }
7695 } 7692 }
7696 7693
7697 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 7694 static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
7698 { 7695 {
7699 struct perf_event *child_event, *next; 7696 struct perf_event *child_event, *next;
7700 struct perf_event_context *child_ctx, *clone_ctx = NULL; 7697 struct perf_event_context *child_ctx, *clone_ctx = NULL;
7701 unsigned long flags; 7698 unsigned long flags;
7702 7699
7703 if (likely(!child->perf_event_ctxp[ctxn])) { 7700 if (likely(!child->perf_event_ctxp[ctxn])) {
7704 perf_event_task(child, NULL, 0); 7701 perf_event_task(child, NULL, 0);
7705 return; 7702 return;
7706 } 7703 }
7707 7704
7708 local_irq_save(flags); 7705 local_irq_save(flags);
7709 /* 7706 /*
7710 * We can't reschedule here because interrupts are disabled, 7707 * We can't reschedule here because interrupts are disabled,
7711 * and either child is current or it is a task that can't be 7708 * and either child is current or it is a task that can't be
7712 * scheduled, so we are now safe from rescheduling changing 7709 * scheduled, so we are now safe from rescheduling changing
7713 * our context. 7710 * our context.
7714 */ 7711 */
7715 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]); 7712 child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
7716 7713
7717 /* 7714 /*
7718 * Take the context lock here so that if find_get_context is 7715 * Take the context lock here so that if find_get_context is
7719 * reading child->perf_event_ctxp, we wait until it has 7716 * reading child->perf_event_ctxp, we wait until it has
7720 * incremented the context's refcount before we do put_ctx below. 7717 * incremented the context's refcount before we do put_ctx below.
7721 */ 7718 */
7722 raw_spin_lock(&child_ctx->lock); 7719 raw_spin_lock(&child_ctx->lock);
7723 task_ctx_sched_out(child_ctx); 7720 task_ctx_sched_out(child_ctx);
7724 child->perf_event_ctxp[ctxn] = NULL; 7721 child->perf_event_ctxp[ctxn] = NULL;
7725 7722
7726 /* 7723 /*
7727 * If this context is a clone; unclone it so it can't get 7724 * If this context is a clone; unclone it so it can't get
7728 * swapped to another process while we're removing all 7725 * swapped to another process while we're removing all
7729 * the events from it. 7726 * the events from it.
7730 */ 7727 */
7731 clone_ctx = unclone_ctx(child_ctx); 7728 clone_ctx = unclone_ctx(child_ctx);
7732 update_context_time(child_ctx); 7729 update_context_time(child_ctx);
7733 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 7730 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7734 7731
7735 if (clone_ctx) 7732 if (clone_ctx)
7736 put_ctx(clone_ctx); 7733 put_ctx(clone_ctx);
7737 7734
7738 /* 7735 /*
7739 * Report the task dead after unscheduling the events so that we 7736 * Report the task dead after unscheduling the events so that we
7740 * won't get any samples after PERF_RECORD_EXIT. We can however still 7737 * won't get any samples after PERF_RECORD_EXIT. We can however still
7741 * get a few PERF_RECORD_READ events. 7738 * get a few PERF_RECORD_READ events.
7742 */ 7739 */
7743 perf_event_task(child, child_ctx, 0); 7740 perf_event_task(child, child_ctx, 0);
7744 7741
7745 /* 7742 /*
7746 * We can recurse on the same lock type through: 7743 * We can recurse on the same lock type through:
7747 * 7744 *
7748 * __perf_event_exit_task() 7745 * __perf_event_exit_task()
7749 * sync_child_event() 7746 * sync_child_event()
7750 * put_event() 7747 * put_event()
7751 * mutex_lock(&ctx->mutex) 7748 * mutex_lock(&ctx->mutex)
7752 * 7749 *
7753 * But since its the parent context it won't be the same instance. 7750 * But since its the parent context it won't be the same instance.
7754 */ 7751 */
7755 mutex_lock(&child_ctx->mutex); 7752 mutex_lock(&child_ctx->mutex);
7756 7753
7757 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) 7754 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
7758 __perf_event_exit_task(child_event, child_ctx, child); 7755 __perf_event_exit_task(child_event, child_ctx, child);
7759 7756
7760 mutex_unlock(&child_ctx->mutex); 7757 mutex_unlock(&child_ctx->mutex);
7761 7758
7762 put_ctx(child_ctx); 7759 put_ctx(child_ctx);
7763 } 7760 }
7764 7761
7765 /* 7762 /*
7766 * When a child task exits, feed back event values to parent events. 7763 * When a child task exits, feed back event values to parent events.
7767 */ 7764 */
7768 void perf_event_exit_task(struct task_struct *child) 7765 void perf_event_exit_task(struct task_struct *child)
7769 { 7766 {
7770 struct perf_event *event, *tmp; 7767 struct perf_event *event, *tmp;
7771 int ctxn; 7768 int ctxn;
7772 7769
7773 mutex_lock(&child->perf_event_mutex); 7770 mutex_lock(&child->perf_event_mutex);
7774 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 7771 list_for_each_entry_safe(event, tmp, &child->perf_event_list,
7775 owner_entry) { 7772 owner_entry) {
7776 list_del_init(&event->owner_entry); 7773 list_del_init(&event->owner_entry);
7777 7774
7778 /* 7775 /*
7779 * Ensure the list deletion is visible before we clear 7776 * Ensure the list deletion is visible before we clear
7780 * the owner, closes a race against perf_release() where 7777 * the owner, closes a race against perf_release() where
7781 * we need to serialize on the owner->perf_event_mutex. 7778 * we need to serialize on the owner->perf_event_mutex.
7782 */ 7779 */
7783 smp_wmb(); 7780 smp_wmb();
7784 event->owner = NULL; 7781 event->owner = NULL;
7785 } 7782 }
7786 mutex_unlock(&child->perf_event_mutex); 7783 mutex_unlock(&child->perf_event_mutex);
7787 7784
7788 for_each_task_context_nr(ctxn) 7785 for_each_task_context_nr(ctxn)
7789 perf_event_exit_task_context(child, ctxn); 7786 perf_event_exit_task_context(child, ctxn);
7790 } 7787 }
7791 7788
7792 static void perf_free_event(struct perf_event *event, 7789 static void perf_free_event(struct perf_event *event,
7793 struct perf_event_context *ctx) 7790 struct perf_event_context *ctx)
7794 { 7791 {
7795 struct perf_event *parent = event->parent; 7792 struct perf_event *parent = event->parent;
7796 7793
7797 if (WARN_ON_ONCE(!parent)) 7794 if (WARN_ON_ONCE(!parent))
7798 return; 7795 return;
7799 7796
7800 mutex_lock(&parent->child_mutex); 7797 mutex_lock(&parent->child_mutex);
7801 list_del_init(&event->child_list); 7798 list_del_init(&event->child_list);
7802 mutex_unlock(&parent->child_mutex); 7799 mutex_unlock(&parent->child_mutex);
7803 7800
7804 put_event(parent); 7801 put_event(parent);
7805 7802
7806 perf_group_detach(event); 7803 perf_group_detach(event);
7807 list_del_event(event, ctx); 7804 list_del_event(event, ctx);
7808 free_event(event); 7805 free_event(event);
7809 } 7806 }
7810 7807
7811 /* 7808 /*
7812 * free an unexposed, unused context as created by inheritance by 7809 * free an unexposed, unused context as created by inheritance by
7813 * perf_event_init_task below, used by fork() in case of fail. 7810 * perf_event_init_task below, used by fork() in case of fail.
7814 */ 7811 */
7815 void perf_event_free_task(struct task_struct *task) 7812 void perf_event_free_task(struct task_struct *task)
7816 { 7813 {
7817 struct perf_event_context *ctx; 7814 struct perf_event_context *ctx;
7818 struct perf_event *event, *tmp; 7815 struct perf_event *event, *tmp;
7819 int ctxn; 7816 int ctxn;
7820 7817
7821 for_each_task_context_nr(ctxn) { 7818 for_each_task_context_nr(ctxn) {
7822 ctx = task->perf_event_ctxp[ctxn]; 7819 ctx = task->perf_event_ctxp[ctxn];
7823 if (!ctx) 7820 if (!ctx)
7824 continue; 7821 continue;
7825 7822
7826 mutex_lock(&ctx->mutex); 7823 mutex_lock(&ctx->mutex);
7827 again: 7824 again:
7828 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, 7825 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
7829 group_entry) 7826 group_entry)
7830 perf_free_event(event, ctx); 7827 perf_free_event(event, ctx);
7831 7828
7832 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, 7829 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
7833 group_entry) 7830 group_entry)
7834 perf_free_event(event, ctx); 7831 perf_free_event(event, ctx);
7835 7832
7836 if (!list_empty(&ctx->pinned_groups) || 7833 if (!list_empty(&ctx->pinned_groups) ||
7837 !list_empty(&ctx->flexible_groups)) 7834 !list_empty(&ctx->flexible_groups))
7838 goto again; 7835 goto again;
7839 7836
7840 mutex_unlock(&ctx->mutex); 7837 mutex_unlock(&ctx->mutex);
7841 7838
7842 put_ctx(ctx); 7839 put_ctx(ctx);
7843 } 7840 }
7844 } 7841 }
7845 7842
7846 void perf_event_delayed_put(struct task_struct *task) 7843 void perf_event_delayed_put(struct task_struct *task)
7847 { 7844 {
7848 int ctxn; 7845 int ctxn;
7849 7846
7850 for_each_task_context_nr(ctxn) 7847 for_each_task_context_nr(ctxn)
7851 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 7848 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
7852 } 7849 }
7853 7850
7854 /* 7851 /*
7855 * inherit a event from parent task to child task: 7852 * inherit a event from parent task to child task:
7856 */ 7853 */
7857 static struct perf_event * 7854 static struct perf_event *
7858 inherit_event(struct perf_event *parent_event, 7855 inherit_event(struct perf_event *parent_event,
7859 struct task_struct *parent, 7856 struct task_struct *parent,
7860 struct perf_event_context *parent_ctx, 7857 struct perf_event_context *parent_ctx,
7861 struct task_struct *child, 7858 struct task_struct *child,
7862 struct perf_event *group_leader, 7859 struct perf_event *group_leader,
7863 struct perf_event_context *child_ctx) 7860 struct perf_event_context *child_ctx)
7864 { 7861 {
7865 enum perf_event_active_state parent_state = parent_event->state; 7862 enum perf_event_active_state parent_state = parent_event->state;
7866 struct perf_event *child_event; 7863 struct perf_event *child_event;
7867 unsigned long flags; 7864 unsigned long flags;
7868 7865
7869 /* 7866 /*
7870 * Instead of creating recursive hierarchies of events, 7867 * Instead of creating recursive hierarchies of events,
7871 * we link inherited events back to the original parent, 7868 * we link inherited events back to the original parent,
7872 * which has a filp for sure, which we use as the reference 7869 * which has a filp for sure, which we use as the reference
7873 * count: 7870 * count:
7874 */ 7871 */
7875 if (parent_event->parent) 7872 if (parent_event->parent)
7876 parent_event = parent_event->parent; 7873 parent_event = parent_event->parent;
7877 7874
7878 child_event = perf_event_alloc(&parent_event->attr, 7875 child_event = perf_event_alloc(&parent_event->attr,
7879 parent_event->cpu, 7876 parent_event->cpu,
7880 child, 7877 child,
7881 group_leader, parent_event, 7878 group_leader, parent_event,
7882 NULL, NULL); 7879 NULL, NULL);
7883 if (IS_ERR(child_event)) 7880 if (IS_ERR(child_event))
7884 return child_event; 7881 return child_event;
7885 7882
7886 if (is_orphaned_event(parent_event) || 7883 if (is_orphaned_event(parent_event) ||
7887 !atomic_long_inc_not_zero(&parent_event->refcount)) { 7884 !atomic_long_inc_not_zero(&parent_event->refcount)) {
7888 free_event(child_event); 7885 free_event(child_event);
7889 return NULL; 7886 return NULL;
7890 } 7887 }
7891 7888
7892 get_ctx(child_ctx); 7889 get_ctx(child_ctx);
7893 7890
7894 /* 7891 /*
7895 * Make the child state follow the state of the parent event, 7892 * Make the child state follow the state of the parent event,
7896 * not its attr.disabled bit. We hold the parent's mutex, 7893 * not its attr.disabled bit. We hold the parent's mutex,
7897 * so we won't race with perf_event_{en, dis}able_family. 7894 * so we won't race with perf_event_{en, dis}able_family.
7898 */ 7895 */
7899 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 7896 if (parent_state >= PERF_EVENT_STATE_INACTIVE)
7900 child_event->state = PERF_EVENT_STATE_INACTIVE; 7897 child_event->state = PERF_EVENT_STATE_INACTIVE;
7901 else 7898 else
7902 child_event->state = PERF_EVENT_STATE_OFF; 7899 child_event->state = PERF_EVENT_STATE_OFF;
7903 7900
7904 if (parent_event->attr.freq) { 7901 if (parent_event->attr.freq) {
7905 u64 sample_period = parent_event->hw.sample_period; 7902 u64 sample_period = parent_event->hw.sample_period;
7906 struct hw_perf_event *hwc = &child_event->hw; 7903 struct hw_perf_event *hwc = &child_event->hw;
7907 7904
7908 hwc->sample_period = sample_period; 7905 hwc->sample_period = sample_period;
7909 hwc->last_period = sample_period; 7906 hwc->last_period = sample_period;
7910 7907
7911 local64_set(&hwc->period_left, sample_period); 7908 local64_set(&hwc->period_left, sample_period);
7912 } 7909 }
7913 7910
7914 child_event->ctx = child_ctx; 7911 child_event->ctx = child_ctx;
7915 child_event->overflow_handler = parent_event->overflow_handler; 7912 child_event->overflow_handler = parent_event->overflow_handler;
7916 child_event->overflow_handler_context 7913 child_event->overflow_handler_context
7917 = parent_event->overflow_handler_context; 7914 = parent_event->overflow_handler_context;
7918 7915
7919 /* 7916 /*
7920 * Precalculate sample_data sizes 7917 * Precalculate sample_data sizes
7921 */ 7918 */
7922 perf_event__header_size(child_event); 7919 perf_event__header_size(child_event);
7923 perf_event__id_header_size(child_event); 7920 perf_event__id_header_size(child_event);
7924 7921
7925 /* 7922 /*
7926 * Link it up in the child's context: 7923 * Link it up in the child's context:
7927 */ 7924 */
7928 raw_spin_lock_irqsave(&child_ctx->lock, flags); 7925 raw_spin_lock_irqsave(&child_ctx->lock, flags);
7929 add_event_to_ctx(child_event, child_ctx); 7926 add_event_to_ctx(child_event, child_ctx);
7930 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 7927 raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7931 7928
7932 /* 7929 /*
7933 * Link this into the parent event's child list 7930 * Link this into the parent event's child list
7934 */ 7931 */
7935 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 7932 WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7936 mutex_lock(&parent_event->child_mutex); 7933 mutex_lock(&parent_event->child_mutex);
7937 list_add_tail(&child_event->child_list, &parent_event->child_list); 7934 list_add_tail(&child_event->child_list, &parent_event->child_list);
7938 mutex_unlock(&parent_event->child_mutex); 7935 mutex_unlock(&parent_event->child_mutex);
7939 7936
7940 return child_event; 7937 return child_event;
7941 } 7938 }
7942 7939
7943 static int inherit_group(struct perf_event *parent_event, 7940 static int inherit_group(struct perf_event *parent_event,
7944 struct task_struct *parent, 7941 struct task_struct *parent,
7945 struct perf_event_context *parent_ctx, 7942 struct perf_event_context *parent_ctx,
7946 struct task_struct *child, 7943 struct task_struct *child,
7947 struct perf_event_context *child_ctx) 7944 struct perf_event_context *child_ctx)
7948 { 7945 {
7949 struct perf_event *leader; 7946 struct perf_event *leader;
7950 struct perf_event *sub; 7947 struct perf_event *sub;
7951 struct perf_event *child_ctr; 7948 struct perf_event *child_ctr;
7952 7949
7953 leader = inherit_event(parent_event, parent, parent_ctx, 7950 leader = inherit_event(parent_event, parent, parent_ctx,
7954 child, NULL, child_ctx); 7951 child, NULL, child_ctx);
7955 if (IS_ERR(leader)) 7952 if (IS_ERR(leader))
7956 return PTR_ERR(leader); 7953 return PTR_ERR(leader);
7957 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) { 7954 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
7958 child_ctr = inherit_event(sub, parent, parent_ctx, 7955 child_ctr = inherit_event(sub, parent, parent_ctx,
7959 child, leader, child_ctx); 7956 child, leader, child_ctx);
7960 if (IS_ERR(child_ctr)) 7957 if (IS_ERR(child_ctr))
7961 return PTR_ERR(child_ctr); 7958 return PTR_ERR(child_ctr);
7962 } 7959 }
7963 return 0; 7960 return 0;
7964 } 7961 }
7965 7962
7966 static int 7963 static int
7967 inherit_task_group(struct perf_event *event, struct task_struct *parent, 7964 inherit_task_group(struct perf_event *event, struct task_struct *parent,
7968 struct perf_event_context *parent_ctx, 7965 struct perf_event_context *parent_ctx,
7969 struct task_struct *child, int ctxn, 7966 struct task_struct *child, int ctxn,
7970 int *inherited_all) 7967 int *inherited_all)
7971 { 7968 {
7972 int ret; 7969 int ret;
7973 struct perf_event_context *child_ctx; 7970 struct perf_event_context *child_ctx;
7974 7971
7975 if (!event->attr.inherit) { 7972 if (!event->attr.inherit) {
7976 *inherited_all = 0; 7973 *inherited_all = 0;
7977 return 0; 7974 return 0;
7978 } 7975 }
7979 7976
7980 child_ctx = child->perf_event_ctxp[ctxn]; 7977 child_ctx = child->perf_event_ctxp[ctxn];
7981 if (!child_ctx) { 7978 if (!child_ctx) {
7982 /* 7979 /*
7983 * This is executed from the parent task context, so 7980 * This is executed from the parent task context, so
7984 * inherit events that have been marked for cloning. 7981 * inherit events that have been marked for cloning.
7985 * First allocate and initialize a context for the 7982 * First allocate and initialize a context for the
7986 * child. 7983 * child.
7987 */ 7984 */
7988 7985
7989 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 7986 child_ctx = alloc_perf_context(parent_ctx->pmu, child);
7990 if (!child_ctx) 7987 if (!child_ctx)
7991 return -ENOMEM; 7988 return -ENOMEM;
7992 7989
7993 child->perf_event_ctxp[ctxn] = child_ctx; 7990 child->perf_event_ctxp[ctxn] = child_ctx;
7994 } 7991 }
7995 7992
7996 ret = inherit_group(event, parent, parent_ctx, 7993 ret = inherit_group(event, parent, parent_ctx,
7997 child, child_ctx); 7994 child, child_ctx);
7998 7995
7999 if (ret) 7996 if (ret)
8000 *inherited_all = 0; 7997 *inherited_all = 0;
8001 7998
8002 return ret; 7999 return ret;
8003 } 8000 }
8004 8001
8005 /* 8002 /*
8006 * Initialize the perf_event context in task_struct 8003 * Initialize the perf_event context in task_struct
8007 */ 8004 */
8008 static int perf_event_init_context(struct task_struct *child, int ctxn) 8005 static int perf_event_init_context(struct task_struct *child, int ctxn)
8009 { 8006 {
8010 struct perf_event_context *child_ctx, *parent_ctx; 8007 struct perf_event_context *child_ctx, *parent_ctx;
8011 struct perf_event_context *cloned_ctx; 8008 struct perf_event_context *cloned_ctx;
8012 struct perf_event *event; 8009 struct perf_event *event;
8013 struct task_struct *parent = current; 8010 struct task_struct *parent = current;
8014 int inherited_all = 1; 8011 int inherited_all = 1;
8015 unsigned long flags; 8012 unsigned long flags;
8016 int ret = 0; 8013 int ret = 0;
8017 8014
8018 if (likely(!parent->perf_event_ctxp[ctxn])) 8015 if (likely(!parent->perf_event_ctxp[ctxn]))
8019 return 0; 8016 return 0;
8020 8017
8021 /* 8018 /*
8022 * If the parent's context is a clone, pin it so it won't get 8019 * If the parent's context is a clone, pin it so it won't get
8023 * swapped under us. 8020 * swapped under us.
8024 */ 8021 */
8025 parent_ctx = perf_pin_task_context(parent, ctxn); 8022 parent_ctx = perf_pin_task_context(parent, ctxn);
8026 if (!parent_ctx) 8023 if (!parent_ctx)
8027 return 0; 8024 return 0;
8028 8025
8029 /* 8026 /*
8030 * No need to check if parent_ctx != NULL here; since we saw 8027 * No need to check if parent_ctx != NULL here; since we saw
8031 * it non-NULL earlier, the only reason for it to become NULL 8028 * it non-NULL earlier, the only reason for it to become NULL
8032 * is if we exit, and since we're currently in the middle of 8029 * is if we exit, and since we're currently in the middle of
8033 * a fork we can't be exiting at the same time. 8030 * a fork we can't be exiting at the same time.
8034 */ 8031 */
8035 8032
8036 /* 8033 /*
8037 * Lock the parent list. No need to lock the child - not PID 8034 * Lock the parent list. No need to lock the child - not PID
8038 * hashed yet and not running, so nobody can access it. 8035 * hashed yet and not running, so nobody can access it.
8039 */ 8036 */
8040 mutex_lock(&parent_ctx->mutex); 8037 mutex_lock(&parent_ctx->mutex);
8041 8038
8042 /* 8039 /*
8043 * We dont have to disable NMIs - we are only looking at 8040 * We dont have to disable NMIs - we are only looking at
8044 * the list, not manipulating it: 8041 * the list, not manipulating it:
8045 */ 8042 */
8046 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) { 8043 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
8047 ret = inherit_task_group(event, parent, parent_ctx, 8044 ret = inherit_task_group(event, parent, parent_ctx,
8048 child, ctxn, &inherited_all); 8045 child, ctxn, &inherited_all);
8049 if (ret) 8046 if (ret)
8050 break; 8047 break;
8051 } 8048 }
8052 8049
8053 /* 8050 /*
8054 * We can't hold ctx->lock when iterating the ->flexible_group list due 8051 * We can't hold ctx->lock when iterating the ->flexible_group list due
8055 * to allocations, but we need to prevent rotation because 8052 * to allocations, but we need to prevent rotation because
8056 * rotate_ctx() will change the list from interrupt context. 8053 * rotate_ctx() will change the list from interrupt context.
8057 */ 8054 */
8058 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 8055 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8059 parent_ctx->rotate_disable = 1; 8056 parent_ctx->rotate_disable = 1;
8060 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 8057 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8061 8058
8062 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) { 8059 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
8063 ret = inherit_task_group(event, parent, parent_ctx, 8060 ret = inherit_task_group(event, parent, parent_ctx,
8064 child, ctxn, &inherited_all); 8061 child, ctxn, &inherited_all);
8065 if (ret) 8062 if (ret)
8066 break; 8063 break;
8067 } 8064 }
8068 8065
8069 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 8066 raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8070 parent_ctx->rotate_disable = 0; 8067 parent_ctx->rotate_disable = 0;
8071 8068
8072 child_ctx = child->perf_event_ctxp[ctxn]; 8069 child_ctx = child->perf_event_ctxp[ctxn];
8073 8070
8074 if (child_ctx && inherited_all) { 8071 if (child_ctx && inherited_all) {
8075 /* 8072 /*
8076 * Mark the child context as a clone of the parent 8073 * Mark the child context as a clone of the parent
8077 * context, or of whatever the parent is a clone of. 8074 * context, or of whatever the parent is a clone of.
8078 * 8075 *
8079 * Note that if the parent is a clone, the holding of 8076 * Note that if the parent is a clone, the holding of
8080 * parent_ctx->lock avoids it from being uncloned. 8077 * parent_ctx->lock avoids it from being uncloned.
8081 */ 8078 */
8082 cloned_ctx = parent_ctx->parent_ctx; 8079 cloned_ctx = parent_ctx->parent_ctx;
8083 if (cloned_ctx) { 8080 if (cloned_ctx) {
8084 child_ctx->parent_ctx = cloned_ctx; 8081 child_ctx->parent_ctx = cloned_ctx;
8085 child_ctx->parent_gen = parent_ctx->parent_gen; 8082 child_ctx->parent_gen = parent_ctx->parent_gen;
8086 } else { 8083 } else {
8087 child_ctx->parent_ctx = parent_ctx; 8084 child_ctx->parent_ctx = parent_ctx;
8088 child_ctx->parent_gen = parent_ctx->generation; 8085 child_ctx->parent_gen = parent_ctx->generation;
8089 } 8086 }
8090 get_ctx(child_ctx->parent_ctx); 8087 get_ctx(child_ctx->parent_ctx);
8091 } 8088 }
8092 8089
8093 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 8090 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8094 mutex_unlock(&parent_ctx->mutex); 8091 mutex_unlock(&parent_ctx->mutex);
8095 8092
8096 perf_unpin_context(parent_ctx); 8093 perf_unpin_context(parent_ctx);
8097 put_ctx(parent_ctx); 8094 put_ctx(parent_ctx);
8098 8095
8099 return ret; 8096 return ret;
8100 } 8097 }
8101 8098
8102 /* 8099 /*
8103 * Initialize the perf_event context in task_struct 8100 * Initialize the perf_event context in task_struct
8104 */ 8101 */
8105 int perf_event_init_task(struct task_struct *child) 8102 int perf_event_init_task(struct task_struct *child)
8106 { 8103 {
8107 int ctxn, ret; 8104 int ctxn, ret;
8108 8105
8109 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 8106 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
8110 mutex_init(&child->perf_event_mutex); 8107 mutex_init(&child->perf_event_mutex);
8111 INIT_LIST_HEAD(&child->perf_event_list); 8108 INIT_LIST_HEAD(&child->perf_event_list);
8112 8109
8113 for_each_task_context_nr(ctxn) { 8110 for_each_task_context_nr(ctxn) {
8114 ret = perf_event_init_context(child, ctxn); 8111 ret = perf_event_init_context(child, ctxn);
8115 if (ret) { 8112 if (ret) {
8116 perf_event_free_task(child); 8113 perf_event_free_task(child);
8117 return ret; 8114 return ret;
8118 } 8115 }
8119 } 8116 }
8120 8117
8121 return 0; 8118 return 0;
8122 } 8119 }
8123 8120
8124 static void __init perf_event_init_all_cpus(void) 8121 static void __init perf_event_init_all_cpus(void)
8125 { 8122 {
8126 struct swevent_htable *swhash; 8123 struct swevent_htable *swhash;
8127 int cpu; 8124 int cpu;
8128 8125
8129 for_each_possible_cpu(cpu) { 8126 for_each_possible_cpu(cpu) {
8130 swhash = &per_cpu(swevent_htable, cpu); 8127 swhash = &per_cpu(swevent_htable, cpu);
8131 mutex_init(&swhash->hlist_mutex); 8128 mutex_init(&swhash->hlist_mutex);
8132 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu)); 8129 INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
8133 } 8130 }
8134 } 8131 }
8135 8132
8136 static void perf_event_init_cpu(int cpu) 8133 static void perf_event_init_cpu(int cpu)
8137 { 8134 {
8138 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 8135 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8139 8136
8140 mutex_lock(&swhash->hlist_mutex); 8137 mutex_lock(&swhash->hlist_mutex);
8141 swhash->online = true; 8138 swhash->online = true;
8142 if (swhash->hlist_refcount > 0) { 8139 if (swhash->hlist_refcount > 0) {
8143 struct swevent_hlist *hlist; 8140 struct swevent_hlist *hlist;
8144 8141
8145 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 8142 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
8146 WARN_ON(!hlist); 8143 WARN_ON(!hlist);
8147 rcu_assign_pointer(swhash->swevent_hlist, hlist); 8144 rcu_assign_pointer(swhash->swevent_hlist, hlist);
8148 } 8145 }
8149 mutex_unlock(&swhash->hlist_mutex); 8146 mutex_unlock(&swhash->hlist_mutex);
8150 } 8147 }
8151 8148
8152 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC 8149 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
8153 static void perf_pmu_rotate_stop(struct pmu *pmu) 8150 static void perf_pmu_rotate_stop(struct pmu *pmu)
8154 { 8151 {
8155 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 8152 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
8156 8153
8157 WARN_ON(!irqs_disabled()); 8154 WARN_ON(!irqs_disabled());
8158 8155
8159 list_del_init(&cpuctx->rotation_list); 8156 list_del_init(&cpuctx->rotation_list);
8160 } 8157 }
8161 8158
8162 static void __perf_event_exit_context(void *__info) 8159 static void __perf_event_exit_context(void *__info)
8163 { 8160 {
8164 struct remove_event re = { .detach_group = true }; 8161 struct remove_event re = { .detach_group = true };
8165 struct perf_event_context *ctx = __info; 8162 struct perf_event_context *ctx = __info;
8166 8163
8167 perf_pmu_rotate_stop(ctx->pmu); 8164 perf_pmu_rotate_stop(ctx->pmu);
8168 8165
8169 rcu_read_lock(); 8166 rcu_read_lock();
8170 list_for_each_entry_rcu(re.event, &ctx->event_list, event_entry) 8167 list_for_each_entry_rcu(re.event, &ctx->event_list, event_entry)
8171 __perf_remove_from_context(&re); 8168 __perf_remove_from_context(&re);
8172 rcu_read_unlock(); 8169 rcu_read_unlock();
8173 } 8170 }
8174 8171
8175 static void perf_event_exit_cpu_context(int cpu) 8172 static void perf_event_exit_cpu_context(int cpu)
8176 { 8173 {
8177 struct perf_event_context *ctx; 8174 struct perf_event_context *ctx;
8178 struct pmu *pmu; 8175 struct pmu *pmu;
8179 int idx; 8176 int idx;
8180 8177
8181 idx = srcu_read_lock(&pmus_srcu); 8178 idx = srcu_read_lock(&pmus_srcu);
8182 list_for_each_entry_rcu(pmu, &pmus, entry) { 8179 list_for_each_entry_rcu(pmu, &pmus, entry) {
8183 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx; 8180 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
8184 8181
8185 mutex_lock(&ctx->mutex); 8182 mutex_lock(&ctx->mutex);
8186 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 8183 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
8187 mutex_unlock(&ctx->mutex); 8184 mutex_unlock(&ctx->mutex);
8188 } 8185 }
8189 srcu_read_unlock(&pmus_srcu, idx); 8186 srcu_read_unlock(&pmus_srcu, idx);
8190 } 8187 }
8191 8188
8192 static void perf_event_exit_cpu(int cpu) 8189 static void perf_event_exit_cpu(int cpu)
8193 { 8190 {
8194 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 8191 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8195 8192
8196 perf_event_exit_cpu_context(cpu); 8193 perf_event_exit_cpu_context(cpu);
8197 8194
8198 mutex_lock(&swhash->hlist_mutex); 8195 mutex_lock(&swhash->hlist_mutex);
8199 swhash->online = false; 8196 swhash->online = false;
8200 swevent_hlist_release(swhash); 8197 swevent_hlist_release(swhash);
8201 mutex_unlock(&swhash->hlist_mutex); 8198 mutex_unlock(&swhash->hlist_mutex);
8202 } 8199 }
8203 #else 8200 #else
8204 static inline void perf_event_exit_cpu(int cpu) { } 8201 static inline void perf_event_exit_cpu(int cpu) { }
8205 #endif 8202 #endif
8206 8203
8207 static int 8204 static int
8208 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 8205 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
8209 { 8206 {
8210 int cpu; 8207 int cpu;
8211 8208
8212 for_each_online_cpu(cpu) 8209 for_each_online_cpu(cpu)
8213 perf_event_exit_cpu(cpu); 8210 perf_event_exit_cpu(cpu);
8214 8211
8215 return NOTIFY_OK; 8212 return NOTIFY_OK;
8216 } 8213 }
8217 8214
8218 /* 8215 /*
8219 * Run the perf reboot notifier at the very last possible moment so that 8216 * Run the perf reboot notifier at the very last possible moment so that
8220 * the generic watchdog code runs as long as possible. 8217 * the generic watchdog code runs as long as possible.
8221 */ 8218 */
8222 static struct notifier_block perf_reboot_notifier = { 8219 static struct notifier_block perf_reboot_notifier = {
8223 .notifier_call = perf_reboot, 8220 .notifier_call = perf_reboot,
8224 .priority = INT_MIN, 8221 .priority = INT_MIN,
8225 }; 8222 };
8226 8223
8227 static int 8224 static int
8228 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 8225 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
8229 { 8226 {
8230 unsigned int cpu = (long)hcpu; 8227 unsigned int cpu = (long)hcpu;
8231 8228
8232 switch (action & ~CPU_TASKS_FROZEN) { 8229 switch (action & ~CPU_TASKS_FROZEN) {
8233 8230
8234 case CPU_UP_PREPARE: 8231 case CPU_UP_PREPARE:
8235 case CPU_DOWN_FAILED: 8232 case CPU_DOWN_FAILED:
8236 perf_event_init_cpu(cpu); 8233 perf_event_init_cpu(cpu);
8237 break; 8234 break;
8238 8235
8239 case CPU_UP_CANCELED: 8236 case CPU_UP_CANCELED:
8240 case CPU_DOWN_PREPARE: 8237 case CPU_DOWN_PREPARE:
8241 perf_event_exit_cpu(cpu); 8238 perf_event_exit_cpu(cpu);
8242 break; 8239 break;
8243 default: 8240 default:
8244 break; 8241 break;
8245 } 8242 }
8246 8243
8247 return NOTIFY_OK; 8244 return NOTIFY_OK;
8248 } 8245 }
8249 8246
8250 void __init perf_event_init(void) 8247 void __init perf_event_init(void)
8251 { 8248 {
8252 int ret; 8249 int ret;
8253 8250
8254 idr_init(&pmu_idr); 8251 idr_init(&pmu_idr);
8255 8252
8256 perf_event_init_all_cpus(); 8253 perf_event_init_all_cpus();
8257 init_srcu_struct(&pmus_srcu); 8254 init_srcu_struct(&pmus_srcu);
8258 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 8255 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
8259 perf_pmu_register(&perf_cpu_clock, NULL, -1); 8256 perf_pmu_register(&perf_cpu_clock, NULL, -1);
8260 perf_pmu_register(&perf_task_clock, NULL, -1); 8257 perf_pmu_register(&perf_task_clock, NULL, -1);
8261 perf_tp_register(); 8258 perf_tp_register();
8262 perf_cpu_notifier(perf_cpu_notify); 8259 perf_cpu_notifier(perf_cpu_notify);
8263 register_reboot_notifier(&perf_reboot_notifier); 8260 register_reboot_notifier(&perf_reboot_notifier);
8264 8261
8265 ret = init_hw_breakpoint(); 8262 ret = init_hw_breakpoint();
8266 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 8263 WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
8267 8264
8268 /* do not patch jump label more than once per second */ 8265 /* do not patch jump label more than once per second */
8269 jump_label_rate_limit(&perf_sched_events, HZ); 8266 jump_label_rate_limit(&perf_sched_events, HZ);
8270 8267
8271 /* 8268 /*
8272 * Build time assertion that we keep the data_head at the intended 8269 * Build time assertion that we keep the data_head at the intended
8273 * location. IOW, validation we got the __reserved[] size right. 8270 * location. IOW, validation we got the __reserved[] size right.
8274 */ 8271 */
8275 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 8272 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
8276 != 1024); 8273 != 1024);
8277 } 8274 }
8278 8275
8279 static int __init perf_event_sysfs_init(void) 8276 static int __init perf_event_sysfs_init(void)
8280 { 8277 {
8281 struct pmu *pmu; 8278 struct pmu *pmu;
8282 int ret; 8279 int ret;
8283 8280
8284 mutex_lock(&pmus_lock); 8281 mutex_lock(&pmus_lock);
8285 8282
8286 ret = bus_register(&pmu_bus); 8283 ret = bus_register(&pmu_bus);
8287 if (ret) 8284 if (ret)
8288 goto unlock; 8285 goto unlock;
8289 8286
8290 list_for_each_entry(pmu, &pmus, entry) { 8287 list_for_each_entry(pmu, &pmus, entry) {
8291 if (!pmu->name || pmu->type < 0) 8288 if (!pmu->name || pmu->type < 0)
8292 continue; 8289 continue;
8293 8290
8294 ret = pmu_dev_alloc(pmu); 8291 ret = pmu_dev_alloc(pmu);
8295 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 8292 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
8296 } 8293 }
8297 pmu_bus_running = 1; 8294 pmu_bus_running = 1;
8298 ret = 0; 8295 ret = 0;
8299 8296
8300 unlock: 8297 unlock:
8301 mutex_unlock(&pmus_lock); 8298 mutex_unlock(&pmus_lock);
8302 8299
8303 return ret; 8300 return ret;
8304 } 8301 }
8305 device_initcall(perf_event_sysfs_init); 8302 device_initcall(perf_event_sysfs_init);
8306 8303
8307 #ifdef CONFIG_CGROUP_PERF 8304 #ifdef CONFIG_CGROUP_PERF
8308 static struct cgroup_subsys_state * 8305 static struct cgroup_subsys_state *
8309 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 8306 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
8310 { 8307 {
8311 struct perf_cgroup *jc; 8308 struct perf_cgroup *jc;
8312 8309
8313 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 8310 jc = kzalloc(sizeof(*jc), GFP_KERNEL);
8314 if (!jc) 8311 if (!jc)
8315 return ERR_PTR(-ENOMEM); 8312 return ERR_PTR(-ENOMEM);
8316 8313
8317 jc->info = alloc_percpu(struct perf_cgroup_info); 8314 jc->info = alloc_percpu(struct perf_cgroup_info);
8318 if (!jc->info) { 8315 if (!jc->info) {
8319 kfree(jc); 8316 kfree(jc);
8320 return ERR_PTR(-ENOMEM); 8317 return ERR_PTR(-ENOMEM);
8321 } 8318 }
8322 8319
8323 return &jc->css; 8320 return &jc->css;
8324 } 8321 }
8325 8322
8326 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 8323 static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
8327 { 8324 {
8328 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 8325 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
8329 8326
8330 free_percpu(jc->info); 8327 free_percpu(jc->info);
8331 kfree(jc); 8328 kfree(jc);
8332 } 8329 }
8333 8330
8334 static int __perf_cgroup_move(void *info) 8331 static int __perf_cgroup_move(void *info)
8335 { 8332 {
8336 struct task_struct *task = info; 8333 struct task_struct *task = info;
8337 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 8334 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
8338 return 0; 8335 return 0;
8339 } 8336 }
8340 8337
8341 static void perf_cgroup_attach(struct cgroup_subsys_state *css, 8338 static void perf_cgroup_attach(struct cgroup_subsys_state *css,
8342 struct cgroup_taskset *tset) 8339 struct cgroup_taskset *tset)
8343 { 8340 {
8344 struct task_struct *task; 8341 struct task_struct *task;
8345 8342
8346 cgroup_taskset_for_each(task, tset) 8343 cgroup_taskset_for_each(task, tset)
8347 task_function_call(task, __perf_cgroup_move, task); 8344 task_function_call(task, __perf_cgroup_move, task);
8348 } 8345 }
8349 8346
8350 static void perf_cgroup_exit(struct cgroup_subsys_state *css, 8347 static void perf_cgroup_exit(struct cgroup_subsys_state *css,
8351 struct cgroup_subsys_state *old_css, 8348 struct cgroup_subsys_state *old_css,
8352 struct task_struct *task) 8349 struct task_struct *task)
8353 { 8350 {
8354 /* 8351 /*
8355 * cgroup_exit() is called in the copy_process() failure path. 8352 * cgroup_exit() is called in the copy_process() failure path.
8356 * Ignore this case since the task hasn't ran yet, this avoids 8353 * Ignore this case since the task hasn't ran yet, this avoids
8357 * trying to poke a half freed task state from generic code. 8354 * trying to poke a half freed task state from generic code.
8358 */ 8355 */
8359 if (!(task->flags & PF_EXITING)) 8356 if (!(task->flags & PF_EXITING))
8360 return; 8357 return;
8361 8358
8362 task_function_call(task, __perf_cgroup_move, task); 8359 task_function_call(task, __perf_cgroup_move, task);
8363 } 8360 }
8364 8361
8365 struct cgroup_subsys perf_event_cgrp_subsys = { 8362 struct cgroup_subsys perf_event_cgrp_subsys = {
8366 .css_alloc = perf_cgroup_css_alloc, 8363 .css_alloc = perf_cgroup_css_alloc,
8367 .css_free = perf_cgroup_css_free, 8364 .css_free = perf_cgroup_css_free,
8368 .exit = perf_cgroup_exit, 8365 .exit = perf_cgroup_exit,
tools/perf/builtin-annotate.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * builtin-annotate.c 2 * builtin-annotate.c
3 * 3 *
4 * Builtin annotate command: Analyze the perf.data input file, 4 * Builtin annotate command: Analyze the perf.data input file,
5 * look up and read DSOs and symbol information and display 5 * look up and read DSOs and symbol information and display
6 * a histogram of results, along various sorting keys. 6 * a histogram of results, along various sorting keys.
7 */ 7 */
8 #include "builtin.h" 8 #include "builtin.h"
9 9
10 #include "util/util.h" 10 #include "util/util.h"
11 #include "util/color.h" 11 #include "util/color.h"
12 #include <linux/list.h> 12 #include <linux/list.h>
13 #include "util/cache.h" 13 #include "util/cache.h"
14 #include <linux/rbtree.h> 14 #include <linux/rbtree.h>
15 #include "util/symbol.h" 15 #include "util/symbol.h"
16 16
17 #include "perf.h" 17 #include "perf.h"
18 #include "util/debug.h" 18 #include "util/debug.h"
19 19
20 #include "util/evlist.h" 20 #include "util/evlist.h"
21 #include "util/evsel.h" 21 #include "util/evsel.h"
22 #include "util/annotate.h" 22 #include "util/annotate.h"
23 #include "util/event.h" 23 #include "util/event.h"
24 #include "util/parse-options.h" 24 #include "util/parse-options.h"
25 #include "util/parse-events.h" 25 #include "util/parse-events.h"
26 #include "util/thread.h" 26 #include "util/thread.h"
27 #include "util/sort.h" 27 #include "util/sort.h"
28 #include "util/hist.h" 28 #include "util/hist.h"
29 #include "util/session.h" 29 #include "util/session.h"
30 #include "util/tool.h" 30 #include "util/tool.h"
31 #include "util/data.h" 31 #include "util/data.h"
32 #include "arch/common.h" 32 #include "arch/common.h"
33 33
34 #include <dlfcn.h> 34 #include <dlfcn.h>
35 #include <linux/bitmap.h> 35 #include <linux/bitmap.h>
36 36
37 struct perf_annotate { 37 struct perf_annotate {
38 struct perf_tool tool; 38 struct perf_tool tool;
39 struct perf_session *session; 39 struct perf_session *session;
40 bool use_tui, use_stdio, use_gtk; 40 bool use_tui, use_stdio, use_gtk;
41 bool full_paths; 41 bool full_paths;
42 bool print_line; 42 bool print_line;
43 bool skip_missing; 43 bool skip_missing;
44 const char *sym_hist_filter; 44 const char *sym_hist_filter;
45 const char *cpu_list; 45 const char *cpu_list;
46 DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS); 46 DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS);
47 }; 47 };
48 48
49 static int perf_evsel__add_sample(struct perf_evsel *evsel, 49 static int perf_evsel__add_sample(struct perf_evsel *evsel,
50 struct perf_sample *sample __maybe_unused, 50 struct perf_sample *sample __maybe_unused,
51 struct addr_location *al, 51 struct addr_location *al,
52 struct perf_annotate *ann) 52 struct perf_annotate *ann)
53 { 53 {
54 struct hists *hists = evsel__hists(evsel); 54 struct hists *hists = evsel__hists(evsel);
55 struct hist_entry *he; 55 struct hist_entry *he;
56 int ret; 56 int ret;
57 57
58 if (ann->sym_hist_filter != NULL && 58 if (ann->sym_hist_filter != NULL &&
59 (al->sym == NULL || 59 (al->sym == NULL ||
60 strcmp(ann->sym_hist_filter, al->sym->name) != 0)) { 60 strcmp(ann->sym_hist_filter, al->sym->name) != 0)) {
61 /* We're only interested in a symbol named sym_hist_filter */ 61 /* We're only interested in a symbol named sym_hist_filter */
62 if (al->sym != NULL) { 62 if (al->sym != NULL) {
63 rb_erase(&al->sym->rb_node, 63 rb_erase(&al->sym->rb_node,
64 &al->map->dso->symbols[al->map->type]); 64 &al->map->dso->symbols[al->map->type]);
65 symbol__delete(al->sym); 65 symbol__delete(al->sym);
66 } 66 }
67 return 0; 67 return 0;
68 } 68 }
69 69
70 he = __hists__add_entry(hists, al, NULL, NULL, NULL, 1, 1, 0, true); 70 he = __hists__add_entry(hists, al, NULL, NULL, NULL, 1, 1, 0, true);
71 if (he == NULL) 71 if (he == NULL)
72 return -ENOMEM; 72 return -ENOMEM;
73 73
74 ret = hist_entry__inc_addr_samples(he, evsel->idx, al->addr); 74 ret = hist_entry__inc_addr_samples(he, evsel->idx, al->addr);
75 hists__inc_nr_samples(hists, true); 75 hists__inc_nr_samples(hists, true);
76 return ret; 76 return ret;
77 } 77 }
78 78
79 static int process_sample_event(struct perf_tool *tool, 79 static int process_sample_event(struct perf_tool *tool,
80 union perf_event *event, 80 union perf_event *event,
81 struct perf_sample *sample, 81 struct perf_sample *sample,
82 struct perf_evsel *evsel, 82 struct perf_evsel *evsel,
83 struct machine *machine) 83 struct machine *machine)
84 { 84 {
85 struct perf_annotate *ann = container_of(tool, struct perf_annotate, tool); 85 struct perf_annotate *ann = container_of(tool, struct perf_annotate, tool);
86 struct addr_location al; 86 struct addr_location al;
87 87
88 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) { 88 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) {
89 pr_warning("problem processing %d event, skipping it.\n", 89 pr_warning("problem processing %d event, skipping it.\n",
90 event->header.type); 90 event->header.type);
91 return -1; 91 return -1;
92 } 92 }
93 93
94 if (ann->cpu_list && !test_bit(sample->cpu, ann->cpu_bitmap)) 94 if (ann->cpu_list && !test_bit(sample->cpu, ann->cpu_bitmap))
95 return 0; 95 return 0;
96 96
97 if (!al.filtered && perf_evsel__add_sample(evsel, sample, &al, ann)) { 97 if (!al.filtered && perf_evsel__add_sample(evsel, sample, &al, ann)) {
98 pr_warning("problem incrementing symbol count, " 98 pr_warning("problem incrementing symbol count, "
99 "skipping event\n"); 99 "skipping event\n");
100 return -1; 100 return -1;
101 } 101 }
102 102
103 return 0; 103 return 0;
104 } 104 }
105 105
106 static int hist_entry__tty_annotate(struct hist_entry *he, 106 static int hist_entry__tty_annotate(struct hist_entry *he,
107 struct perf_evsel *evsel, 107 struct perf_evsel *evsel,
108 struct perf_annotate *ann) 108 struct perf_annotate *ann)
109 { 109 {
110 return symbol__tty_annotate(he->ms.sym, he->ms.map, evsel, 110 return symbol__tty_annotate(he->ms.sym, he->ms.map, evsel,
111 ann->print_line, ann->full_paths, 0, 0); 111 ann->print_line, ann->full_paths, 0, 0);
112 } 112 }
113 113
114 static void hists__find_annotations(struct hists *hists, 114 static void hists__find_annotations(struct hists *hists,
115 struct perf_evsel *evsel, 115 struct perf_evsel *evsel,
116 struct perf_annotate *ann) 116 struct perf_annotate *ann)
117 { 117 {
118 struct rb_node *nd = rb_first(&hists->entries), *next; 118 struct rb_node *nd = rb_first(&hists->entries), *next;
119 int key = K_RIGHT; 119 int key = K_RIGHT;
120 120
121 while (nd) { 121 while (nd) {
122 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); 122 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
123 struct annotation *notes; 123 struct annotation *notes;
124 124
125 if (he->ms.sym == NULL || he->ms.map->dso->annotate_warned) 125 if (he->ms.sym == NULL || he->ms.map->dso->annotate_warned)
126 goto find_next; 126 goto find_next;
127 127
128 notes = symbol__annotation(he->ms.sym); 128 notes = symbol__annotation(he->ms.sym);
129 if (notes->src == NULL) { 129 if (notes->src == NULL) {
130 find_next: 130 find_next:
131 if (key == K_LEFT) 131 if (key == K_LEFT)
132 nd = rb_prev(nd); 132 nd = rb_prev(nd);
133 else 133 else
134 nd = rb_next(nd); 134 nd = rb_next(nd);
135 continue; 135 continue;
136 } 136 }
137 137
138 if (use_browser == 2) { 138 if (use_browser == 2) {
139 int ret; 139 int ret;
140 int (*annotate)(struct hist_entry *he, 140 int (*annotate)(struct hist_entry *he,
141 struct perf_evsel *evsel, 141 struct perf_evsel *evsel,
142 struct hist_browser_timer *hbt); 142 struct hist_browser_timer *hbt);
143 143
144 annotate = dlsym(perf_gtk_handle, 144 annotate = dlsym(perf_gtk_handle,
145 "hist_entry__gtk_annotate"); 145 "hist_entry__gtk_annotate");
146 if (annotate == NULL) { 146 if (annotate == NULL) {
147 ui__error("GTK browser not found!\n"); 147 ui__error("GTK browser not found!\n");
148 return; 148 return;
149 } 149 }
150 150
151 ret = annotate(he, evsel, NULL); 151 ret = annotate(he, evsel, NULL);
152 if (!ret || !ann->skip_missing) 152 if (!ret || !ann->skip_missing)
153 return; 153 return;
154 154
155 /* skip missing symbols */ 155 /* skip missing symbols */
156 nd = rb_next(nd); 156 nd = rb_next(nd);
157 } else if (use_browser == 1) { 157 } else if (use_browser == 1) {
158 key = hist_entry__tui_annotate(he, evsel, NULL); 158 key = hist_entry__tui_annotate(he, evsel, NULL);
159 switch (key) { 159 switch (key) {
160 case -1: 160 case -1:
161 if (!ann->skip_missing) 161 if (!ann->skip_missing)
162 return; 162 return;
163 /* fall through */ 163 /* fall through */
164 case K_RIGHT: 164 case K_RIGHT:
165 next = rb_next(nd); 165 next = rb_next(nd);
166 break; 166 break;
167 case K_LEFT: 167 case K_LEFT:
168 next = rb_prev(nd); 168 next = rb_prev(nd);
169 break; 169 break;
170 default: 170 default:
171 return; 171 return;
172 } 172 }
173 173
174 if (next != NULL) 174 if (next != NULL)
175 nd = next; 175 nd = next;
176 } else { 176 } else {
177 hist_entry__tty_annotate(he, evsel, ann); 177 hist_entry__tty_annotate(he, evsel, ann);
178 nd = rb_next(nd); 178 nd = rb_next(nd);
179 /* 179 /*
180 * Since we have a hist_entry per IP for the same 180 * Since we have a hist_entry per IP for the same
181 * symbol, free he->ms.sym->src to signal we already 181 * symbol, free he->ms.sym->src to signal we already
182 * processed this symbol. 182 * processed this symbol.
183 */ 183 */
184 zfree(&notes->src); 184 zfree(&notes->src);
185 } 185 }
186 } 186 }
187 } 187 }
188 188
189 static int __cmd_annotate(struct perf_annotate *ann) 189 static int __cmd_annotate(struct perf_annotate *ann)
190 { 190 {
191 int ret; 191 int ret;
192 struct perf_session *session = ann->session; 192 struct perf_session *session = ann->session;
193 struct perf_evsel *pos; 193 struct perf_evsel *pos;
194 u64 total_nr_samples; 194 u64 total_nr_samples;
195 195
196 machines__set_symbol_filter(&session->machines, symbol__annotate_init); 196 machines__set_symbol_filter(&session->machines, symbol__annotate_init);
197 197
198 if (ann->cpu_list) { 198 if (ann->cpu_list) {
199 ret = perf_session__cpu_bitmap(session, ann->cpu_list, 199 ret = perf_session__cpu_bitmap(session, ann->cpu_list,
200 ann->cpu_bitmap); 200 ann->cpu_bitmap);
201 if (ret) 201 if (ret)
202 goto out; 202 goto out;
203 } 203 }
204 204
205 if (!objdump_path) { 205 if (!objdump_path) {
206 ret = perf_session_env__lookup_objdump(&session->header.env); 206 ret = perf_session_env__lookup_objdump(&session->header.env);
207 if (ret) 207 if (ret)
208 goto out; 208 goto out;
209 } 209 }
210 210
211 ret = perf_session__process_events(session, &ann->tool); 211 ret = perf_session__process_events(session, &ann->tool);
212 if (ret) 212 if (ret)
213 goto out; 213 goto out;
214 214
215 if (dump_trace) { 215 if (dump_trace) {
216 perf_session__fprintf_nr_events(session, stdout); 216 perf_session__fprintf_nr_events(session, stdout);
217 perf_evlist__fprintf_nr_events(session->evlist, stdout); 217 perf_evlist__fprintf_nr_events(session->evlist, stdout);
218 goto out; 218 goto out;
219 } 219 }
220 220
221 if (verbose > 3) 221 if (verbose > 3)
222 perf_session__fprintf(session, stdout); 222 perf_session__fprintf(session, stdout);
223 223
224 if (verbose > 2) 224 if (verbose > 2)
225 perf_session__fprintf_dsos(session, stdout); 225 perf_session__fprintf_dsos(session, stdout);
226 226
227 total_nr_samples = 0; 227 total_nr_samples = 0;
228 evlist__for_each(session->evlist, pos) { 228 evlist__for_each(session->evlist, pos) {
229 struct hists *hists = evsel__hists(pos); 229 struct hists *hists = evsel__hists(pos);
230 u32 nr_samples = hists->stats.nr_events[PERF_RECORD_SAMPLE]; 230 u32 nr_samples = hists->stats.nr_events[PERF_RECORD_SAMPLE];
231 231
232 if (nr_samples > 0) { 232 if (nr_samples > 0) {
233 total_nr_samples += nr_samples; 233 total_nr_samples += nr_samples;
234 hists__collapse_resort(hists, NULL); 234 hists__collapse_resort(hists, NULL);
235 hists__output_resort(hists); 235 hists__output_resort(hists, NULL);
236 236
237 if (symbol_conf.event_group && 237 if (symbol_conf.event_group &&
238 !perf_evsel__is_group_leader(pos)) 238 !perf_evsel__is_group_leader(pos))
239 continue; 239 continue;
240 240
241 hists__find_annotations(hists, pos, ann); 241 hists__find_annotations(hists, pos, ann);
242 } 242 }
243 } 243 }
244 244
245 if (total_nr_samples == 0) { 245 if (total_nr_samples == 0) {
246 ui__error("The %s file has no samples!\n", session->file->path); 246 ui__error("The %s file has no samples!\n", session->file->path);
247 goto out; 247 goto out;
248 } 248 }
249 249
250 if (use_browser == 2) { 250 if (use_browser == 2) {
251 void (*show_annotations)(void); 251 void (*show_annotations)(void);
252 252
253 show_annotations = dlsym(perf_gtk_handle, 253 show_annotations = dlsym(perf_gtk_handle,
254 "perf_gtk__show_annotations"); 254 "perf_gtk__show_annotations");
255 if (show_annotations == NULL) { 255 if (show_annotations == NULL) {
256 ui__error("GTK browser not found!\n"); 256 ui__error("GTK browser not found!\n");
257 goto out; 257 goto out;
258 } 258 }
259 show_annotations(); 259 show_annotations();
260 } 260 }
261 261
262 out: 262 out:
263 return ret; 263 return ret;
264 } 264 }
265 265
266 static const char * const annotate_usage[] = { 266 static const char * const annotate_usage[] = {
267 "perf annotate [<options>]", 267 "perf annotate [<options>]",
268 NULL 268 NULL
269 }; 269 };
270 270
271 int cmd_annotate(int argc, const char **argv, const char *prefix __maybe_unused) 271 int cmd_annotate(int argc, const char **argv, const char *prefix __maybe_unused)
272 { 272 {
273 struct perf_annotate annotate = { 273 struct perf_annotate annotate = {
274 .tool = { 274 .tool = {
275 .sample = process_sample_event, 275 .sample = process_sample_event,
276 .mmap = perf_event__process_mmap, 276 .mmap = perf_event__process_mmap,
277 .mmap2 = perf_event__process_mmap2, 277 .mmap2 = perf_event__process_mmap2,
278 .comm = perf_event__process_comm, 278 .comm = perf_event__process_comm,
279 .exit = perf_event__process_exit, 279 .exit = perf_event__process_exit,
280 .fork = perf_event__process_fork, 280 .fork = perf_event__process_fork,
281 .ordered_events = true, 281 .ordered_events = true,
282 .ordering_requires_timestamps = true, 282 .ordering_requires_timestamps = true,
283 }, 283 },
284 }; 284 };
285 struct perf_data_file file = { 285 struct perf_data_file file = {
286 .path = input_name, 286 .path = input_name,
287 .mode = PERF_DATA_MODE_READ, 287 .mode = PERF_DATA_MODE_READ,
288 }; 288 };
289 const struct option options[] = { 289 const struct option options[] = {
290 OPT_STRING('i', "input", &input_name, "file", 290 OPT_STRING('i', "input", &input_name, "file",
291 "input file name"), 291 "input file name"),
292 OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]", 292 OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
293 "only consider symbols in these dsos"), 293 "only consider symbols in these dsos"),
294 OPT_STRING('s', "symbol", &annotate.sym_hist_filter, "symbol", 294 OPT_STRING('s', "symbol", &annotate.sym_hist_filter, "symbol",
295 "symbol to annotate"), 295 "symbol to annotate"),
296 OPT_BOOLEAN('f', "force", &file.force, "don't complain, do it"), 296 OPT_BOOLEAN('f', "force", &file.force, "don't complain, do it"),
297 OPT_INCR('v', "verbose", &verbose, 297 OPT_INCR('v', "verbose", &verbose,
298 "be more verbose (show symbol address, etc)"), 298 "be more verbose (show symbol address, etc)"),
299 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace, 299 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
300 "dump raw trace in ASCII"), 300 "dump raw trace in ASCII"),
301 OPT_BOOLEAN(0, "gtk", &annotate.use_gtk, "Use the GTK interface"), 301 OPT_BOOLEAN(0, "gtk", &annotate.use_gtk, "Use the GTK interface"),
302 OPT_BOOLEAN(0, "tui", &annotate.use_tui, "Use the TUI interface"), 302 OPT_BOOLEAN(0, "tui", &annotate.use_tui, "Use the TUI interface"),
303 OPT_BOOLEAN(0, "stdio", &annotate.use_stdio, "Use the stdio interface"), 303 OPT_BOOLEAN(0, "stdio", &annotate.use_stdio, "Use the stdio interface"),
304 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name, 304 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
305 "file", "vmlinux pathname"), 305 "file", "vmlinux pathname"),
306 OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules, 306 OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules,
307 "load module symbols - WARNING: use only with -k and LIVE kernel"), 307 "load module symbols - WARNING: use only with -k and LIVE kernel"),
308 OPT_BOOLEAN('l', "print-line", &annotate.print_line, 308 OPT_BOOLEAN('l', "print-line", &annotate.print_line,
309 "print matching source lines (may be slow)"), 309 "print matching source lines (may be slow)"),
310 OPT_BOOLEAN('P', "full-paths", &annotate.full_paths, 310 OPT_BOOLEAN('P', "full-paths", &annotate.full_paths,
311 "Don't shorten the displayed pathnames"), 311 "Don't shorten the displayed pathnames"),
312 OPT_BOOLEAN(0, "skip-missing", &annotate.skip_missing, 312 OPT_BOOLEAN(0, "skip-missing", &annotate.skip_missing,
313 "Skip symbols that cannot be annotated"), 313 "Skip symbols that cannot be annotated"),
314 OPT_STRING('C', "cpu", &annotate.cpu_list, "cpu", "list of cpus to profile"), 314 OPT_STRING('C', "cpu", &annotate.cpu_list, "cpu", "list of cpus to profile"),
315 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory", 315 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
316 "Look for files with symbols relative to this directory"), 316 "Look for files with symbols relative to this directory"),
317 OPT_BOOLEAN(0, "source", &symbol_conf.annotate_src, 317 OPT_BOOLEAN(0, "source", &symbol_conf.annotate_src,
318 "Interleave source code with assembly code (default)"), 318 "Interleave source code with assembly code (default)"),
319 OPT_BOOLEAN(0, "asm-raw", &symbol_conf.annotate_asm_raw, 319 OPT_BOOLEAN(0, "asm-raw", &symbol_conf.annotate_asm_raw,
320 "Display raw encoding of assembly instructions (default)"), 320 "Display raw encoding of assembly instructions (default)"),
321 OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style", 321 OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style",
322 "Specify disassembler style (e.g. -M intel for intel syntax)"), 322 "Specify disassembler style (e.g. -M intel for intel syntax)"),
323 OPT_STRING(0, "objdump", &objdump_path, "path", 323 OPT_STRING(0, "objdump", &objdump_path, "path",
324 "objdump binary to use for disassembly and annotations"), 324 "objdump binary to use for disassembly and annotations"),
325 OPT_BOOLEAN(0, "group", &symbol_conf.event_group, 325 OPT_BOOLEAN(0, "group", &symbol_conf.event_group,
326 "Show event group information together"), 326 "Show event group information together"),
327 OPT_END() 327 OPT_END()
328 }; 328 };
329 int ret = hists__init(); 329 int ret = hists__init();
330 330
331 if (ret < 0) 331 if (ret < 0)
332 return ret; 332 return ret;
333 333
334 argc = parse_options(argc, argv, options, annotate_usage, 0); 334 argc = parse_options(argc, argv, options, annotate_usage, 0);
335 335
336 if (annotate.use_stdio) 336 if (annotate.use_stdio)
337 use_browser = 0; 337 use_browser = 0;
338 else if (annotate.use_tui) 338 else if (annotate.use_tui)
339 use_browser = 1; 339 use_browser = 1;
340 else if (annotate.use_gtk) 340 else if (annotate.use_gtk)
341 use_browser = 2; 341 use_browser = 2;
342 342
343 setup_browser(true); 343 setup_browser(true);
344 344
345 annotate.session = perf_session__new(&file, false, &annotate.tool); 345 annotate.session = perf_session__new(&file, false, &annotate.tool);
346 if (annotate.session == NULL) 346 if (annotate.session == NULL)
347 return -1; 347 return -1;
348 348
349 symbol_conf.priv_size = sizeof(struct annotation); 349 symbol_conf.priv_size = sizeof(struct annotation);
350 symbol_conf.try_vmlinux_path = true; 350 symbol_conf.try_vmlinux_path = true;
351 351
352 ret = symbol__init(&annotate.session->header.env); 352 ret = symbol__init(&annotate.session->header.env);
353 if (ret < 0) 353 if (ret < 0)
354 goto out_delete; 354 goto out_delete;
355 355
356 if (setup_sorting() < 0) 356 if (setup_sorting() < 0)
357 usage_with_options(annotate_usage, options); 357 usage_with_options(annotate_usage, options);
358 358
359 if (argc) { 359 if (argc) {
360 /* 360 /*
361 * Special case: if there's an argument left then assume that 361 * Special case: if there's an argument left then assume that
362 * it's a symbol filter: 362 * it's a symbol filter:
363 */ 363 */
364 if (argc > 1) 364 if (argc > 1)
365 usage_with_options(annotate_usage, options); 365 usage_with_options(annotate_usage, options);
366 366
367 annotate.sym_hist_filter = argv[0]; 367 annotate.sym_hist_filter = argv[0];
368 } 368 }
369 369
370 ret = __cmd_annotate(&annotate); 370 ret = __cmd_annotate(&annotate);
371 371
372 out_delete: 372 out_delete:
373 /* 373 /*
374 * Speed up the exit process, for large files this can 374 * Speed up the exit process, for large files this can
375 * take quite a while. 375 * take quite a while.
376 * 376 *
377 * XXX Enable this when using valgrind or if we ever 377 * XXX Enable this when using valgrind or if we ever
378 * librarize this command. 378 * librarize this command.
379 * 379 *
380 * Also experiment with obstacks to see how much speed 380 * Also experiment with obstacks to see how much speed
381 * up we'll get here. 381 * up we'll get here.
382 * 382 *
383 * perf_session__delete(session); 383 * perf_session__delete(session);
384 */ 384 */
385 return ret; 385 return ret;
386 } 386 }
387 387
tools/perf/builtin-diff.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * builtin-diff.c 2 * builtin-diff.c
3 * 3 *
4 * Builtin diff command: Analyze two perf.data input files, look up and read 4 * Builtin diff command: Analyze two perf.data input files, look up and read
5 * DSOs and symbol information, sort them and produce a diff. 5 * DSOs and symbol information, sort them and produce a diff.
6 */ 6 */
7 #include "builtin.h" 7 #include "builtin.h"
8 8
9 #include "util/debug.h" 9 #include "util/debug.h"
10 #include "util/event.h" 10 #include "util/event.h"
11 #include "util/hist.h" 11 #include "util/hist.h"
12 #include "util/evsel.h" 12 #include "util/evsel.h"
13 #include "util/evlist.h" 13 #include "util/evlist.h"
14 #include "util/session.h" 14 #include "util/session.h"
15 #include "util/tool.h" 15 #include "util/tool.h"
16 #include "util/sort.h" 16 #include "util/sort.h"
17 #include "util/symbol.h" 17 #include "util/symbol.h"
18 #include "util/util.h" 18 #include "util/util.h"
19 #include "util/data.h" 19 #include "util/data.h"
20 20
21 #include <stdlib.h> 21 #include <stdlib.h>
22 #include <math.h> 22 #include <math.h>
23 23
24 /* Diff command specific HPP columns. */ 24 /* Diff command specific HPP columns. */
25 enum { 25 enum {
26 PERF_HPP_DIFF__BASELINE, 26 PERF_HPP_DIFF__BASELINE,
27 PERF_HPP_DIFF__PERIOD, 27 PERF_HPP_DIFF__PERIOD,
28 PERF_HPP_DIFF__PERIOD_BASELINE, 28 PERF_HPP_DIFF__PERIOD_BASELINE,
29 PERF_HPP_DIFF__DELTA, 29 PERF_HPP_DIFF__DELTA,
30 PERF_HPP_DIFF__RATIO, 30 PERF_HPP_DIFF__RATIO,
31 PERF_HPP_DIFF__WEIGHTED_DIFF, 31 PERF_HPP_DIFF__WEIGHTED_DIFF,
32 PERF_HPP_DIFF__FORMULA, 32 PERF_HPP_DIFF__FORMULA,
33 33
34 PERF_HPP_DIFF__MAX_INDEX 34 PERF_HPP_DIFF__MAX_INDEX
35 }; 35 };
36 36
37 struct diff_hpp_fmt { 37 struct diff_hpp_fmt {
38 struct perf_hpp_fmt fmt; 38 struct perf_hpp_fmt fmt;
39 int idx; 39 int idx;
40 char *header; 40 char *header;
41 int header_width; 41 int header_width;
42 }; 42 };
43 43
44 struct data__file { 44 struct data__file {
45 struct perf_session *session; 45 struct perf_session *session;
46 struct perf_data_file file; 46 struct perf_data_file file;
47 int idx; 47 int idx;
48 struct hists *hists; 48 struct hists *hists;
49 struct diff_hpp_fmt fmt[PERF_HPP_DIFF__MAX_INDEX]; 49 struct diff_hpp_fmt fmt[PERF_HPP_DIFF__MAX_INDEX];
50 }; 50 };
51 51
52 static struct data__file *data__files; 52 static struct data__file *data__files;
53 static int data__files_cnt; 53 static int data__files_cnt;
54 54
55 #define data__for_each_file_start(i, d, s) \ 55 #define data__for_each_file_start(i, d, s) \
56 for (i = s, d = &data__files[s]; \ 56 for (i = s, d = &data__files[s]; \
57 i < data__files_cnt; \ 57 i < data__files_cnt; \
58 i++, d = &data__files[i]) 58 i++, d = &data__files[i])
59 59
60 #define data__for_each_file(i, d) data__for_each_file_start(i, d, 0) 60 #define data__for_each_file(i, d) data__for_each_file_start(i, d, 0)
61 #define data__for_each_file_new(i, d) data__for_each_file_start(i, d, 1) 61 #define data__for_each_file_new(i, d) data__for_each_file_start(i, d, 1)
62 62
63 static bool force; 63 static bool force;
64 static bool show_period; 64 static bool show_period;
65 static bool show_formula; 65 static bool show_formula;
66 static bool show_baseline_only; 66 static bool show_baseline_only;
67 static unsigned int sort_compute; 67 static unsigned int sort_compute;
68 68
69 static s64 compute_wdiff_w1; 69 static s64 compute_wdiff_w1;
70 static s64 compute_wdiff_w2; 70 static s64 compute_wdiff_w2;
71 71
72 enum { 72 enum {
73 COMPUTE_DELTA, 73 COMPUTE_DELTA,
74 COMPUTE_RATIO, 74 COMPUTE_RATIO,
75 COMPUTE_WEIGHTED_DIFF, 75 COMPUTE_WEIGHTED_DIFF,
76 COMPUTE_MAX, 76 COMPUTE_MAX,
77 }; 77 };
78 78
79 const char *compute_names[COMPUTE_MAX] = { 79 const char *compute_names[COMPUTE_MAX] = {
80 [COMPUTE_DELTA] = "delta", 80 [COMPUTE_DELTA] = "delta",
81 [COMPUTE_RATIO] = "ratio", 81 [COMPUTE_RATIO] = "ratio",
82 [COMPUTE_WEIGHTED_DIFF] = "wdiff", 82 [COMPUTE_WEIGHTED_DIFF] = "wdiff",
83 }; 83 };
84 84
85 static int compute; 85 static int compute;
86 86
87 static int compute_2_hpp[COMPUTE_MAX] = { 87 static int compute_2_hpp[COMPUTE_MAX] = {
88 [COMPUTE_DELTA] = PERF_HPP_DIFF__DELTA, 88 [COMPUTE_DELTA] = PERF_HPP_DIFF__DELTA,
89 [COMPUTE_RATIO] = PERF_HPP_DIFF__RATIO, 89 [COMPUTE_RATIO] = PERF_HPP_DIFF__RATIO,
90 [COMPUTE_WEIGHTED_DIFF] = PERF_HPP_DIFF__WEIGHTED_DIFF, 90 [COMPUTE_WEIGHTED_DIFF] = PERF_HPP_DIFF__WEIGHTED_DIFF,
91 }; 91 };
92 92
93 #define MAX_COL_WIDTH 70 93 #define MAX_COL_WIDTH 70
94 94
95 static struct header_column { 95 static struct header_column {
96 const char *name; 96 const char *name;
97 int width; 97 int width;
98 } columns[PERF_HPP_DIFF__MAX_INDEX] = { 98 } columns[PERF_HPP_DIFF__MAX_INDEX] = {
99 [PERF_HPP_DIFF__BASELINE] = { 99 [PERF_HPP_DIFF__BASELINE] = {
100 .name = "Baseline", 100 .name = "Baseline",
101 }, 101 },
102 [PERF_HPP_DIFF__PERIOD] = { 102 [PERF_HPP_DIFF__PERIOD] = {
103 .name = "Period", 103 .name = "Period",
104 .width = 14, 104 .width = 14,
105 }, 105 },
106 [PERF_HPP_DIFF__PERIOD_BASELINE] = { 106 [PERF_HPP_DIFF__PERIOD_BASELINE] = {
107 .name = "Base period", 107 .name = "Base period",
108 .width = 14, 108 .width = 14,
109 }, 109 },
110 [PERF_HPP_DIFF__DELTA] = { 110 [PERF_HPP_DIFF__DELTA] = {
111 .name = "Delta", 111 .name = "Delta",
112 .width = 7, 112 .width = 7,
113 }, 113 },
114 [PERF_HPP_DIFF__RATIO] = { 114 [PERF_HPP_DIFF__RATIO] = {
115 .name = "Ratio", 115 .name = "Ratio",
116 .width = 14, 116 .width = 14,
117 }, 117 },
118 [PERF_HPP_DIFF__WEIGHTED_DIFF] = { 118 [PERF_HPP_DIFF__WEIGHTED_DIFF] = {
119 .name = "Weighted diff", 119 .name = "Weighted diff",
120 .width = 14, 120 .width = 14,
121 }, 121 },
122 [PERF_HPP_DIFF__FORMULA] = { 122 [PERF_HPP_DIFF__FORMULA] = {
123 .name = "Formula", 123 .name = "Formula",
124 .width = MAX_COL_WIDTH, 124 .width = MAX_COL_WIDTH,
125 } 125 }
126 }; 126 };
127 127
128 static int setup_compute_opt_wdiff(char *opt) 128 static int setup_compute_opt_wdiff(char *opt)
129 { 129 {
130 char *w1_str = opt; 130 char *w1_str = opt;
131 char *w2_str; 131 char *w2_str;
132 132
133 int ret = -EINVAL; 133 int ret = -EINVAL;
134 134
135 if (!opt) 135 if (!opt)
136 goto out; 136 goto out;
137 137
138 w2_str = strchr(opt, ','); 138 w2_str = strchr(opt, ',');
139 if (!w2_str) 139 if (!w2_str)
140 goto out; 140 goto out;
141 141
142 *w2_str++ = 0x0; 142 *w2_str++ = 0x0;
143 if (!*w2_str) 143 if (!*w2_str)
144 goto out; 144 goto out;
145 145
146 compute_wdiff_w1 = strtol(w1_str, NULL, 10); 146 compute_wdiff_w1 = strtol(w1_str, NULL, 10);
147 compute_wdiff_w2 = strtol(w2_str, NULL, 10); 147 compute_wdiff_w2 = strtol(w2_str, NULL, 10);
148 148
149 if (!compute_wdiff_w1 || !compute_wdiff_w2) 149 if (!compute_wdiff_w1 || !compute_wdiff_w2)
150 goto out; 150 goto out;
151 151
152 pr_debug("compute wdiff w1(%" PRId64 ") w2(%" PRId64 ")\n", 152 pr_debug("compute wdiff w1(%" PRId64 ") w2(%" PRId64 ")\n",
153 compute_wdiff_w1, compute_wdiff_w2); 153 compute_wdiff_w1, compute_wdiff_w2);
154 154
155 ret = 0; 155 ret = 0;
156 156
157 out: 157 out:
158 if (ret) 158 if (ret)
159 pr_err("Failed: wrong weight data, use 'wdiff:w1,w2'\n"); 159 pr_err("Failed: wrong weight data, use 'wdiff:w1,w2'\n");
160 160
161 return ret; 161 return ret;
162 } 162 }
163 163
164 static int setup_compute_opt(char *opt) 164 static int setup_compute_opt(char *opt)
165 { 165 {
166 if (compute == COMPUTE_WEIGHTED_DIFF) 166 if (compute == COMPUTE_WEIGHTED_DIFF)
167 return setup_compute_opt_wdiff(opt); 167 return setup_compute_opt_wdiff(opt);
168 168
169 if (opt) { 169 if (opt) {
170 pr_err("Failed: extra option specified '%s'", opt); 170 pr_err("Failed: extra option specified '%s'", opt);
171 return -EINVAL; 171 return -EINVAL;
172 } 172 }
173 173
174 return 0; 174 return 0;
175 } 175 }
176 176
177 static int setup_compute(const struct option *opt, const char *str, 177 static int setup_compute(const struct option *opt, const char *str,
178 int unset __maybe_unused) 178 int unset __maybe_unused)
179 { 179 {
180 int *cp = (int *) opt->value; 180 int *cp = (int *) opt->value;
181 char *cstr = (char *) str; 181 char *cstr = (char *) str;
182 char buf[50]; 182 char buf[50];
183 unsigned i; 183 unsigned i;
184 char *option; 184 char *option;
185 185
186 if (!str) { 186 if (!str) {
187 *cp = COMPUTE_DELTA; 187 *cp = COMPUTE_DELTA;
188 return 0; 188 return 0;
189 } 189 }
190 190
191 option = strchr(str, ':'); 191 option = strchr(str, ':');
192 if (option) { 192 if (option) {
193 unsigned len = option++ - str; 193 unsigned len = option++ - str;
194 194
195 /* 195 /*
196 * The str data are not writeable, so we need 196 * The str data are not writeable, so we need
197 * to use another buffer. 197 * to use another buffer.
198 */ 198 */
199 199
200 /* No option value is longer. */ 200 /* No option value is longer. */
201 if (len >= sizeof(buf)) 201 if (len >= sizeof(buf))
202 return -EINVAL; 202 return -EINVAL;
203 203
204 strncpy(buf, str, len); 204 strncpy(buf, str, len);
205 buf[len] = 0x0; 205 buf[len] = 0x0;
206 cstr = buf; 206 cstr = buf;
207 } 207 }
208 208
209 for (i = 0; i < COMPUTE_MAX; i++) 209 for (i = 0; i < COMPUTE_MAX; i++)
210 if (!strcmp(cstr, compute_names[i])) { 210 if (!strcmp(cstr, compute_names[i])) {
211 *cp = i; 211 *cp = i;
212 return setup_compute_opt(option); 212 return setup_compute_opt(option);
213 } 213 }
214 214
215 pr_err("Failed: '%s' is not computation method " 215 pr_err("Failed: '%s' is not computation method "
216 "(use 'delta','ratio' or 'wdiff')\n", str); 216 "(use 'delta','ratio' or 'wdiff')\n", str);
217 return -EINVAL; 217 return -EINVAL;
218 } 218 }
219 219
220 static double period_percent(struct hist_entry *he, u64 period) 220 static double period_percent(struct hist_entry *he, u64 period)
221 { 221 {
222 u64 total = hists__total_period(he->hists); 222 u64 total = hists__total_period(he->hists);
223 223
224 return (period * 100.0) / total; 224 return (period * 100.0) / total;
225 } 225 }
226 226
227 static double compute_delta(struct hist_entry *he, struct hist_entry *pair) 227 static double compute_delta(struct hist_entry *he, struct hist_entry *pair)
228 { 228 {
229 double old_percent = period_percent(he, he->stat.period); 229 double old_percent = period_percent(he, he->stat.period);
230 double new_percent = period_percent(pair, pair->stat.period); 230 double new_percent = period_percent(pair, pair->stat.period);
231 231
232 pair->diff.period_ratio_delta = new_percent - old_percent; 232 pair->diff.period_ratio_delta = new_percent - old_percent;
233 pair->diff.computed = true; 233 pair->diff.computed = true;
234 return pair->diff.period_ratio_delta; 234 return pair->diff.period_ratio_delta;
235 } 235 }
236 236
237 static double compute_ratio(struct hist_entry *he, struct hist_entry *pair) 237 static double compute_ratio(struct hist_entry *he, struct hist_entry *pair)
238 { 238 {
239 double old_period = he->stat.period ?: 1; 239 double old_period = he->stat.period ?: 1;
240 double new_period = pair->stat.period; 240 double new_period = pair->stat.period;
241 241
242 pair->diff.computed = true; 242 pair->diff.computed = true;
243 pair->diff.period_ratio = new_period / old_period; 243 pair->diff.period_ratio = new_period / old_period;
244 return pair->diff.period_ratio; 244 return pair->diff.period_ratio;
245 } 245 }
246 246
247 static s64 compute_wdiff(struct hist_entry *he, struct hist_entry *pair) 247 static s64 compute_wdiff(struct hist_entry *he, struct hist_entry *pair)
248 { 248 {
249 u64 old_period = he->stat.period; 249 u64 old_period = he->stat.period;
250 u64 new_period = pair->stat.period; 250 u64 new_period = pair->stat.period;
251 251
252 pair->diff.computed = true; 252 pair->diff.computed = true;
253 pair->diff.wdiff = new_period * compute_wdiff_w2 - 253 pair->diff.wdiff = new_period * compute_wdiff_w2 -
254 old_period * compute_wdiff_w1; 254 old_period * compute_wdiff_w1;
255 255
256 return pair->diff.wdiff; 256 return pair->diff.wdiff;
257 } 257 }
258 258
259 static int formula_delta(struct hist_entry *he, struct hist_entry *pair, 259 static int formula_delta(struct hist_entry *he, struct hist_entry *pair,
260 char *buf, size_t size) 260 char *buf, size_t size)
261 { 261 {
262 u64 he_total = he->hists->stats.total_period; 262 u64 he_total = he->hists->stats.total_period;
263 u64 pair_total = pair->hists->stats.total_period; 263 u64 pair_total = pair->hists->stats.total_period;
264 264
265 if (symbol_conf.filter_relative) { 265 if (symbol_conf.filter_relative) {
266 he_total = he->hists->stats.total_non_filtered_period; 266 he_total = he->hists->stats.total_non_filtered_period;
267 pair_total = pair->hists->stats.total_non_filtered_period; 267 pair_total = pair->hists->stats.total_non_filtered_period;
268 } 268 }
269 return scnprintf(buf, size, 269 return scnprintf(buf, size,
270 "(%" PRIu64 " * 100 / %" PRIu64 ") - " 270 "(%" PRIu64 " * 100 / %" PRIu64 ") - "
271 "(%" PRIu64 " * 100 / %" PRIu64 ")", 271 "(%" PRIu64 " * 100 / %" PRIu64 ")",
272 pair->stat.period, pair_total, 272 pair->stat.period, pair_total,
273 he->stat.period, he_total); 273 he->stat.period, he_total);
274 } 274 }
275 275
276 static int formula_ratio(struct hist_entry *he, struct hist_entry *pair, 276 static int formula_ratio(struct hist_entry *he, struct hist_entry *pair,
277 char *buf, size_t size) 277 char *buf, size_t size)
278 { 278 {
279 double old_period = he->stat.period; 279 double old_period = he->stat.period;
280 double new_period = pair->stat.period; 280 double new_period = pair->stat.period;
281 281
282 return scnprintf(buf, size, "%.0F / %.0F", new_period, old_period); 282 return scnprintf(buf, size, "%.0F / %.0F", new_period, old_period);
283 } 283 }
284 284
285 static int formula_wdiff(struct hist_entry *he, struct hist_entry *pair, 285 static int formula_wdiff(struct hist_entry *he, struct hist_entry *pair,
286 char *buf, size_t size) 286 char *buf, size_t size)
287 { 287 {
288 u64 old_period = he->stat.period; 288 u64 old_period = he->stat.period;
289 u64 new_period = pair->stat.period; 289 u64 new_period = pair->stat.period;
290 290
291 return scnprintf(buf, size, 291 return scnprintf(buf, size,
292 "(%" PRIu64 " * " "%" PRId64 ") - (%" PRIu64 " * " "%" PRId64 ")", 292 "(%" PRIu64 " * " "%" PRId64 ") - (%" PRIu64 " * " "%" PRId64 ")",
293 new_period, compute_wdiff_w2, old_period, compute_wdiff_w1); 293 new_period, compute_wdiff_w2, old_period, compute_wdiff_w1);
294 } 294 }
295 295
296 static int formula_fprintf(struct hist_entry *he, struct hist_entry *pair, 296 static int formula_fprintf(struct hist_entry *he, struct hist_entry *pair,
297 char *buf, size_t size) 297 char *buf, size_t size)
298 { 298 {
299 switch (compute) { 299 switch (compute) {
300 case COMPUTE_DELTA: 300 case COMPUTE_DELTA:
301 return formula_delta(he, pair, buf, size); 301 return formula_delta(he, pair, buf, size);
302 case COMPUTE_RATIO: 302 case COMPUTE_RATIO:
303 return formula_ratio(he, pair, buf, size); 303 return formula_ratio(he, pair, buf, size);
304 case COMPUTE_WEIGHTED_DIFF: 304 case COMPUTE_WEIGHTED_DIFF:
305 return formula_wdiff(he, pair, buf, size); 305 return formula_wdiff(he, pair, buf, size);
306 default: 306 default:
307 BUG_ON(1); 307 BUG_ON(1);
308 } 308 }
309 309
310 return -1; 310 return -1;
311 } 311 }
312 312
313 static int hists__add_entry(struct hists *hists, 313 static int hists__add_entry(struct hists *hists,
314 struct addr_location *al, u64 period, 314 struct addr_location *al, u64 period,
315 u64 weight, u64 transaction) 315 u64 weight, u64 transaction)
316 { 316 {
317 if (__hists__add_entry(hists, al, NULL, NULL, NULL, period, weight, 317 if (__hists__add_entry(hists, al, NULL, NULL, NULL, period, weight,
318 transaction, true) != NULL) 318 transaction, true) != NULL)
319 return 0; 319 return 0;
320 return -ENOMEM; 320 return -ENOMEM;
321 } 321 }
322 322
323 static int diff__process_sample_event(struct perf_tool *tool __maybe_unused, 323 static int diff__process_sample_event(struct perf_tool *tool __maybe_unused,
324 union perf_event *event, 324 union perf_event *event,
325 struct perf_sample *sample, 325 struct perf_sample *sample,
326 struct perf_evsel *evsel, 326 struct perf_evsel *evsel,
327 struct machine *machine) 327 struct machine *machine)
328 { 328 {
329 struct addr_location al; 329 struct addr_location al;
330 struct hists *hists = evsel__hists(evsel); 330 struct hists *hists = evsel__hists(evsel);
331 331
332 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) { 332 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) {
333 pr_warning("problem processing %d event, skipping it.\n", 333 pr_warning("problem processing %d event, skipping it.\n",
334 event->header.type); 334 event->header.type);
335 return -1; 335 return -1;
336 } 336 }
337 337
338 if (hists__add_entry(hists, &al, sample->period, 338 if (hists__add_entry(hists, &al, sample->period,
339 sample->weight, sample->transaction)) { 339 sample->weight, sample->transaction)) {
340 pr_warning("problem incrementing symbol period, skipping event\n"); 340 pr_warning("problem incrementing symbol period, skipping event\n");
341 return -1; 341 return -1;
342 } 342 }
343 343
344 /* 344 /*
345 * The total_period is updated here before going to the output 345 * The total_period is updated here before going to the output
346 * tree since normally only the baseline hists will call 346 * tree since normally only the baseline hists will call
347 * hists__output_resort() and precompute needs the total 347 * hists__output_resort() and precompute needs the total
348 * period in order to sort entries by percentage delta. 348 * period in order to sort entries by percentage delta.
349 */ 349 */
350 hists->stats.total_period += sample->period; 350 hists->stats.total_period += sample->period;
351 if (!al.filtered) 351 if (!al.filtered)
352 hists->stats.total_non_filtered_period += sample->period; 352 hists->stats.total_non_filtered_period += sample->period;
353 353
354 return 0; 354 return 0;
355 } 355 }
356 356
357 static struct perf_tool tool = { 357 static struct perf_tool tool = {
358 .sample = diff__process_sample_event, 358 .sample = diff__process_sample_event,
359 .mmap = perf_event__process_mmap, 359 .mmap = perf_event__process_mmap,
360 .mmap2 = perf_event__process_mmap2, 360 .mmap2 = perf_event__process_mmap2,
361 .comm = perf_event__process_comm, 361 .comm = perf_event__process_comm,
362 .exit = perf_event__process_exit, 362 .exit = perf_event__process_exit,
363 .fork = perf_event__process_fork, 363 .fork = perf_event__process_fork,
364 .lost = perf_event__process_lost, 364 .lost = perf_event__process_lost,
365 .ordered_events = true, 365 .ordered_events = true,
366 .ordering_requires_timestamps = true, 366 .ordering_requires_timestamps = true,
367 }; 367 };
368 368
369 static struct perf_evsel *evsel_match(struct perf_evsel *evsel, 369 static struct perf_evsel *evsel_match(struct perf_evsel *evsel,
370 struct perf_evlist *evlist) 370 struct perf_evlist *evlist)
371 { 371 {
372 struct perf_evsel *e; 372 struct perf_evsel *e;
373 373
374 evlist__for_each(evlist, e) { 374 evlist__for_each(evlist, e) {
375 if (perf_evsel__match2(evsel, e)) 375 if (perf_evsel__match2(evsel, e))
376 return e; 376 return e;
377 } 377 }
378 378
379 return NULL; 379 return NULL;
380 } 380 }
381 381
382 static void perf_evlist__collapse_resort(struct perf_evlist *evlist) 382 static void perf_evlist__collapse_resort(struct perf_evlist *evlist)
383 { 383 {
384 struct perf_evsel *evsel; 384 struct perf_evsel *evsel;
385 385
386 evlist__for_each(evlist, evsel) { 386 evlist__for_each(evlist, evsel) {
387 struct hists *hists = evsel__hists(evsel); 387 struct hists *hists = evsel__hists(evsel);
388 388
389 hists__collapse_resort(hists, NULL); 389 hists__collapse_resort(hists, NULL);
390 } 390 }
391 } 391 }
392 392
393 static struct hist_entry* 393 static struct hist_entry*
394 get_pair_data(struct hist_entry *he, struct data__file *d) 394 get_pair_data(struct hist_entry *he, struct data__file *d)
395 { 395 {
396 if (hist_entry__has_pairs(he)) { 396 if (hist_entry__has_pairs(he)) {
397 struct hist_entry *pair; 397 struct hist_entry *pair;
398 398
399 list_for_each_entry(pair, &he->pairs.head, pairs.node) 399 list_for_each_entry(pair, &he->pairs.head, pairs.node)
400 if (pair->hists == d->hists) 400 if (pair->hists == d->hists)
401 return pair; 401 return pair;
402 } 402 }
403 403
404 return NULL; 404 return NULL;
405 } 405 }
406 406
407 static struct hist_entry* 407 static struct hist_entry*
408 get_pair_fmt(struct hist_entry *he, struct diff_hpp_fmt *dfmt) 408 get_pair_fmt(struct hist_entry *he, struct diff_hpp_fmt *dfmt)
409 { 409 {
410 void *ptr = dfmt - dfmt->idx; 410 void *ptr = dfmt - dfmt->idx;
411 struct data__file *d = container_of(ptr, struct data__file, fmt); 411 struct data__file *d = container_of(ptr, struct data__file, fmt);
412 412
413 return get_pair_data(he, d); 413 return get_pair_data(he, d);
414 } 414 }
415 415
416 static void hists__baseline_only(struct hists *hists) 416 static void hists__baseline_only(struct hists *hists)
417 { 417 {
418 struct rb_root *root; 418 struct rb_root *root;
419 struct rb_node *next; 419 struct rb_node *next;
420 420
421 if (sort__need_collapse) 421 if (sort__need_collapse)
422 root = &hists->entries_collapsed; 422 root = &hists->entries_collapsed;
423 else 423 else
424 root = hists->entries_in; 424 root = hists->entries_in;
425 425
426 next = rb_first(root); 426 next = rb_first(root);
427 while (next != NULL) { 427 while (next != NULL) {
428 struct hist_entry *he = rb_entry(next, struct hist_entry, rb_node_in); 428 struct hist_entry *he = rb_entry(next, struct hist_entry, rb_node_in);
429 429
430 next = rb_next(&he->rb_node_in); 430 next = rb_next(&he->rb_node_in);
431 if (!hist_entry__next_pair(he)) { 431 if (!hist_entry__next_pair(he)) {
432 rb_erase(&he->rb_node_in, root); 432 rb_erase(&he->rb_node_in, root);
433 hist_entry__free(he); 433 hist_entry__free(he);
434 } 434 }
435 } 435 }
436 } 436 }
437 437
438 static void hists__precompute(struct hists *hists) 438 static void hists__precompute(struct hists *hists)
439 { 439 {
440 struct rb_root *root; 440 struct rb_root *root;
441 struct rb_node *next; 441 struct rb_node *next;
442 442
443 if (sort__need_collapse) 443 if (sort__need_collapse)
444 root = &hists->entries_collapsed; 444 root = &hists->entries_collapsed;
445 else 445 else
446 root = hists->entries_in; 446 root = hists->entries_in;
447 447
448 next = rb_first(root); 448 next = rb_first(root);
449 while (next != NULL) { 449 while (next != NULL) {
450 struct hist_entry *he, *pair; 450 struct hist_entry *he, *pair;
451 451
452 he = rb_entry(next, struct hist_entry, rb_node_in); 452 he = rb_entry(next, struct hist_entry, rb_node_in);
453 next = rb_next(&he->rb_node_in); 453 next = rb_next(&he->rb_node_in);
454 454
455 pair = get_pair_data(he, &data__files[sort_compute]); 455 pair = get_pair_data(he, &data__files[sort_compute]);
456 if (!pair) 456 if (!pair)
457 continue; 457 continue;
458 458
459 switch (compute) { 459 switch (compute) {
460 case COMPUTE_DELTA: 460 case COMPUTE_DELTA:
461 compute_delta(he, pair); 461 compute_delta(he, pair);
462 break; 462 break;
463 case COMPUTE_RATIO: 463 case COMPUTE_RATIO:
464 compute_ratio(he, pair); 464 compute_ratio(he, pair);
465 break; 465 break;
466 case COMPUTE_WEIGHTED_DIFF: 466 case COMPUTE_WEIGHTED_DIFF:
467 compute_wdiff(he, pair); 467 compute_wdiff(he, pair);
468 break; 468 break;
469 default: 469 default:
470 BUG_ON(1); 470 BUG_ON(1);
471 } 471 }
472 } 472 }
473 } 473 }
474 474
475 static int64_t cmp_doubles(double l, double r) 475 static int64_t cmp_doubles(double l, double r)
476 { 476 {
477 if (l > r) 477 if (l > r)
478 return -1; 478 return -1;
479 else if (l < r) 479 else if (l < r)
480 return 1; 480 return 1;
481 else 481 else
482 return 0; 482 return 0;
483 } 483 }
484 484
485 static int64_t 485 static int64_t
486 __hist_entry__cmp_compute(struct hist_entry *left, struct hist_entry *right, 486 __hist_entry__cmp_compute(struct hist_entry *left, struct hist_entry *right,
487 int c) 487 int c)
488 { 488 {
489 switch (c) { 489 switch (c) {
490 case COMPUTE_DELTA: 490 case COMPUTE_DELTA:
491 { 491 {
492 double l = left->diff.period_ratio_delta; 492 double l = left->diff.period_ratio_delta;
493 double r = right->diff.period_ratio_delta; 493 double r = right->diff.period_ratio_delta;
494 494
495 return cmp_doubles(l, r); 495 return cmp_doubles(l, r);
496 } 496 }
497 case COMPUTE_RATIO: 497 case COMPUTE_RATIO:
498 { 498 {
499 double l = left->diff.period_ratio; 499 double l = left->diff.period_ratio;
500 double r = right->diff.period_ratio; 500 double r = right->diff.period_ratio;
501 501
502 return cmp_doubles(l, r); 502 return cmp_doubles(l, r);
503 } 503 }
504 case COMPUTE_WEIGHTED_DIFF: 504 case COMPUTE_WEIGHTED_DIFF:
505 { 505 {
506 s64 l = left->diff.wdiff; 506 s64 l = left->diff.wdiff;
507 s64 r = right->diff.wdiff; 507 s64 r = right->diff.wdiff;
508 508
509 return r - l; 509 return r - l;
510 } 510 }
511 default: 511 default:
512 BUG_ON(1); 512 BUG_ON(1);
513 } 513 }
514 514
515 return 0; 515 return 0;
516 } 516 }
517 517
518 static int64_t 518 static int64_t
519 hist_entry__cmp_compute(struct hist_entry *left, struct hist_entry *right, 519 hist_entry__cmp_compute(struct hist_entry *left, struct hist_entry *right,
520 int c) 520 int c)
521 { 521 {
522 bool pairs_left = hist_entry__has_pairs(left); 522 bool pairs_left = hist_entry__has_pairs(left);
523 bool pairs_right = hist_entry__has_pairs(right); 523 bool pairs_right = hist_entry__has_pairs(right);
524 struct hist_entry *p_right, *p_left; 524 struct hist_entry *p_right, *p_left;
525 525
526 if (!pairs_left && !pairs_right) 526 if (!pairs_left && !pairs_right)
527 return 0; 527 return 0;
528 528
529 if (!pairs_left || !pairs_right) 529 if (!pairs_left || !pairs_right)
530 return pairs_left ? -1 : 1; 530 return pairs_left ? -1 : 1;
531 531
532 p_left = get_pair_data(left, &data__files[sort_compute]); 532 p_left = get_pair_data(left, &data__files[sort_compute]);
533 p_right = get_pair_data(right, &data__files[sort_compute]); 533 p_right = get_pair_data(right, &data__files[sort_compute]);
534 534
535 if (!p_left && !p_right) 535 if (!p_left && !p_right)
536 return 0; 536 return 0;
537 537
538 if (!p_left || !p_right) 538 if (!p_left || !p_right)
539 return p_left ? -1 : 1; 539 return p_left ? -1 : 1;
540 540
541 /* 541 /*
542 * We have 2 entries of same kind, let's 542 * We have 2 entries of same kind, let's
543 * make the data comparison. 543 * make the data comparison.
544 */ 544 */
545 return __hist_entry__cmp_compute(p_left, p_right, c); 545 return __hist_entry__cmp_compute(p_left, p_right, c);
546 } 546 }
547 547
548 static int64_t
549 hist_entry__cmp_nop(struct hist_entry *left __maybe_unused,
550 struct hist_entry *right __maybe_unused)
551 {
552 return 0;
553 }
554
555 static int64_t
556 hist_entry__cmp_baseline(struct hist_entry *left, struct hist_entry *right)
557 {
558 if (sort_compute)
559 return 0;
560
561 if (left->stat.period == right->stat.period)
562 return 0;
563 return left->stat.period > right->stat.period ? 1 : -1;
564 }
565
566 static int64_t
567 hist_entry__cmp_delta(struct hist_entry *left, struct hist_entry *right)
568 {
569 return hist_entry__cmp_compute(right, left, COMPUTE_DELTA);
570 }
571
572 static int64_t
573 hist_entry__cmp_ratio(struct hist_entry *left, struct hist_entry *right)
574 {
575 return hist_entry__cmp_compute(right, left, COMPUTE_RATIO);
576 }
577
578 static int64_t
579 hist_entry__cmp_wdiff(struct hist_entry *left, struct hist_entry *right)
580 {
581 return hist_entry__cmp_compute(right, left, COMPUTE_WEIGHTED_DIFF);
582 }
583
548 static void insert_hist_entry_by_compute(struct rb_root *root, 584 static void insert_hist_entry_by_compute(struct rb_root *root,
549 struct hist_entry *he, 585 struct hist_entry *he,
550 int c) 586 int c)
551 { 587 {
552 struct rb_node **p = &root->rb_node; 588 struct rb_node **p = &root->rb_node;
553 struct rb_node *parent = NULL; 589 struct rb_node *parent = NULL;
554 struct hist_entry *iter; 590 struct hist_entry *iter;
555 591
556 while (*p != NULL) { 592 while (*p != NULL) {
557 parent = *p; 593 parent = *p;
558 iter = rb_entry(parent, struct hist_entry, rb_node); 594 iter = rb_entry(parent, struct hist_entry, rb_node);
559 if (hist_entry__cmp_compute(he, iter, c) < 0) 595 if (hist_entry__cmp_compute(he, iter, c) < 0)
560 p = &(*p)->rb_left; 596 p = &(*p)->rb_left;
561 else 597 else
562 p = &(*p)->rb_right; 598 p = &(*p)->rb_right;
563 } 599 }
564 600
565 rb_link_node(&he->rb_node, parent, p); 601 rb_link_node(&he->rb_node, parent, p);
566 rb_insert_color(&he->rb_node, root); 602 rb_insert_color(&he->rb_node, root);
567 } 603 }
568 604
569 static void hists__compute_resort(struct hists *hists) 605 static void hists__compute_resort(struct hists *hists)
570 { 606 {
571 struct rb_root *root; 607 struct rb_root *root;
572 struct rb_node *next; 608 struct rb_node *next;
573 609
574 if (sort__need_collapse) 610 if (sort__need_collapse)
575 root = &hists->entries_collapsed; 611 root = &hists->entries_collapsed;
576 else 612 else
577 root = hists->entries_in; 613 root = hists->entries_in;
578 614
579 hists->entries = RB_ROOT; 615 hists->entries = RB_ROOT;
580 next = rb_first(root); 616 next = rb_first(root);
581 617
582 hists__reset_stats(hists); 618 hists__reset_stats(hists);
583 hists__reset_col_len(hists); 619 hists__reset_col_len(hists);
584 620
585 while (next != NULL) { 621 while (next != NULL) {
586 struct hist_entry *he; 622 struct hist_entry *he;
587 623
588 he = rb_entry(next, struct hist_entry, rb_node_in); 624 he = rb_entry(next, struct hist_entry, rb_node_in);
589 next = rb_next(&he->rb_node_in); 625 next = rb_next(&he->rb_node_in);
590 626
591 insert_hist_entry_by_compute(&hists->entries, he, compute); 627 insert_hist_entry_by_compute(&hists->entries, he, compute);
592 hists__inc_stats(hists, he); 628 hists__inc_stats(hists, he);
593 629
594 if (!he->filtered) 630 if (!he->filtered)
595 hists__calc_col_len(hists, he); 631 hists__calc_col_len(hists, he);
596 } 632 }
597 } 633 }
598 634
599 static void hists__process(struct hists *hists) 635 static void hists__process(struct hists *hists)
600 { 636 {
601 if (show_baseline_only) 637 if (show_baseline_only)
602 hists__baseline_only(hists); 638 hists__baseline_only(hists);
603 639
604 if (sort_compute) { 640 if (sort_compute) {
605 hists__precompute(hists); 641 hists__precompute(hists);
606 hists__compute_resort(hists); 642 hists__compute_resort(hists);
607 } else { 643 } else {
608 hists__output_resort(hists); 644 hists__output_resort(hists, NULL);
609 } 645 }
610 646
611 hists__fprintf(hists, true, 0, 0, 0, stdout); 647 hists__fprintf(hists, true, 0, 0, 0, stdout);
612 } 648 }
613 649
614 static void data__fprintf(void) 650 static void data__fprintf(void)
615 { 651 {
616 struct data__file *d; 652 struct data__file *d;
617 int i; 653 int i;
618 654
619 fprintf(stdout, "# Data files:\n"); 655 fprintf(stdout, "# Data files:\n");
620 656
621 data__for_each_file(i, d) 657 data__for_each_file(i, d)
622 fprintf(stdout, "# [%d] %s %s\n", 658 fprintf(stdout, "# [%d] %s %s\n",
623 d->idx, d->file.path, 659 d->idx, d->file.path,
624 !d->idx ? "(Baseline)" : ""); 660 !d->idx ? "(Baseline)" : "");
625 661
626 fprintf(stdout, "#\n"); 662 fprintf(stdout, "#\n");
627 } 663 }
628 664
629 static void data_process(void) 665 static void data_process(void)
630 { 666 {
631 struct perf_evlist *evlist_base = data__files[0].session->evlist; 667 struct perf_evlist *evlist_base = data__files[0].session->evlist;
632 struct perf_evsel *evsel_base; 668 struct perf_evsel *evsel_base;
633 bool first = true; 669 bool first = true;
634 670
635 evlist__for_each(evlist_base, evsel_base) { 671 evlist__for_each(evlist_base, evsel_base) {
636 struct hists *hists_base = evsel__hists(evsel_base); 672 struct hists *hists_base = evsel__hists(evsel_base);
637 struct data__file *d; 673 struct data__file *d;
638 int i; 674 int i;
639 675
640 data__for_each_file_new(i, d) { 676 data__for_each_file_new(i, d) {
641 struct perf_evlist *evlist = d->session->evlist; 677 struct perf_evlist *evlist = d->session->evlist;
642 struct perf_evsel *evsel; 678 struct perf_evsel *evsel;
643 struct hists *hists; 679 struct hists *hists;
644 680
645 evsel = evsel_match(evsel_base, evlist); 681 evsel = evsel_match(evsel_base, evlist);
646 if (!evsel) 682 if (!evsel)
647 continue; 683 continue;
648 684
649 hists = evsel__hists(evsel); 685 hists = evsel__hists(evsel);
650 d->hists = hists; 686 d->hists = hists;
651 687
652 hists__match(hists_base, hists); 688 hists__match(hists_base, hists);
653 689
654 if (!show_baseline_only) 690 if (!show_baseline_only)
655 hists__link(hists_base, hists); 691 hists__link(hists_base, hists);
656 } 692 }
657 693
658 fprintf(stdout, "%s# Event '%s'\n#\n", first ? "" : "\n", 694 fprintf(stdout, "%s# Event '%s'\n#\n", first ? "" : "\n",
659 perf_evsel__name(evsel_base)); 695 perf_evsel__name(evsel_base));
660 696
661 first = false; 697 first = false;
662 698
663 if (verbose || data__files_cnt > 2) 699 if (verbose || data__files_cnt > 2)
664 data__fprintf(); 700 data__fprintf();
665 701
666 hists__process(hists_base); 702 hists__process(hists_base);
667 } 703 }
668 } 704 }
669 705
670 static void data__free(struct data__file *d) 706 static void data__free(struct data__file *d)
671 { 707 {
672 int col; 708 int col;
673 709
674 for (col = 0; col < PERF_HPP_DIFF__MAX_INDEX; col++) { 710 for (col = 0; col < PERF_HPP_DIFF__MAX_INDEX; col++) {
675 struct diff_hpp_fmt *fmt = &d->fmt[col]; 711 struct diff_hpp_fmt *fmt = &d->fmt[col];
676 712
677 zfree(&fmt->header); 713 zfree(&fmt->header);
678 } 714 }
679 } 715 }
680 716
681 static int __cmd_diff(void) 717 static int __cmd_diff(void)
682 { 718 {
683 struct data__file *d; 719 struct data__file *d;
684 int ret = -EINVAL, i; 720 int ret = -EINVAL, i;
685 721
686 data__for_each_file(i, d) { 722 data__for_each_file(i, d) {
687 d->session = perf_session__new(&d->file, false, &tool); 723 d->session = perf_session__new(&d->file, false, &tool);
688 if (!d->session) { 724 if (!d->session) {
689 pr_err("Failed to open %s\n", d->file.path); 725 pr_err("Failed to open %s\n", d->file.path);
690 ret = -1; 726 ret = -1;
691 goto out_delete; 727 goto out_delete;
692 } 728 }
693 729
694 ret = perf_session__process_events(d->session, &tool); 730 ret = perf_session__process_events(d->session, &tool);
695 if (ret) { 731 if (ret) {
696 pr_err("Failed to process %s\n", d->file.path); 732 pr_err("Failed to process %s\n", d->file.path);
697 goto out_delete; 733 goto out_delete;
698 } 734 }
699 735
700 perf_evlist__collapse_resort(d->session->evlist); 736 perf_evlist__collapse_resort(d->session->evlist);
701 } 737 }
702 738
703 data_process(); 739 data_process();
704 740
705 out_delete: 741 out_delete:
706 data__for_each_file(i, d) { 742 data__for_each_file(i, d) {
707 if (d->session) 743 if (d->session)
708 perf_session__delete(d->session); 744 perf_session__delete(d->session);
709 745
710 data__free(d); 746 data__free(d);
711 } 747 }
712 748
713 free(data__files); 749 free(data__files);
714 return ret; 750 return ret;
715 } 751 }
716 752
717 static const char * const diff_usage[] = { 753 static const char * const diff_usage[] = {
718 "perf diff [<options>] [old_file] [new_file]", 754 "perf diff [<options>] [old_file] [new_file]",
719 NULL, 755 NULL,
720 }; 756 };
721 757
722 static const struct option options[] = { 758 static const struct option options[] = {
723 OPT_INCR('v', "verbose", &verbose, 759 OPT_INCR('v', "verbose", &verbose,
724 "be more verbose (show symbol address, etc)"), 760 "be more verbose (show symbol address, etc)"),
725 OPT_BOOLEAN('b', "baseline-only", &show_baseline_only, 761 OPT_BOOLEAN('b', "baseline-only", &show_baseline_only,
726 "Show only items with match in baseline"), 762 "Show only items with match in baseline"),
727 OPT_CALLBACK('c', "compute", &compute, 763 OPT_CALLBACK('c', "compute", &compute,
728 "delta,ratio,wdiff:w1,w2 (default delta)", 764 "delta,ratio,wdiff:w1,w2 (default delta)",
729 "Entries differential computation selection", 765 "Entries differential computation selection",
730 setup_compute), 766 setup_compute),
731 OPT_BOOLEAN('p', "period", &show_period, 767 OPT_BOOLEAN('p', "period", &show_period,
732 "Show period values."), 768 "Show period values."),
733 OPT_BOOLEAN('F', "formula", &show_formula, 769 OPT_BOOLEAN('F', "formula", &show_formula,
734 "Show formula."), 770 "Show formula."),
735 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace, 771 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
736 "dump raw trace in ASCII"), 772 "dump raw trace in ASCII"),
737 OPT_BOOLEAN('f', "force", &force, "don't complain, do it"), 773 OPT_BOOLEAN('f', "force", &force, "don't complain, do it"),
738 OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules, 774 OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules,
739 "load module symbols - WARNING: use only with -k and LIVE kernel"), 775 "load module symbols - WARNING: use only with -k and LIVE kernel"),
740 OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]", 776 OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
741 "only consider symbols in these dsos"), 777 "only consider symbols in these dsos"),
742 OPT_STRING('C', "comms", &symbol_conf.comm_list_str, "comm[,comm...]", 778 OPT_STRING('C', "comms", &symbol_conf.comm_list_str, "comm[,comm...]",
743 "only consider symbols in these comms"), 779 "only consider symbols in these comms"),
744 OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]", 780 OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]",
745 "only consider these symbols"), 781 "only consider these symbols"),
746 OPT_STRING('s', "sort", &sort_order, "key[,key2...]", 782 OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
747 "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..." 783 "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..."
748 " Please refer the man page for the complete list."), 784 " Please refer the man page for the complete list."),
749 OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator", 785 OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator",
750 "separator for columns, no spaces will be added between " 786 "separator for columns, no spaces will be added between "
751 "columns '.' is reserved."), 787 "columns '.' is reserved."),
752 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory", 788 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
753 "Look for files with symbols relative to this directory"), 789 "Look for files with symbols relative to this directory"),
754 OPT_UINTEGER('o', "order", &sort_compute, "Specify compute sorting."), 790 OPT_UINTEGER('o', "order", &sort_compute, "Specify compute sorting."),
755 OPT_CALLBACK(0, "percentage", NULL, "relative|absolute", 791 OPT_CALLBACK(0, "percentage", NULL, "relative|absolute",
756 "How to display percentage of filtered entries", parse_filter_percentage), 792 "How to display percentage of filtered entries", parse_filter_percentage),
757 OPT_END() 793 OPT_END()
758 }; 794 };
759 795
760 static double baseline_percent(struct hist_entry *he) 796 static double baseline_percent(struct hist_entry *he)
761 { 797 {
762 u64 total = hists__total_period(he->hists); 798 u64 total = hists__total_period(he->hists);
763 799
764 return 100.0 * he->stat.period / total; 800 return 100.0 * he->stat.period / total;
765 } 801 }
766 802
767 static int hpp__color_baseline(struct perf_hpp_fmt *fmt, 803 static int hpp__color_baseline(struct perf_hpp_fmt *fmt,
768 struct perf_hpp *hpp, struct hist_entry *he) 804 struct perf_hpp *hpp, struct hist_entry *he)
769 { 805 {
770 struct diff_hpp_fmt *dfmt = 806 struct diff_hpp_fmt *dfmt =
771 container_of(fmt, struct diff_hpp_fmt, fmt); 807 container_of(fmt, struct diff_hpp_fmt, fmt);
772 double percent = baseline_percent(he); 808 double percent = baseline_percent(he);
773 char pfmt[20] = " "; 809 char pfmt[20] = " ";
774 810
775 if (!he->dummy) { 811 if (!he->dummy) {
776 scnprintf(pfmt, 20, "%%%d.2f%%%%", dfmt->header_width - 1); 812 scnprintf(pfmt, 20, "%%%d.2f%%%%", dfmt->header_width - 1);
777 return percent_color_snprintf(hpp->buf, hpp->size, 813 return percent_color_snprintf(hpp->buf, hpp->size,
778 pfmt, percent); 814 pfmt, percent);
779 } else 815 } else
780 return scnprintf(hpp->buf, hpp->size, "%*s", 816 return scnprintf(hpp->buf, hpp->size, "%*s",
781 dfmt->header_width, pfmt); 817 dfmt->header_width, pfmt);
782 } 818 }
783 819
784 static int hpp__entry_baseline(struct hist_entry *he, char *buf, size_t size) 820 static int hpp__entry_baseline(struct hist_entry *he, char *buf, size_t size)
785 { 821 {
786 double percent = baseline_percent(he); 822 double percent = baseline_percent(he);
787 const char *fmt = symbol_conf.field_sep ? "%.2f" : "%6.2f%%"; 823 const char *fmt = symbol_conf.field_sep ? "%.2f" : "%6.2f%%";
788 int ret = 0; 824 int ret = 0;
789 825
790 if (!he->dummy) 826 if (!he->dummy)
791 ret = scnprintf(buf, size, fmt, percent); 827 ret = scnprintf(buf, size, fmt, percent);
792 828
793 return ret; 829 return ret;
794 } 830 }
795 831
796 static int __hpp__color_compare(struct perf_hpp_fmt *fmt, 832 static int __hpp__color_compare(struct perf_hpp_fmt *fmt,
797 struct perf_hpp *hpp, struct hist_entry *he, 833 struct perf_hpp *hpp, struct hist_entry *he,
798 int comparison_method) 834 int comparison_method)
799 { 835 {
800 struct diff_hpp_fmt *dfmt = 836 struct diff_hpp_fmt *dfmt =
801 container_of(fmt, struct diff_hpp_fmt, fmt); 837 container_of(fmt, struct diff_hpp_fmt, fmt);
802 struct hist_entry *pair = get_pair_fmt(he, dfmt); 838 struct hist_entry *pair = get_pair_fmt(he, dfmt);
803 double diff; 839 double diff;
804 s64 wdiff; 840 s64 wdiff;
805 char pfmt[20] = " "; 841 char pfmt[20] = " ";
806 842
807 if (!pair) 843 if (!pair)
808 goto dummy_print; 844 goto dummy_print;
809 845
810 switch (comparison_method) { 846 switch (comparison_method) {
811 case COMPUTE_DELTA: 847 case COMPUTE_DELTA:
812 if (pair->diff.computed) 848 if (pair->diff.computed)
813 diff = pair->diff.period_ratio_delta; 849 diff = pair->diff.period_ratio_delta;
814 else 850 else
815 diff = compute_delta(he, pair); 851 diff = compute_delta(he, pair);
816 852
817 if (fabs(diff) < 0.01) 853 if (fabs(diff) < 0.01)
818 goto dummy_print; 854 goto dummy_print;
819 scnprintf(pfmt, 20, "%%%+d.2f%%%%", dfmt->header_width - 1); 855 scnprintf(pfmt, 20, "%%%+d.2f%%%%", dfmt->header_width - 1);
820 return percent_color_snprintf(hpp->buf, hpp->size, 856 return percent_color_snprintf(hpp->buf, hpp->size,
821 pfmt, diff); 857 pfmt, diff);
822 case COMPUTE_RATIO: 858 case COMPUTE_RATIO:
823 if (he->dummy) 859 if (he->dummy)
824 goto dummy_print; 860 goto dummy_print;
825 if (pair->diff.computed) 861 if (pair->diff.computed)
826 diff = pair->diff.period_ratio; 862 diff = pair->diff.period_ratio;
827 else 863 else
828 diff = compute_ratio(he, pair); 864 diff = compute_ratio(he, pair);
829 865
830 scnprintf(pfmt, 20, "%%%d.6f", dfmt->header_width); 866 scnprintf(pfmt, 20, "%%%d.6f", dfmt->header_width);
831 return value_color_snprintf(hpp->buf, hpp->size, 867 return value_color_snprintf(hpp->buf, hpp->size,
832 pfmt, diff); 868 pfmt, diff);
833 case COMPUTE_WEIGHTED_DIFF: 869 case COMPUTE_WEIGHTED_DIFF:
834 if (he->dummy) 870 if (he->dummy)
835 goto dummy_print; 871 goto dummy_print;
836 if (pair->diff.computed) 872 if (pair->diff.computed)
837 wdiff = pair->diff.wdiff; 873 wdiff = pair->diff.wdiff;
838 else 874 else
839 wdiff = compute_wdiff(he, pair); 875 wdiff = compute_wdiff(he, pair);
840 876
841 scnprintf(pfmt, 20, "%%14ld", dfmt->header_width); 877 scnprintf(pfmt, 20, "%%14ld", dfmt->header_width);
842 return color_snprintf(hpp->buf, hpp->size, 878 return color_snprintf(hpp->buf, hpp->size,
843 get_percent_color(wdiff), 879 get_percent_color(wdiff),
844 pfmt, wdiff); 880 pfmt, wdiff);
845 default: 881 default:
846 BUG_ON(1); 882 BUG_ON(1);
847 } 883 }
848 dummy_print: 884 dummy_print:
849 return scnprintf(hpp->buf, hpp->size, "%*s", 885 return scnprintf(hpp->buf, hpp->size, "%*s",
850 dfmt->header_width, pfmt); 886 dfmt->header_width, pfmt);
851 } 887 }
852 888
853 static int hpp__color_delta(struct perf_hpp_fmt *fmt, 889 static int hpp__color_delta(struct perf_hpp_fmt *fmt,
854 struct perf_hpp *hpp, struct hist_entry *he) 890 struct perf_hpp *hpp, struct hist_entry *he)
855 { 891 {
856 return __hpp__color_compare(fmt, hpp, he, COMPUTE_DELTA); 892 return __hpp__color_compare(fmt, hpp, he, COMPUTE_DELTA);
857 } 893 }
858 894
859 static int hpp__color_ratio(struct perf_hpp_fmt *fmt, 895 static int hpp__color_ratio(struct perf_hpp_fmt *fmt,
860 struct perf_hpp *hpp, struct hist_entry *he) 896 struct perf_hpp *hpp, struct hist_entry *he)
861 { 897 {
862 return __hpp__color_compare(fmt, hpp, he, COMPUTE_RATIO); 898 return __hpp__color_compare(fmt, hpp, he, COMPUTE_RATIO);
863 } 899 }
864 900
865 static int hpp__color_wdiff(struct perf_hpp_fmt *fmt, 901 static int hpp__color_wdiff(struct perf_hpp_fmt *fmt,
866 struct perf_hpp *hpp, struct hist_entry *he) 902 struct perf_hpp *hpp, struct hist_entry *he)
867 { 903 {
868 return __hpp__color_compare(fmt, hpp, he, COMPUTE_WEIGHTED_DIFF); 904 return __hpp__color_compare(fmt, hpp, he, COMPUTE_WEIGHTED_DIFF);
869 } 905 }
870 906
871 static void 907 static void
872 hpp__entry_unpair(struct hist_entry *he, int idx, char *buf, size_t size) 908 hpp__entry_unpair(struct hist_entry *he, int idx, char *buf, size_t size)
873 { 909 {
874 switch (idx) { 910 switch (idx) {
875 case PERF_HPP_DIFF__PERIOD_BASELINE: 911 case PERF_HPP_DIFF__PERIOD_BASELINE:
876 scnprintf(buf, size, "%" PRIu64, he->stat.period); 912 scnprintf(buf, size, "%" PRIu64, he->stat.period);
877 break; 913 break;
878 914
879 default: 915 default:
880 break; 916 break;
881 } 917 }
882 } 918 }
883 919
884 static void 920 static void
885 hpp__entry_pair(struct hist_entry *he, struct hist_entry *pair, 921 hpp__entry_pair(struct hist_entry *he, struct hist_entry *pair,
886 int idx, char *buf, size_t size) 922 int idx, char *buf, size_t size)
887 { 923 {
888 double diff; 924 double diff;
889 double ratio; 925 double ratio;
890 s64 wdiff; 926 s64 wdiff;
891 927
892 switch (idx) { 928 switch (idx) {
893 case PERF_HPP_DIFF__DELTA: 929 case PERF_HPP_DIFF__DELTA:
894 if (pair->diff.computed) 930 if (pair->diff.computed)
895 diff = pair->diff.period_ratio_delta; 931 diff = pair->diff.period_ratio_delta;
896 else 932 else
897 diff = compute_delta(he, pair); 933 diff = compute_delta(he, pair);
898 934
899 if (fabs(diff) >= 0.01) 935 if (fabs(diff) >= 0.01)
900 scnprintf(buf, size, "%+4.2F%%", diff); 936 scnprintf(buf, size, "%+4.2F%%", diff);
901 break; 937 break;
902 938
903 case PERF_HPP_DIFF__RATIO: 939 case PERF_HPP_DIFF__RATIO:
904 /* No point for ratio number if we are dummy.. */ 940 /* No point for ratio number if we are dummy.. */
905 if (he->dummy) 941 if (he->dummy)
906 break; 942 break;
907 943
908 if (pair->diff.computed) 944 if (pair->diff.computed)
909 ratio = pair->diff.period_ratio; 945 ratio = pair->diff.period_ratio;
910 else 946 else
911 ratio = compute_ratio(he, pair); 947 ratio = compute_ratio(he, pair);
912 948
913 if (ratio > 0.0) 949 if (ratio > 0.0)
914 scnprintf(buf, size, "%14.6F", ratio); 950 scnprintf(buf, size, "%14.6F", ratio);
915 break; 951 break;
916 952
917 case PERF_HPP_DIFF__WEIGHTED_DIFF: 953 case PERF_HPP_DIFF__WEIGHTED_DIFF:
918 /* No point for wdiff number if we are dummy.. */ 954 /* No point for wdiff number if we are dummy.. */
919 if (he->dummy) 955 if (he->dummy)
920 break; 956 break;
921 957
922 if (pair->diff.computed) 958 if (pair->diff.computed)
923 wdiff = pair->diff.wdiff; 959 wdiff = pair->diff.wdiff;
924 else 960 else
925 wdiff = compute_wdiff(he, pair); 961 wdiff = compute_wdiff(he, pair);
926 962
927 if (wdiff != 0) 963 if (wdiff != 0)
928 scnprintf(buf, size, "%14ld", wdiff); 964 scnprintf(buf, size, "%14ld", wdiff);
929 break; 965 break;
930 966
931 case PERF_HPP_DIFF__FORMULA: 967 case PERF_HPP_DIFF__FORMULA:
932 formula_fprintf(he, pair, buf, size); 968 formula_fprintf(he, pair, buf, size);
933 break; 969 break;
934 970
935 case PERF_HPP_DIFF__PERIOD: 971 case PERF_HPP_DIFF__PERIOD:
936 scnprintf(buf, size, "%" PRIu64, pair->stat.period); 972 scnprintf(buf, size, "%" PRIu64, pair->stat.period);
937 break; 973 break;
938 974
939 default: 975 default:
940 BUG_ON(1); 976 BUG_ON(1);
941 }; 977 };
942 } 978 }
943 979
944 static void 980 static void
945 __hpp__entry_global(struct hist_entry *he, struct diff_hpp_fmt *dfmt, 981 __hpp__entry_global(struct hist_entry *he, struct diff_hpp_fmt *dfmt,
946 char *buf, size_t size) 982 char *buf, size_t size)
947 { 983 {
948 struct hist_entry *pair = get_pair_fmt(he, dfmt); 984 struct hist_entry *pair = get_pair_fmt(he, dfmt);
949 int idx = dfmt->idx; 985 int idx = dfmt->idx;
950 986
951 /* baseline is special */ 987 /* baseline is special */
952 if (idx == PERF_HPP_DIFF__BASELINE) 988 if (idx == PERF_HPP_DIFF__BASELINE)
953 hpp__entry_baseline(he, buf, size); 989 hpp__entry_baseline(he, buf, size);
954 else { 990 else {
955 if (pair) 991 if (pair)
956 hpp__entry_pair(he, pair, idx, buf, size); 992 hpp__entry_pair(he, pair, idx, buf, size);
957 else 993 else
958 hpp__entry_unpair(he, idx, buf, size); 994 hpp__entry_unpair(he, idx, buf, size);
959 } 995 }
960 } 996 }
961 997
962 static int hpp__entry_global(struct perf_hpp_fmt *_fmt, struct perf_hpp *hpp, 998 static int hpp__entry_global(struct perf_hpp_fmt *_fmt, struct perf_hpp *hpp,
963 struct hist_entry *he) 999 struct hist_entry *he)
964 { 1000 {
965 struct diff_hpp_fmt *dfmt = 1001 struct diff_hpp_fmt *dfmt =
966 container_of(_fmt, struct diff_hpp_fmt, fmt); 1002 container_of(_fmt, struct diff_hpp_fmt, fmt);
967 char buf[MAX_COL_WIDTH] = " "; 1003 char buf[MAX_COL_WIDTH] = " ";
968 1004
969 __hpp__entry_global(he, dfmt, buf, MAX_COL_WIDTH); 1005 __hpp__entry_global(he, dfmt, buf, MAX_COL_WIDTH);
970 1006
971 if (symbol_conf.field_sep) 1007 if (symbol_conf.field_sep)
972 return scnprintf(hpp->buf, hpp->size, "%s", buf); 1008 return scnprintf(hpp->buf, hpp->size, "%s", buf);
973 else 1009 else
974 return scnprintf(hpp->buf, hpp->size, "%*s", 1010 return scnprintf(hpp->buf, hpp->size, "%*s",
975 dfmt->header_width, buf); 1011 dfmt->header_width, buf);
976 } 1012 }
977 1013
978 static int hpp__header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 1014 static int hpp__header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
979 struct perf_evsel *evsel __maybe_unused) 1015 struct perf_evsel *evsel __maybe_unused)
980 { 1016 {
981 struct diff_hpp_fmt *dfmt = 1017 struct diff_hpp_fmt *dfmt =
982 container_of(fmt, struct diff_hpp_fmt, fmt); 1018 container_of(fmt, struct diff_hpp_fmt, fmt);
983 1019
984 BUG_ON(!dfmt->header); 1020 BUG_ON(!dfmt->header);
985 return scnprintf(hpp->buf, hpp->size, dfmt->header); 1021 return scnprintf(hpp->buf, hpp->size, dfmt->header);
986 } 1022 }
987 1023
988 static int hpp__width(struct perf_hpp_fmt *fmt, 1024 static int hpp__width(struct perf_hpp_fmt *fmt,
989 struct perf_hpp *hpp __maybe_unused, 1025 struct perf_hpp *hpp __maybe_unused,
990 struct perf_evsel *evsel __maybe_unused) 1026 struct perf_evsel *evsel __maybe_unused)
991 { 1027 {
992 struct diff_hpp_fmt *dfmt = 1028 struct diff_hpp_fmt *dfmt =
993 container_of(fmt, struct diff_hpp_fmt, fmt); 1029 container_of(fmt, struct diff_hpp_fmt, fmt);
994 1030
995 BUG_ON(dfmt->header_width <= 0); 1031 BUG_ON(dfmt->header_width <= 0);
996 return dfmt->header_width; 1032 return dfmt->header_width;
997 } 1033 }
998 1034
999 static void init_header(struct data__file *d, struct diff_hpp_fmt *dfmt) 1035 static void init_header(struct data__file *d, struct diff_hpp_fmt *dfmt)
1000 { 1036 {
1001 #define MAX_HEADER_NAME 100 1037 #define MAX_HEADER_NAME 100
1002 char buf_indent[MAX_HEADER_NAME]; 1038 char buf_indent[MAX_HEADER_NAME];
1003 char buf[MAX_HEADER_NAME]; 1039 char buf[MAX_HEADER_NAME];
1004 const char *header = NULL; 1040 const char *header = NULL;
1005 int width = 0; 1041 int width = 0;
1006 1042
1007 BUG_ON(dfmt->idx >= PERF_HPP_DIFF__MAX_INDEX); 1043 BUG_ON(dfmt->idx >= PERF_HPP_DIFF__MAX_INDEX);
1008 header = columns[dfmt->idx].name; 1044 header = columns[dfmt->idx].name;
1009 width = columns[dfmt->idx].width; 1045 width = columns[dfmt->idx].width;
1010 1046
1011 /* Only our defined HPP fmts should appear here. */ 1047 /* Only our defined HPP fmts should appear here. */
1012 BUG_ON(!header); 1048 BUG_ON(!header);
1013 1049
1014 if (data__files_cnt > 2) 1050 if (data__files_cnt > 2)
1015 scnprintf(buf, MAX_HEADER_NAME, "%s/%d", header, d->idx); 1051 scnprintf(buf, MAX_HEADER_NAME, "%s/%d", header, d->idx);
1016 1052
1017 #define NAME (data__files_cnt > 2 ? buf : header) 1053 #define NAME (data__files_cnt > 2 ? buf : header)
1018 dfmt->header_width = width; 1054 dfmt->header_width = width;
1019 width = (int) strlen(NAME); 1055 width = (int) strlen(NAME);
1020 if (dfmt->header_width < width) 1056 if (dfmt->header_width < width)
1021 dfmt->header_width = width; 1057 dfmt->header_width = width;
1022 1058
1023 scnprintf(buf_indent, MAX_HEADER_NAME, "%*s", 1059 scnprintf(buf_indent, MAX_HEADER_NAME, "%*s",
1024 dfmt->header_width, NAME); 1060 dfmt->header_width, NAME);
1025 1061
1026 dfmt->header = strdup(buf_indent); 1062 dfmt->header = strdup(buf_indent);
1027 #undef MAX_HEADER_NAME 1063 #undef MAX_HEADER_NAME
1028 #undef NAME 1064 #undef NAME
1029 } 1065 }
1030 1066
1031 static void data__hpp_register(struct data__file *d, int idx) 1067 static void data__hpp_register(struct data__file *d, int idx)
1032 { 1068 {
1033 struct diff_hpp_fmt *dfmt = &d->fmt[idx]; 1069 struct diff_hpp_fmt *dfmt = &d->fmt[idx];
1034 struct perf_hpp_fmt *fmt = &dfmt->fmt; 1070 struct perf_hpp_fmt *fmt = &dfmt->fmt;
1035 1071
1036 dfmt->idx = idx; 1072 dfmt->idx = idx;
1037 1073
1038 fmt->header = hpp__header; 1074 fmt->header = hpp__header;
1039 fmt->width = hpp__width; 1075 fmt->width = hpp__width;
1040 fmt->entry = hpp__entry_global; 1076 fmt->entry = hpp__entry_global;
1077 fmt->cmp = hist_entry__cmp_nop;
1078 fmt->collapse = hist_entry__cmp_nop;
1041 1079
1042 /* TODO more colors */ 1080 /* TODO more colors */
1043 switch (idx) { 1081 switch (idx) {
1044 case PERF_HPP_DIFF__BASELINE: 1082 case PERF_HPP_DIFF__BASELINE:
1045 fmt->color = hpp__color_baseline; 1083 fmt->color = hpp__color_baseline;
1084 fmt->sort = hist_entry__cmp_baseline;
1046 break; 1085 break;
1047 case PERF_HPP_DIFF__DELTA: 1086 case PERF_HPP_DIFF__DELTA:
1048 fmt->color = hpp__color_delta; 1087 fmt->color = hpp__color_delta;
1088 fmt->sort = hist_entry__cmp_delta;
1049 break; 1089 break;
1050 case PERF_HPP_DIFF__RATIO: 1090 case PERF_HPP_DIFF__RATIO:
1051 fmt->color = hpp__color_ratio; 1091 fmt->color = hpp__color_ratio;
1092 fmt->sort = hist_entry__cmp_ratio;
1052 break; 1093 break;
1053 case PERF_HPP_DIFF__WEIGHTED_DIFF: 1094 case PERF_HPP_DIFF__WEIGHTED_DIFF:
1054 fmt->color = hpp__color_wdiff; 1095 fmt->color = hpp__color_wdiff;
1096 fmt->sort = hist_entry__cmp_wdiff;
1055 break; 1097 break;
1056 default: 1098 default:
1099 fmt->sort = hist_entry__cmp_nop;
1057 break; 1100 break;
1058 } 1101 }
1059 1102
1060 init_header(d, dfmt); 1103 init_header(d, dfmt);
1061 perf_hpp__column_register(fmt); 1104 perf_hpp__column_register(fmt);
1105 perf_hpp__register_sort_field(fmt);
1062 } 1106 }
1063 1107
1064 static void ui_init(void) 1108 static void ui_init(void)
1065 { 1109 {
1066 struct data__file *d; 1110 struct data__file *d;
1067 int i; 1111 int i;
1068 1112
1069 data__for_each_file(i, d) { 1113 data__for_each_file(i, d) {
1070 1114
1071 /* 1115 /*
1072 * Baseline or compute realted columns: 1116 * Baseline or compute realted columns:
1073 * 1117 *
1074 * PERF_HPP_DIFF__BASELINE 1118 * PERF_HPP_DIFF__BASELINE
1075 * PERF_HPP_DIFF__DELTA 1119 * PERF_HPP_DIFF__DELTA
1076 * PERF_HPP_DIFF__RATIO 1120 * PERF_HPP_DIFF__RATIO
1077 * PERF_HPP_DIFF__WEIGHTED_DIFF 1121 * PERF_HPP_DIFF__WEIGHTED_DIFF
1078 */ 1122 */
1079 data__hpp_register(d, i ? compute_2_hpp[compute] : 1123 data__hpp_register(d, i ? compute_2_hpp[compute] :
1080 PERF_HPP_DIFF__BASELINE); 1124 PERF_HPP_DIFF__BASELINE);
1081 1125
1082 /* 1126 /*
1083 * And the rest: 1127 * And the rest:
1084 * 1128 *
1085 * PERF_HPP_DIFF__FORMULA 1129 * PERF_HPP_DIFF__FORMULA
1086 * PERF_HPP_DIFF__PERIOD 1130 * PERF_HPP_DIFF__PERIOD
1087 * PERF_HPP_DIFF__PERIOD_BASELINE 1131 * PERF_HPP_DIFF__PERIOD_BASELINE
1088 */ 1132 */
1089 if (show_formula && i) 1133 if (show_formula && i)
1090 data__hpp_register(d, PERF_HPP_DIFF__FORMULA); 1134 data__hpp_register(d, PERF_HPP_DIFF__FORMULA);
1091 1135
1092 if (show_period) 1136 if (show_period)
1093 data__hpp_register(d, i ? PERF_HPP_DIFF__PERIOD : 1137 data__hpp_register(d, i ? PERF_HPP_DIFF__PERIOD :
1094 PERF_HPP_DIFF__PERIOD_BASELINE); 1138 PERF_HPP_DIFF__PERIOD_BASELINE);
1095 } 1139 }
1096 } 1140 }
1097 1141
1098 static int data_init(int argc, const char **argv) 1142 static int data_init(int argc, const char **argv)
1099 { 1143 {
1100 struct data__file *d; 1144 struct data__file *d;
1101 static const char *defaults[] = { 1145 static const char *defaults[] = {
1102 "perf.data.old", 1146 "perf.data.old",
1103 "perf.data", 1147 "perf.data",
1104 }; 1148 };
1105 bool use_default = true; 1149 bool use_default = true;
1106 int i; 1150 int i;
1107 1151
1108 data__files_cnt = 2; 1152 data__files_cnt = 2;
1109 1153
1110 if (argc) { 1154 if (argc) {
1111 if (argc == 1) 1155 if (argc == 1)
1112 defaults[1] = argv[0]; 1156 defaults[1] = argv[0];
1113 else { 1157 else {
1114 data__files_cnt = argc; 1158 data__files_cnt = argc;
1115 use_default = false; 1159 use_default = false;
1116 } 1160 }
1117 } else if (perf_guest) { 1161 } else if (perf_guest) {
1118 defaults[0] = "perf.data.host"; 1162 defaults[0] = "perf.data.host";
1119 defaults[1] = "perf.data.guest"; 1163 defaults[1] = "perf.data.guest";
1120 } 1164 }
1121 1165
1122 if (sort_compute >= (unsigned int) data__files_cnt) { 1166 if (sort_compute >= (unsigned int) data__files_cnt) {
1123 pr_err("Order option out of limit.\n"); 1167 pr_err("Order option out of limit.\n");
1124 return -EINVAL; 1168 return -EINVAL;
1125 } 1169 }
1126 1170
1127 data__files = zalloc(sizeof(*data__files) * data__files_cnt); 1171 data__files = zalloc(sizeof(*data__files) * data__files_cnt);
1128 if (!data__files) 1172 if (!data__files)
1129 return -ENOMEM; 1173 return -ENOMEM;
1130 1174
1131 data__for_each_file(i, d) { 1175 data__for_each_file(i, d) {
1132 struct perf_data_file *file = &d->file; 1176 struct perf_data_file *file = &d->file;
1133 1177
1134 file->path = use_default ? defaults[i] : argv[i]; 1178 file->path = use_default ? defaults[i] : argv[i];
1135 file->mode = PERF_DATA_MODE_READ, 1179 file->mode = PERF_DATA_MODE_READ,
1136 file->force = force, 1180 file->force = force,
1137 1181
1138 d->idx = i; 1182 d->idx = i;
1139 } 1183 }
1140 1184
1141 return 0; 1185 return 0;
1142 } 1186 }
1143 1187
1144 int cmd_diff(int argc, const char **argv, const char *prefix __maybe_unused) 1188 int cmd_diff(int argc, const char **argv, const char *prefix __maybe_unused)
1145 { 1189 {
1146 int ret = hists__init(); 1190 int ret = hists__init();
1147 1191
1148 if (ret < 0) 1192 if (ret < 0)
1149 return ret; 1193 return ret;
1150 1194
1151 perf_config(perf_default_config, NULL); 1195 perf_config(perf_default_config, NULL);
1152 1196
1153 argc = parse_options(argc, argv, options, diff_usage, 0); 1197 argc = parse_options(argc, argv, options, diff_usage, 0);
1154 1198
1155 if (symbol__init(NULL) < 0) 1199 if (symbol__init(NULL) < 0)
1156 return -1; 1200 return -1;
1157 1201
1158 if (data_init(argc, argv) < 0) 1202 if (data_init(argc, argv) < 0)
1159 return -1; 1203 return -1;
1160 1204
1161 ui_init(); 1205 ui_init();
1162 1206
1163 sort__mode = SORT_MODE__DIFF; 1207 sort__mode = SORT_MODE__DIFF;
1164 1208
1165 if (setup_sorting() < 0) 1209 if (setup_sorting() < 0)
1166 usage_with_options(diff_usage, options); 1210 usage_with_options(diff_usage, options);
1167 1211
1168 setup_pager(); 1212 setup_pager();
1169 1213
1170 sort__setup_elide(NULL); 1214 sort__setup_elide(NULL);
1171 1215
1172 return __cmd_diff(); 1216 return __cmd_diff();
1173 } 1217 }
1174 1218
tools/perf/builtin-list.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * builtin-list.c 2 * builtin-list.c
3 * 3 *
4 * Builtin list command: list all event types 4 * Builtin list command: list all event types
5 * 5 *
6 * Copyright (C) 2009, Thomas Gleixner <tglx@linutronix.de> 6 * Copyright (C) 2009, Thomas Gleixner <tglx@linutronix.de>
7 * Copyright (C) 2008-2009, Red Hat Inc, Ingo Molnar <mingo@redhat.com> 7 * Copyright (C) 2008-2009, Red Hat Inc, Ingo Molnar <mingo@redhat.com>
8 * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com> 8 * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
9 */ 9 */
10 #include "builtin.h" 10 #include "builtin.h"
11 11
12 #include "perf.h" 12 #include "perf.h"
13 13
14 #include "util/parse-events.h" 14 #include "util/parse-events.h"
15 #include "util/cache.h" 15 #include "util/cache.h"
16 #include "util/pmu.h" 16 #include "util/pmu.h"
17 #include "util/parse-options.h" 17 #include "util/parse-options.h"
18 18
19 int cmd_list(int argc, const char **argv, const char *prefix __maybe_unused) 19 int cmd_list(int argc, const char **argv, const char *prefix __maybe_unused)
20 { 20 {
21 int i; 21 int i;
22 const struct option list_options[] = { 22 bool raw_dump = false;
23 struct option list_options[] = {
24 OPT_BOOLEAN(0, "raw-dump", &raw_dump, "Dump raw events"),
23 OPT_END() 25 OPT_END()
24 }; 26 };
25 const char * const list_usage[] = { 27 const char * const list_usage[] = {
26 "perf list [hw|sw|cache|tracepoint|pmu|event_glob]", 28 "perf list [hw|sw|cache|tracepoint|pmu|event_glob]",
27 NULL 29 NULL
28 }; 30 };
29 31
32 set_option_flag(list_options, 0, "raw-dump", PARSE_OPT_HIDDEN);
33
30 argc = parse_options(argc, argv, list_options, list_usage, 34 argc = parse_options(argc, argv, list_options, list_usage,
31 PARSE_OPT_STOP_AT_NON_OPTION); 35 PARSE_OPT_STOP_AT_NON_OPTION);
32 36
33 setup_pager(); 37 setup_pager();
34 38
39 if (raw_dump) {
40 print_events(NULL, true);
41 return 0;
42 }
43
35 if (argc == 0) { 44 if (argc == 0) {
36 print_events(NULL, false); 45 print_events(NULL, false);
37 return 0; 46 return 0;
38 } 47 }
39 48
40 for (i = 0; i < argc; ++i) { 49 for (i = 0; i < argc; ++i) {
41 if (i) 50 if (i)
42 putchar('\n'); 51 putchar('\n');
43 if (strncmp(argv[i], "tracepoint", 10) == 0) 52 if (strncmp(argv[i], "tracepoint", 10) == 0)
44 print_tracepoint_events(NULL, NULL, false); 53 print_tracepoint_events(NULL, NULL, false);
45 else if (strcmp(argv[i], "hw") == 0 || 54 else if (strcmp(argv[i], "hw") == 0 ||
46 strcmp(argv[i], "hardware") == 0) 55 strcmp(argv[i], "hardware") == 0)
47 print_events_type(PERF_TYPE_HARDWARE); 56 print_events_type(PERF_TYPE_HARDWARE);
48 else if (strcmp(argv[i], "sw") == 0 || 57 else if (strcmp(argv[i], "sw") == 0 ||
49 strcmp(argv[i], "software") == 0) 58 strcmp(argv[i], "software") == 0)
50 print_events_type(PERF_TYPE_SOFTWARE); 59 print_events_type(PERF_TYPE_SOFTWARE);
51 else if (strcmp(argv[i], "cache") == 0 || 60 else if (strcmp(argv[i], "cache") == 0 ||
52 strcmp(argv[i], "hwcache") == 0) 61 strcmp(argv[i], "hwcache") == 0)
53 print_hwcache_events(NULL, false); 62 print_hwcache_events(NULL, false);
54 else if (strcmp(argv[i], "pmu") == 0) 63 else if (strcmp(argv[i], "pmu") == 0)
55 print_pmu_events(NULL, false); 64 print_pmu_events(NULL, false);
56 else if (strcmp(argv[i], "--raw-dump") == 0)
57 print_events(NULL, true);
58 else { 65 else {
59 char *sep = strchr(argv[i], ':'), *s; 66 char *sep = strchr(argv[i], ':'), *s;
60 int sep_idx; 67 int sep_idx;
61 68
62 if (sep == NULL) { 69 if (sep == NULL) {
63 print_events(argv[i], false); 70 print_events(argv[i], false);
64 continue; 71 continue;
65 } 72 }
66 sep_idx = sep - argv[i]; 73 sep_idx = sep - argv[i];
67 s = strdup(argv[i]); 74 s = strdup(argv[i]);
68 if (s == NULL) 75 if (s == NULL)
69 return -1; 76 return -1;
70 77
71 s[sep_idx] = '\0'; 78 s[sep_idx] = '\0';
72 print_tracepoint_events(s, s + sep_idx + 1, false); 79 print_tracepoint_events(s, s + sep_idx + 1, false);
73 free(s); 80 free(s);
74 } 81 }
75 } 82 }
76 return 0; 83 return 0;
tools/perf/builtin-report.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * builtin-report.c 2 * builtin-report.c
3 * 3 *
4 * Builtin report command: Analyze the perf.data input file, 4 * Builtin report command: Analyze the perf.data input file,
5 * look up and read DSOs and symbol information and display 5 * look up and read DSOs and symbol information and display
6 * a histogram of results, along various sorting keys. 6 * a histogram of results, along various sorting keys.
7 */ 7 */
8 #include "builtin.h" 8 #include "builtin.h"
9 9
10 #include "util/util.h" 10 #include "util/util.h"
11 #include "util/cache.h" 11 #include "util/cache.h"
12 12
13 #include "util/annotate.h" 13 #include "util/annotate.h"
14 #include "util/color.h" 14 #include "util/color.h"
15 #include <linux/list.h> 15 #include <linux/list.h>
16 #include <linux/rbtree.h> 16 #include <linux/rbtree.h>
17 #include "util/symbol.h" 17 #include "util/symbol.h"
18 #include "util/callchain.h" 18 #include "util/callchain.h"
19 #include "util/strlist.h" 19 #include "util/strlist.h"
20 #include "util/values.h" 20 #include "util/values.h"
21 21
22 #include "perf.h" 22 #include "perf.h"
23 #include "util/debug.h" 23 #include "util/debug.h"
24 #include "util/evlist.h" 24 #include "util/evlist.h"
25 #include "util/evsel.h" 25 #include "util/evsel.h"
26 #include "util/header.h" 26 #include "util/header.h"
27 #include "util/session.h" 27 #include "util/session.h"
28 #include "util/tool.h" 28 #include "util/tool.h"
29 29
30 #include "util/parse-options.h" 30 #include "util/parse-options.h"
31 #include "util/parse-events.h" 31 #include "util/parse-events.h"
32 32
33 #include "util/thread.h" 33 #include "util/thread.h"
34 #include "util/sort.h" 34 #include "util/sort.h"
35 #include "util/hist.h" 35 #include "util/hist.h"
36 #include "util/data.h" 36 #include "util/data.h"
37 #include "arch/common.h" 37 #include "arch/common.h"
38 38
39 #include <dlfcn.h> 39 #include <dlfcn.h>
40 #include <linux/bitmap.h> 40 #include <linux/bitmap.h>
41 41
42 struct report { 42 struct report {
43 struct perf_tool tool; 43 struct perf_tool tool;
44 struct perf_session *session; 44 struct perf_session *session;
45 bool force, use_tui, use_gtk, use_stdio; 45 bool force, use_tui, use_gtk, use_stdio;
46 bool hide_unresolved; 46 bool hide_unresolved;
47 bool dont_use_callchains; 47 bool dont_use_callchains;
48 bool show_full_info; 48 bool show_full_info;
49 bool show_threads; 49 bool show_threads;
50 bool inverted_callchain; 50 bool inverted_callchain;
51 bool mem_mode; 51 bool mem_mode;
52 bool header; 52 bool header;
53 bool header_only; 53 bool header_only;
54 int max_stack; 54 int max_stack;
55 struct perf_read_values show_threads_values; 55 struct perf_read_values show_threads_values;
56 const char *pretty_printing_style; 56 const char *pretty_printing_style;
57 const char *cpu_list; 57 const char *cpu_list;
58 const char *symbol_filter_str; 58 const char *symbol_filter_str;
59 float min_percent; 59 float min_percent;
60 u64 nr_entries; 60 u64 nr_entries;
61 u64 queue_size; 61 u64 queue_size;
62 DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS); 62 DECLARE_BITMAP(cpu_bitmap, MAX_NR_CPUS);
63 }; 63 };
64 64
65 static int report__config(const char *var, const char *value, void *cb) 65 static int report__config(const char *var, const char *value, void *cb)
66 { 66 {
67 struct report *rep = cb; 67 struct report *rep = cb;
68 68
69 if (!strcmp(var, "report.group")) { 69 if (!strcmp(var, "report.group")) {
70 symbol_conf.event_group = perf_config_bool(var, value); 70 symbol_conf.event_group = perf_config_bool(var, value);
71 return 0; 71 return 0;
72 } 72 }
73 if (!strcmp(var, "report.percent-limit")) { 73 if (!strcmp(var, "report.percent-limit")) {
74 rep->min_percent = strtof(value, NULL); 74 rep->min_percent = strtof(value, NULL);
75 return 0; 75 return 0;
76 } 76 }
77 if (!strcmp(var, "report.children")) { 77 if (!strcmp(var, "report.children")) {
78 symbol_conf.cumulate_callchain = perf_config_bool(var, value); 78 symbol_conf.cumulate_callchain = perf_config_bool(var, value);
79 return 0; 79 return 0;
80 } 80 }
81 if (!strcmp(var, "report.queue-size")) { 81 if (!strcmp(var, "report.queue-size")) {
82 rep->queue_size = perf_config_u64(var, value); 82 rep->queue_size = perf_config_u64(var, value);
83 return 0; 83 return 0;
84 } 84 }
85 85
86 return perf_default_config(var, value, cb); 86 return perf_default_config(var, value, cb);
87 } 87 }
88 88
89 static void report__inc_stats(struct report *rep, struct hist_entry *he) 89 static void report__inc_stats(struct report *rep, struct hist_entry *he)
90 { 90 {
91 /* 91 /*
92 * The @he is either of a newly created one or an existing one 92 * The @he is either of a newly created one or an existing one
93 * merging current sample. We only want to count a new one so 93 * merging current sample. We only want to count a new one so
94 * checking ->nr_events being 1. 94 * checking ->nr_events being 1.
95 */ 95 */
96 if (he->stat.nr_events == 1) 96 if (he->stat.nr_events == 1)
97 rep->nr_entries++; 97 rep->nr_entries++;
98 } 98 }
99 99
100 static int hist_iter__report_callback(struct hist_entry_iter *iter, 100 static int hist_iter__report_callback(struct hist_entry_iter *iter,
101 struct addr_location *al, bool single, 101 struct addr_location *al, bool single,
102 void *arg) 102 void *arg)
103 { 103 {
104 int err = 0; 104 int err = 0;
105 struct report *rep = arg; 105 struct report *rep = arg;
106 struct hist_entry *he = iter->he; 106 struct hist_entry *he = iter->he;
107 struct perf_evsel *evsel = iter->evsel; 107 struct perf_evsel *evsel = iter->evsel;
108 struct mem_info *mi; 108 struct mem_info *mi;
109 struct branch_info *bi; 109 struct branch_info *bi;
110 110
111 report__inc_stats(rep, he); 111 report__inc_stats(rep, he);
112 112
113 if (!ui__has_annotation()) 113 if (!ui__has_annotation())
114 return 0; 114 return 0;
115 115
116 if (sort__mode == SORT_MODE__BRANCH) { 116 if (sort__mode == SORT_MODE__BRANCH) {
117 bi = he->branch_info; 117 bi = he->branch_info;
118 err = addr_map_symbol__inc_samples(&bi->from, evsel->idx); 118 err = addr_map_symbol__inc_samples(&bi->from, evsel->idx);
119 if (err) 119 if (err)
120 goto out; 120 goto out;
121 121
122 err = addr_map_symbol__inc_samples(&bi->to, evsel->idx); 122 err = addr_map_symbol__inc_samples(&bi->to, evsel->idx);
123 123
124 } else if (rep->mem_mode) { 124 } else if (rep->mem_mode) {
125 mi = he->mem_info; 125 mi = he->mem_info;
126 err = addr_map_symbol__inc_samples(&mi->daddr, evsel->idx); 126 err = addr_map_symbol__inc_samples(&mi->daddr, evsel->idx);
127 if (err) 127 if (err)
128 goto out; 128 goto out;
129 129
130 err = hist_entry__inc_addr_samples(he, evsel->idx, al->addr); 130 err = hist_entry__inc_addr_samples(he, evsel->idx, al->addr);
131 131
132 } else if (symbol_conf.cumulate_callchain) { 132 } else if (symbol_conf.cumulate_callchain) {
133 if (single) 133 if (single)
134 err = hist_entry__inc_addr_samples(he, evsel->idx, 134 err = hist_entry__inc_addr_samples(he, evsel->idx,
135 al->addr); 135 al->addr);
136 } else { 136 } else {
137 err = hist_entry__inc_addr_samples(he, evsel->idx, al->addr); 137 err = hist_entry__inc_addr_samples(he, evsel->idx, al->addr);
138 } 138 }
139 139
140 out: 140 out:
141 return err; 141 return err;
142 } 142 }
143 143
144 static int process_sample_event(struct perf_tool *tool, 144 static int process_sample_event(struct perf_tool *tool,
145 union perf_event *event, 145 union perf_event *event,
146 struct perf_sample *sample, 146 struct perf_sample *sample,
147 struct perf_evsel *evsel, 147 struct perf_evsel *evsel,
148 struct machine *machine) 148 struct machine *machine)
149 { 149 {
150 struct report *rep = container_of(tool, struct report, tool); 150 struct report *rep = container_of(tool, struct report, tool);
151 struct addr_location al; 151 struct addr_location al;
152 struct hist_entry_iter iter = { 152 struct hist_entry_iter iter = {
153 .hide_unresolved = rep->hide_unresolved, 153 .hide_unresolved = rep->hide_unresolved,
154 .add_entry_cb = hist_iter__report_callback, 154 .add_entry_cb = hist_iter__report_callback,
155 }; 155 };
156 int ret; 156 int ret;
157 157
158 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) { 158 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) {
159 pr_debug("problem processing %d event, skipping it.\n", 159 pr_debug("problem processing %d event, skipping it.\n",
160 event->header.type); 160 event->header.type);
161 return -1; 161 return -1;
162 } 162 }
163 163
164 if (rep->hide_unresolved && al.sym == NULL) 164 if (rep->hide_unresolved && al.sym == NULL)
165 return 0; 165 return 0;
166 166
167 if (rep->cpu_list && !test_bit(sample->cpu, rep->cpu_bitmap)) 167 if (rep->cpu_list && !test_bit(sample->cpu, rep->cpu_bitmap))
168 return 0; 168 return 0;
169 169
170 if (sort__mode == SORT_MODE__BRANCH) 170 if (sort__mode == SORT_MODE__BRANCH)
171 iter.ops = &hist_iter_branch; 171 iter.ops = &hist_iter_branch;
172 else if (rep->mem_mode) 172 else if (rep->mem_mode)
173 iter.ops = &hist_iter_mem; 173 iter.ops = &hist_iter_mem;
174 else if (symbol_conf.cumulate_callchain) 174 else if (symbol_conf.cumulate_callchain)
175 iter.ops = &hist_iter_cumulative; 175 iter.ops = &hist_iter_cumulative;
176 else 176 else
177 iter.ops = &hist_iter_normal; 177 iter.ops = &hist_iter_normal;
178 178
179 if (al.map != NULL) 179 if (al.map != NULL)
180 al.map->dso->hit = 1; 180 al.map->dso->hit = 1;
181 181
182 ret = hist_entry_iter__add(&iter, &al, evsel, sample, rep->max_stack, 182 ret = hist_entry_iter__add(&iter, &al, evsel, sample, rep->max_stack,
183 rep); 183 rep);
184 if (ret < 0) 184 if (ret < 0)
185 pr_debug("problem adding hist entry, skipping event\n"); 185 pr_debug("problem adding hist entry, skipping event\n");
186 186
187 return ret; 187 return ret;
188 } 188 }
189 189
190 static int process_read_event(struct perf_tool *tool, 190 static int process_read_event(struct perf_tool *tool,
191 union perf_event *event, 191 union perf_event *event,
192 struct perf_sample *sample __maybe_unused, 192 struct perf_sample *sample __maybe_unused,
193 struct perf_evsel *evsel, 193 struct perf_evsel *evsel,
194 struct machine *machine __maybe_unused) 194 struct machine *machine __maybe_unused)
195 { 195 {
196 struct report *rep = container_of(tool, struct report, tool); 196 struct report *rep = container_of(tool, struct report, tool);
197 197
198 if (rep->show_threads) { 198 if (rep->show_threads) {
199 const char *name = evsel ? perf_evsel__name(evsel) : "unknown"; 199 const char *name = evsel ? perf_evsel__name(evsel) : "unknown";
200 perf_read_values_add_value(&rep->show_threads_values, 200 perf_read_values_add_value(&rep->show_threads_values,
201 event->read.pid, event->read.tid, 201 event->read.pid, event->read.tid,
202 event->read.id, 202 event->read.id,
203 name, 203 name,
204 event->read.value); 204 event->read.value);
205 } 205 }
206 206
207 dump_printf(": %d %d %s %" PRIu64 "\n", event->read.pid, event->read.tid, 207 dump_printf(": %d %d %s %" PRIu64 "\n", event->read.pid, event->read.tid,
208 evsel ? perf_evsel__name(evsel) : "FAIL", 208 evsel ? perf_evsel__name(evsel) : "FAIL",
209 event->read.value); 209 event->read.value);
210 210
211 return 0; 211 return 0;
212 } 212 }
213 213
214 /* For pipe mode, sample_type is not currently set */ 214 /* For pipe mode, sample_type is not currently set */
215 static int report__setup_sample_type(struct report *rep) 215 static int report__setup_sample_type(struct report *rep)
216 { 216 {
217 struct perf_session *session = rep->session; 217 struct perf_session *session = rep->session;
218 u64 sample_type = perf_evlist__combined_sample_type(session->evlist); 218 u64 sample_type = perf_evlist__combined_sample_type(session->evlist);
219 bool is_pipe = perf_data_file__is_pipe(session->file); 219 bool is_pipe = perf_data_file__is_pipe(session->file);
220 220
221 if (!is_pipe && !(sample_type & PERF_SAMPLE_CALLCHAIN)) { 221 if (!is_pipe && !(sample_type & PERF_SAMPLE_CALLCHAIN)) {
222 if (sort__has_parent) { 222 if (sort__has_parent) {
223 ui__error("Selected --sort parent, but no " 223 ui__error("Selected --sort parent, but no "
224 "callchain data. Did you call " 224 "callchain data. Did you call "
225 "'perf record' without -g?\n"); 225 "'perf record' without -g?\n");
226 return -EINVAL; 226 return -EINVAL;
227 } 227 }
228 if (symbol_conf.use_callchain) { 228 if (symbol_conf.use_callchain) {
229 ui__error("Selected -g or --branch-history but no " 229 ui__error("Selected -g or --branch-history but no "
230 "callchain data. Did\n" 230 "callchain data. Did\n"
231 "you call 'perf record' without -g?\n"); 231 "you call 'perf record' without -g?\n");
232 return -1; 232 return -1;
233 } 233 }
234 } else if (!rep->dont_use_callchains && 234 } else if (!rep->dont_use_callchains &&
235 callchain_param.mode != CHAIN_NONE && 235 callchain_param.mode != CHAIN_NONE &&
236 !symbol_conf.use_callchain) { 236 !symbol_conf.use_callchain) {
237 symbol_conf.use_callchain = true; 237 symbol_conf.use_callchain = true;
238 if (callchain_register_param(&callchain_param) < 0) { 238 if (callchain_register_param(&callchain_param) < 0) {
239 ui__error("Can't register callchain params.\n"); 239 ui__error("Can't register callchain params.\n");
240 return -EINVAL; 240 return -EINVAL;
241 } 241 }
242 } 242 }
243 243
244 if (symbol_conf.cumulate_callchain) { 244 if (symbol_conf.cumulate_callchain) {
245 /* Silently ignore if callchain is missing */ 245 /* Silently ignore if callchain is missing */
246 if (!(sample_type & PERF_SAMPLE_CALLCHAIN)) { 246 if (!(sample_type & PERF_SAMPLE_CALLCHAIN)) {
247 symbol_conf.cumulate_callchain = false; 247 symbol_conf.cumulate_callchain = false;
248 perf_hpp__cancel_cumulate(); 248 perf_hpp__cancel_cumulate();
249 } 249 }
250 } 250 }
251 251
252 if (sort__mode == SORT_MODE__BRANCH) { 252 if (sort__mode == SORT_MODE__BRANCH) {
253 if (!is_pipe && 253 if (!is_pipe &&
254 !(sample_type & PERF_SAMPLE_BRANCH_STACK)) { 254 !(sample_type & PERF_SAMPLE_BRANCH_STACK)) {
255 ui__error("Selected -b but no branch data. " 255 ui__error("Selected -b but no branch data. "
256 "Did you call perf record without -b?\n"); 256 "Did you call perf record without -b?\n");
257 return -1; 257 return -1;
258 } 258 }
259 } 259 }
260 260
261 if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain) { 261 if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain) {
262 if ((sample_type & PERF_SAMPLE_REGS_USER) && 262 if ((sample_type & PERF_SAMPLE_REGS_USER) &&
263 (sample_type & PERF_SAMPLE_STACK_USER)) 263 (sample_type & PERF_SAMPLE_STACK_USER))
264 callchain_param.record_mode = CALLCHAIN_DWARF; 264 callchain_param.record_mode = CALLCHAIN_DWARF;
265 else 265 else
266 callchain_param.record_mode = CALLCHAIN_FP; 266 callchain_param.record_mode = CALLCHAIN_FP;
267 } 267 }
268 return 0; 268 return 0;
269 } 269 }
270 270
271 static void sig_handler(int sig __maybe_unused) 271 static void sig_handler(int sig __maybe_unused)
272 { 272 {
273 session_done = 1; 273 session_done = 1;
274 } 274 }
275 275
276 static size_t hists__fprintf_nr_sample_events(struct hists *hists, struct report *rep, 276 static size_t hists__fprintf_nr_sample_events(struct hists *hists, struct report *rep,
277 const char *evname, FILE *fp) 277 const char *evname, FILE *fp)
278 { 278 {
279 size_t ret; 279 size_t ret;
280 char unit; 280 char unit;
281 unsigned long nr_samples = hists->stats.nr_events[PERF_RECORD_SAMPLE]; 281 unsigned long nr_samples = hists->stats.nr_events[PERF_RECORD_SAMPLE];
282 u64 nr_events = hists->stats.total_period; 282 u64 nr_events = hists->stats.total_period;
283 struct perf_evsel *evsel = hists_to_evsel(hists); 283 struct perf_evsel *evsel = hists_to_evsel(hists);
284 char buf[512]; 284 char buf[512];
285 size_t size = sizeof(buf); 285 size_t size = sizeof(buf);
286 286
287 if (symbol_conf.filter_relative) { 287 if (symbol_conf.filter_relative) {
288 nr_samples = hists->stats.nr_non_filtered_samples; 288 nr_samples = hists->stats.nr_non_filtered_samples;
289 nr_events = hists->stats.total_non_filtered_period; 289 nr_events = hists->stats.total_non_filtered_period;
290 } 290 }
291 291
292 if (perf_evsel__is_group_event(evsel)) { 292 if (perf_evsel__is_group_event(evsel)) {
293 struct perf_evsel *pos; 293 struct perf_evsel *pos;
294 294
295 perf_evsel__group_desc(evsel, buf, size); 295 perf_evsel__group_desc(evsel, buf, size);
296 evname = buf; 296 evname = buf;
297 297
298 for_each_group_member(pos, evsel) { 298 for_each_group_member(pos, evsel) {
299 const struct hists *pos_hists = evsel__hists(pos); 299 const struct hists *pos_hists = evsel__hists(pos);
300 300
301 if (symbol_conf.filter_relative) { 301 if (symbol_conf.filter_relative) {
302 nr_samples += pos_hists->stats.nr_non_filtered_samples; 302 nr_samples += pos_hists->stats.nr_non_filtered_samples;
303 nr_events += pos_hists->stats.total_non_filtered_period; 303 nr_events += pos_hists->stats.total_non_filtered_period;
304 } else { 304 } else {
305 nr_samples += pos_hists->stats.nr_events[PERF_RECORD_SAMPLE]; 305 nr_samples += pos_hists->stats.nr_events[PERF_RECORD_SAMPLE];
306 nr_events += pos_hists->stats.total_period; 306 nr_events += pos_hists->stats.total_period;
307 } 307 }
308 } 308 }
309 } 309 }
310 310
311 nr_samples = convert_unit(nr_samples, &unit); 311 nr_samples = convert_unit(nr_samples, &unit);
312 ret = fprintf(fp, "# Samples: %lu%c", nr_samples, unit); 312 ret = fprintf(fp, "# Samples: %lu%c", nr_samples, unit);
313 if (evname != NULL) 313 if (evname != NULL)
314 ret += fprintf(fp, " of event '%s'", evname); 314 ret += fprintf(fp, " of event '%s'", evname);
315 315
316 if (rep->mem_mode) { 316 if (rep->mem_mode) {
317 ret += fprintf(fp, "\n# Total weight : %" PRIu64, nr_events); 317 ret += fprintf(fp, "\n# Total weight : %" PRIu64, nr_events);
318 ret += fprintf(fp, "\n# Sort order : %s", sort_order); 318 ret += fprintf(fp, "\n# Sort order : %s", sort_order);
319 } else 319 } else
320 ret += fprintf(fp, "\n# Event count (approx.): %" PRIu64, nr_events); 320 ret += fprintf(fp, "\n# Event count (approx.): %" PRIu64, nr_events);
321 return ret + fprintf(fp, "\n#\n"); 321 return ret + fprintf(fp, "\n#\n");
322 } 322 }
323 323
324 static int perf_evlist__tty_browse_hists(struct perf_evlist *evlist, 324 static int perf_evlist__tty_browse_hists(struct perf_evlist *evlist,
325 struct report *rep, 325 struct report *rep,
326 const char *help) 326 const char *help)
327 { 327 {
328 struct perf_evsel *pos; 328 struct perf_evsel *pos;
329 329
330 evlist__for_each(evlist, pos) { 330 evlist__for_each(evlist, pos) {
331 struct hists *hists = evsel__hists(pos); 331 struct hists *hists = evsel__hists(pos);
332 const char *evname = perf_evsel__name(pos); 332 const char *evname = perf_evsel__name(pos);
333 333
334 if (symbol_conf.event_group && 334 if (symbol_conf.event_group &&
335 !perf_evsel__is_group_leader(pos)) 335 !perf_evsel__is_group_leader(pos))
336 continue; 336 continue;
337 337
338 hists__fprintf_nr_sample_events(hists, rep, evname, stdout); 338 hists__fprintf_nr_sample_events(hists, rep, evname, stdout);
339 hists__fprintf(hists, true, 0, 0, rep->min_percent, stdout); 339 hists__fprintf(hists, true, 0, 0, rep->min_percent, stdout);
340 fprintf(stdout, "\n\n"); 340 fprintf(stdout, "\n\n");
341 } 341 }
342 342
343 if (sort_order == default_sort_order && 343 if (sort_order == default_sort_order &&
344 parent_pattern == default_parent_pattern) { 344 parent_pattern == default_parent_pattern) {
345 fprintf(stdout, "#\n# (%s)\n#\n", help); 345 fprintf(stdout, "#\n# (%s)\n#\n", help);
346 346
347 if (rep->show_threads) { 347 if (rep->show_threads) {
348 bool style = !strcmp(rep->pretty_printing_style, "raw"); 348 bool style = !strcmp(rep->pretty_printing_style, "raw");
349 perf_read_values_display(stdout, &rep->show_threads_values, 349 perf_read_values_display(stdout, &rep->show_threads_values,
350 style); 350 style);
351 perf_read_values_destroy(&rep->show_threads_values); 351 perf_read_values_destroy(&rep->show_threads_values);
352 } 352 }
353 } 353 }
354 354
355 return 0; 355 return 0;
356 } 356 }
357 357
358 static void report__warn_kptr_restrict(const struct report *rep) 358 static void report__warn_kptr_restrict(const struct report *rep)
359 { 359 {
360 struct map *kernel_map = rep->session->machines.host.vmlinux_maps[MAP__FUNCTION]; 360 struct map *kernel_map = rep->session->machines.host.vmlinux_maps[MAP__FUNCTION];
361 struct kmap *kernel_kmap = map__kmap(kernel_map); 361 struct kmap *kernel_kmap = map__kmap(kernel_map);
362 362
363 if (kernel_map == NULL || 363 if (kernel_map == NULL ||
364 (kernel_map->dso->hit && 364 (kernel_map->dso->hit &&
365 (kernel_kmap->ref_reloc_sym == NULL || 365 (kernel_kmap->ref_reloc_sym == NULL ||
366 kernel_kmap->ref_reloc_sym->addr == 0))) { 366 kernel_kmap->ref_reloc_sym->addr == 0))) {
367 const char *desc = 367 const char *desc =
368 "As no suitable kallsyms nor vmlinux was found, kernel samples\n" 368 "As no suitable kallsyms nor vmlinux was found, kernel samples\n"
369 "can't be resolved."; 369 "can't be resolved.";
370 370
371 if (kernel_map) { 371 if (kernel_map) {
372 const struct dso *kdso = kernel_map->dso; 372 const struct dso *kdso = kernel_map->dso;
373 if (!RB_EMPTY_ROOT(&kdso->symbols[MAP__FUNCTION])) { 373 if (!RB_EMPTY_ROOT(&kdso->symbols[MAP__FUNCTION])) {
374 desc = "If some relocation was applied (e.g. " 374 desc = "If some relocation was applied (e.g. "
375 "kexec) symbols may be misresolved."; 375 "kexec) symbols may be misresolved.";
376 } 376 }
377 } 377 }
378 378
379 ui__warning( 379 ui__warning(
380 "Kernel address maps (/proc/{kallsyms,modules}) were restricted.\n\n" 380 "Kernel address maps (/proc/{kallsyms,modules}) were restricted.\n\n"
381 "Check /proc/sys/kernel/kptr_restrict before running 'perf record'.\n\n%s\n\n" 381 "Check /proc/sys/kernel/kptr_restrict before running 'perf record'.\n\n%s\n\n"
382 "Samples in kernel modules can't be resolved as well.\n\n", 382 "Samples in kernel modules can't be resolved as well.\n\n",
383 desc); 383 desc);
384 } 384 }
385 } 385 }
386 386
387 static int report__gtk_browse_hists(struct report *rep, const char *help) 387 static int report__gtk_browse_hists(struct report *rep, const char *help)
388 { 388 {
389 int (*hist_browser)(struct perf_evlist *evlist, const char *help, 389 int (*hist_browser)(struct perf_evlist *evlist, const char *help,
390 struct hist_browser_timer *timer, float min_pcnt); 390 struct hist_browser_timer *timer, float min_pcnt);
391 391
392 hist_browser = dlsym(perf_gtk_handle, "perf_evlist__gtk_browse_hists"); 392 hist_browser = dlsym(perf_gtk_handle, "perf_evlist__gtk_browse_hists");
393 393
394 if (hist_browser == NULL) { 394 if (hist_browser == NULL) {
395 ui__error("GTK browser not found!\n"); 395 ui__error("GTK browser not found!\n");
396 return -1; 396 return -1;
397 } 397 }
398 398
399 return hist_browser(rep->session->evlist, help, NULL, rep->min_percent); 399 return hist_browser(rep->session->evlist, help, NULL, rep->min_percent);
400 } 400 }
401 401
402 static int report__browse_hists(struct report *rep) 402 static int report__browse_hists(struct report *rep)
403 { 403 {
404 int ret; 404 int ret;
405 struct perf_session *session = rep->session; 405 struct perf_session *session = rep->session;
406 struct perf_evlist *evlist = session->evlist; 406 struct perf_evlist *evlist = session->evlist;
407 const char *help = "For a higher level overview, try: perf report --sort comm,dso"; 407 const char *help = "For a higher level overview, try: perf report --sort comm,dso";
408 408
409 switch (use_browser) { 409 switch (use_browser) {
410 case 1: 410 case 1:
411 ret = perf_evlist__tui_browse_hists(evlist, help, NULL, 411 ret = perf_evlist__tui_browse_hists(evlist, help, NULL,
412 rep->min_percent, 412 rep->min_percent,
413 &session->header.env); 413 &session->header.env);
414 /* 414 /*
415 * Usually "ret" is the last pressed key, and we only 415 * Usually "ret" is the last pressed key, and we only
416 * care if the key notifies us to switch data file. 416 * care if the key notifies us to switch data file.
417 */ 417 */
418 if (ret != K_SWITCH_INPUT_DATA) 418 if (ret != K_SWITCH_INPUT_DATA)
419 ret = 0; 419 ret = 0;
420 break; 420 break;
421 case 2: 421 case 2:
422 ret = report__gtk_browse_hists(rep, help); 422 ret = report__gtk_browse_hists(rep, help);
423 break; 423 break;
424 default: 424 default:
425 ret = perf_evlist__tty_browse_hists(evlist, rep, help); 425 ret = perf_evlist__tty_browse_hists(evlist, rep, help);
426 break; 426 break;
427 } 427 }
428 428
429 return ret; 429 return ret;
430 } 430 }
431 431
432 static void report__collapse_hists(struct report *rep) 432 static void report__collapse_hists(struct report *rep)
433 { 433 {
434 struct ui_progress prog; 434 struct ui_progress prog;
435 struct perf_evsel *pos; 435 struct perf_evsel *pos;
436 436
437 ui_progress__init(&prog, rep->nr_entries, "Merging related events..."); 437 ui_progress__init(&prog, rep->nr_entries, "Merging related events...");
438 438
439 evlist__for_each(rep->session->evlist, pos) { 439 evlist__for_each(rep->session->evlist, pos) {
440 struct hists *hists = evsel__hists(pos); 440 struct hists *hists = evsel__hists(pos);
441 441
442 if (pos->idx == 0) 442 if (pos->idx == 0)
443 hists->symbol_filter_str = rep->symbol_filter_str; 443 hists->symbol_filter_str = rep->symbol_filter_str;
444 444
445 hists__collapse_resort(hists, &prog); 445 hists__collapse_resort(hists, &prog);
446 446
447 /* Non-group events are considered as leader */ 447 /* Non-group events are considered as leader */
448 if (symbol_conf.event_group && 448 if (symbol_conf.event_group &&
449 !perf_evsel__is_group_leader(pos)) { 449 !perf_evsel__is_group_leader(pos)) {
450 struct hists *leader_hists = evsel__hists(pos->leader); 450 struct hists *leader_hists = evsel__hists(pos->leader);
451 451
452 hists__match(leader_hists, hists); 452 hists__match(leader_hists, hists);
453 hists__link(leader_hists, hists); 453 hists__link(leader_hists, hists);
454 } 454 }
455 } 455 }
456 456
457 ui_progress__finish(); 457 ui_progress__finish();
458 } 458 }
459 459
460 static void report__output_resort(struct report *rep)
461 {
462 struct ui_progress prog;
463 struct perf_evsel *pos;
464
465 ui_progress__init(&prog, rep->nr_entries, "Sorting events for output...");
466
467 evlist__for_each(rep->session->evlist, pos)
468 hists__output_resort(evsel__hists(pos), &prog);
469
470 ui_progress__finish();
471 }
472
460 static int __cmd_report(struct report *rep) 473 static int __cmd_report(struct report *rep)
461 { 474 {
462 int ret; 475 int ret;
463 struct perf_session *session = rep->session; 476 struct perf_session *session = rep->session;
464 struct perf_evsel *pos; 477 struct perf_evsel *pos;
465 struct perf_data_file *file = session->file; 478 struct perf_data_file *file = session->file;
466 479
467 signal(SIGINT, sig_handler); 480 signal(SIGINT, sig_handler);
468 481
469 if (rep->cpu_list) { 482 if (rep->cpu_list) {
470 ret = perf_session__cpu_bitmap(session, rep->cpu_list, 483 ret = perf_session__cpu_bitmap(session, rep->cpu_list,
471 rep->cpu_bitmap); 484 rep->cpu_bitmap);
472 if (ret) 485 if (ret)
473 return ret; 486 return ret;
474 } 487 }
475 488
476 if (rep->show_threads) 489 if (rep->show_threads)
477 perf_read_values_init(&rep->show_threads_values); 490 perf_read_values_init(&rep->show_threads_values);
478 491
479 ret = report__setup_sample_type(rep); 492 ret = report__setup_sample_type(rep);
480 if (ret) 493 if (ret)
481 return ret; 494 return ret;
482 495
483 ret = perf_session__process_events(session, &rep->tool); 496 ret = perf_session__process_events(session, &rep->tool);
484 if (ret) 497 if (ret)
485 return ret; 498 return ret;
486 499
487 report__warn_kptr_restrict(rep); 500 report__warn_kptr_restrict(rep);
488 501
489 if (use_browser == 0) { 502 if (use_browser == 0) {
490 if (verbose > 3) 503 if (verbose > 3)
491 perf_session__fprintf(session, stdout); 504 perf_session__fprintf(session, stdout);
492 505
493 if (verbose > 2) 506 if (verbose > 2)
494 perf_session__fprintf_dsos(session, stdout); 507 perf_session__fprintf_dsos(session, stdout);
495 508
496 if (dump_trace) { 509 if (dump_trace) {
497 perf_session__fprintf_nr_events(session, stdout); 510 perf_session__fprintf_nr_events(session, stdout);
498 perf_evlist__fprintf_nr_events(session->evlist, stdout); 511 perf_evlist__fprintf_nr_events(session->evlist, stdout);
499 return 0; 512 return 0;
500 } 513 }
501 } 514 }
502 515
503 report__collapse_hists(rep); 516 report__collapse_hists(rep);
504 517
505 if (session_done()) 518 if (session_done())
506 return 0; 519 return 0;
507 520
521 /*
522 * recalculate number of entries after collapsing since it
523 * might be changed during the collapse phase.
524 */
525 rep->nr_entries = 0;
526 evlist__for_each(session->evlist, pos)
527 rep->nr_entries += evsel__hists(pos)->nr_entries;
528
508 if (rep->nr_entries == 0) { 529 if (rep->nr_entries == 0) {
509 ui__error("The %s file has no samples!\n", file->path); 530 ui__error("The %s file has no samples!\n", file->path);
510 return 0; 531 return 0;
511 } 532 }
512 533
513 evlist__for_each(session->evlist, pos) 534 report__output_resort(rep);
514 hists__output_resort(evsel__hists(pos));
515 535
516 return report__browse_hists(rep); 536 return report__browse_hists(rep);
517 } 537 }
518 538
519 static int 539 static int
520 report_parse_callchain_opt(const struct option *opt, const char *arg, int unset) 540 report_parse_callchain_opt(const struct option *opt, const char *arg, int unset)
521 { 541 {
522 struct report *rep = (struct report *)opt->value; 542 struct report *rep = (struct report *)opt->value;
523 543
524 /* 544 /*
525 * --no-call-graph 545 * --no-call-graph
526 */ 546 */
527 if (unset) { 547 if (unset) {
528 rep->dont_use_callchains = true; 548 rep->dont_use_callchains = true;
529 return 0; 549 return 0;
530 } 550 }
531 551
532 return parse_callchain_report_opt(arg); 552 return parse_callchain_report_opt(arg);
533 } 553 }
534 554
535 int 555 int
536 report_parse_ignore_callees_opt(const struct option *opt __maybe_unused, 556 report_parse_ignore_callees_opt(const struct option *opt __maybe_unused,
537 const char *arg, int unset __maybe_unused) 557 const char *arg, int unset __maybe_unused)
538 { 558 {
539 if (arg) { 559 if (arg) {
540 int err = regcomp(&ignore_callees_regex, arg, REG_EXTENDED); 560 int err = regcomp(&ignore_callees_regex, arg, REG_EXTENDED);
541 if (err) { 561 if (err) {
542 char buf[BUFSIZ]; 562 char buf[BUFSIZ];
543 regerror(err, &ignore_callees_regex, buf, sizeof(buf)); 563 regerror(err, &ignore_callees_regex, buf, sizeof(buf));
544 pr_err("Invalid --ignore-callees regex: %s\n%s", arg, buf); 564 pr_err("Invalid --ignore-callees regex: %s\n%s", arg, buf);
545 return -1; 565 return -1;
546 } 566 }
547 have_ignore_callees = 1; 567 have_ignore_callees = 1;
548 } 568 }
549 569
550 return 0; 570 return 0;
551 } 571 }
552 572
553 static int 573 static int
554 parse_branch_mode(const struct option *opt __maybe_unused, 574 parse_branch_mode(const struct option *opt __maybe_unused,
555 const char *str __maybe_unused, int unset) 575 const char *str __maybe_unused, int unset)
556 { 576 {
557 int *branch_mode = opt->value; 577 int *branch_mode = opt->value;
558 578
559 *branch_mode = !unset; 579 *branch_mode = !unset;
560 return 0; 580 return 0;
561 } 581 }
562 582
563 static int 583 static int
564 parse_percent_limit(const struct option *opt, const char *str, 584 parse_percent_limit(const struct option *opt, const char *str,
565 int unset __maybe_unused) 585 int unset __maybe_unused)
566 { 586 {
567 struct report *rep = opt->value; 587 struct report *rep = opt->value;
568 588
569 rep->min_percent = strtof(str, NULL); 589 rep->min_percent = strtof(str, NULL);
570 return 0; 590 return 0;
571 } 591 }
572 592
573 int cmd_report(int argc, const char **argv, const char *prefix __maybe_unused) 593 int cmd_report(int argc, const char **argv, const char *prefix __maybe_unused)
574 { 594 {
575 struct perf_session *session; 595 struct perf_session *session;
576 struct stat st; 596 struct stat st;
577 bool has_br_stack = false; 597 bool has_br_stack = false;
578 int branch_mode = -1; 598 int branch_mode = -1;
579 bool branch_call_mode = false; 599 bool branch_call_mode = false;
580 char callchain_default_opt[] = "fractal,0.5,callee"; 600 char callchain_default_opt[] = "fractal,0.5,callee";
581 const char * const report_usage[] = { 601 const char * const report_usage[] = {
582 "perf report [<options>]", 602 "perf report [<options>]",
583 NULL 603 NULL
584 }; 604 };
585 struct report report = { 605 struct report report = {
586 .tool = { 606 .tool = {
587 .sample = process_sample_event, 607 .sample = process_sample_event,
588 .mmap = perf_event__process_mmap, 608 .mmap = perf_event__process_mmap,
589 .mmap2 = perf_event__process_mmap2, 609 .mmap2 = perf_event__process_mmap2,
590 .comm = perf_event__process_comm, 610 .comm = perf_event__process_comm,
591 .exit = perf_event__process_exit, 611 .exit = perf_event__process_exit,
592 .fork = perf_event__process_fork, 612 .fork = perf_event__process_fork,
593 .lost = perf_event__process_lost, 613 .lost = perf_event__process_lost,
594 .read = process_read_event, 614 .read = process_read_event,
595 .attr = perf_event__process_attr, 615 .attr = perf_event__process_attr,
596 .tracing_data = perf_event__process_tracing_data, 616 .tracing_data = perf_event__process_tracing_data,
597 .build_id = perf_event__process_build_id, 617 .build_id = perf_event__process_build_id,
598 .ordered_events = true, 618 .ordered_events = true,
599 .ordering_requires_timestamps = true, 619 .ordering_requires_timestamps = true,
600 }, 620 },
601 .max_stack = PERF_MAX_STACK_DEPTH, 621 .max_stack = PERF_MAX_STACK_DEPTH,
602 .pretty_printing_style = "normal", 622 .pretty_printing_style = "normal",
603 }; 623 };
604 const struct option options[] = { 624 const struct option options[] = {
605 OPT_STRING('i', "input", &input_name, "file", 625 OPT_STRING('i', "input", &input_name, "file",
606 "input file name"), 626 "input file name"),
607 OPT_INCR('v', "verbose", &verbose, 627 OPT_INCR('v', "verbose", &verbose,
608 "be more verbose (show symbol address, etc)"), 628 "be more verbose (show symbol address, etc)"),
609 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace, 629 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
610 "dump raw trace in ASCII"), 630 "dump raw trace in ASCII"),
611 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name, 631 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
612 "file", "vmlinux pathname"), 632 "file", "vmlinux pathname"),
613 OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, 633 OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name,
614 "file", "kallsyms pathname"), 634 "file", "kallsyms pathname"),
615 OPT_BOOLEAN('f', "force", &report.force, "don't complain, do it"), 635 OPT_BOOLEAN('f', "force", &report.force, "don't complain, do it"),
616 OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules, 636 OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules,
617 "load module symbols - WARNING: use only with -k and LIVE kernel"), 637 "load module symbols - WARNING: use only with -k and LIVE kernel"),
618 OPT_BOOLEAN('n', "show-nr-samples", &symbol_conf.show_nr_samples, 638 OPT_BOOLEAN('n', "show-nr-samples", &symbol_conf.show_nr_samples,
619 "Show a column with the number of samples"), 639 "Show a column with the number of samples"),
620 OPT_BOOLEAN('T', "threads", &report.show_threads, 640 OPT_BOOLEAN('T', "threads", &report.show_threads,
621 "Show per-thread event counters"), 641 "Show per-thread event counters"),
622 OPT_STRING(0, "pretty", &report.pretty_printing_style, "key", 642 OPT_STRING(0, "pretty", &report.pretty_printing_style, "key",
623 "pretty printing style key: normal raw"), 643 "pretty printing style key: normal raw"),
624 OPT_BOOLEAN(0, "tui", &report.use_tui, "Use the TUI interface"), 644 OPT_BOOLEAN(0, "tui", &report.use_tui, "Use the TUI interface"),
625 OPT_BOOLEAN(0, "gtk", &report.use_gtk, "Use the GTK2 interface"), 645 OPT_BOOLEAN(0, "gtk", &report.use_gtk, "Use the GTK2 interface"),
626 OPT_BOOLEAN(0, "stdio", &report.use_stdio, 646 OPT_BOOLEAN(0, "stdio", &report.use_stdio,
627 "Use the stdio interface"), 647 "Use the stdio interface"),
628 OPT_BOOLEAN(0, "header", &report.header, "Show data header."), 648 OPT_BOOLEAN(0, "header", &report.header, "Show data header."),
629 OPT_BOOLEAN(0, "header-only", &report.header_only, 649 OPT_BOOLEAN(0, "header-only", &report.header_only,
630 "Show only data header."), 650 "Show only data header."),
631 OPT_STRING('s', "sort", &sort_order, "key[,key2...]", 651 OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
632 "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..." 652 "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..."
633 " Please refer the man page for the complete list."), 653 " Please refer the man page for the complete list."),
634 OPT_STRING('F', "fields", &field_order, "key[,keys...]", 654 OPT_STRING('F', "fields", &field_order, "key[,keys...]",
635 "output field(s): overhead, period, sample plus all of sort keys"), 655 "output field(s): overhead, period, sample plus all of sort keys"),
636 OPT_BOOLEAN(0, "showcpuutilization", &symbol_conf.show_cpu_utilization, 656 OPT_BOOLEAN(0, "showcpuutilization", &symbol_conf.show_cpu_utilization,
637 "Show sample percentage for different cpu modes"), 657 "Show sample percentage for different cpu modes"),
638 OPT_STRING('p', "parent", &parent_pattern, "regex", 658 OPT_STRING('p', "parent", &parent_pattern, "regex",
639 "regex filter to identify parent, see: '--sort parent'"), 659 "regex filter to identify parent, see: '--sort parent'"),
640 OPT_BOOLEAN('x', "exclude-other", &symbol_conf.exclude_other, 660 OPT_BOOLEAN('x', "exclude-other", &symbol_conf.exclude_other,
641 "Only display entries with parent-match"), 661 "Only display entries with parent-match"),
642 OPT_CALLBACK_DEFAULT('g', "call-graph", &report, "output_type,min_percent[,print_limit],call_order[,branch]", 662 OPT_CALLBACK_DEFAULT('g', "call-graph", &report, "output_type,min_percent[,print_limit],call_order[,branch]",
643 "Display callchains using output_type (graph, flat, fractal, or none) , min percent threshold, optional print limit, callchain order, key (function or address), add branches. " 663 "Display callchains using output_type (graph, flat, fractal, or none) , min percent threshold, optional print limit, callchain order, key (function or address), add branches. "
644 "Default: fractal,0.5,callee,function", &report_parse_callchain_opt, callchain_default_opt), 664 "Default: fractal,0.5,callee,function", &report_parse_callchain_opt, callchain_default_opt),
645 OPT_BOOLEAN(0, "children", &symbol_conf.cumulate_callchain, 665 OPT_BOOLEAN(0, "children", &symbol_conf.cumulate_callchain,
646 "Accumulate callchains of children and show total overhead as well"), 666 "Accumulate callchains of children and show total overhead as well"),
647 OPT_INTEGER(0, "max-stack", &report.max_stack, 667 OPT_INTEGER(0, "max-stack", &report.max_stack,
648 "Set the maximum stack depth when parsing the callchain, " 668 "Set the maximum stack depth when parsing the callchain, "
649 "anything beyond the specified depth will be ignored. " 669 "anything beyond the specified depth will be ignored. "
650 "Default: " __stringify(PERF_MAX_STACK_DEPTH)), 670 "Default: " __stringify(PERF_MAX_STACK_DEPTH)),
651 OPT_BOOLEAN('G', "inverted", &report.inverted_callchain, 671 OPT_BOOLEAN('G', "inverted", &report.inverted_callchain,
652 "alias for inverted call graph"), 672 "alias for inverted call graph"),
653 OPT_CALLBACK(0, "ignore-callees", NULL, "regex", 673 OPT_CALLBACK(0, "ignore-callees", NULL, "regex",
654 "ignore callees of these functions in call graphs", 674 "ignore callees of these functions in call graphs",
655 report_parse_ignore_callees_opt), 675 report_parse_ignore_callees_opt),
656 OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]", 676 OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
657 "only consider symbols in these dsos"), 677 "only consider symbols in these dsos"),
658 OPT_STRING('c', "comms", &symbol_conf.comm_list_str, "comm[,comm...]", 678 OPT_STRING('c', "comms", &symbol_conf.comm_list_str, "comm[,comm...]",
659 "only consider symbols in these comms"), 679 "only consider symbols in these comms"),
660 OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]", 680 OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]",
661 "only consider these symbols"), 681 "only consider these symbols"),
662 OPT_STRING(0, "symbol-filter", &report.symbol_filter_str, "filter", 682 OPT_STRING(0, "symbol-filter", &report.symbol_filter_str, "filter",
663 "only show symbols that (partially) match with this filter"), 683 "only show symbols that (partially) match with this filter"),
664 OPT_STRING('w', "column-widths", &symbol_conf.col_width_list_str, 684 OPT_STRING('w', "column-widths", &symbol_conf.col_width_list_str,
665 "width[,width...]", 685 "width[,width...]",
666 "don't try to adjust column width, use these fixed values"), 686 "don't try to adjust column width, use these fixed values"),
667 OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator", 687 OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator",
668 "separator for columns, no spaces will be added between " 688 "separator for columns, no spaces will be added between "
669 "columns '.' is reserved."), 689 "columns '.' is reserved."),
670 OPT_BOOLEAN('U', "hide-unresolved", &report.hide_unresolved, 690 OPT_BOOLEAN('U', "hide-unresolved", &report.hide_unresolved,
671 "Only display entries resolved to a symbol"), 691 "Only display entries resolved to a symbol"),
672 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory", 692 OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
673 "Look for files with symbols relative to this directory"), 693 "Look for files with symbols relative to this directory"),
674 OPT_STRING('C', "cpu", &report.cpu_list, "cpu", 694 OPT_STRING('C', "cpu", &report.cpu_list, "cpu",
675 "list of cpus to profile"), 695 "list of cpus to profile"),
676 OPT_BOOLEAN('I', "show-info", &report.show_full_info, 696 OPT_BOOLEAN('I', "show-info", &report.show_full_info,
677 "Display extended information about perf.data file"), 697 "Display extended information about perf.data file"),
678 OPT_BOOLEAN(0, "source", &symbol_conf.annotate_src, 698 OPT_BOOLEAN(0, "source", &symbol_conf.annotate_src,
679 "Interleave source code with assembly code (default)"), 699 "Interleave source code with assembly code (default)"),
680 OPT_BOOLEAN(0, "asm-raw", &symbol_conf.annotate_asm_raw, 700 OPT_BOOLEAN(0, "asm-raw", &symbol_conf.annotate_asm_raw,
681 "Display raw encoding of assembly instructions (default)"), 701 "Display raw encoding of assembly instructions (default)"),
682 OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style", 702 OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style",
683 "Specify disassembler style (e.g. -M intel for intel syntax)"), 703 "Specify disassembler style (e.g. -M intel for intel syntax)"),
684 OPT_BOOLEAN(0, "show-total-period", &symbol_conf.show_total_period, 704 OPT_BOOLEAN(0, "show-total-period", &symbol_conf.show_total_period,
685 "Show a column with the sum of periods"), 705 "Show a column with the sum of periods"),
686 OPT_BOOLEAN(0, "group", &symbol_conf.event_group, 706 OPT_BOOLEAN(0, "group", &symbol_conf.event_group,
687 "Show event group information together"), 707 "Show event group information together"),
688 OPT_CALLBACK_NOOPT('b', "branch-stack", &branch_mode, "", 708 OPT_CALLBACK_NOOPT('b', "branch-stack", &branch_mode, "",
689 "use branch records for per branch histogram filling", 709 "use branch records for per branch histogram filling",
690 parse_branch_mode), 710 parse_branch_mode),
691 OPT_BOOLEAN(0, "branch-history", &branch_call_mode, 711 OPT_BOOLEAN(0, "branch-history", &branch_call_mode,
692 "add last branch records to call history"), 712 "add last branch records to call history"),
693 OPT_STRING(0, "objdump", &objdump_path, "path", 713 OPT_STRING(0, "objdump", &objdump_path, "path",
694 "objdump binary to use for disassembly and annotations"), 714 "objdump binary to use for disassembly and annotations"),
695 OPT_BOOLEAN(0, "demangle", &symbol_conf.demangle, 715 OPT_BOOLEAN(0, "demangle", &symbol_conf.demangle,
696 "Disable symbol demangling"), 716 "Disable symbol demangling"),
697 OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel, 717 OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel,
698 "Enable kernel symbol demangling"), 718 "Enable kernel symbol demangling"),
699 OPT_BOOLEAN(0, "mem-mode", &report.mem_mode, "mem access profile"), 719 OPT_BOOLEAN(0, "mem-mode", &report.mem_mode, "mem access profile"),
700 OPT_CALLBACK(0, "percent-limit", &report, "percent", 720 OPT_CALLBACK(0, "percent-limit", &report, "percent",
701 "Don't show entries under that percent", parse_percent_limit), 721 "Don't show entries under that percent", parse_percent_limit),
702 OPT_CALLBACK(0, "percentage", NULL, "relative|absolute", 722 OPT_CALLBACK(0, "percentage", NULL, "relative|absolute",
703 "how to display percentage of filtered entries", parse_filter_percentage), 723 "how to display percentage of filtered entries", parse_filter_percentage),
704 OPT_END() 724 OPT_END()
705 }; 725 };
706 struct perf_data_file file = { 726 struct perf_data_file file = {
707 .mode = PERF_DATA_MODE_READ, 727 .mode = PERF_DATA_MODE_READ,
708 }; 728 };
709 int ret = hists__init(); 729 int ret = hists__init();
710 730
711 if (ret < 0) 731 if (ret < 0)
712 return ret; 732 return ret;
713 733
714 perf_config(report__config, &report); 734 perf_config(report__config, &report);
715 735
716 argc = parse_options(argc, argv, options, report_usage, 0); 736 argc = parse_options(argc, argv, options, report_usage, 0);
717 737
718 if (report.use_stdio) 738 if (report.use_stdio)
719 use_browser = 0; 739 use_browser = 0;
720 else if (report.use_tui) 740 else if (report.use_tui)
721 use_browser = 1; 741 use_browser = 1;
722 else if (report.use_gtk) 742 else if (report.use_gtk)
723 use_browser = 2; 743 use_browser = 2;
724 744
725 if (report.inverted_callchain) 745 if (report.inverted_callchain)
726 callchain_param.order = ORDER_CALLER; 746 callchain_param.order = ORDER_CALLER;
727 747
728 if (!input_name || !strlen(input_name)) { 748 if (!input_name || !strlen(input_name)) {
729 if (!fstat(STDIN_FILENO, &st) && S_ISFIFO(st.st_mode)) 749 if (!fstat(STDIN_FILENO, &st) && S_ISFIFO(st.st_mode))
730 input_name = "-"; 750 input_name = "-";
731 else 751 else
732 input_name = "perf.data"; 752 input_name = "perf.data";
733 } 753 }
734 754
735 file.path = input_name; 755 file.path = input_name;
736 file.force = report.force; 756 file.force = report.force;
737 757
738 repeat: 758 repeat:
739 session = perf_session__new(&file, false, &report.tool); 759 session = perf_session__new(&file, false, &report.tool);
740 if (session == NULL) 760 if (session == NULL)
741 return -1; 761 return -1;
742 762
743 if (report.queue_size) { 763 if (report.queue_size) {
744 ordered_events__set_alloc_size(&session->ordered_events, 764 ordered_events__set_alloc_size(&session->ordered_events,
745 report.queue_size); 765 report.queue_size);
746 } 766 }
747 767
748 report.session = session; 768 report.session = session;
749 769
750 has_br_stack = perf_header__has_feat(&session->header, 770 has_br_stack = perf_header__has_feat(&session->header,
751 HEADER_BRANCH_STACK); 771 HEADER_BRANCH_STACK);
752 772
753 /* 773 /*
754 * Branch mode is a tristate: 774 * Branch mode is a tristate:
755 * -1 means default, so decide based on the file having branch data. 775 * -1 means default, so decide based on the file having branch data.
756 * 0/1 means the user chose a mode. 776 * 0/1 means the user chose a mode.
757 */ 777 */
758 if (((branch_mode == -1 && has_br_stack) || branch_mode == 1) && 778 if (((branch_mode == -1 && has_br_stack) || branch_mode == 1) &&
759 branch_call_mode == -1) { 779 branch_call_mode == -1) {
760 sort__mode = SORT_MODE__BRANCH; 780 sort__mode = SORT_MODE__BRANCH;
761 symbol_conf.cumulate_callchain = false; 781 symbol_conf.cumulate_callchain = false;
762 } 782 }
763 if (branch_call_mode) { 783 if (branch_call_mode) {
764 callchain_param.key = CCKEY_ADDRESS; 784 callchain_param.key = CCKEY_ADDRESS;
765 callchain_param.branch_callstack = 1; 785 callchain_param.branch_callstack = 1;
766 symbol_conf.use_callchain = true; 786 symbol_conf.use_callchain = true;
767 callchain_register_param(&callchain_param); 787 callchain_register_param(&callchain_param);
768 if (sort_order == NULL) 788 if (sort_order == NULL)
769 sort_order = "srcline,symbol,dso"; 789 sort_order = "srcline,symbol,dso";
770 } 790 }
771 791
772 if (report.mem_mode) { 792 if (report.mem_mode) {
773 if (sort__mode == SORT_MODE__BRANCH) { 793 if (sort__mode == SORT_MODE__BRANCH) {
774 pr_err("branch and mem mode incompatible\n"); 794 pr_err("branch and mem mode incompatible\n");
775 goto error; 795 goto error;
776 } 796 }
777 sort__mode = SORT_MODE__MEMORY; 797 sort__mode = SORT_MODE__MEMORY;
778 symbol_conf.cumulate_callchain = false; 798 symbol_conf.cumulate_callchain = false;
779 } 799 }
780 800
781 if (setup_sorting() < 0) { 801 if (setup_sorting() < 0) {
782 if (sort_order) 802 if (sort_order)
783 parse_options_usage(report_usage, options, "s", 1); 803 parse_options_usage(report_usage, options, "s", 1);
784 if (field_order) 804 if (field_order)
785 parse_options_usage(sort_order ? NULL : report_usage, 805 parse_options_usage(sort_order ? NULL : report_usage,
786 options, "F", 1); 806 options, "F", 1);
787 goto error; 807 goto error;
788 } 808 }
789 809
790 /* Force tty output for header output. */ 810 /* Force tty output for header output. */
791 if (report.header || report.header_only) 811 if (report.header || report.header_only)
792 use_browser = 0; 812 use_browser = 0;
793 813
794 if (strcmp(input_name, "-") != 0) 814 if (strcmp(input_name, "-") != 0)
795 setup_browser(true); 815 setup_browser(true);
796 else 816 else
797 use_browser = 0; 817 use_browser = 0;
798 818
799 if (report.header || report.header_only) { 819 if (report.header || report.header_only) {
800 perf_session__fprintf_info(session, stdout, 820 perf_session__fprintf_info(session, stdout,
801 report.show_full_info); 821 report.show_full_info);
802 if (report.header_only) 822 if (report.header_only)
803 return 0; 823 return 0;
804 } else if (use_browser == 0) { 824 } else if (use_browser == 0) {
805 fputs("# To display the perf.data header info, please use --header/--header-only options.\n#\n", 825 fputs("# To display the perf.data header info, please use --header/--header-only options.\n#\n",
806 stdout); 826 stdout);
807 } 827 }
808 828
809 /* 829 /*
810 * Only in the TUI browser we are doing integrated annotation, 830 * Only in the TUI browser we are doing integrated annotation,
811 * so don't allocate extra space that won't be used in the stdio 831 * so don't allocate extra space that won't be used in the stdio
812 * implementation. 832 * implementation.
813 */ 833 */
814 if (ui__has_annotation()) { 834 if (ui__has_annotation()) {
815 symbol_conf.priv_size = sizeof(struct annotation); 835 symbol_conf.priv_size = sizeof(struct annotation);
816 machines__set_symbol_filter(&session->machines, 836 machines__set_symbol_filter(&session->machines,
817 symbol__annotate_init); 837 symbol__annotate_init);
818 /* 838 /*
819 * For searching by name on the "Browse map details". 839 * For searching by name on the "Browse map details".
820 * providing it only in verbose mode not to bloat too 840 * providing it only in verbose mode not to bloat too
821 * much struct symbol. 841 * much struct symbol.
822 */ 842 */
823 if (verbose) { 843 if (verbose) {
824 /* 844 /*
825 * XXX: Need to provide a less kludgy way to ask for 845 * XXX: Need to provide a less kludgy way to ask for
826 * more space per symbol, the u32 is for the index on 846 * more space per symbol, the u32 is for the index on
827 * the ui browser. 847 * the ui browser.
828 * See symbol__browser_index. 848 * See symbol__browser_index.
829 */ 849 */
830 symbol_conf.priv_size += sizeof(u32); 850 symbol_conf.priv_size += sizeof(u32);
831 symbol_conf.sort_by_name = true; 851 symbol_conf.sort_by_name = true;
832 } 852 }
833 } 853 }
834 854
835 if (symbol__init(&session->header.env) < 0) 855 if (symbol__init(&session->header.env) < 0)
836 goto error; 856 goto error;
837 857
838 if (argc) { 858 if (argc) {
839 /* 859 /*
840 * Special case: if there's an argument left then assume that 860 * Special case: if there's an argument left then assume that
841 * it's a symbol filter: 861 * it's a symbol filter:
842 */ 862 */
843 if (argc > 1) 863 if (argc > 1)
844 usage_with_options(report_usage, options); 864 usage_with_options(report_usage, options);
845 865
846 report.symbol_filter_str = argv[0]; 866 report.symbol_filter_str = argv[0];
847 } 867 }
848 868
849 sort__setup_elide(stdout); 869 sort__setup_elide(stdout);
850 870
851 ret = __cmd_report(&report); 871 ret = __cmd_report(&report);
852 if (ret == K_SWITCH_INPUT_DATA) { 872 if (ret == K_SWITCH_INPUT_DATA) {
853 perf_session__delete(session); 873 perf_session__delete(session);
854 goto repeat; 874 goto repeat;
855 } else 875 } else
856 ret = 0; 876 ret = 0;
857 877
858 error: 878 error:
859 perf_session__delete(session); 879 perf_session__delete(session);
860 return ret; 880 return ret;
861 } 881 }
tools/perf/builtin-top.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * builtin-top.c 2 * builtin-top.c
3 * 3 *
4 * Builtin top command: Display a continuously updated profile of 4 * Builtin top command: Display a continuously updated profile of
5 * any workload, CPU or specific PID. 5 * any workload, CPU or specific PID.
6 * 6 *
7 * Copyright (C) 2008, Red Hat Inc, Ingo Molnar <mingo@redhat.com> 7 * Copyright (C) 2008, Red Hat Inc, Ingo Molnar <mingo@redhat.com>
8 * 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com> 8 * 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
9 * 9 *
10 * Improvements and fixes by: 10 * Improvements and fixes by:
11 * 11 *
12 * Arjan van de Ven <arjan@linux.intel.com> 12 * Arjan van de Ven <arjan@linux.intel.com>
13 * Yanmin Zhang <yanmin.zhang@intel.com> 13 * Yanmin Zhang <yanmin.zhang@intel.com>
14 * Wu Fengguang <fengguang.wu@intel.com> 14 * Wu Fengguang <fengguang.wu@intel.com>
15 * Mike Galbraith <efault@gmx.de> 15 * Mike Galbraith <efault@gmx.de>
16 * Paul Mackerras <paulus@samba.org> 16 * Paul Mackerras <paulus@samba.org>
17 * 17 *
18 * Released under the GPL v2. (and only v2, not any later version) 18 * Released under the GPL v2. (and only v2, not any later version)
19 */ 19 */
20 #include "builtin.h" 20 #include "builtin.h"
21 21
22 #include "perf.h" 22 #include "perf.h"
23 23
24 #include "util/annotate.h" 24 #include "util/annotate.h"
25 #include "util/cache.h" 25 #include "util/cache.h"
26 #include "util/color.h" 26 #include "util/color.h"
27 #include "util/evlist.h" 27 #include "util/evlist.h"
28 #include "util/evsel.h" 28 #include "util/evsel.h"
29 #include "util/machine.h" 29 #include "util/machine.h"
30 #include "util/session.h" 30 #include "util/session.h"
31 #include "util/symbol.h" 31 #include "util/symbol.h"
32 #include "util/thread.h" 32 #include "util/thread.h"
33 #include "util/thread_map.h" 33 #include "util/thread_map.h"
34 #include "util/top.h" 34 #include "util/top.h"
35 #include "util/util.h" 35 #include "util/util.h"
36 #include <linux/rbtree.h> 36 #include <linux/rbtree.h>
37 #include "util/parse-options.h" 37 #include "util/parse-options.h"
38 #include "util/parse-events.h" 38 #include "util/parse-events.h"
39 #include "util/cpumap.h" 39 #include "util/cpumap.h"
40 #include "util/xyarray.h" 40 #include "util/xyarray.h"
41 #include "util/sort.h" 41 #include "util/sort.h"
42 #include "util/intlist.h" 42 #include "util/intlist.h"
43 #include "arch/common.h" 43 #include "arch/common.h"
44 44
45 #include "util/debug.h" 45 #include "util/debug.h"
46 46
47 #include <assert.h> 47 #include <assert.h>
48 #include <elf.h> 48 #include <elf.h>
49 #include <fcntl.h> 49 #include <fcntl.h>
50 50
51 #include <stdio.h> 51 #include <stdio.h>
52 #include <termios.h> 52 #include <termios.h>
53 #include <unistd.h> 53 #include <unistd.h>
54 #include <inttypes.h> 54 #include <inttypes.h>
55 55
56 #include <errno.h> 56 #include <errno.h>
57 #include <time.h> 57 #include <time.h>
58 #include <sched.h> 58 #include <sched.h>
59 59
60 #include <sys/syscall.h> 60 #include <sys/syscall.h>
61 #include <sys/ioctl.h> 61 #include <sys/ioctl.h>
62 #include <poll.h> 62 #include <poll.h>
63 #include <sys/prctl.h> 63 #include <sys/prctl.h>
64 #include <sys/wait.h> 64 #include <sys/wait.h>
65 #include <sys/uio.h> 65 #include <sys/uio.h>
66 #include <sys/utsname.h> 66 #include <sys/utsname.h>
67 #include <sys/mman.h> 67 #include <sys/mman.h>
68 68
69 #include <linux/unistd.h> 69 #include <linux/unistd.h>
70 #include <linux/types.h> 70 #include <linux/types.h>
71 71
72 static volatile int done; 72 static volatile int done;
73 73
74 #define HEADER_LINE_NR 5 74 #define HEADER_LINE_NR 5
75 75
76 static void perf_top__update_print_entries(struct perf_top *top) 76 static void perf_top__update_print_entries(struct perf_top *top)
77 { 77 {
78 top->print_entries = top->winsize.ws_row - HEADER_LINE_NR; 78 top->print_entries = top->winsize.ws_row - HEADER_LINE_NR;
79 } 79 }
80 80
81 static void perf_top__sig_winch(int sig __maybe_unused, 81 static void perf_top__sig_winch(int sig __maybe_unused,
82 siginfo_t *info __maybe_unused, void *arg) 82 siginfo_t *info __maybe_unused, void *arg)
83 { 83 {
84 struct perf_top *top = arg; 84 struct perf_top *top = arg;
85 85
86 get_term_dimensions(&top->winsize); 86 get_term_dimensions(&top->winsize);
87 perf_top__update_print_entries(top); 87 perf_top__update_print_entries(top);
88 } 88 }
89 89
90 static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) 90 static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he)
91 { 91 {
92 struct symbol *sym; 92 struct symbol *sym;
93 struct annotation *notes; 93 struct annotation *notes;
94 struct map *map; 94 struct map *map;
95 int err = -1; 95 int err = -1;
96 96
97 if (!he || !he->ms.sym) 97 if (!he || !he->ms.sym)
98 return -1; 98 return -1;
99 99
100 sym = he->ms.sym; 100 sym = he->ms.sym;
101 map = he->ms.map; 101 map = he->ms.map;
102 102
103 /* 103 /*
104 * We can't annotate with just /proc/kallsyms 104 * We can't annotate with just /proc/kallsyms
105 */ 105 */
106 if (map->dso->symtab_type == DSO_BINARY_TYPE__KALLSYMS && 106 if (map->dso->symtab_type == DSO_BINARY_TYPE__KALLSYMS &&
107 !dso__is_kcore(map->dso)) { 107 !dso__is_kcore(map->dso)) {
108 pr_err("Can't annotate %s: No vmlinux file was found in the " 108 pr_err("Can't annotate %s: No vmlinux file was found in the "
109 "path\n", sym->name); 109 "path\n", sym->name);
110 sleep(1); 110 sleep(1);
111 return -1; 111 return -1;
112 } 112 }
113 113
114 notes = symbol__annotation(sym); 114 notes = symbol__annotation(sym);
115 if (notes->src != NULL) { 115 if (notes->src != NULL) {
116 pthread_mutex_lock(&notes->lock); 116 pthread_mutex_lock(&notes->lock);
117 goto out_assign; 117 goto out_assign;
118 } 118 }
119 119
120 pthread_mutex_lock(&notes->lock); 120 pthread_mutex_lock(&notes->lock);
121 121
122 if (symbol__alloc_hist(sym) < 0) { 122 if (symbol__alloc_hist(sym) < 0) {
123 pthread_mutex_unlock(&notes->lock); 123 pthread_mutex_unlock(&notes->lock);
124 pr_err("Not enough memory for annotating '%s' symbol!\n", 124 pr_err("Not enough memory for annotating '%s' symbol!\n",
125 sym->name); 125 sym->name);
126 sleep(1); 126 sleep(1);
127 return err; 127 return err;
128 } 128 }
129 129
130 err = symbol__annotate(sym, map, 0); 130 err = symbol__annotate(sym, map, 0);
131 if (err == 0) { 131 if (err == 0) {
132 out_assign: 132 out_assign:
133 top->sym_filter_entry = he; 133 top->sym_filter_entry = he;
134 } 134 }
135 135
136 pthread_mutex_unlock(&notes->lock); 136 pthread_mutex_unlock(&notes->lock);
137 return err; 137 return err;
138 } 138 }
139 139
140 static void __zero_source_counters(struct hist_entry *he) 140 static void __zero_source_counters(struct hist_entry *he)
141 { 141 {
142 struct symbol *sym = he->ms.sym; 142 struct symbol *sym = he->ms.sym;
143 symbol__annotate_zero_histograms(sym); 143 symbol__annotate_zero_histograms(sym);
144 } 144 }
145 145
146 static void ui__warn_map_erange(struct map *map, struct symbol *sym, u64 ip) 146 static void ui__warn_map_erange(struct map *map, struct symbol *sym, u64 ip)
147 { 147 {
148 struct utsname uts; 148 struct utsname uts;
149 int err = uname(&uts); 149 int err = uname(&uts);
150 150
151 ui__warning("Out of bounds address found:\n\n" 151 ui__warning("Out of bounds address found:\n\n"
152 "Addr: %" PRIx64 "\n" 152 "Addr: %" PRIx64 "\n"
153 "DSO: %s %c\n" 153 "DSO: %s %c\n"
154 "Map: %" PRIx64 "-%" PRIx64 "\n" 154 "Map: %" PRIx64 "-%" PRIx64 "\n"
155 "Symbol: %" PRIx64 "-%" PRIx64 " %c %s\n" 155 "Symbol: %" PRIx64 "-%" PRIx64 " %c %s\n"
156 "Arch: %s\n" 156 "Arch: %s\n"
157 "Kernel: %s\n" 157 "Kernel: %s\n"
158 "Tools: %s\n\n" 158 "Tools: %s\n\n"
159 "Not all samples will be on the annotation output.\n\n" 159 "Not all samples will be on the annotation output.\n\n"
160 "Please report to linux-kernel@vger.kernel.org\n", 160 "Please report to linux-kernel@vger.kernel.org\n",
161 ip, map->dso->long_name, dso__symtab_origin(map->dso), 161 ip, map->dso->long_name, dso__symtab_origin(map->dso),
162 map->start, map->end, sym->start, sym->end, 162 map->start, map->end, sym->start, sym->end,
163 sym->binding == STB_GLOBAL ? 'g' : 163 sym->binding == STB_GLOBAL ? 'g' :
164 sym->binding == STB_LOCAL ? 'l' : 'w', sym->name, 164 sym->binding == STB_LOCAL ? 'l' : 'w', sym->name,
165 err ? "[unknown]" : uts.machine, 165 err ? "[unknown]" : uts.machine,
166 err ? "[unknown]" : uts.release, perf_version_string); 166 err ? "[unknown]" : uts.release, perf_version_string);
167 if (use_browser <= 0) 167 if (use_browser <= 0)
168 sleep(5); 168 sleep(5);
169 169
170 map->erange_warned = true; 170 map->erange_warned = true;
171 } 171 }
172 172
173 static void perf_top__record_precise_ip(struct perf_top *top, 173 static void perf_top__record_precise_ip(struct perf_top *top,
174 struct hist_entry *he, 174 struct hist_entry *he,
175 int counter, u64 ip) 175 int counter, u64 ip)
176 { 176 {
177 struct annotation *notes; 177 struct annotation *notes;
178 struct symbol *sym; 178 struct symbol *sym;
179 int err = 0; 179 int err = 0;
180 180
181 if (he == NULL || he->ms.sym == NULL || 181 if (he == NULL || he->ms.sym == NULL ||
182 ((top->sym_filter_entry == NULL || 182 ((top->sym_filter_entry == NULL ||
183 top->sym_filter_entry->ms.sym != he->ms.sym) && use_browser != 1)) 183 top->sym_filter_entry->ms.sym != he->ms.sym) && use_browser != 1))
184 return; 184 return;
185 185
186 sym = he->ms.sym; 186 sym = he->ms.sym;
187 notes = symbol__annotation(sym); 187 notes = symbol__annotation(sym);
188 188
189 if (pthread_mutex_trylock(&notes->lock)) 189 if (pthread_mutex_trylock(&notes->lock))
190 return; 190 return;
191 191
192 ip = he->ms.map->map_ip(he->ms.map, ip); 192 ip = he->ms.map->map_ip(he->ms.map, ip);
193 193
194 if (ui__has_annotation()) 194 if (ui__has_annotation())
195 err = hist_entry__inc_addr_samples(he, counter, ip); 195 err = hist_entry__inc_addr_samples(he, counter, ip);
196 196
197 pthread_mutex_unlock(&notes->lock); 197 pthread_mutex_unlock(&notes->lock);
198 198
199 /* 199 /*
200 * This function is now called with he->hists->lock held. 200 * This function is now called with he->hists->lock held.
201 * Release it before going to sleep. 201 * Release it before going to sleep.
202 */ 202 */
203 pthread_mutex_unlock(&he->hists->lock); 203 pthread_mutex_unlock(&he->hists->lock);
204 204
205 if (err == -ERANGE && !he->ms.map->erange_warned) 205 if (err == -ERANGE && !he->ms.map->erange_warned)
206 ui__warn_map_erange(he->ms.map, sym, ip); 206 ui__warn_map_erange(he->ms.map, sym, ip);
207 else if (err == -ENOMEM) { 207 else if (err == -ENOMEM) {
208 pr_err("Not enough memory for annotating '%s' symbol!\n", 208 pr_err("Not enough memory for annotating '%s' symbol!\n",
209 sym->name); 209 sym->name);
210 sleep(1); 210 sleep(1);
211 } 211 }
212 212
213 pthread_mutex_lock(&he->hists->lock); 213 pthread_mutex_lock(&he->hists->lock);
214 } 214 }
215 215
216 static void perf_top__show_details(struct perf_top *top) 216 static void perf_top__show_details(struct perf_top *top)
217 { 217 {
218 struct hist_entry *he = top->sym_filter_entry; 218 struct hist_entry *he = top->sym_filter_entry;
219 struct annotation *notes; 219 struct annotation *notes;
220 struct symbol *symbol; 220 struct symbol *symbol;
221 int more; 221 int more;
222 222
223 if (!he) 223 if (!he)
224 return; 224 return;
225 225
226 symbol = he->ms.sym; 226 symbol = he->ms.sym;
227 notes = symbol__annotation(symbol); 227 notes = symbol__annotation(symbol);
228 228
229 pthread_mutex_lock(&notes->lock); 229 pthread_mutex_lock(&notes->lock);
230 230
231 if (notes->src == NULL) 231 if (notes->src == NULL)
232 goto out_unlock; 232 goto out_unlock;
233 233
234 printf("Showing %s for %s\n", perf_evsel__name(top->sym_evsel), symbol->name); 234 printf("Showing %s for %s\n", perf_evsel__name(top->sym_evsel), symbol->name);
235 printf(" Events Pcnt (>=%d%%)\n", top->sym_pcnt_filter); 235 printf(" Events Pcnt (>=%d%%)\n", top->sym_pcnt_filter);
236 236
237 more = symbol__annotate_printf(symbol, he->ms.map, top->sym_evsel, 237 more = symbol__annotate_printf(symbol, he->ms.map, top->sym_evsel,
238 0, top->sym_pcnt_filter, top->print_entries, 4); 238 0, top->sym_pcnt_filter, top->print_entries, 4);
239 if (top->zero) 239 if (top->zero)
240 symbol__annotate_zero_histogram(symbol, top->sym_evsel->idx); 240 symbol__annotate_zero_histogram(symbol, top->sym_evsel->idx);
241 else 241 else
242 symbol__annotate_decay_histogram(symbol, top->sym_evsel->idx); 242 symbol__annotate_decay_histogram(symbol, top->sym_evsel->idx);
243 if (more != 0) 243 if (more != 0)
244 printf("%d lines not displayed, maybe increase display entries [e]\n", more); 244 printf("%d lines not displayed, maybe increase display entries [e]\n", more);
245 out_unlock: 245 out_unlock:
246 pthread_mutex_unlock(&notes->lock); 246 pthread_mutex_unlock(&notes->lock);
247 } 247 }
248 248
249 static void perf_top__print_sym_table(struct perf_top *top) 249 static void perf_top__print_sym_table(struct perf_top *top)
250 { 250 {
251 char bf[160]; 251 char bf[160];
252 int printed = 0; 252 int printed = 0;
253 const int win_width = top->winsize.ws_col - 1; 253 const int win_width = top->winsize.ws_col - 1;
254 struct hists *hists = evsel__hists(top->sym_evsel); 254 struct hists *hists = evsel__hists(top->sym_evsel);
255 255
256 puts(CONSOLE_CLEAR); 256 puts(CONSOLE_CLEAR);
257 257
258 perf_top__header_snprintf(top, bf, sizeof(bf)); 258 perf_top__header_snprintf(top, bf, sizeof(bf));
259 printf("%s\n", bf); 259 printf("%s\n", bf);
260 260
261 perf_top__reset_sample_counters(top); 261 perf_top__reset_sample_counters(top);
262 262
263 printf("%-*.*s\n", win_width, win_width, graph_dotted_line); 263 printf("%-*.*s\n", win_width, win_width, graph_dotted_line);
264 264
265 if (hists->stats.nr_lost_warned != 265 if (hists->stats.nr_lost_warned !=
266 hists->stats.nr_events[PERF_RECORD_LOST]) { 266 hists->stats.nr_events[PERF_RECORD_LOST]) {
267 hists->stats.nr_lost_warned = 267 hists->stats.nr_lost_warned =
268 hists->stats.nr_events[PERF_RECORD_LOST]; 268 hists->stats.nr_events[PERF_RECORD_LOST];
269 color_fprintf(stdout, PERF_COLOR_RED, 269 color_fprintf(stdout, PERF_COLOR_RED,
270 "WARNING: LOST %d chunks, Check IO/CPU overload", 270 "WARNING: LOST %d chunks, Check IO/CPU overload",
271 hists->stats.nr_lost_warned); 271 hists->stats.nr_lost_warned);
272 ++printed; 272 ++printed;
273 } 273 }
274 274
275 if (top->sym_filter_entry) { 275 if (top->sym_filter_entry) {
276 perf_top__show_details(top); 276 perf_top__show_details(top);
277 return; 277 return;
278 } 278 }
279 279
280 if (top->zero) { 280 if (top->zero) {
281 hists__delete_entries(hists); 281 hists__delete_entries(hists);
282 } else { 282 } else {
283 hists__decay_entries(hists, top->hide_user_symbols, 283 hists__decay_entries(hists, top->hide_user_symbols,
284 top->hide_kernel_symbols); 284 top->hide_kernel_symbols);
285 } 285 }
286 286
287 hists__collapse_resort(hists, NULL); 287 hists__collapse_resort(hists, NULL);
288 hists__output_resort(hists); 288 hists__output_resort(hists, NULL);
289 289
290 hists__output_recalc_col_len(hists, top->print_entries - printed); 290 hists__output_recalc_col_len(hists, top->print_entries - printed);
291 putchar('\n'); 291 putchar('\n');
292 hists__fprintf(hists, false, top->print_entries - printed, win_width, 292 hists__fprintf(hists, false, top->print_entries - printed, win_width,
293 top->min_percent, stdout); 293 top->min_percent, stdout);
294 } 294 }
295 295
296 static void prompt_integer(int *target, const char *msg) 296 static void prompt_integer(int *target, const char *msg)
297 { 297 {
298 char *buf = malloc(0), *p; 298 char *buf = malloc(0), *p;
299 size_t dummy = 0; 299 size_t dummy = 0;
300 int tmp; 300 int tmp;
301 301
302 fprintf(stdout, "\n%s: ", msg); 302 fprintf(stdout, "\n%s: ", msg);
303 if (getline(&buf, &dummy, stdin) < 0) 303 if (getline(&buf, &dummy, stdin) < 0)
304 return; 304 return;
305 305
306 p = strchr(buf, '\n'); 306 p = strchr(buf, '\n');
307 if (p) 307 if (p)
308 *p = 0; 308 *p = 0;
309 309
310 p = buf; 310 p = buf;
311 while(*p) { 311 while(*p) {
312 if (!isdigit(*p)) 312 if (!isdigit(*p))
313 goto out_free; 313 goto out_free;
314 p++; 314 p++;
315 } 315 }
316 tmp = strtoul(buf, NULL, 10); 316 tmp = strtoul(buf, NULL, 10);
317 *target = tmp; 317 *target = tmp;
318 out_free: 318 out_free:
319 free(buf); 319 free(buf);
320 } 320 }
321 321
322 static void prompt_percent(int *target, const char *msg) 322 static void prompt_percent(int *target, const char *msg)
323 { 323 {
324 int tmp = 0; 324 int tmp = 0;
325 325
326 prompt_integer(&tmp, msg); 326 prompt_integer(&tmp, msg);
327 if (tmp >= 0 && tmp <= 100) 327 if (tmp >= 0 && tmp <= 100)
328 *target = tmp; 328 *target = tmp;
329 } 329 }
330 330
331 static void perf_top__prompt_symbol(struct perf_top *top, const char *msg) 331 static void perf_top__prompt_symbol(struct perf_top *top, const char *msg)
332 { 332 {
333 char *buf = malloc(0), *p; 333 char *buf = malloc(0), *p;
334 struct hist_entry *syme = top->sym_filter_entry, *n, *found = NULL; 334 struct hist_entry *syme = top->sym_filter_entry, *n, *found = NULL;
335 struct hists *hists = evsel__hists(top->sym_evsel); 335 struct hists *hists = evsel__hists(top->sym_evsel);
336 struct rb_node *next; 336 struct rb_node *next;
337 size_t dummy = 0; 337 size_t dummy = 0;
338 338
339 /* zero counters of active symbol */ 339 /* zero counters of active symbol */
340 if (syme) { 340 if (syme) {
341 __zero_source_counters(syme); 341 __zero_source_counters(syme);
342 top->sym_filter_entry = NULL; 342 top->sym_filter_entry = NULL;
343 } 343 }
344 344
345 fprintf(stdout, "\n%s: ", msg); 345 fprintf(stdout, "\n%s: ", msg);
346 if (getline(&buf, &dummy, stdin) < 0) 346 if (getline(&buf, &dummy, stdin) < 0)
347 goto out_free; 347 goto out_free;
348 348
349 p = strchr(buf, '\n'); 349 p = strchr(buf, '\n');
350 if (p) 350 if (p)
351 *p = 0; 351 *p = 0;
352 352
353 next = rb_first(&hists->entries); 353 next = rb_first(&hists->entries);
354 while (next) { 354 while (next) {
355 n = rb_entry(next, struct hist_entry, rb_node); 355 n = rb_entry(next, struct hist_entry, rb_node);
356 if (n->ms.sym && !strcmp(buf, n->ms.sym->name)) { 356 if (n->ms.sym && !strcmp(buf, n->ms.sym->name)) {
357 found = n; 357 found = n;
358 break; 358 break;
359 } 359 }
360 next = rb_next(&n->rb_node); 360 next = rb_next(&n->rb_node);
361 } 361 }
362 362
363 if (!found) { 363 if (!found) {
364 fprintf(stderr, "Sorry, %s is not active.\n", buf); 364 fprintf(stderr, "Sorry, %s is not active.\n", buf);
365 sleep(1); 365 sleep(1);
366 } else 366 } else
367 perf_top__parse_source(top, found); 367 perf_top__parse_source(top, found);
368 368
369 out_free: 369 out_free:
370 free(buf); 370 free(buf);
371 } 371 }
372 372
373 static void perf_top__print_mapped_keys(struct perf_top *top) 373 static void perf_top__print_mapped_keys(struct perf_top *top)
374 { 374 {
375 char *name = NULL; 375 char *name = NULL;
376 376
377 if (top->sym_filter_entry) { 377 if (top->sym_filter_entry) {
378 struct symbol *sym = top->sym_filter_entry->ms.sym; 378 struct symbol *sym = top->sym_filter_entry->ms.sym;
379 name = sym->name; 379 name = sym->name;
380 } 380 }
381 381
382 fprintf(stdout, "\nMapped keys:\n"); 382 fprintf(stdout, "\nMapped keys:\n");
383 fprintf(stdout, "\t[d] display refresh delay. \t(%d)\n", top->delay_secs); 383 fprintf(stdout, "\t[d] display refresh delay. \t(%d)\n", top->delay_secs);
384 fprintf(stdout, "\t[e] display entries (lines). \t(%d)\n", top->print_entries); 384 fprintf(stdout, "\t[e] display entries (lines). \t(%d)\n", top->print_entries);
385 385
386 if (top->evlist->nr_entries > 1) 386 if (top->evlist->nr_entries > 1)
387 fprintf(stdout, "\t[E] active event counter. \t(%s)\n", perf_evsel__name(top->sym_evsel)); 387 fprintf(stdout, "\t[E] active event counter. \t(%s)\n", perf_evsel__name(top->sym_evsel));
388 388
389 fprintf(stdout, "\t[f] profile display filter (count). \t(%d)\n", top->count_filter); 389 fprintf(stdout, "\t[f] profile display filter (count). \t(%d)\n", top->count_filter);
390 390
391 fprintf(stdout, "\t[F] annotate display filter (percent). \t(%d%%)\n", top->sym_pcnt_filter); 391 fprintf(stdout, "\t[F] annotate display filter (percent). \t(%d%%)\n", top->sym_pcnt_filter);
392 fprintf(stdout, "\t[s] annotate symbol. \t(%s)\n", name?: "NULL"); 392 fprintf(stdout, "\t[s] annotate symbol. \t(%s)\n", name?: "NULL");
393 fprintf(stdout, "\t[S] stop annotation.\n"); 393 fprintf(stdout, "\t[S] stop annotation.\n");
394 394
395 fprintf(stdout, 395 fprintf(stdout,
396 "\t[K] hide kernel_symbols symbols. \t(%s)\n", 396 "\t[K] hide kernel_symbols symbols. \t(%s)\n",
397 top->hide_kernel_symbols ? "yes" : "no"); 397 top->hide_kernel_symbols ? "yes" : "no");
398 fprintf(stdout, 398 fprintf(stdout,
399 "\t[U] hide user symbols. \t(%s)\n", 399 "\t[U] hide user symbols. \t(%s)\n",
400 top->hide_user_symbols ? "yes" : "no"); 400 top->hide_user_symbols ? "yes" : "no");
401 fprintf(stdout, "\t[z] toggle sample zeroing. \t(%d)\n", top->zero ? 1 : 0); 401 fprintf(stdout, "\t[z] toggle sample zeroing. \t(%d)\n", top->zero ? 1 : 0);
402 fprintf(stdout, "\t[qQ] quit.\n"); 402 fprintf(stdout, "\t[qQ] quit.\n");
403 } 403 }
404 404
405 static int perf_top__key_mapped(struct perf_top *top, int c) 405 static int perf_top__key_mapped(struct perf_top *top, int c)
406 { 406 {
407 switch (c) { 407 switch (c) {
408 case 'd': 408 case 'd':
409 case 'e': 409 case 'e':
410 case 'f': 410 case 'f':
411 case 'z': 411 case 'z':
412 case 'q': 412 case 'q':
413 case 'Q': 413 case 'Q':
414 case 'K': 414 case 'K':
415 case 'U': 415 case 'U':
416 case 'F': 416 case 'F':
417 case 's': 417 case 's':
418 case 'S': 418 case 'S':
419 return 1; 419 return 1;
420 case 'E': 420 case 'E':
421 return top->evlist->nr_entries > 1 ? 1 : 0; 421 return top->evlist->nr_entries > 1 ? 1 : 0;
422 default: 422 default:
423 break; 423 break;
424 } 424 }
425 425
426 return 0; 426 return 0;
427 } 427 }
428 428
429 static bool perf_top__handle_keypress(struct perf_top *top, int c) 429 static bool perf_top__handle_keypress(struct perf_top *top, int c)
430 { 430 {
431 bool ret = true; 431 bool ret = true;
432 432
433 if (!perf_top__key_mapped(top, c)) { 433 if (!perf_top__key_mapped(top, c)) {
434 struct pollfd stdin_poll = { .fd = 0, .events = POLLIN }; 434 struct pollfd stdin_poll = { .fd = 0, .events = POLLIN };
435 struct termios save; 435 struct termios save;
436 436
437 perf_top__print_mapped_keys(top); 437 perf_top__print_mapped_keys(top);
438 fprintf(stdout, "\nEnter selection, or unmapped key to continue: "); 438 fprintf(stdout, "\nEnter selection, or unmapped key to continue: ");
439 fflush(stdout); 439 fflush(stdout);
440 440
441 set_term_quiet_input(&save); 441 set_term_quiet_input(&save);
442 442
443 poll(&stdin_poll, 1, -1); 443 poll(&stdin_poll, 1, -1);
444 c = getc(stdin); 444 c = getc(stdin);
445 445
446 tcsetattr(0, TCSAFLUSH, &save); 446 tcsetattr(0, TCSAFLUSH, &save);
447 if (!perf_top__key_mapped(top, c)) 447 if (!perf_top__key_mapped(top, c))
448 return ret; 448 return ret;
449 } 449 }
450 450
451 switch (c) { 451 switch (c) {
452 case 'd': 452 case 'd':
453 prompt_integer(&top->delay_secs, "Enter display delay"); 453 prompt_integer(&top->delay_secs, "Enter display delay");
454 if (top->delay_secs < 1) 454 if (top->delay_secs < 1)
455 top->delay_secs = 1; 455 top->delay_secs = 1;
456 break; 456 break;
457 case 'e': 457 case 'e':
458 prompt_integer(&top->print_entries, "Enter display entries (lines)"); 458 prompt_integer(&top->print_entries, "Enter display entries (lines)");
459 if (top->print_entries == 0) { 459 if (top->print_entries == 0) {
460 struct sigaction act = { 460 struct sigaction act = {
461 .sa_sigaction = perf_top__sig_winch, 461 .sa_sigaction = perf_top__sig_winch,
462 .sa_flags = SA_SIGINFO, 462 .sa_flags = SA_SIGINFO,
463 }; 463 };
464 perf_top__sig_winch(SIGWINCH, NULL, top); 464 perf_top__sig_winch(SIGWINCH, NULL, top);
465 sigaction(SIGWINCH, &act, NULL); 465 sigaction(SIGWINCH, &act, NULL);
466 } else { 466 } else {
467 signal(SIGWINCH, SIG_DFL); 467 signal(SIGWINCH, SIG_DFL);
468 } 468 }
469 break; 469 break;
470 case 'E': 470 case 'E':
471 if (top->evlist->nr_entries > 1) { 471 if (top->evlist->nr_entries > 1) {
472 /* Select 0 as the default event: */ 472 /* Select 0 as the default event: */
473 int counter = 0; 473 int counter = 0;
474 474
475 fprintf(stderr, "\nAvailable events:"); 475 fprintf(stderr, "\nAvailable events:");
476 476
477 evlist__for_each(top->evlist, top->sym_evsel) 477 evlist__for_each(top->evlist, top->sym_evsel)
478 fprintf(stderr, "\n\t%d %s", top->sym_evsel->idx, perf_evsel__name(top->sym_evsel)); 478 fprintf(stderr, "\n\t%d %s", top->sym_evsel->idx, perf_evsel__name(top->sym_evsel));
479 479
480 prompt_integer(&counter, "Enter details event counter"); 480 prompt_integer(&counter, "Enter details event counter");
481 481
482 if (counter >= top->evlist->nr_entries) { 482 if (counter >= top->evlist->nr_entries) {
483 top->sym_evsel = perf_evlist__first(top->evlist); 483 top->sym_evsel = perf_evlist__first(top->evlist);
484 fprintf(stderr, "Sorry, no such event, using %s.\n", perf_evsel__name(top->sym_evsel)); 484 fprintf(stderr, "Sorry, no such event, using %s.\n", perf_evsel__name(top->sym_evsel));
485 sleep(1); 485 sleep(1);
486 break; 486 break;
487 } 487 }
488 evlist__for_each(top->evlist, top->sym_evsel) 488 evlist__for_each(top->evlist, top->sym_evsel)
489 if (top->sym_evsel->idx == counter) 489 if (top->sym_evsel->idx == counter)
490 break; 490 break;
491 } else 491 } else
492 top->sym_evsel = perf_evlist__first(top->evlist); 492 top->sym_evsel = perf_evlist__first(top->evlist);
493 break; 493 break;
494 case 'f': 494 case 'f':
495 prompt_integer(&top->count_filter, "Enter display event count filter"); 495 prompt_integer(&top->count_filter, "Enter display event count filter");
496 break; 496 break;
497 case 'F': 497 case 'F':
498 prompt_percent(&top->sym_pcnt_filter, 498 prompt_percent(&top->sym_pcnt_filter,
499 "Enter details display event filter (percent)"); 499 "Enter details display event filter (percent)");
500 break; 500 break;
501 case 'K': 501 case 'K':
502 top->hide_kernel_symbols = !top->hide_kernel_symbols; 502 top->hide_kernel_symbols = !top->hide_kernel_symbols;
503 break; 503 break;
504 case 'q': 504 case 'q':
505 case 'Q': 505 case 'Q':
506 printf("exiting.\n"); 506 printf("exiting.\n");
507 if (top->dump_symtab) 507 if (top->dump_symtab)
508 perf_session__fprintf_dsos(top->session, stderr); 508 perf_session__fprintf_dsos(top->session, stderr);
509 ret = false; 509 ret = false;
510 break; 510 break;
511 case 's': 511 case 's':
512 perf_top__prompt_symbol(top, "Enter details symbol"); 512 perf_top__prompt_symbol(top, "Enter details symbol");
513 break; 513 break;
514 case 'S': 514 case 'S':
515 if (!top->sym_filter_entry) 515 if (!top->sym_filter_entry)
516 break; 516 break;
517 else { 517 else {
518 struct hist_entry *syme = top->sym_filter_entry; 518 struct hist_entry *syme = top->sym_filter_entry;
519 519
520 top->sym_filter_entry = NULL; 520 top->sym_filter_entry = NULL;
521 __zero_source_counters(syme); 521 __zero_source_counters(syme);
522 } 522 }
523 break; 523 break;
524 case 'U': 524 case 'U':
525 top->hide_user_symbols = !top->hide_user_symbols; 525 top->hide_user_symbols = !top->hide_user_symbols;
526 break; 526 break;
527 case 'z': 527 case 'z':
528 top->zero = !top->zero; 528 top->zero = !top->zero;
529 break; 529 break;
530 default: 530 default:
531 break; 531 break;
532 } 532 }
533 533
534 return ret; 534 return ret;
535 } 535 }
536 536
537 static void perf_top__sort_new_samples(void *arg) 537 static void perf_top__sort_new_samples(void *arg)
538 { 538 {
539 struct perf_top *t = arg; 539 struct perf_top *t = arg;
540 struct hists *hists; 540 struct hists *hists;
541 541
542 perf_top__reset_sample_counters(t); 542 perf_top__reset_sample_counters(t);
543 543
544 if (t->evlist->selected != NULL) 544 if (t->evlist->selected != NULL)
545 t->sym_evsel = t->evlist->selected; 545 t->sym_evsel = t->evlist->selected;
546 546
547 hists = evsel__hists(t->sym_evsel); 547 hists = evsel__hists(t->sym_evsel);
548 548
549 if (t->zero) { 549 if (t->zero) {
550 hists__delete_entries(hists); 550 hists__delete_entries(hists);
551 } else { 551 } else {
552 hists__decay_entries(hists, t->hide_user_symbols, 552 hists__decay_entries(hists, t->hide_user_symbols,
553 t->hide_kernel_symbols); 553 t->hide_kernel_symbols);
554 } 554 }
555 555
556 hists__collapse_resort(hists, NULL); 556 hists__collapse_resort(hists, NULL);
557 hists__output_resort(hists); 557 hists__output_resort(hists, NULL);
558 } 558 }
559 559
560 static void *display_thread_tui(void *arg) 560 static void *display_thread_tui(void *arg)
561 { 561 {
562 struct perf_evsel *pos; 562 struct perf_evsel *pos;
563 struct perf_top *top = arg; 563 struct perf_top *top = arg;
564 const char *help = "For a higher level overview, try: perf top --sort comm,dso"; 564 const char *help = "For a higher level overview, try: perf top --sort comm,dso";
565 struct hist_browser_timer hbt = { 565 struct hist_browser_timer hbt = {
566 .timer = perf_top__sort_new_samples, 566 .timer = perf_top__sort_new_samples,
567 .arg = top, 567 .arg = top,
568 .refresh = top->delay_secs, 568 .refresh = top->delay_secs,
569 }; 569 };
570 570
571 perf_top__sort_new_samples(top); 571 perf_top__sort_new_samples(top);
572 572
573 /* 573 /*
574 * Initialize the uid_filter_str, in the future the TUI will allow 574 * Initialize the uid_filter_str, in the future the TUI will allow
575 * Zooming in/out UIDs. For now juse use whatever the user passed 575 * Zooming in/out UIDs. For now juse use whatever the user passed
576 * via --uid. 576 * via --uid.
577 */ 577 */
578 evlist__for_each(top->evlist, pos) { 578 evlist__for_each(top->evlist, pos) {
579 struct hists *hists = evsel__hists(pos); 579 struct hists *hists = evsel__hists(pos);
580 hists->uid_filter_str = top->record_opts.target.uid_str; 580 hists->uid_filter_str = top->record_opts.target.uid_str;
581 } 581 }
582 582
583 perf_evlist__tui_browse_hists(top->evlist, help, &hbt, top->min_percent, 583 perf_evlist__tui_browse_hists(top->evlist, help, &hbt, top->min_percent,
584 &top->session->header.env); 584 &top->session->header.env);
585 585
586 done = 1; 586 done = 1;
587 return NULL; 587 return NULL;
588 } 588 }
589 589
590 static void display_sig(int sig __maybe_unused) 590 static void display_sig(int sig __maybe_unused)
591 { 591 {
592 done = 1; 592 done = 1;
593 } 593 }
594 594
595 static void display_setup_sig(void) 595 static void display_setup_sig(void)
596 { 596 {
597 signal(SIGSEGV, display_sig); 597 signal(SIGSEGV, display_sig);
598 signal(SIGFPE, display_sig); 598 signal(SIGFPE, display_sig);
599 signal(SIGINT, display_sig); 599 signal(SIGINT, display_sig);
600 signal(SIGQUIT, display_sig); 600 signal(SIGQUIT, display_sig);
601 signal(SIGTERM, display_sig); 601 signal(SIGTERM, display_sig);
602 } 602 }
603 603
604 static void *display_thread(void *arg) 604 static void *display_thread(void *arg)
605 { 605 {
606 struct pollfd stdin_poll = { .fd = 0, .events = POLLIN }; 606 struct pollfd stdin_poll = { .fd = 0, .events = POLLIN };
607 struct termios save; 607 struct termios save;
608 struct perf_top *top = arg; 608 struct perf_top *top = arg;
609 int delay_msecs, c; 609 int delay_msecs, c;
610 610
611 display_setup_sig(); 611 display_setup_sig();
612 pthread__unblock_sigwinch(); 612 pthread__unblock_sigwinch();
613 repeat: 613 repeat:
614 delay_msecs = top->delay_secs * 1000; 614 delay_msecs = top->delay_secs * 1000;
615 set_term_quiet_input(&save); 615 set_term_quiet_input(&save);
616 /* trash return*/ 616 /* trash return*/
617 getc(stdin); 617 getc(stdin);
618 618
619 while (!done) { 619 while (!done) {
620 perf_top__print_sym_table(top); 620 perf_top__print_sym_table(top);
621 /* 621 /*
622 * Either timeout expired or we got an EINTR due to SIGWINCH, 622 * Either timeout expired or we got an EINTR due to SIGWINCH,
623 * refresh screen in both cases. 623 * refresh screen in both cases.
624 */ 624 */
625 switch (poll(&stdin_poll, 1, delay_msecs)) { 625 switch (poll(&stdin_poll, 1, delay_msecs)) {
626 case 0: 626 case 0:
627 continue; 627 continue;
628 case -1: 628 case -1:
629 if (errno == EINTR) 629 if (errno == EINTR)
630 continue; 630 continue;
631 /* Fall trhu */ 631 /* Fall trhu */
632 default: 632 default:
633 c = getc(stdin); 633 c = getc(stdin);
634 tcsetattr(0, TCSAFLUSH, &save); 634 tcsetattr(0, TCSAFLUSH, &save);
635 635
636 if (perf_top__handle_keypress(top, c)) 636 if (perf_top__handle_keypress(top, c))
637 goto repeat; 637 goto repeat;
638 done = 1; 638 done = 1;
639 } 639 }
640 } 640 }
641 641
642 tcsetattr(0, TCSAFLUSH, &save); 642 tcsetattr(0, TCSAFLUSH, &save);
643 return NULL; 643 return NULL;
644 } 644 }
645 645
646 static int symbol_filter(struct map *map, struct symbol *sym) 646 static int symbol_filter(struct map *map, struct symbol *sym)
647 { 647 {
648 const char *name = sym->name; 648 const char *name = sym->name;
649 649
650 if (!map->dso->kernel) 650 if (!map->dso->kernel)
651 return 0; 651 return 0;
652 /* 652 /*
653 * ppc64 uses function descriptors and appends a '.' to the 653 * ppc64 uses function descriptors and appends a '.' to the
654 * start of every instruction address. Remove it. 654 * start of every instruction address. Remove it.
655 */ 655 */
656 if (name[0] == '.') 656 if (name[0] == '.')
657 name++; 657 name++;
658 658
659 if (!strcmp(name, "_text") || 659 if (!strcmp(name, "_text") ||
660 !strcmp(name, "_etext") || 660 !strcmp(name, "_etext") ||
661 !strcmp(name, "_sinittext") || 661 !strcmp(name, "_sinittext") ||
662 !strncmp("init_module", name, 11) || 662 !strncmp("init_module", name, 11) ||
663 !strncmp("cleanup_module", name, 14) || 663 !strncmp("cleanup_module", name, 14) ||
664 strstr(name, "_text_start") || 664 strstr(name, "_text_start") ||
665 strstr(name, "_text_end")) 665 strstr(name, "_text_end"))
666 return 1; 666 return 1;
667 667
668 if (symbol__is_idle(sym)) 668 if (symbol__is_idle(sym))
669 sym->ignore = true; 669 sym->ignore = true;
670 670
671 return 0; 671 return 0;
672 } 672 }
673 673
674 static int hist_iter__top_callback(struct hist_entry_iter *iter, 674 static int hist_iter__top_callback(struct hist_entry_iter *iter,
675 struct addr_location *al, bool single, 675 struct addr_location *al, bool single,
676 void *arg) 676 void *arg)
677 { 677 {
678 struct perf_top *top = arg; 678 struct perf_top *top = arg;
679 struct hist_entry *he = iter->he; 679 struct hist_entry *he = iter->he;
680 struct perf_evsel *evsel = iter->evsel; 680 struct perf_evsel *evsel = iter->evsel;
681 681
682 if (sort__has_sym && single) { 682 if (sort__has_sym && single) {
683 u64 ip = al->addr; 683 u64 ip = al->addr;
684 684
685 if (al->map) 685 if (al->map)
686 ip = al->map->unmap_ip(al->map, ip); 686 ip = al->map->unmap_ip(al->map, ip);
687 687
688 perf_top__record_precise_ip(top, he, evsel->idx, ip); 688 perf_top__record_precise_ip(top, he, evsel->idx, ip);
689 } 689 }
690 690
691 return 0; 691 return 0;
692 } 692 }
693 693
694 static void perf_event__process_sample(struct perf_tool *tool, 694 static void perf_event__process_sample(struct perf_tool *tool,
695 const union perf_event *event, 695 const union perf_event *event,
696 struct perf_evsel *evsel, 696 struct perf_evsel *evsel,
697 struct perf_sample *sample, 697 struct perf_sample *sample,
698 struct machine *machine) 698 struct machine *machine)
699 { 699 {
700 struct perf_top *top = container_of(tool, struct perf_top, tool); 700 struct perf_top *top = container_of(tool, struct perf_top, tool);
701 struct addr_location al; 701 struct addr_location al;
702 int err; 702 int err;
703 703
704 if (!machine && perf_guest) { 704 if (!machine && perf_guest) {
705 static struct intlist *seen; 705 static struct intlist *seen;
706 706
707 if (!seen) 707 if (!seen)
708 seen = intlist__new(NULL); 708 seen = intlist__new(NULL);
709 709
710 if (!intlist__has_entry(seen, sample->pid)) { 710 if (!intlist__has_entry(seen, sample->pid)) {
711 pr_err("Can't find guest [%d]'s kernel information\n", 711 pr_err("Can't find guest [%d]'s kernel information\n",
712 sample->pid); 712 sample->pid);
713 intlist__add(seen, sample->pid); 713 intlist__add(seen, sample->pid);
714 } 714 }
715 return; 715 return;
716 } 716 }
717 717
718 if (!machine) { 718 if (!machine) {
719 pr_err("%u unprocessable samples recorded.\r", 719 pr_err("%u unprocessable samples recorded.\r",
720 top->session->stats.nr_unprocessable_samples++); 720 top->session->stats.nr_unprocessable_samples++);
721 return; 721 return;
722 } 722 }
723 723
724 if (event->header.misc & PERF_RECORD_MISC_EXACT_IP) 724 if (event->header.misc & PERF_RECORD_MISC_EXACT_IP)
725 top->exact_samples++; 725 top->exact_samples++;
726 726
727 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0) 727 if (perf_event__preprocess_sample(event, machine, &al, sample) < 0)
728 return; 728 return;
729 729
730 if (!top->kptr_restrict_warned && 730 if (!top->kptr_restrict_warned &&
731 symbol_conf.kptr_restrict && 731 symbol_conf.kptr_restrict &&
732 al.cpumode == PERF_RECORD_MISC_KERNEL) { 732 al.cpumode == PERF_RECORD_MISC_KERNEL) {
733 ui__warning( 733 ui__warning(
734 "Kernel address maps (/proc/{kallsyms,modules}) are restricted.\n\n" 734 "Kernel address maps (/proc/{kallsyms,modules}) are restricted.\n\n"
735 "Check /proc/sys/kernel/kptr_restrict.\n\n" 735 "Check /proc/sys/kernel/kptr_restrict.\n\n"
736 "Kernel%s samples will not be resolved.\n", 736 "Kernel%s samples will not be resolved.\n",
737 !RB_EMPTY_ROOT(&al.map->dso->symbols[MAP__FUNCTION]) ? 737 !RB_EMPTY_ROOT(&al.map->dso->symbols[MAP__FUNCTION]) ?
738 " modules" : ""); 738 " modules" : "");
739 if (use_browser <= 0) 739 if (use_browser <= 0)
740 sleep(5); 740 sleep(5);
741 top->kptr_restrict_warned = true; 741 top->kptr_restrict_warned = true;
742 } 742 }
743 743
744 if (al.sym == NULL) { 744 if (al.sym == NULL) {
745 const char *msg = "Kernel samples will not be resolved.\n"; 745 const char *msg = "Kernel samples will not be resolved.\n";
746 /* 746 /*
747 * As we do lazy loading of symtabs we only will know if the 747 * As we do lazy loading of symtabs we only will know if the
748 * specified vmlinux file is invalid when we actually have a 748 * specified vmlinux file is invalid when we actually have a
749 * hit in kernel space and then try to load it. So if we get 749 * hit in kernel space and then try to load it. So if we get
750 * here and there are _no_ symbols in the DSO backing the 750 * here and there are _no_ symbols in the DSO backing the
751 * kernel map, bail out. 751 * kernel map, bail out.
752 * 752 *
753 * We may never get here, for instance, if we use -K/ 753 * We may never get here, for instance, if we use -K/
754 * --hide-kernel-symbols, even if the user specifies an 754 * --hide-kernel-symbols, even if the user specifies an
755 * invalid --vmlinux ;-) 755 * invalid --vmlinux ;-)
756 */ 756 */
757 if (!top->kptr_restrict_warned && !top->vmlinux_warned && 757 if (!top->kptr_restrict_warned && !top->vmlinux_warned &&
758 al.map == machine->vmlinux_maps[MAP__FUNCTION] && 758 al.map == machine->vmlinux_maps[MAP__FUNCTION] &&
759 RB_EMPTY_ROOT(&al.map->dso->symbols[MAP__FUNCTION])) { 759 RB_EMPTY_ROOT(&al.map->dso->symbols[MAP__FUNCTION])) {
760 if (symbol_conf.vmlinux_name) { 760 if (symbol_conf.vmlinux_name) {
761 ui__warning("The %s file can't be used.\n%s", 761 ui__warning("The %s file can't be used.\n%s",
762 symbol_conf.vmlinux_name, msg); 762 symbol_conf.vmlinux_name, msg);
763 } else { 763 } else {
764 ui__warning("A vmlinux file was not found.\n%s", 764 ui__warning("A vmlinux file was not found.\n%s",
765 msg); 765 msg);
766 } 766 }
767 767
768 if (use_browser <= 0) 768 if (use_browser <= 0)
769 sleep(5); 769 sleep(5);
770 top->vmlinux_warned = true; 770 top->vmlinux_warned = true;
771 } 771 }
772 } 772 }
773 773
774 if (al.sym == NULL || !al.sym->ignore) { 774 if (al.sym == NULL || !al.sym->ignore) {
775 struct hists *hists = evsel__hists(evsel); 775 struct hists *hists = evsel__hists(evsel);
776 struct hist_entry_iter iter = { 776 struct hist_entry_iter iter = {
777 .add_entry_cb = hist_iter__top_callback, 777 .add_entry_cb = hist_iter__top_callback,
778 }; 778 };
779 779
780 if (symbol_conf.cumulate_callchain) 780 if (symbol_conf.cumulate_callchain)
781 iter.ops = &hist_iter_cumulative; 781 iter.ops = &hist_iter_cumulative;
782 else 782 else
783 iter.ops = &hist_iter_normal; 783 iter.ops = &hist_iter_normal;
784 784
785 pthread_mutex_lock(&hists->lock); 785 pthread_mutex_lock(&hists->lock);
786 786
787 err = hist_entry_iter__add(&iter, &al, evsel, sample, 787 err = hist_entry_iter__add(&iter, &al, evsel, sample,
788 top->max_stack, top); 788 top->max_stack, top);
789 if (err < 0) 789 if (err < 0)
790 pr_err("Problem incrementing symbol period, skipping event\n"); 790 pr_err("Problem incrementing symbol period, skipping event\n");
791 791
792 pthread_mutex_unlock(&hists->lock); 792 pthread_mutex_unlock(&hists->lock);
793 } 793 }
794 794
795 return; 795 return;
796 } 796 }
797 797
798 static void perf_top__mmap_read_idx(struct perf_top *top, int idx) 798 static void perf_top__mmap_read_idx(struct perf_top *top, int idx)
799 { 799 {
800 struct perf_sample sample; 800 struct perf_sample sample;
801 struct perf_evsel *evsel; 801 struct perf_evsel *evsel;
802 struct perf_session *session = top->session; 802 struct perf_session *session = top->session;
803 union perf_event *event; 803 union perf_event *event;
804 struct machine *machine; 804 struct machine *machine;
805 u8 origin; 805 u8 origin;
806 int ret; 806 int ret;
807 807
808 while ((event = perf_evlist__mmap_read(top->evlist, idx)) != NULL) { 808 while ((event = perf_evlist__mmap_read(top->evlist, idx)) != NULL) {
809 ret = perf_evlist__parse_sample(top->evlist, event, &sample); 809 ret = perf_evlist__parse_sample(top->evlist, event, &sample);
810 if (ret) { 810 if (ret) {
811 pr_err("Can't parse sample, err = %d\n", ret); 811 pr_err("Can't parse sample, err = %d\n", ret);
812 goto next_event; 812 goto next_event;
813 } 813 }
814 814
815 evsel = perf_evlist__id2evsel(session->evlist, sample.id); 815 evsel = perf_evlist__id2evsel(session->evlist, sample.id);
816 assert(evsel != NULL); 816 assert(evsel != NULL);
817 817
818 origin = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; 818 origin = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
819 819
820 if (event->header.type == PERF_RECORD_SAMPLE) 820 if (event->header.type == PERF_RECORD_SAMPLE)
821 ++top->samples; 821 ++top->samples;
822 822
823 switch (origin) { 823 switch (origin) {
824 case PERF_RECORD_MISC_USER: 824 case PERF_RECORD_MISC_USER:
825 ++top->us_samples; 825 ++top->us_samples;
826 if (top->hide_user_symbols) 826 if (top->hide_user_symbols)
827 goto next_event; 827 goto next_event;
828 machine = &session->machines.host; 828 machine = &session->machines.host;
829 break; 829 break;
830 case PERF_RECORD_MISC_KERNEL: 830 case PERF_RECORD_MISC_KERNEL:
831 ++top->kernel_samples; 831 ++top->kernel_samples;
832 if (top->hide_kernel_symbols) 832 if (top->hide_kernel_symbols)
833 goto next_event; 833 goto next_event;
834 machine = &session->machines.host; 834 machine = &session->machines.host;
835 break; 835 break;
836 case PERF_RECORD_MISC_GUEST_KERNEL: 836 case PERF_RECORD_MISC_GUEST_KERNEL:
837 ++top->guest_kernel_samples; 837 ++top->guest_kernel_samples;
838 machine = perf_session__find_machine(session, 838 machine = perf_session__find_machine(session,
839 sample.pid); 839 sample.pid);
840 break; 840 break;
841 case PERF_RECORD_MISC_GUEST_USER: 841 case PERF_RECORD_MISC_GUEST_USER:
842 ++top->guest_us_samples; 842 ++top->guest_us_samples;
843 /* 843 /*
844 * TODO: we don't process guest user from host side 844 * TODO: we don't process guest user from host side
845 * except simple counting. 845 * except simple counting.
846 */ 846 */
847 /* Fall thru */ 847 /* Fall thru */
848 default: 848 default:
849 goto next_event; 849 goto next_event;
850 } 850 }
851 851
852 852
853 if (event->header.type == PERF_RECORD_SAMPLE) { 853 if (event->header.type == PERF_RECORD_SAMPLE) {
854 perf_event__process_sample(&top->tool, event, evsel, 854 perf_event__process_sample(&top->tool, event, evsel,
855 &sample, machine); 855 &sample, machine);
856 } else if (event->header.type < PERF_RECORD_MAX) { 856 } else if (event->header.type < PERF_RECORD_MAX) {
857 hists__inc_nr_events(evsel__hists(evsel), event->header.type); 857 hists__inc_nr_events(evsel__hists(evsel), event->header.type);
858 machine__process_event(machine, event, &sample); 858 machine__process_event(machine, event, &sample);
859 } else 859 } else
860 ++session->stats.nr_unknown_events; 860 ++session->stats.nr_unknown_events;
861 next_event: 861 next_event:
862 perf_evlist__mmap_consume(top->evlist, idx); 862 perf_evlist__mmap_consume(top->evlist, idx);
863 } 863 }
864 } 864 }
865 865
866 static void perf_top__mmap_read(struct perf_top *top) 866 static void perf_top__mmap_read(struct perf_top *top)
867 { 867 {
868 int i; 868 int i;
869 869
870 for (i = 0; i < top->evlist->nr_mmaps; i++) 870 for (i = 0; i < top->evlist->nr_mmaps; i++)
871 perf_top__mmap_read_idx(top, i); 871 perf_top__mmap_read_idx(top, i);
872 } 872 }
873 873
874 static int perf_top__start_counters(struct perf_top *top) 874 static int perf_top__start_counters(struct perf_top *top)
875 { 875 {
876 char msg[512]; 876 char msg[512];
877 struct perf_evsel *counter; 877 struct perf_evsel *counter;
878 struct perf_evlist *evlist = top->evlist; 878 struct perf_evlist *evlist = top->evlist;
879 struct record_opts *opts = &top->record_opts; 879 struct record_opts *opts = &top->record_opts;
880 880
881 perf_evlist__config(evlist, opts); 881 perf_evlist__config(evlist, opts);
882 882
883 evlist__for_each(evlist, counter) { 883 evlist__for_each(evlist, counter) {
884 try_again: 884 try_again:
885 if (perf_evsel__open(counter, top->evlist->cpus, 885 if (perf_evsel__open(counter, top->evlist->cpus,
886 top->evlist->threads) < 0) { 886 top->evlist->threads) < 0) {
887 if (perf_evsel__fallback(counter, errno, msg, sizeof(msg))) { 887 if (perf_evsel__fallback(counter, errno, msg, sizeof(msg))) {
888 if (verbose) 888 if (verbose)
889 ui__warning("%s\n", msg); 889 ui__warning("%s\n", msg);
890 goto try_again; 890 goto try_again;
891 } 891 }
892 892
893 perf_evsel__open_strerror(counter, &opts->target, 893 perf_evsel__open_strerror(counter, &opts->target,
894 errno, msg, sizeof(msg)); 894 errno, msg, sizeof(msg));
895 ui__error("%s\n", msg); 895 ui__error("%s\n", msg);
896 goto out_err; 896 goto out_err;
897 } 897 }
898 } 898 }
899 899
900 if (perf_evlist__mmap(evlist, opts->mmap_pages, false) < 0) { 900 if (perf_evlist__mmap(evlist, opts->mmap_pages, false) < 0) {
901 ui__error("Failed to mmap with %d (%s)\n", 901 ui__error("Failed to mmap with %d (%s)\n",
902 errno, strerror_r(errno, msg, sizeof(msg))); 902 errno, strerror_r(errno, msg, sizeof(msg)));
903 goto out_err; 903 goto out_err;
904 } 904 }
905 905
906 return 0; 906 return 0;
907 907
908 out_err: 908 out_err:
909 return -1; 909 return -1;
910 } 910 }
911 911
912 static int perf_top__setup_sample_type(struct perf_top *top __maybe_unused) 912 static int perf_top__setup_sample_type(struct perf_top *top __maybe_unused)
913 { 913 {
914 if (!sort__has_sym) { 914 if (!sort__has_sym) {
915 if (symbol_conf.use_callchain) { 915 if (symbol_conf.use_callchain) {
916 ui__error("Selected -g but \"sym\" not present in --sort/-s."); 916 ui__error("Selected -g but \"sym\" not present in --sort/-s.");
917 return -EINVAL; 917 return -EINVAL;
918 } 918 }
919 } else if (callchain_param.mode != CHAIN_NONE) { 919 } else if (callchain_param.mode != CHAIN_NONE) {
920 if (callchain_register_param(&callchain_param) < 0) { 920 if (callchain_register_param(&callchain_param) < 0) {
921 ui__error("Can't register callchain params.\n"); 921 ui__error("Can't register callchain params.\n");
922 return -EINVAL; 922 return -EINVAL;
923 } 923 }
924 } 924 }
925 925
926 return 0; 926 return 0;
927 } 927 }
928 928
929 static int __cmd_top(struct perf_top *top) 929 static int __cmd_top(struct perf_top *top)
930 { 930 {
931 struct record_opts *opts = &top->record_opts; 931 struct record_opts *opts = &top->record_opts;
932 pthread_t thread; 932 pthread_t thread;
933 int ret; 933 int ret;
934 934
935 top->session = perf_session__new(NULL, false, NULL); 935 top->session = perf_session__new(NULL, false, NULL);
936 if (top->session == NULL) 936 if (top->session == NULL)
937 return -1; 937 return -1;
938 938
939 machines__set_symbol_filter(&top->session->machines, symbol_filter); 939 machines__set_symbol_filter(&top->session->machines, symbol_filter);
940 940
941 if (!objdump_path) { 941 if (!objdump_path) {
942 ret = perf_session_env__lookup_objdump(&top->session->header.env); 942 ret = perf_session_env__lookup_objdump(&top->session->header.env);
943 if (ret) 943 if (ret)
944 goto out_delete; 944 goto out_delete;
945 } 945 }
946 946
947 ret = perf_top__setup_sample_type(top); 947 ret = perf_top__setup_sample_type(top);
948 if (ret) 948 if (ret)
949 goto out_delete; 949 goto out_delete;
950 950
951 machine__synthesize_threads(&top->session->machines.host, &opts->target, 951 machine__synthesize_threads(&top->session->machines.host, &opts->target,
952 top->evlist->threads, false); 952 top->evlist->threads, false);
953 ret = perf_top__start_counters(top); 953 ret = perf_top__start_counters(top);
954 if (ret) 954 if (ret)
955 goto out_delete; 955 goto out_delete;
956 956
957 top->session->evlist = top->evlist; 957 top->session->evlist = top->evlist;
958 perf_session__set_id_hdr_size(top->session); 958 perf_session__set_id_hdr_size(top->session);
959 959
960 /* 960 /*
961 * When perf is starting the traced process, all the events (apart from 961 * When perf is starting the traced process, all the events (apart from
962 * group members) have enable_on_exec=1 set, so don't spoil it by 962 * group members) have enable_on_exec=1 set, so don't spoil it by
963 * prematurely enabling them. 963 * prematurely enabling them.
964 * 964 *
965 * XXX 'top' still doesn't start workloads like record, trace, but should, 965 * XXX 'top' still doesn't start workloads like record, trace, but should,
966 * so leave the check here. 966 * so leave the check here.
967 */ 967 */
968 if (!target__none(&opts->target)) 968 if (!target__none(&opts->target))
969 perf_evlist__enable(top->evlist); 969 perf_evlist__enable(top->evlist);
970 970
971 /* Wait for a minimal set of events before starting the snapshot */ 971 /* Wait for a minimal set of events before starting the snapshot */
972 perf_evlist__poll(top->evlist, 100); 972 perf_evlist__poll(top->evlist, 100);
973 973
974 perf_top__mmap_read(top); 974 perf_top__mmap_read(top);
975 975
976 ret = -1; 976 ret = -1;
977 if (pthread_create(&thread, NULL, (use_browser > 0 ? display_thread_tui : 977 if (pthread_create(&thread, NULL, (use_browser > 0 ? display_thread_tui :
978 display_thread), top)) { 978 display_thread), top)) {
979 ui__error("Could not create display thread.\n"); 979 ui__error("Could not create display thread.\n");
980 goto out_delete; 980 goto out_delete;
981 } 981 }
982 982
983 if (top->realtime_prio) { 983 if (top->realtime_prio) {
984 struct sched_param param; 984 struct sched_param param;
985 985
986 param.sched_priority = top->realtime_prio; 986 param.sched_priority = top->realtime_prio;
987 if (sched_setscheduler(0, SCHED_FIFO, &param)) { 987 if (sched_setscheduler(0, SCHED_FIFO, &param)) {
988 ui__error("Could not set realtime priority.\n"); 988 ui__error("Could not set realtime priority.\n");
989 goto out_join; 989 goto out_join;
990 } 990 }
991 } 991 }
992 992
993 while (!done) { 993 while (!done) {
994 u64 hits = top->samples; 994 u64 hits = top->samples;
995 995
996 perf_top__mmap_read(top); 996 perf_top__mmap_read(top);
997 997
998 if (hits == top->samples) 998 if (hits == top->samples)
999 ret = perf_evlist__poll(top->evlist, 100); 999 ret = perf_evlist__poll(top->evlist, 100);
1000 } 1000 }
1001 1001
1002 ret = 0; 1002 ret = 0;
1003 out_join: 1003 out_join:
1004 pthread_join(thread, NULL); 1004 pthread_join(thread, NULL);
1005 out_delete: 1005 out_delete:
1006 perf_session__delete(top->session); 1006 perf_session__delete(top->session);
1007 top->session = NULL; 1007 top->session = NULL;
1008 1008
1009 return ret; 1009 return ret;
1010 } 1010 }
1011 1011
1012 static int 1012 static int
1013 callchain_opt(const struct option *opt, const char *arg, int unset) 1013 callchain_opt(const struct option *opt, const char *arg, int unset)
1014 { 1014 {
1015 symbol_conf.use_callchain = true; 1015 symbol_conf.use_callchain = true;
1016 return record_callchain_opt(opt, arg, unset); 1016 return record_callchain_opt(opt, arg, unset);
1017 } 1017 }
1018 1018
1019 static int 1019 static int
1020 parse_callchain_opt(const struct option *opt, const char *arg, int unset) 1020 parse_callchain_opt(const struct option *opt, const char *arg, int unset)
1021 { 1021 {
1022 symbol_conf.use_callchain = true; 1022 symbol_conf.use_callchain = true;
1023 return record_parse_callchain_opt(opt, arg, unset); 1023 return record_parse_callchain_opt(opt, arg, unset);
1024 } 1024 }
1025 1025
1026 static int perf_top_config(const char *var, const char *value, void *cb) 1026 static int perf_top_config(const char *var, const char *value, void *cb)
1027 { 1027 {
1028 if (!strcmp(var, "top.call-graph")) 1028 if (!strcmp(var, "top.call-graph"))
1029 var = "call-graph.record-mode"; /* fall-through */ 1029 var = "call-graph.record-mode"; /* fall-through */
1030 if (!strcmp(var, "top.children")) { 1030 if (!strcmp(var, "top.children")) {
1031 symbol_conf.cumulate_callchain = perf_config_bool(var, value); 1031 symbol_conf.cumulate_callchain = perf_config_bool(var, value);
1032 return 0; 1032 return 0;
1033 } 1033 }
1034 1034
1035 return perf_default_config(var, value, cb); 1035 return perf_default_config(var, value, cb);
1036 } 1036 }
1037 1037
1038 static int 1038 static int
1039 parse_percent_limit(const struct option *opt, const char *arg, 1039 parse_percent_limit(const struct option *opt, const char *arg,
1040 int unset __maybe_unused) 1040 int unset __maybe_unused)
1041 { 1041 {
1042 struct perf_top *top = opt->value; 1042 struct perf_top *top = opt->value;
1043 1043
1044 top->min_percent = strtof(arg, NULL); 1044 top->min_percent = strtof(arg, NULL);
1045 return 0; 1045 return 0;
1046 } 1046 }
1047 1047
1048 int cmd_top(int argc, const char **argv, const char *prefix __maybe_unused) 1048 int cmd_top(int argc, const char **argv, const char *prefix __maybe_unused)
1049 { 1049 {
1050 char errbuf[BUFSIZ]; 1050 char errbuf[BUFSIZ];
1051 struct perf_top top = { 1051 struct perf_top top = {
1052 .count_filter = 5, 1052 .count_filter = 5,
1053 .delay_secs = 2, 1053 .delay_secs = 2,
1054 .record_opts = { 1054 .record_opts = {
1055 .mmap_pages = UINT_MAX, 1055 .mmap_pages = UINT_MAX,
1056 .user_freq = UINT_MAX, 1056 .user_freq = UINT_MAX,
1057 .user_interval = ULLONG_MAX, 1057 .user_interval = ULLONG_MAX,
1058 .freq = 4000, /* 4 KHz */ 1058 .freq = 4000, /* 4 KHz */
1059 .target = { 1059 .target = {
1060 .uses_mmap = true, 1060 .uses_mmap = true,
1061 }, 1061 },
1062 }, 1062 },
1063 .max_stack = PERF_MAX_STACK_DEPTH, 1063 .max_stack = PERF_MAX_STACK_DEPTH,
1064 .sym_pcnt_filter = 5, 1064 .sym_pcnt_filter = 5,
1065 }; 1065 };
1066 struct record_opts *opts = &top.record_opts; 1066 struct record_opts *opts = &top.record_opts;
1067 struct target *target = &opts->target; 1067 struct target *target = &opts->target;
1068 const struct option options[] = { 1068 const struct option options[] = {
1069 OPT_CALLBACK('e', "event", &top.evlist, "event", 1069 OPT_CALLBACK('e', "event", &top.evlist, "event",
1070 "event selector. use 'perf list' to list available events", 1070 "event selector. use 'perf list' to list available events",
1071 parse_events_option), 1071 parse_events_option),
1072 OPT_U64('c', "count", &opts->user_interval, "event period to sample"), 1072 OPT_U64('c', "count", &opts->user_interval, "event period to sample"),
1073 OPT_STRING('p', "pid", &target->pid, "pid", 1073 OPT_STRING('p', "pid", &target->pid, "pid",
1074 "profile events on existing process id"), 1074 "profile events on existing process id"),
1075 OPT_STRING('t', "tid", &target->tid, "tid", 1075 OPT_STRING('t', "tid", &target->tid, "tid",
1076 "profile events on existing thread id"), 1076 "profile events on existing thread id"),
1077 OPT_BOOLEAN('a', "all-cpus", &target->system_wide, 1077 OPT_BOOLEAN('a', "all-cpus", &target->system_wide,
1078 "system-wide collection from all CPUs"), 1078 "system-wide collection from all CPUs"),
1079 OPT_STRING('C', "cpu", &target->cpu_list, "cpu", 1079 OPT_STRING('C', "cpu", &target->cpu_list, "cpu",
1080 "list of cpus to monitor"), 1080 "list of cpus to monitor"),
1081 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name, 1081 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
1082 "file", "vmlinux pathname"), 1082 "file", "vmlinux pathname"),
1083 OPT_BOOLEAN(0, "ignore-vmlinux", &symbol_conf.ignore_vmlinux, 1083 OPT_BOOLEAN(0, "ignore-vmlinux", &symbol_conf.ignore_vmlinux,
1084 "don't load vmlinux even if found"), 1084 "don't load vmlinux even if found"),
1085 OPT_BOOLEAN('K', "hide_kernel_symbols", &top.hide_kernel_symbols, 1085 OPT_BOOLEAN('K', "hide_kernel_symbols", &top.hide_kernel_symbols,
1086 "hide kernel symbols"), 1086 "hide kernel symbols"),
1087 OPT_CALLBACK('m', "mmap-pages", &opts->mmap_pages, "pages", 1087 OPT_CALLBACK('m', "mmap-pages", &opts->mmap_pages, "pages",
1088 "number of mmap data pages", 1088 "number of mmap data pages",
1089 perf_evlist__parse_mmap_pages), 1089 perf_evlist__parse_mmap_pages),
1090 OPT_INTEGER('r', "realtime", &top.realtime_prio, 1090 OPT_INTEGER('r', "realtime", &top.realtime_prio,
1091 "collect data with this RT SCHED_FIFO priority"), 1091 "collect data with this RT SCHED_FIFO priority"),
1092 OPT_INTEGER('d', "delay", &top.delay_secs, 1092 OPT_INTEGER('d', "delay", &top.delay_secs,
1093 "number of seconds to delay between refreshes"), 1093 "number of seconds to delay between refreshes"),
1094 OPT_BOOLEAN('D', "dump-symtab", &top.dump_symtab, 1094 OPT_BOOLEAN('D', "dump-symtab", &top.dump_symtab,
1095 "dump the symbol table used for profiling"), 1095 "dump the symbol table used for profiling"),
1096 OPT_INTEGER('f', "count-filter", &top.count_filter, 1096 OPT_INTEGER('f', "count-filter", &top.count_filter,
1097 "only display functions with more events than this"), 1097 "only display functions with more events than this"),
1098 OPT_BOOLEAN(0, "group", &opts->group, 1098 OPT_BOOLEAN(0, "group", &opts->group,
1099 "put the counters into a counter group"), 1099 "put the counters into a counter group"),
1100 OPT_BOOLEAN('i', "no-inherit", &opts->no_inherit, 1100 OPT_BOOLEAN('i', "no-inherit", &opts->no_inherit,
1101 "child tasks do not inherit counters"), 1101 "child tasks do not inherit counters"),
1102 OPT_STRING(0, "sym-annotate", &top.sym_filter, "symbol name", 1102 OPT_STRING(0, "sym-annotate", &top.sym_filter, "symbol name",
1103 "symbol to annotate"), 1103 "symbol to annotate"),
1104 OPT_BOOLEAN('z', "zero", &top.zero, "zero history across updates"), 1104 OPT_BOOLEAN('z', "zero", &top.zero, "zero history across updates"),
1105 OPT_UINTEGER('F', "freq", &opts->user_freq, "profile at this frequency"), 1105 OPT_UINTEGER('F', "freq", &opts->user_freq, "profile at this frequency"),
1106 OPT_INTEGER('E', "entries", &top.print_entries, 1106 OPT_INTEGER('E', "entries", &top.print_entries,
1107 "display this many functions"), 1107 "display this many functions"),
1108 OPT_BOOLEAN('U', "hide_user_symbols", &top.hide_user_symbols, 1108 OPT_BOOLEAN('U', "hide_user_symbols", &top.hide_user_symbols,
1109 "hide user symbols"), 1109 "hide user symbols"),
1110 OPT_BOOLEAN(0, "tui", &top.use_tui, "Use the TUI interface"), 1110 OPT_BOOLEAN(0, "tui", &top.use_tui, "Use the TUI interface"),
1111 OPT_BOOLEAN(0, "stdio", &top.use_stdio, "Use the stdio interface"), 1111 OPT_BOOLEAN(0, "stdio", &top.use_stdio, "Use the stdio interface"),
1112 OPT_INCR('v', "verbose", &verbose, 1112 OPT_INCR('v', "verbose", &verbose,
1113 "be more verbose (show counter open errors, etc)"), 1113 "be more verbose (show counter open errors, etc)"),
1114 OPT_STRING('s', "sort", &sort_order, "key[,key2...]", 1114 OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
1115 "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..." 1115 "sort by key(s): pid, comm, dso, symbol, parent, cpu, srcline, ..."
1116 " Please refer the man page for the complete list."), 1116 " Please refer the man page for the complete list."),
1117 OPT_STRING(0, "fields", &field_order, "key[,keys...]", 1117 OPT_STRING(0, "fields", &field_order, "key[,keys...]",
1118 "output field(s): overhead, period, sample plus all of sort keys"), 1118 "output field(s): overhead, period, sample plus all of sort keys"),
1119 OPT_BOOLEAN('n', "show-nr-samples", &symbol_conf.show_nr_samples, 1119 OPT_BOOLEAN('n', "show-nr-samples", &symbol_conf.show_nr_samples,
1120 "Show a column with the number of samples"), 1120 "Show a column with the number of samples"),
1121 OPT_CALLBACK_NOOPT('g', NULL, &top.record_opts, 1121 OPT_CALLBACK_NOOPT('g', NULL, &top.record_opts,
1122 NULL, "enables call-graph recording", 1122 NULL, "enables call-graph recording",
1123 &callchain_opt), 1123 &callchain_opt),
1124 OPT_CALLBACK(0, "call-graph", &top.record_opts, 1124 OPT_CALLBACK(0, "call-graph", &top.record_opts,
1125 "mode[,dump_size]", record_callchain_help, 1125 "mode[,dump_size]", record_callchain_help,
1126 &parse_callchain_opt), 1126 &parse_callchain_opt),
1127 OPT_BOOLEAN(0, "children", &symbol_conf.cumulate_callchain, 1127 OPT_BOOLEAN(0, "children", &symbol_conf.cumulate_callchain,
1128 "Accumulate callchains of children and show total overhead as well"), 1128 "Accumulate callchains of children and show total overhead as well"),
1129 OPT_INTEGER(0, "max-stack", &top.max_stack, 1129 OPT_INTEGER(0, "max-stack", &top.max_stack,
1130 "Set the maximum stack depth when parsing the callchain. " 1130 "Set the maximum stack depth when parsing the callchain. "
1131 "Default: " __stringify(PERF_MAX_STACK_DEPTH)), 1131 "Default: " __stringify(PERF_MAX_STACK_DEPTH)),
1132 OPT_CALLBACK(0, "ignore-callees", NULL, "regex", 1132 OPT_CALLBACK(0, "ignore-callees", NULL, "regex",
1133 "ignore callees of these functions in call graphs", 1133 "ignore callees of these functions in call graphs",
1134 report_parse_ignore_callees_opt), 1134 report_parse_ignore_callees_opt),
1135 OPT_BOOLEAN(0, "show-total-period", &symbol_conf.show_total_period, 1135 OPT_BOOLEAN(0, "show-total-period", &symbol_conf.show_total_period,
1136 "Show a column with the sum of periods"), 1136 "Show a column with the sum of periods"),
1137 OPT_STRING(0, "dsos", &symbol_conf.dso_list_str, "dso[,dso...]", 1137 OPT_STRING(0, "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
1138 "only consider symbols in these dsos"), 1138 "only consider symbols in these dsos"),
1139 OPT_STRING(0, "comms", &symbol_conf.comm_list_str, "comm[,comm...]", 1139 OPT_STRING(0, "comms", &symbol_conf.comm_list_str, "comm[,comm...]",
1140 "only consider symbols in these comms"), 1140 "only consider symbols in these comms"),
1141 OPT_STRING(0, "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]", 1141 OPT_STRING(0, "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]",
1142 "only consider these symbols"), 1142 "only consider these symbols"),
1143 OPT_BOOLEAN(0, "source", &symbol_conf.annotate_src, 1143 OPT_BOOLEAN(0, "source", &symbol_conf.annotate_src,
1144 "Interleave source code with assembly code (default)"), 1144 "Interleave source code with assembly code (default)"),
1145 OPT_BOOLEAN(0, "asm-raw", &symbol_conf.annotate_asm_raw, 1145 OPT_BOOLEAN(0, "asm-raw", &symbol_conf.annotate_asm_raw,
1146 "Display raw encoding of assembly instructions (default)"), 1146 "Display raw encoding of assembly instructions (default)"),
1147 OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel, 1147 OPT_BOOLEAN(0, "demangle-kernel", &symbol_conf.demangle_kernel,
1148 "Enable kernel symbol demangling"), 1148 "Enable kernel symbol demangling"),
1149 OPT_STRING(0, "objdump", &objdump_path, "path", 1149 OPT_STRING(0, "objdump", &objdump_path, "path",
1150 "objdump binary to use for disassembly and annotations"), 1150 "objdump binary to use for disassembly and annotations"),
1151 OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style", 1151 OPT_STRING('M', "disassembler-style", &disassembler_style, "disassembler style",
1152 "Specify disassembler style (e.g. -M intel for intel syntax)"), 1152 "Specify disassembler style (e.g. -M intel for intel syntax)"),
1153 OPT_STRING('u', "uid", &target->uid_str, "user", "user to profile"), 1153 OPT_STRING('u', "uid", &target->uid_str, "user", "user to profile"),
1154 OPT_CALLBACK(0, "percent-limit", &top, "percent", 1154 OPT_CALLBACK(0, "percent-limit", &top, "percent",
1155 "Don't show entries under that percent", parse_percent_limit), 1155 "Don't show entries under that percent", parse_percent_limit),
1156 OPT_CALLBACK(0, "percentage", NULL, "relative|absolute", 1156 OPT_CALLBACK(0, "percentage", NULL, "relative|absolute",
1157 "How to display percentage of filtered entries", parse_filter_percentage), 1157 "How to display percentage of filtered entries", parse_filter_percentage),
1158 OPT_STRING('w', "column-widths", &symbol_conf.col_width_list_str, 1158 OPT_STRING('w', "column-widths", &symbol_conf.col_width_list_str,
1159 "width[,width...]", 1159 "width[,width...]",
1160 "don't try to adjust column width, use these fixed values"), 1160 "don't try to adjust column width, use these fixed values"),
1161 OPT_END() 1161 OPT_END()
1162 }; 1162 };
1163 const char * const top_usage[] = { 1163 const char * const top_usage[] = {
1164 "perf top [<options>]", 1164 "perf top [<options>]",
1165 NULL 1165 NULL
1166 }; 1166 };
1167 int status = hists__init(); 1167 int status = hists__init();
1168 1168
1169 if (status < 0) 1169 if (status < 0)
1170 return status; 1170 return status;
1171 1171
1172 top.evlist = perf_evlist__new(); 1172 top.evlist = perf_evlist__new();
1173 if (top.evlist == NULL) 1173 if (top.evlist == NULL)
1174 return -ENOMEM; 1174 return -ENOMEM;
1175 1175
1176 perf_config(perf_top_config, &top); 1176 perf_config(perf_top_config, &top);
1177 1177
1178 argc = parse_options(argc, argv, options, top_usage, 0); 1178 argc = parse_options(argc, argv, options, top_usage, 0);
1179 if (argc) 1179 if (argc)
1180 usage_with_options(top_usage, options); 1180 usage_with_options(top_usage, options);
1181 1181
1182 sort__mode = SORT_MODE__TOP; 1182 sort__mode = SORT_MODE__TOP;
1183 /* display thread wants entries to be collapsed in a different tree */ 1183 /* display thread wants entries to be collapsed in a different tree */
1184 sort__need_collapse = 1; 1184 sort__need_collapse = 1;
1185 1185
1186 if (setup_sorting() < 0) { 1186 if (setup_sorting() < 0) {
1187 if (sort_order) 1187 if (sort_order)
1188 parse_options_usage(top_usage, options, "s", 1); 1188 parse_options_usage(top_usage, options, "s", 1);
1189 if (field_order) 1189 if (field_order)
1190 parse_options_usage(sort_order ? NULL : top_usage, 1190 parse_options_usage(sort_order ? NULL : top_usage,
1191 options, "fields", 0); 1191 options, "fields", 0);
1192 goto out_delete_evlist; 1192 goto out_delete_evlist;
1193 } 1193 }
1194 1194
1195 if (top.use_stdio) 1195 if (top.use_stdio)
1196 use_browser = 0; 1196 use_browser = 0;
1197 else if (top.use_tui) 1197 else if (top.use_tui)
1198 use_browser = 1; 1198 use_browser = 1;
1199 1199
1200 setup_browser(false); 1200 setup_browser(false);
1201 1201
1202 status = target__validate(target); 1202 status = target__validate(target);
1203 if (status) { 1203 if (status) {
1204 target__strerror(target, status, errbuf, BUFSIZ); 1204 target__strerror(target, status, errbuf, BUFSIZ);
1205 ui__warning("%s\n", errbuf); 1205 ui__warning("%s\n", errbuf);
1206 } 1206 }
1207 1207
1208 status = target__parse_uid(target); 1208 status = target__parse_uid(target);
1209 if (status) { 1209 if (status) {
1210 int saved_errno = errno; 1210 int saved_errno = errno;
1211 1211
1212 target__strerror(target, status, errbuf, BUFSIZ); 1212 target__strerror(target, status, errbuf, BUFSIZ);
1213 ui__error("%s\n", errbuf); 1213 ui__error("%s\n", errbuf);
1214 1214
1215 status = -saved_errno; 1215 status = -saved_errno;
1216 goto out_delete_evlist; 1216 goto out_delete_evlist;
1217 } 1217 }
1218 1218
1219 if (target__none(target)) 1219 if (target__none(target))
1220 target->system_wide = true; 1220 target->system_wide = true;
1221 1221
1222 if (perf_evlist__create_maps(top.evlist, target) < 0) 1222 if (perf_evlist__create_maps(top.evlist, target) < 0)
1223 usage_with_options(top_usage, options); 1223 usage_with_options(top_usage, options);
1224 1224
1225 if (!top.evlist->nr_entries && 1225 if (!top.evlist->nr_entries &&
1226 perf_evlist__add_default(top.evlist) < 0) { 1226 perf_evlist__add_default(top.evlist) < 0) {
1227 ui__error("Not enough memory for event selector list\n"); 1227 ui__error("Not enough memory for event selector list\n");
1228 goto out_delete_evlist; 1228 goto out_delete_evlist;
1229 } 1229 }
1230 1230
1231 symbol_conf.nr_events = top.evlist->nr_entries; 1231 symbol_conf.nr_events = top.evlist->nr_entries;
1232 1232
1233 if (top.delay_secs < 1) 1233 if (top.delay_secs < 1)
1234 top.delay_secs = 1; 1234 top.delay_secs = 1;
1235 1235
1236 if (record_opts__config(opts)) { 1236 if (record_opts__config(opts)) {
1237 status = -EINVAL; 1237 status = -EINVAL;
1238 goto out_delete_evlist; 1238 goto out_delete_evlist;
1239 } 1239 }
1240 1240
1241 top.sym_evsel = perf_evlist__first(top.evlist); 1241 top.sym_evsel = perf_evlist__first(top.evlist);
1242 1242
1243 if (!symbol_conf.use_callchain) { 1243 if (!symbol_conf.use_callchain) {
1244 symbol_conf.cumulate_callchain = false; 1244 symbol_conf.cumulate_callchain = false;
1245 perf_hpp__cancel_cumulate(); 1245 perf_hpp__cancel_cumulate();
1246 } 1246 }
1247 1247
1248 symbol_conf.priv_size = sizeof(struct annotation); 1248 symbol_conf.priv_size = sizeof(struct annotation);
1249 1249
1250 symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL); 1250 symbol_conf.try_vmlinux_path = (symbol_conf.vmlinux_name == NULL);
1251 if (symbol__init(NULL) < 0) 1251 if (symbol__init(NULL) < 0)
1252 return -1; 1252 return -1;
1253 1253
1254 sort__setup_elide(stdout); 1254 sort__setup_elide(stdout);
1255 1255
1256 get_term_dimensions(&top.winsize); 1256 get_term_dimensions(&top.winsize);
1257 if (top.print_entries == 0) { 1257 if (top.print_entries == 0) {
1258 struct sigaction act = { 1258 struct sigaction act = {
1259 .sa_sigaction = perf_top__sig_winch, 1259 .sa_sigaction = perf_top__sig_winch,
1260 .sa_flags = SA_SIGINFO, 1260 .sa_flags = SA_SIGINFO,
1261 }; 1261 };
1262 perf_top__update_print_entries(&top); 1262 perf_top__update_print_entries(&top);
1263 sigaction(SIGWINCH, &act, NULL); 1263 sigaction(SIGWINCH, &act, NULL);
1264 } 1264 }
1265 1265
1266 status = __cmd_top(&top); 1266 status = __cmd_top(&top);
1267 1267
1268 out_delete_evlist: 1268 out_delete_evlist:
1269 perf_evlist__delete(top.evlist); 1269 perf_evlist__delete(top.evlist);
1270 1270
1271 return status; 1271 return status;
1272 } 1272 }
1273 1273
tools/perf/tests/hists_cumulate.c
Diff comments   View file @ ddb321a
1 #include "perf.h" 1 #include "perf.h"
2 #include "util/debug.h" 2 #include "util/debug.h"
3 #include "util/symbol.h" 3 #include "util/symbol.h"
4 #include "util/sort.h" 4 #include "util/sort.h"
5 #include "util/evsel.h" 5 #include "util/evsel.h"
6 #include "util/evlist.h" 6 #include "util/evlist.h"
7 #include "util/machine.h" 7 #include "util/machine.h"
8 #include "util/thread.h" 8 #include "util/thread.h"
9 #include "util/parse-events.h" 9 #include "util/parse-events.h"
10 #include "tests/tests.h" 10 #include "tests/tests.h"
11 #include "tests/hists_common.h" 11 #include "tests/hists_common.h"
12 12
13 struct sample { 13 struct sample {
14 u32 pid; 14 u32 pid;
15 u64 ip; 15 u64 ip;
16 struct thread *thread; 16 struct thread *thread;
17 struct map *map; 17 struct map *map;
18 struct symbol *sym; 18 struct symbol *sym;
19 }; 19 };
20 20
21 /* For the numbers, see hists_common.c */ 21 /* For the numbers, see hists_common.c */
22 static struct sample fake_samples[] = { 22 static struct sample fake_samples[] = {
23 /* perf [kernel] schedule() */ 23 /* perf [kernel] schedule() */
24 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_KERNEL_SCHEDULE, }, 24 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_KERNEL_SCHEDULE, },
25 /* perf [perf] main() */ 25 /* perf [perf] main() */
26 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_MAIN, }, 26 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_MAIN, },
27 /* perf [perf] cmd_record() */ 27 /* perf [perf] cmd_record() */
28 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_CMD_RECORD, }, 28 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_CMD_RECORD, },
29 /* perf [libc] malloc() */ 29 /* perf [libc] malloc() */
30 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_MALLOC, }, 30 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_MALLOC, },
31 /* perf [libc] free() */ 31 /* perf [libc] free() */
32 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_FREE, }, 32 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_FREE, },
33 /* perf [perf] main() */ 33 /* perf [perf] main() */
34 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_MAIN, }, 34 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_MAIN, },
35 /* perf [kernel] page_fault() */ 35 /* perf [kernel] page_fault() */
36 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_KERNEL_PAGE_FAULT, }, 36 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_KERNEL_PAGE_FAULT, },
37 /* bash [bash] main() */ 37 /* bash [bash] main() */
38 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_MAIN, }, 38 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_MAIN, },
39 /* bash [bash] xmalloc() */ 39 /* bash [bash] xmalloc() */
40 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_XMALLOC, }, 40 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_XMALLOC, },
41 /* bash [kernel] page_fault() */ 41 /* bash [kernel] page_fault() */
42 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_KERNEL_PAGE_FAULT, }, 42 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_KERNEL_PAGE_FAULT, },
43 }; 43 };
44 44
45 /* 45 /*
46 * Will be casted to struct ip_callchain which has all 64 bit entries 46 * Will be casted to struct ip_callchain which has all 64 bit entries
47 * of nr and ips[]. 47 * of nr and ips[].
48 */ 48 */
49 static u64 fake_callchains[][10] = { 49 static u64 fake_callchains[][10] = {
50 /* schedule => run_command => main */ 50 /* schedule => run_command => main */
51 { 3, FAKE_IP_KERNEL_SCHEDULE, FAKE_IP_PERF_RUN_COMMAND, FAKE_IP_PERF_MAIN, }, 51 { 3, FAKE_IP_KERNEL_SCHEDULE, FAKE_IP_PERF_RUN_COMMAND, FAKE_IP_PERF_MAIN, },
52 /* main */ 52 /* main */
53 { 1, FAKE_IP_PERF_MAIN, }, 53 { 1, FAKE_IP_PERF_MAIN, },
54 /* cmd_record => run_command => main */ 54 /* cmd_record => run_command => main */
55 { 3, FAKE_IP_PERF_CMD_RECORD, FAKE_IP_PERF_RUN_COMMAND, FAKE_IP_PERF_MAIN, }, 55 { 3, FAKE_IP_PERF_CMD_RECORD, FAKE_IP_PERF_RUN_COMMAND, FAKE_IP_PERF_MAIN, },
56 /* malloc => cmd_record => run_command => main */ 56 /* malloc => cmd_record => run_command => main */
57 { 4, FAKE_IP_LIBC_MALLOC, FAKE_IP_PERF_CMD_RECORD, FAKE_IP_PERF_RUN_COMMAND, 57 { 4, FAKE_IP_LIBC_MALLOC, FAKE_IP_PERF_CMD_RECORD, FAKE_IP_PERF_RUN_COMMAND,
58 FAKE_IP_PERF_MAIN, }, 58 FAKE_IP_PERF_MAIN, },
59 /* free => cmd_record => run_command => main */ 59 /* free => cmd_record => run_command => main */
60 { 4, FAKE_IP_LIBC_FREE, FAKE_IP_PERF_CMD_RECORD, FAKE_IP_PERF_RUN_COMMAND, 60 { 4, FAKE_IP_LIBC_FREE, FAKE_IP_PERF_CMD_RECORD, FAKE_IP_PERF_RUN_COMMAND,
61 FAKE_IP_PERF_MAIN, }, 61 FAKE_IP_PERF_MAIN, },
62 /* main */ 62 /* main */
63 { 1, FAKE_IP_PERF_MAIN, }, 63 { 1, FAKE_IP_PERF_MAIN, },
64 /* page_fault => sys_perf_event_open => run_command => main */ 64 /* page_fault => sys_perf_event_open => run_command => main */
65 { 4, FAKE_IP_KERNEL_PAGE_FAULT, FAKE_IP_KERNEL_SYS_PERF_EVENT_OPEN, 65 { 4, FAKE_IP_KERNEL_PAGE_FAULT, FAKE_IP_KERNEL_SYS_PERF_EVENT_OPEN,
66 FAKE_IP_PERF_RUN_COMMAND, FAKE_IP_PERF_MAIN, }, 66 FAKE_IP_PERF_RUN_COMMAND, FAKE_IP_PERF_MAIN, },
67 /* main */ 67 /* main */
68 { 1, FAKE_IP_BASH_MAIN, }, 68 { 1, FAKE_IP_BASH_MAIN, },
69 /* xmalloc => malloc => xmalloc => malloc => xmalloc => main */ 69 /* xmalloc => malloc => xmalloc => malloc => xmalloc => main */
70 { 6, FAKE_IP_BASH_XMALLOC, FAKE_IP_LIBC_MALLOC, FAKE_IP_BASH_XMALLOC, 70 { 6, FAKE_IP_BASH_XMALLOC, FAKE_IP_LIBC_MALLOC, FAKE_IP_BASH_XMALLOC,
71 FAKE_IP_LIBC_MALLOC, FAKE_IP_BASH_XMALLOC, FAKE_IP_BASH_MAIN, }, 71 FAKE_IP_LIBC_MALLOC, FAKE_IP_BASH_XMALLOC, FAKE_IP_BASH_MAIN, },
72 /* page_fault => malloc => main */ 72 /* page_fault => malloc => main */
73 { 3, FAKE_IP_KERNEL_PAGE_FAULT, FAKE_IP_LIBC_MALLOC, FAKE_IP_BASH_MAIN, }, 73 { 3, FAKE_IP_KERNEL_PAGE_FAULT, FAKE_IP_LIBC_MALLOC, FAKE_IP_BASH_MAIN, },
74 }; 74 };
75 75
76 static int add_hist_entries(struct hists *hists, struct machine *machine) 76 static int add_hist_entries(struct hists *hists, struct machine *machine)
77 { 77 {
78 struct addr_location al; 78 struct addr_location al;
79 struct perf_evsel *evsel = hists_to_evsel(hists); 79 struct perf_evsel *evsel = hists_to_evsel(hists);
80 struct perf_sample sample = { .period = 1000, }; 80 struct perf_sample sample = { .period = 1000, };
81 size_t i; 81 size_t i;
82 82
83 for (i = 0; i < ARRAY_SIZE(fake_samples); i++) { 83 for (i = 0; i < ARRAY_SIZE(fake_samples); i++) {
84 const union perf_event event = { 84 const union perf_event event = {
85 .header = { 85 .header = {
86 .misc = PERF_RECORD_MISC_USER, 86 .misc = PERF_RECORD_MISC_USER,
87 }, 87 },
88 }; 88 };
89 struct hist_entry_iter iter = { 89 struct hist_entry_iter iter = {
90 .hide_unresolved = false, 90 .hide_unresolved = false,
91 }; 91 };
92 92
93 if (symbol_conf.cumulate_callchain) 93 if (symbol_conf.cumulate_callchain)
94 iter.ops = &hist_iter_cumulative; 94 iter.ops = &hist_iter_cumulative;
95 else 95 else
96 iter.ops = &hist_iter_normal; 96 iter.ops = &hist_iter_normal;
97 97
98 sample.pid = fake_samples[i].pid; 98 sample.pid = fake_samples[i].pid;
99 sample.tid = fake_samples[i].pid; 99 sample.tid = fake_samples[i].pid;
100 sample.ip = fake_samples[i].ip; 100 sample.ip = fake_samples[i].ip;
101 sample.callchain = (struct ip_callchain *)fake_callchains[i]; 101 sample.callchain = (struct ip_callchain *)fake_callchains[i];
102 102
103 if (perf_event__preprocess_sample(&event, machine, &al, 103 if (perf_event__preprocess_sample(&event, machine, &al,
104 &sample) < 0) 104 &sample) < 0)
105 goto out; 105 goto out;
106 106
107 if (hist_entry_iter__add(&iter, &al, evsel, &sample, 107 if (hist_entry_iter__add(&iter, &al, evsel, &sample,
108 PERF_MAX_STACK_DEPTH, NULL) < 0) 108 PERF_MAX_STACK_DEPTH, NULL) < 0)
109 goto out; 109 goto out;
110 110
111 fake_samples[i].thread = al.thread; 111 fake_samples[i].thread = al.thread;
112 fake_samples[i].map = al.map; 112 fake_samples[i].map = al.map;
113 fake_samples[i].sym = al.sym; 113 fake_samples[i].sym = al.sym;
114 } 114 }
115 115
116 return TEST_OK; 116 return TEST_OK;
117 117
118 out: 118 out:
119 pr_debug("Not enough memory for adding a hist entry\n"); 119 pr_debug("Not enough memory for adding a hist entry\n");
120 return TEST_FAIL; 120 return TEST_FAIL;
121 } 121 }
122 122
123 static void del_hist_entries(struct hists *hists) 123 static void del_hist_entries(struct hists *hists)
124 { 124 {
125 struct hist_entry *he; 125 struct hist_entry *he;
126 struct rb_root *root_in; 126 struct rb_root *root_in;
127 struct rb_root *root_out; 127 struct rb_root *root_out;
128 struct rb_node *node; 128 struct rb_node *node;
129 129
130 if (sort__need_collapse) 130 if (sort__need_collapse)
131 root_in = &hists->entries_collapsed; 131 root_in = &hists->entries_collapsed;
132 else 132 else
133 root_in = hists->entries_in; 133 root_in = hists->entries_in;
134 134
135 root_out = &hists->entries; 135 root_out = &hists->entries;
136 136
137 while (!RB_EMPTY_ROOT(root_out)) { 137 while (!RB_EMPTY_ROOT(root_out)) {
138 node = rb_first(root_out); 138 node = rb_first(root_out);
139 139
140 he = rb_entry(node, struct hist_entry, rb_node); 140 he = rb_entry(node, struct hist_entry, rb_node);
141 rb_erase(node, root_out); 141 rb_erase(node, root_out);
142 rb_erase(&he->rb_node_in, root_in); 142 rb_erase(&he->rb_node_in, root_in);
143 hist_entry__free(he); 143 hist_entry__free(he);
144 } 144 }
145 } 145 }
146 146
147 typedef int (*test_fn_t)(struct perf_evsel *, struct machine *); 147 typedef int (*test_fn_t)(struct perf_evsel *, struct machine *);
148 148
149 #define COMM(he) (thread__comm_str(he->thread)) 149 #define COMM(he) (thread__comm_str(he->thread))
150 #define DSO(he) (he->ms.map->dso->short_name) 150 #define DSO(he) (he->ms.map->dso->short_name)
151 #define SYM(he) (he->ms.sym->name) 151 #define SYM(he) (he->ms.sym->name)
152 #define CPU(he) (he->cpu) 152 #define CPU(he) (he->cpu)
153 #define PID(he) (he->thread->tid) 153 #define PID(he) (he->thread->tid)
154 #define DEPTH(he) (he->callchain->max_depth) 154 #define DEPTH(he) (he->callchain->max_depth)
155 #define CDSO(cl) (cl->ms.map->dso->short_name) 155 #define CDSO(cl) (cl->ms.map->dso->short_name)
156 #define CSYM(cl) (cl->ms.sym->name) 156 #define CSYM(cl) (cl->ms.sym->name)
157 157
158 struct result { 158 struct result {
159 u64 children; 159 u64 children;
160 u64 self; 160 u64 self;
161 const char *comm; 161 const char *comm;
162 const char *dso; 162 const char *dso;
163 const char *sym; 163 const char *sym;
164 }; 164 };
165 165
166 struct callchain_result { 166 struct callchain_result {
167 u64 nr; 167 u64 nr;
168 struct { 168 struct {
169 const char *dso; 169 const char *dso;
170 const char *sym; 170 const char *sym;
171 } node[10]; 171 } node[10];
172 }; 172 };
173 173
174 static int do_test(struct hists *hists, struct result *expected, size_t nr_expected, 174 static int do_test(struct hists *hists, struct result *expected, size_t nr_expected,
175 struct callchain_result *expected_callchain, size_t nr_callchain) 175 struct callchain_result *expected_callchain, size_t nr_callchain)
176 { 176 {
177 char buf[32]; 177 char buf[32];
178 size_t i, c; 178 size_t i, c;
179 struct hist_entry *he; 179 struct hist_entry *he;
180 struct rb_root *root; 180 struct rb_root *root;
181 struct rb_node *node; 181 struct rb_node *node;
182 struct callchain_node *cnode; 182 struct callchain_node *cnode;
183 struct callchain_list *clist; 183 struct callchain_list *clist;
184 184
185 /* 185 /*
186 * adding and deleting hist entries must be done outside of this 186 * adding and deleting hist entries must be done outside of this
187 * function since TEST_ASSERT_VAL() returns in case of failure. 187 * function since TEST_ASSERT_VAL() returns in case of failure.
188 */ 188 */
189 hists__collapse_resort(hists, NULL); 189 hists__collapse_resort(hists, NULL);
190 hists__output_resort(hists); 190 hists__output_resort(hists, NULL);
191 191
192 if (verbose > 2) { 192 if (verbose > 2) {
193 pr_info("use callchain: %d, cumulate callchain: %d\n", 193 pr_info("use callchain: %d, cumulate callchain: %d\n",
194 symbol_conf.use_callchain, 194 symbol_conf.use_callchain,
195 symbol_conf.cumulate_callchain); 195 symbol_conf.cumulate_callchain);
196 print_hists_out(hists); 196 print_hists_out(hists);
197 } 197 }
198 198
199 root = &hists->entries; 199 root = &hists->entries;
200 for (node = rb_first(root), i = 0; 200 for (node = rb_first(root), i = 0;
201 node && (he = rb_entry(node, struct hist_entry, rb_node)); 201 node && (he = rb_entry(node, struct hist_entry, rb_node));
202 node = rb_next(node), i++) { 202 node = rb_next(node), i++) {
203 scnprintf(buf, sizeof(buf), "Invalid hist entry #%zd", i); 203 scnprintf(buf, sizeof(buf), "Invalid hist entry #%zd", i);
204 204
205 TEST_ASSERT_VAL("Incorrect number of hist entry", 205 TEST_ASSERT_VAL("Incorrect number of hist entry",
206 i < nr_expected); 206 i < nr_expected);
207 TEST_ASSERT_VAL(buf, he->stat.period == expected[i].self && 207 TEST_ASSERT_VAL(buf, he->stat.period == expected[i].self &&
208 !strcmp(COMM(he), expected[i].comm) && 208 !strcmp(COMM(he), expected[i].comm) &&
209 !strcmp(DSO(he), expected[i].dso) && 209 !strcmp(DSO(he), expected[i].dso) &&
210 !strcmp(SYM(he), expected[i].sym)); 210 !strcmp(SYM(he), expected[i].sym));
211 211
212 if (symbol_conf.cumulate_callchain) 212 if (symbol_conf.cumulate_callchain)
213 TEST_ASSERT_VAL(buf, he->stat_acc->period == expected[i].children); 213 TEST_ASSERT_VAL(buf, he->stat_acc->period == expected[i].children);
214 214
215 if (!symbol_conf.use_callchain) 215 if (!symbol_conf.use_callchain)
216 continue; 216 continue;
217 217
218 /* check callchain entries */ 218 /* check callchain entries */
219 root = &he->callchain->node.rb_root; 219 root = &he->callchain->node.rb_root;
220 cnode = rb_entry(rb_first(root), struct callchain_node, rb_node); 220 cnode = rb_entry(rb_first(root), struct callchain_node, rb_node);
221 221
222 c = 0; 222 c = 0;
223 list_for_each_entry(clist, &cnode->val, list) { 223 list_for_each_entry(clist, &cnode->val, list) {
224 scnprintf(buf, sizeof(buf), "Invalid callchain entry #%zd/%zd", i, c); 224 scnprintf(buf, sizeof(buf), "Invalid callchain entry #%zd/%zd", i, c);
225 225
226 TEST_ASSERT_VAL("Incorrect number of callchain entry", 226 TEST_ASSERT_VAL("Incorrect number of callchain entry",
227 c < expected_callchain[i].nr); 227 c < expected_callchain[i].nr);
228 TEST_ASSERT_VAL(buf, 228 TEST_ASSERT_VAL(buf,
229 !strcmp(CDSO(clist), expected_callchain[i].node[c].dso) && 229 !strcmp(CDSO(clist), expected_callchain[i].node[c].dso) &&
230 !strcmp(CSYM(clist), expected_callchain[i].node[c].sym)); 230 !strcmp(CSYM(clist), expected_callchain[i].node[c].sym));
231 c++; 231 c++;
232 } 232 }
233 /* TODO: handle multiple child nodes properly */ 233 /* TODO: handle multiple child nodes properly */
234 TEST_ASSERT_VAL("Incorrect number of callchain entry", 234 TEST_ASSERT_VAL("Incorrect number of callchain entry",
235 c <= expected_callchain[i].nr); 235 c <= expected_callchain[i].nr);
236 } 236 }
237 TEST_ASSERT_VAL("Incorrect number of hist entry", 237 TEST_ASSERT_VAL("Incorrect number of hist entry",
238 i == nr_expected); 238 i == nr_expected);
239 TEST_ASSERT_VAL("Incorrect number of callchain entry", 239 TEST_ASSERT_VAL("Incorrect number of callchain entry",
240 !symbol_conf.use_callchain || nr_expected == nr_callchain); 240 !symbol_conf.use_callchain || nr_expected == nr_callchain);
241 return 0; 241 return 0;
242 } 242 }
243 243
244 /* NO callchain + NO children */ 244 /* NO callchain + NO children */
245 static int test1(struct perf_evsel *evsel, struct machine *machine) 245 static int test1(struct perf_evsel *evsel, struct machine *machine)
246 { 246 {
247 int err; 247 int err;
248 struct hists *hists = evsel__hists(evsel); 248 struct hists *hists = evsel__hists(evsel);
249 /* 249 /*
250 * expected output: 250 * expected output:
251 * 251 *
252 * Overhead Command Shared Object Symbol 252 * Overhead Command Shared Object Symbol
253 * ======== ======= ============= ============== 253 * ======== ======= ============= ==============
254 * 20.00% perf perf [.] main 254 * 20.00% perf perf [.] main
255 * 10.00% bash [kernel] [k] page_fault 255 * 10.00% bash [kernel] [k] page_fault
256 * 10.00% bash bash [.] main 256 * 10.00% bash bash [.] main
257 * 10.00% bash bash [.] xmalloc 257 * 10.00% bash bash [.] xmalloc
258 * 10.00% perf [kernel] [k] page_fault 258 * 10.00% perf [kernel] [k] page_fault
259 * 10.00% perf [kernel] [k] schedule 259 * 10.00% perf [kernel] [k] schedule
260 * 10.00% perf libc [.] free 260 * 10.00% perf libc [.] free
261 * 10.00% perf libc [.] malloc 261 * 10.00% perf libc [.] malloc
262 * 10.00% perf perf [.] cmd_record 262 * 10.00% perf perf [.] cmd_record
263 */ 263 */
264 struct result expected[] = { 264 struct result expected[] = {
265 { 0, 2000, "perf", "perf", "main" }, 265 { 0, 2000, "perf", "perf", "main" },
266 { 0, 1000, "bash", "[kernel]", "page_fault" }, 266 { 0, 1000, "bash", "[kernel]", "page_fault" },
267 { 0, 1000, "bash", "bash", "main" }, 267 { 0, 1000, "bash", "bash", "main" },
268 { 0, 1000, "bash", "bash", "xmalloc" }, 268 { 0, 1000, "bash", "bash", "xmalloc" },
269 { 0, 1000, "perf", "[kernel]", "page_fault" }, 269 { 0, 1000, "perf", "[kernel]", "page_fault" },
270 { 0, 1000, "perf", "[kernel]", "schedule" }, 270 { 0, 1000, "perf", "[kernel]", "schedule" },
271 { 0, 1000, "perf", "libc", "free" }, 271 { 0, 1000, "perf", "libc", "free" },
272 { 0, 1000, "perf", "libc", "malloc" }, 272 { 0, 1000, "perf", "libc", "malloc" },
273 { 0, 1000, "perf", "perf", "cmd_record" }, 273 { 0, 1000, "perf", "perf", "cmd_record" },
274 }; 274 };
275 275
276 symbol_conf.use_callchain = false; 276 symbol_conf.use_callchain = false;
277 symbol_conf.cumulate_callchain = false; 277 symbol_conf.cumulate_callchain = false;
278 278
279 setup_sorting(); 279 setup_sorting();
280 callchain_register_param(&callchain_param); 280 callchain_register_param(&callchain_param);
281 281
282 err = add_hist_entries(hists, machine); 282 err = add_hist_entries(hists, machine);
283 if (err < 0) 283 if (err < 0)
284 goto out; 284 goto out;
285 285
286 err = do_test(hists, expected, ARRAY_SIZE(expected), NULL, 0); 286 err = do_test(hists, expected, ARRAY_SIZE(expected), NULL, 0);
287 287
288 out: 288 out:
289 del_hist_entries(hists); 289 del_hist_entries(hists);
290 reset_output_field(); 290 reset_output_field();
291 return err; 291 return err;
292 } 292 }
293 293
294 /* callcain + NO children */ 294 /* callcain + NO children */
295 static int test2(struct perf_evsel *evsel, struct machine *machine) 295 static int test2(struct perf_evsel *evsel, struct machine *machine)
296 { 296 {
297 int err; 297 int err;
298 struct hists *hists = evsel__hists(evsel); 298 struct hists *hists = evsel__hists(evsel);
299 /* 299 /*
300 * expected output: 300 * expected output:
301 * 301 *
302 * Overhead Command Shared Object Symbol 302 * Overhead Command Shared Object Symbol
303 * ======== ======= ============= ============== 303 * ======== ======= ============= ==============
304 * 20.00% perf perf [.] main 304 * 20.00% perf perf [.] main
305 * | 305 * |
306 * --- main 306 * --- main
307 * 307 *
308 * 10.00% bash [kernel] [k] page_fault 308 * 10.00% bash [kernel] [k] page_fault
309 * | 309 * |
310 * --- page_fault 310 * --- page_fault
311 * malloc 311 * malloc
312 * main 312 * main
313 * 313 *
314 * 10.00% bash bash [.] main 314 * 10.00% bash bash [.] main
315 * | 315 * |
316 * --- main 316 * --- main
317 * 317 *
318 * 10.00% bash bash [.] xmalloc 318 * 10.00% bash bash [.] xmalloc
319 * | 319 * |
320 * --- xmalloc 320 * --- xmalloc
321 * malloc 321 * malloc
322 * xmalloc <--- NOTE: there's a cycle 322 * xmalloc <--- NOTE: there's a cycle
323 * malloc 323 * malloc
324 * xmalloc 324 * xmalloc
325 * main 325 * main
326 * 326 *
327 * 10.00% perf [kernel] [k] page_fault 327 * 10.00% perf [kernel] [k] page_fault
328 * | 328 * |
329 * --- page_fault 329 * --- page_fault
330 * sys_perf_event_open 330 * sys_perf_event_open
331 * run_command 331 * run_command
332 * main 332 * main
333 * 333 *
334 * 10.00% perf [kernel] [k] schedule 334 * 10.00% perf [kernel] [k] schedule
335 * | 335 * |
336 * --- schedule 336 * --- schedule
337 * run_command 337 * run_command
338 * main 338 * main
339 * 339 *
340 * 10.00% perf libc [.] free 340 * 10.00% perf libc [.] free
341 * | 341 * |
342 * --- free 342 * --- free
343 * cmd_record 343 * cmd_record
344 * run_command 344 * run_command
345 * main 345 * main
346 * 346 *
347 * 10.00% perf libc [.] malloc 347 * 10.00% perf libc [.] malloc
348 * | 348 * |
349 * --- malloc 349 * --- malloc
350 * cmd_record 350 * cmd_record
351 * run_command 351 * run_command
352 * main 352 * main
353 * 353 *
354 * 10.00% perf perf [.] cmd_record 354 * 10.00% perf perf [.] cmd_record
355 * | 355 * |
356 * --- cmd_record 356 * --- cmd_record
357 * run_command 357 * run_command
358 * main 358 * main
359 * 359 *
360 */ 360 */
361 struct result expected[] = { 361 struct result expected[] = {
362 { 0, 2000, "perf", "perf", "main" }, 362 { 0, 2000, "perf", "perf", "main" },
363 { 0, 1000, "bash", "[kernel]", "page_fault" }, 363 { 0, 1000, "bash", "[kernel]", "page_fault" },
364 { 0, 1000, "bash", "bash", "main" }, 364 { 0, 1000, "bash", "bash", "main" },
365 { 0, 1000, "bash", "bash", "xmalloc" }, 365 { 0, 1000, "bash", "bash", "xmalloc" },
366 { 0, 1000, "perf", "[kernel]", "page_fault" }, 366 { 0, 1000, "perf", "[kernel]", "page_fault" },
367 { 0, 1000, "perf", "[kernel]", "schedule" }, 367 { 0, 1000, "perf", "[kernel]", "schedule" },
368 { 0, 1000, "perf", "libc", "free" }, 368 { 0, 1000, "perf", "libc", "free" },
369 { 0, 1000, "perf", "libc", "malloc" }, 369 { 0, 1000, "perf", "libc", "malloc" },
370 { 0, 1000, "perf", "perf", "cmd_record" }, 370 { 0, 1000, "perf", "perf", "cmd_record" },
371 }; 371 };
372 struct callchain_result expected_callchain[] = { 372 struct callchain_result expected_callchain[] = {
373 { 373 {
374 1, { { "perf", "main" }, }, 374 1, { { "perf", "main" }, },
375 }, 375 },
376 { 376 {
377 3, { { "[kernel]", "page_fault" }, 377 3, { { "[kernel]", "page_fault" },
378 { "libc", "malloc" }, 378 { "libc", "malloc" },
379 { "bash", "main" }, }, 379 { "bash", "main" }, },
380 }, 380 },
381 { 381 {
382 1, { { "bash", "main" }, }, 382 1, { { "bash", "main" }, },
383 }, 383 },
384 { 384 {
385 6, { { "bash", "xmalloc" }, 385 6, { { "bash", "xmalloc" },
386 { "libc", "malloc" }, 386 { "libc", "malloc" },
387 { "bash", "xmalloc" }, 387 { "bash", "xmalloc" },
388 { "libc", "malloc" }, 388 { "libc", "malloc" },
389 { "bash", "xmalloc" }, 389 { "bash", "xmalloc" },
390 { "bash", "main" }, }, 390 { "bash", "main" }, },
391 }, 391 },
392 { 392 {
393 4, { { "[kernel]", "page_fault" }, 393 4, { { "[kernel]", "page_fault" },
394 { "[kernel]", "sys_perf_event_open" }, 394 { "[kernel]", "sys_perf_event_open" },
395 { "perf", "run_command" }, 395 { "perf", "run_command" },
396 { "perf", "main" }, }, 396 { "perf", "main" }, },
397 }, 397 },
398 { 398 {
399 3, { { "[kernel]", "schedule" }, 399 3, { { "[kernel]", "schedule" },
400 { "perf", "run_command" }, 400 { "perf", "run_command" },
401 { "perf", "main" }, }, 401 { "perf", "main" }, },
402 }, 402 },
403 { 403 {
404 4, { { "libc", "free" }, 404 4, { { "libc", "free" },
405 { "perf", "cmd_record" }, 405 { "perf", "cmd_record" },
406 { "perf", "run_command" }, 406 { "perf", "run_command" },
407 { "perf", "main" }, }, 407 { "perf", "main" }, },
408 }, 408 },
409 { 409 {
410 4, { { "libc", "malloc" }, 410 4, { { "libc", "malloc" },
411 { "perf", "cmd_record" }, 411 { "perf", "cmd_record" },
412 { "perf", "run_command" }, 412 { "perf", "run_command" },
413 { "perf", "main" }, }, 413 { "perf", "main" }, },
414 }, 414 },
415 { 415 {
416 3, { { "perf", "cmd_record" }, 416 3, { { "perf", "cmd_record" },
417 { "perf", "run_command" }, 417 { "perf", "run_command" },
418 { "perf", "main" }, }, 418 { "perf", "main" }, },
419 }, 419 },
420 }; 420 };
421 421
422 symbol_conf.use_callchain = true; 422 symbol_conf.use_callchain = true;
423 symbol_conf.cumulate_callchain = false; 423 symbol_conf.cumulate_callchain = false;
424 424
425 setup_sorting(); 425 setup_sorting();
426 callchain_register_param(&callchain_param); 426 callchain_register_param(&callchain_param);
427 427
428 err = add_hist_entries(hists, machine); 428 err = add_hist_entries(hists, machine);
429 if (err < 0) 429 if (err < 0)
430 goto out; 430 goto out;
431 431
432 err = do_test(hists, expected, ARRAY_SIZE(expected), 432 err = do_test(hists, expected, ARRAY_SIZE(expected),
433 expected_callchain, ARRAY_SIZE(expected_callchain)); 433 expected_callchain, ARRAY_SIZE(expected_callchain));
434 434
435 out: 435 out:
436 del_hist_entries(hists); 436 del_hist_entries(hists);
437 reset_output_field(); 437 reset_output_field();
438 return err; 438 return err;
439 } 439 }
440 440
441 /* NO callchain + children */ 441 /* NO callchain + children */
442 static int test3(struct perf_evsel *evsel, struct machine *machine) 442 static int test3(struct perf_evsel *evsel, struct machine *machine)
443 { 443 {
444 int err; 444 int err;
445 struct hists *hists = evsel__hists(evsel); 445 struct hists *hists = evsel__hists(evsel);
446 /* 446 /*
447 * expected output: 447 * expected output:
448 * 448 *
449 * Children Self Command Shared Object Symbol 449 * Children Self Command Shared Object Symbol
450 * ======== ======== ======= ============= ======================= 450 * ======== ======== ======= ============= =======================
451 * 70.00% 20.00% perf perf [.] main 451 * 70.00% 20.00% perf perf [.] main
452 * 50.00% 0.00% perf perf [.] run_command 452 * 50.00% 0.00% perf perf [.] run_command
453 * 30.00% 10.00% bash bash [.] main 453 * 30.00% 10.00% bash bash [.] main
454 * 30.00% 10.00% perf perf [.] cmd_record 454 * 30.00% 10.00% perf perf [.] cmd_record
455 * 20.00% 0.00% bash libc [.] malloc 455 * 20.00% 0.00% bash libc [.] malloc
456 * 10.00% 10.00% bash [kernel] [k] page_fault 456 * 10.00% 10.00% bash [kernel] [k] page_fault
457 * 10.00% 10.00% perf [kernel] [k] schedule 457 * 10.00% 10.00% bash bash [.] xmalloc
458 * 10.00% 0.00% perf [kernel] [k] sys_perf_event_open
459 * 10.00% 10.00% perf [kernel] [k] page_fault 458 * 10.00% 10.00% perf [kernel] [k] page_fault
460 * 10.00% 10.00% perf libc [.] free
461 * 10.00% 10.00% perf libc [.] malloc 459 * 10.00% 10.00% perf libc [.] malloc
462 * 10.00% 10.00% bash bash [.] xmalloc 460 * 10.00% 10.00% perf [kernel] [k] schedule
461 * 10.00% 10.00% perf libc [.] free
462 * 10.00% 0.00% perf [kernel] [k] sys_perf_event_open
463 */ 463 */
464 struct result expected[] = { 464 struct result expected[] = {
465 { 7000, 2000, "perf", "perf", "main" }, 465 { 7000, 2000, "perf", "perf", "main" },
466 { 5000, 0, "perf", "perf", "run_command" }, 466 { 5000, 0, "perf", "perf", "run_command" },
467 { 3000, 1000, "bash", "bash", "main" }, 467 { 3000, 1000, "bash", "bash", "main" },
468 { 3000, 1000, "perf", "perf", "cmd_record" }, 468 { 3000, 1000, "perf", "perf", "cmd_record" },
469 { 2000, 0, "bash", "libc", "malloc" }, 469 { 2000, 0, "bash", "libc", "malloc" },
470 { 1000, 1000, "bash", "[kernel]", "page_fault" }, 470 { 1000, 1000, "bash", "[kernel]", "page_fault" },
471 { 1000, 1000, "perf", "[kernel]", "schedule" }, 471 { 1000, 1000, "bash", "bash", "xmalloc" },
472 { 1000, 0, "perf", "[kernel]", "sys_perf_event_open" },
473 { 1000, 1000, "perf", "[kernel]", "page_fault" }, 472 { 1000, 1000, "perf", "[kernel]", "page_fault" },
473 { 1000, 1000, "perf", "[kernel]", "schedule" },
474 { 1000, 1000, "perf", "libc", "free" }, 474 { 1000, 1000, "perf", "libc", "free" },
475 { 1000, 1000, "perf", "libc", "malloc" }, 475 { 1000, 1000, "perf", "libc", "malloc" },
476 { 1000, 1000, "bash", "bash", "xmalloc" }, 476 { 1000, 0, "perf", "[kernel]", "sys_perf_event_open" },
477 }; 477 };
478 478
479 symbol_conf.use_callchain = false; 479 symbol_conf.use_callchain = false;
480 symbol_conf.cumulate_callchain = true; 480 symbol_conf.cumulate_callchain = true;
481 481
482 setup_sorting(); 482 setup_sorting();
483 callchain_register_param(&callchain_param); 483 callchain_register_param(&callchain_param);
484 484
485 err = add_hist_entries(hists, machine); 485 err = add_hist_entries(hists, machine);
486 if (err < 0) 486 if (err < 0)
487 goto out; 487 goto out;
488 488
489 err = do_test(hists, expected, ARRAY_SIZE(expected), NULL, 0); 489 err = do_test(hists, expected, ARRAY_SIZE(expected), NULL, 0);
490 490
491 out: 491 out:
492 del_hist_entries(hists); 492 del_hist_entries(hists);
493 reset_output_field(); 493 reset_output_field();
494 return err; 494 return err;
495 } 495 }
496 496
497 /* callchain + children */ 497 /* callchain + children */
498 static int test4(struct perf_evsel *evsel, struct machine *machine) 498 static int test4(struct perf_evsel *evsel, struct machine *machine)
499 { 499 {
500 int err; 500 int err;
501 struct hists *hists = evsel__hists(evsel); 501 struct hists *hists = evsel__hists(evsel);
502 /* 502 /*
503 * expected output: 503 * expected output:
504 * 504 *
505 * Children Self Command Shared Object Symbol 505 * Children Self Command Shared Object Symbol
506 * ======== ======== ======= ============= ======================= 506 * ======== ======== ======= ============= =======================
507 * 70.00% 20.00% perf perf [.] main 507 * 70.00% 20.00% perf perf [.] main
508 * | 508 * |
509 * --- main 509 * --- main
510 * 510 *
511 * 50.00% 0.00% perf perf [.] run_command 511 * 50.00% 0.00% perf perf [.] run_command
512 * | 512 * |
513 * --- run_command 513 * --- run_command
514 * main 514 * main
515 * 515 *
516 * 30.00% 10.00% bash bash [.] main 516 * 30.00% 10.00% bash bash [.] main
517 * | 517 * |
518 * --- main 518 * --- main
519 * 519 *
520 * 30.00% 10.00% perf perf [.] cmd_record 520 * 30.00% 10.00% perf perf [.] cmd_record
521 * | 521 * |
522 * --- cmd_record 522 * --- cmd_record
523 * run_command 523 * run_command
524 * main 524 * main
525 * 525 *
526 * 20.00% 0.00% bash libc [.] malloc 526 * 20.00% 0.00% bash libc [.] malloc
527 * | 527 * |
528 * --- malloc 528 * --- malloc
529 * | 529 * |
530 * |--50.00%-- xmalloc 530 * |--50.00%-- xmalloc
531 * | main 531 * | main
532 * --50.00%-- main 532 * --50.00%-- main
533 * 533 *
534 * 10.00% 10.00% bash [kernel] [k] page_fault 534 * 10.00% 10.00% bash [kernel] [k] page_fault
535 * | 535 * |
536 * --- page_fault 536 * --- page_fault
537 * malloc 537 * malloc
538 * main 538 * main
539 * 539 *
540 * 10.00% 10.00% perf [kernel] [k] schedule 540 * 10.00% 10.00% bash bash [.] xmalloc
541 * | 541 * |
542 * --- schedule 542 * --- xmalloc
543 * run_command 543 * malloc
544 * xmalloc <--- NOTE: there's a cycle
545 * malloc
546 * xmalloc
544 * main 547 * main
545 * 548 *
546 * 10.00% 0.00% perf [kernel] [k] sys_perf_event_open 549 * 10.00% 0.00% perf [kernel] [k] sys_perf_event_open
547 * | 550 * |
548 * --- sys_perf_event_open 551 * --- sys_perf_event_open
549 * run_command 552 * run_command
550 * main 553 * main
551 * 554 *
552 * 10.00% 10.00% perf [kernel] [k] page_fault 555 * 10.00% 10.00% perf [kernel] [k] page_fault
553 * | 556 * |
554 * --- page_fault 557 * --- page_fault
555 * sys_perf_event_open 558 * sys_perf_event_open
556 * run_command 559 * run_command
557 * main 560 * main
558 * 561 *
562 * 10.00% 10.00% perf [kernel] [k] schedule
563 * |
564 * --- schedule
565 * run_command
566 * main
567 *
559 * 10.00% 10.00% perf libc [.] free 568 * 10.00% 10.00% perf libc [.] free
560 * | 569 * |
561 * --- free 570 * --- free
562 * cmd_record 571 * cmd_record
563 * run_command 572 * run_command
564 * main 573 * main
565 * 574 *
566 * 10.00% 10.00% perf libc [.] malloc 575 * 10.00% 10.00% perf libc [.] malloc
567 * | 576 * |
568 * --- malloc 577 * --- malloc
569 * cmd_record 578 * cmd_record
570 * run_command 579 * run_command
571 * main 580 * main
572 * 581 *
573 * 10.00% 10.00% bash bash [.] xmalloc
574 * |
575 * --- xmalloc
576 * malloc
577 * xmalloc <--- NOTE: there's a cycle
578 * malloc
579 * xmalloc
580 * main
581 *
582 */ 582 */
583 struct result expected[] = { 583 struct result expected[] = {
584 { 7000, 2000, "perf", "perf", "main" }, 584 { 7000, 2000, "perf", "perf", "main" },
585 { 5000, 0, "perf", "perf", "run_command" }, 585 { 5000, 0, "perf", "perf", "run_command" },
586 { 3000, 1000, "bash", "bash", "main" }, 586 { 3000, 1000, "bash", "bash", "main" },
587 { 3000, 1000, "perf", "perf", "cmd_record" }, 587 { 3000, 1000, "perf", "perf", "cmd_record" },
588 { 2000, 0, "bash", "libc", "malloc" }, 588 { 2000, 0, "bash", "libc", "malloc" },
589 { 1000, 1000, "bash", "[kernel]", "page_fault" }, 589 { 1000, 1000, "bash", "[kernel]", "page_fault" },
590 { 1000, 1000, "perf", "[kernel]", "schedule" }, 590 { 1000, 1000, "bash", "bash", "xmalloc" },
591 { 1000, 0, "perf", "[kernel]", "sys_perf_event_open" }, 591 { 1000, 0, "perf", "[kernel]", "sys_perf_event_open" },
592 { 1000, 1000, "perf", "[kernel]", "page_fault" }, 592 { 1000, 1000, "perf", "[kernel]", "page_fault" },
593 { 1000, 1000, "perf", "[kernel]", "schedule" },
593 { 1000, 1000, "perf", "libc", "free" }, 594 { 1000, 1000, "perf", "libc", "free" },
594 { 1000, 1000, "perf", "libc", "malloc" }, 595 { 1000, 1000, "perf", "libc", "malloc" },
595 { 1000, 1000, "bash", "bash", "xmalloc" },
596 }; 596 };
597 struct callchain_result expected_callchain[] = { 597 struct callchain_result expected_callchain[] = {
598 { 598 {
599 1, { { "perf", "main" }, }, 599 1, { { "perf", "main" }, },
600 }, 600 },
601 { 601 {
602 2, { { "perf", "run_command" }, 602 2, { { "perf", "run_command" },
603 { "perf", "main" }, }, 603 { "perf", "main" }, },
604 }, 604 },
605 { 605 {
606 1, { { "bash", "main" }, }, 606 1, { { "bash", "main" }, },
607 }, 607 },
608 { 608 {
609 3, { { "perf", "cmd_record" }, 609 3, { { "perf", "cmd_record" },
610 { "perf", "run_command" }, 610 { "perf", "run_command" },
611 { "perf", "main" }, }, 611 { "perf", "main" }, },
612 }, 612 },
613 { 613 {
614 4, { { "libc", "malloc" }, 614 4, { { "libc", "malloc" },
615 { "bash", "xmalloc" }, 615 { "bash", "xmalloc" },
616 { "bash", "main" }, 616 { "bash", "main" },
617 { "bash", "main" }, }, 617 { "bash", "main" }, },
618 }, 618 },
619 { 619 {
620 3, { { "[kernel]", "page_fault" }, 620 3, { { "[kernel]", "page_fault" },
621 { "libc", "malloc" }, 621 { "libc", "malloc" },
622 { "bash", "main" }, }, 622 { "bash", "main" }, },
623 }, 623 },
624 { 624 {
625 3, { { "[kernel]", "schedule" }, 625 6, { { "bash", "xmalloc" },
626 { "perf", "run_command" }, 626 { "libc", "malloc" },
627 { "perf", "main" }, }, 627 { "bash", "xmalloc" },
628 { "libc", "malloc" },
629 { "bash", "xmalloc" },
630 { "bash", "main" }, },
628 }, 631 },
629 { 632 {
630 3, { { "[kernel]", "sys_perf_event_open" }, 633 3, { { "[kernel]", "sys_perf_event_open" },
631 { "perf", "run_command" }, 634 { "perf", "run_command" },
632 { "perf", "main" }, }, 635 { "perf", "main" }, },
633 }, 636 },
634 { 637 {
635 4, { { "[kernel]", "page_fault" }, 638 4, { { "[kernel]", "page_fault" },
636 { "[kernel]", "sys_perf_event_open" }, 639 { "[kernel]", "sys_perf_event_open" },
637 { "perf", "run_command" }, 640 { "perf", "run_command" },
638 { "perf", "main" }, }, 641 { "perf", "main" }, },
639 }, 642 },
640 { 643 {
644 3, { { "[kernel]", "schedule" },
645 { "perf", "run_command" },
646 { "perf", "main" }, },
647 },
648 {
641 4, { { "libc", "free" }, 649 4, { { "libc", "free" },
642 { "perf", "cmd_record" }, 650 { "perf", "cmd_record" },
643 { "perf", "run_command" }, 651 { "perf", "run_command" },
644 { "perf", "main" }, }, 652 { "perf", "main" }, },
645 }, 653 },
646 { 654 {
647 4, { { "libc", "malloc" }, 655 4, { { "libc", "malloc" },
648 { "perf", "cmd_record" }, 656 { "perf", "cmd_record" },
649 { "perf", "run_command" }, 657 { "perf", "run_command" },
650 { "perf", "main" }, }, 658 { "perf", "main" }, },
651 },
652 {
653 6, { { "bash", "xmalloc" },
654 { "libc", "malloc" },
655 { "bash", "xmalloc" },
656 { "libc", "malloc" },
657 { "bash", "xmalloc" },
658 { "bash", "main" }, },
659 }, 659 },
660 }; 660 };
661 661
662 symbol_conf.use_callchain = true; 662 symbol_conf.use_callchain = true;
663 symbol_conf.cumulate_callchain = true; 663 symbol_conf.cumulate_callchain = true;
664 664
665 setup_sorting(); 665 setup_sorting();
666 callchain_register_param(&callchain_param); 666 callchain_register_param(&callchain_param);
667 667
668 err = add_hist_entries(hists, machine); 668 err = add_hist_entries(hists, machine);
669 if (err < 0) 669 if (err < 0)
670 goto out; 670 goto out;
671 671
672 err = do_test(hists, expected, ARRAY_SIZE(expected), 672 err = do_test(hists, expected, ARRAY_SIZE(expected),
673 expected_callchain, ARRAY_SIZE(expected_callchain)); 673 expected_callchain, ARRAY_SIZE(expected_callchain));
674 674
675 out: 675 out:
676 del_hist_entries(hists); 676 del_hist_entries(hists);
677 reset_output_field(); 677 reset_output_field();
678 return err; 678 return err;
679 } 679 }
680 680
681 int test__hists_cumulate(void) 681 int test__hists_cumulate(void)
682 { 682 {
683 int err = TEST_FAIL; 683 int err = TEST_FAIL;
684 struct machines machines; 684 struct machines machines;
685 struct machine *machine; 685 struct machine *machine;
686 struct perf_evsel *evsel; 686 struct perf_evsel *evsel;
687 struct perf_evlist *evlist = perf_evlist__new(); 687 struct perf_evlist *evlist = perf_evlist__new();
688 size_t i; 688 size_t i;
689 test_fn_t testcases[] = { 689 test_fn_t testcases[] = {
690 test1, 690 test1,
691 test2, 691 test2,
692 test3, 692 test3,
693 test4, 693 test4,
694 }; 694 };
695 695
696 TEST_ASSERT_VAL("No memory", evlist); 696 TEST_ASSERT_VAL("No memory", evlist);
697 697
698 err = parse_events(evlist, "cpu-clock"); 698 err = parse_events(evlist, "cpu-clock");
699 if (err) 699 if (err)
700 goto out; 700 goto out;
701 701
702 machines__init(&machines); 702 machines__init(&machines);
703 703
704 /* setup threads/dso/map/symbols also */ 704 /* setup threads/dso/map/symbols also */
705 machine = setup_fake_machine(&machines); 705 machine = setup_fake_machine(&machines);
706 if (!machine) 706 if (!machine)
tools/perf/tests/hists_filter.c
Diff comments   View file @ ddb321a
1 #include "perf.h" 1 #include "perf.h"
2 #include "util/debug.h" 2 #include "util/debug.h"
3 #include "util/symbol.h" 3 #include "util/symbol.h"
4 #include "util/sort.h" 4 #include "util/sort.h"
5 #include "util/evsel.h" 5 #include "util/evsel.h"
6 #include "util/evlist.h" 6 #include "util/evlist.h"
7 #include "util/machine.h" 7 #include "util/machine.h"
8 #include "util/thread.h" 8 #include "util/thread.h"
9 #include "util/parse-events.h" 9 #include "util/parse-events.h"
10 #include "tests/tests.h" 10 #include "tests/tests.h"
11 #include "tests/hists_common.h" 11 #include "tests/hists_common.h"
12 12
13 struct sample { 13 struct sample {
14 u32 pid; 14 u32 pid;
15 u64 ip; 15 u64 ip;
16 struct thread *thread; 16 struct thread *thread;
17 struct map *map; 17 struct map *map;
18 struct symbol *sym; 18 struct symbol *sym;
19 }; 19 };
20 20
21 /* For the numbers, see hists_common.c */ 21 /* For the numbers, see hists_common.c */
22 static struct sample fake_samples[] = { 22 static struct sample fake_samples[] = {
23 /* perf [kernel] schedule() */ 23 /* perf [kernel] schedule() */
24 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_KERNEL_SCHEDULE, }, 24 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_KERNEL_SCHEDULE, },
25 /* perf [perf] main() */ 25 /* perf [perf] main() */
26 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_MAIN, }, 26 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_MAIN, },
27 /* perf [libc] malloc() */ 27 /* perf [libc] malloc() */
28 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_MALLOC, }, 28 { .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_MALLOC, },
29 /* perf [perf] main() */ 29 /* perf [perf] main() */
30 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_MAIN, }, /* will be merged */ 30 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_MAIN, }, /* will be merged */
31 /* perf [perf] cmd_record() */ 31 /* perf [perf] cmd_record() */
32 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_CMD_RECORD, }, 32 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_CMD_RECORD, },
33 /* perf [kernel] page_fault() */ 33 /* perf [kernel] page_fault() */
34 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_KERNEL_PAGE_FAULT, }, 34 { .pid = FAKE_PID_PERF2, .ip = FAKE_IP_KERNEL_PAGE_FAULT, },
35 /* bash [bash] main() */ 35 /* bash [bash] main() */
36 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_MAIN, }, 36 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_MAIN, },
37 /* bash [bash] xmalloc() */ 37 /* bash [bash] xmalloc() */
38 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_XMALLOC, }, 38 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_XMALLOC, },
39 /* bash [libc] malloc() */ 39 /* bash [libc] malloc() */
40 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_LIBC_MALLOC, }, 40 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_LIBC_MALLOC, },
41 /* bash [kernel] page_fault() */ 41 /* bash [kernel] page_fault() */
42 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_KERNEL_PAGE_FAULT, }, 42 { .pid = FAKE_PID_BASH, .ip = FAKE_IP_KERNEL_PAGE_FAULT, },
43 }; 43 };
44 44
45 static int add_hist_entries(struct perf_evlist *evlist, 45 static int add_hist_entries(struct perf_evlist *evlist,
46 struct machine *machine) 46 struct machine *machine)
47 { 47 {
48 struct perf_evsel *evsel; 48 struct perf_evsel *evsel;
49 struct addr_location al; 49 struct addr_location al;
50 struct perf_sample sample = { .period = 100, }; 50 struct perf_sample sample = { .period = 100, };
51 size_t i; 51 size_t i;
52 52
53 /* 53 /*
54 * each evsel will have 10 samples but the 4th sample 54 * each evsel will have 10 samples but the 4th sample
55 * (perf [perf] main) will be collapsed to an existing entry 55 * (perf [perf] main) will be collapsed to an existing entry
56 * so total 9 entries will be in the tree. 56 * so total 9 entries will be in the tree.
57 */ 57 */
58 evlist__for_each(evlist, evsel) { 58 evlist__for_each(evlist, evsel) {
59 for (i = 0; i < ARRAY_SIZE(fake_samples); i++) { 59 for (i = 0; i < ARRAY_SIZE(fake_samples); i++) {
60 const union perf_event event = { 60 const union perf_event event = {
61 .header = { 61 .header = {
62 .misc = PERF_RECORD_MISC_USER, 62 .misc = PERF_RECORD_MISC_USER,
63 }, 63 },
64 }; 64 };
65 struct hist_entry_iter iter = { 65 struct hist_entry_iter iter = {
66 .ops = &hist_iter_normal, 66 .ops = &hist_iter_normal,
67 .hide_unresolved = false, 67 .hide_unresolved = false,
68 }; 68 };
69 struct hists *hists = evsel__hists(evsel); 69 struct hists *hists = evsel__hists(evsel);
70 70
71 /* make sure it has no filter at first */ 71 /* make sure it has no filter at first */
72 hists->thread_filter = NULL; 72 hists->thread_filter = NULL;
73 hists->dso_filter = NULL; 73 hists->dso_filter = NULL;
74 hists->symbol_filter_str = NULL; 74 hists->symbol_filter_str = NULL;
75 75
76 sample.pid = fake_samples[i].pid; 76 sample.pid = fake_samples[i].pid;
77 sample.tid = fake_samples[i].pid; 77 sample.tid = fake_samples[i].pid;
78 sample.ip = fake_samples[i].ip; 78 sample.ip = fake_samples[i].ip;
79 79
80 if (perf_event__preprocess_sample(&event, machine, &al, 80 if (perf_event__preprocess_sample(&event, machine, &al,
81 &sample) < 0) 81 &sample) < 0)
82 goto out; 82 goto out;
83 83
84 if (hist_entry_iter__add(&iter, &al, evsel, &sample, 84 if (hist_entry_iter__add(&iter, &al, evsel, &sample,
85 PERF_MAX_STACK_DEPTH, NULL) < 0) 85 PERF_MAX_STACK_DEPTH, NULL) < 0)
86 goto out; 86 goto out;
87 87
88 fake_samples[i].thread = al.thread; 88 fake_samples[i].thread = al.thread;
89 fake_samples[i].map = al.map; 89 fake_samples[i].map = al.map;
90 fake_samples[i].sym = al.sym; 90 fake_samples[i].sym = al.sym;
91 } 91 }
92 } 92 }
93 93
94 return 0; 94 return 0;
95 95
96 out: 96 out:
97 pr_debug("Not enough memory for adding a hist entry\n"); 97 pr_debug("Not enough memory for adding a hist entry\n");
98 return TEST_FAIL; 98 return TEST_FAIL;
99 } 99 }
100 100
101 int test__hists_filter(void) 101 int test__hists_filter(void)
102 { 102 {
103 int err = TEST_FAIL; 103 int err = TEST_FAIL;
104 struct machines machines; 104 struct machines machines;
105 struct machine *machine; 105 struct machine *machine;
106 struct perf_evsel *evsel; 106 struct perf_evsel *evsel;
107 struct perf_evlist *evlist = perf_evlist__new(); 107 struct perf_evlist *evlist = perf_evlist__new();
108 108
109 TEST_ASSERT_VAL("No memory", evlist); 109 TEST_ASSERT_VAL("No memory", evlist);
110 110
111 err = parse_events(evlist, "cpu-clock"); 111 err = parse_events(evlist, "cpu-clock");
112 if (err) 112 if (err)
113 goto out; 113 goto out;
114 err = parse_events(evlist, "task-clock"); 114 err = parse_events(evlist, "task-clock");
115 if (err) 115 if (err)
116 goto out; 116 goto out;
117 117
118 /* default sort order (comm,dso,sym) will be used */ 118 /* default sort order (comm,dso,sym) will be used */
119 if (setup_sorting() < 0) 119 if (setup_sorting() < 0)
120 goto out; 120 goto out;
121 121
122 machines__init(&machines); 122 machines__init(&machines);
123 123
124 /* setup threads/dso/map/symbols also */ 124 /* setup threads/dso/map/symbols also */
125 machine = setup_fake_machine(&machines); 125 machine = setup_fake_machine(&machines);
126 if (!machine) 126 if (!machine)
127 goto out; 127 goto out;
128 128
129 if (verbose > 1) 129 if (verbose > 1)
130 machine__fprintf(machine, stderr); 130 machine__fprintf(machine, stderr);
131 131
132 /* process sample events */ 132 /* process sample events */
133 err = add_hist_entries(evlist, machine); 133 err = add_hist_entries(evlist, machine);
134 if (err < 0) 134 if (err < 0)
135 goto out; 135 goto out;
136 136
137 evlist__for_each(evlist, evsel) { 137 evlist__for_each(evlist, evsel) {
138 struct hists *hists = evsel__hists(evsel); 138 struct hists *hists = evsel__hists(evsel);
139 139
140 hists__collapse_resort(hists, NULL); 140 hists__collapse_resort(hists, NULL);
141 hists__output_resort(hists); 141 hists__output_resort(hists, NULL);
142 142
143 if (verbose > 2) { 143 if (verbose > 2) {
144 pr_info("Normal histogram\n"); 144 pr_info("Normal histogram\n");
145 print_hists_out(hists); 145 print_hists_out(hists);
146 } 146 }
147 147
148 TEST_ASSERT_VAL("Invalid nr samples", 148 TEST_ASSERT_VAL("Invalid nr samples",
149 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10); 149 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10);
150 TEST_ASSERT_VAL("Invalid nr hist entries", 150 TEST_ASSERT_VAL("Invalid nr hist entries",
151 hists->nr_entries == 9); 151 hists->nr_entries == 9);
152 TEST_ASSERT_VAL("Invalid total period", 152 TEST_ASSERT_VAL("Invalid total period",
153 hists->stats.total_period == 1000); 153 hists->stats.total_period == 1000);
154 TEST_ASSERT_VAL("Unmatched nr samples", 154 TEST_ASSERT_VAL("Unmatched nr samples",
155 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 155 hists->stats.nr_events[PERF_RECORD_SAMPLE] ==
156 hists->stats.nr_non_filtered_samples); 156 hists->stats.nr_non_filtered_samples);
157 TEST_ASSERT_VAL("Unmatched nr hist entries", 157 TEST_ASSERT_VAL("Unmatched nr hist entries",
158 hists->nr_entries == hists->nr_non_filtered_entries); 158 hists->nr_entries == hists->nr_non_filtered_entries);
159 TEST_ASSERT_VAL("Unmatched total period", 159 TEST_ASSERT_VAL("Unmatched total period",
160 hists->stats.total_period == 160 hists->stats.total_period ==
161 hists->stats.total_non_filtered_period); 161 hists->stats.total_non_filtered_period);
162 162
163 /* now applying thread filter for 'bash' */ 163 /* now applying thread filter for 'bash' */
164 hists->thread_filter = fake_samples[9].thread; 164 hists->thread_filter = fake_samples[9].thread;
165 hists__filter_by_thread(hists); 165 hists__filter_by_thread(hists);
166 166
167 if (verbose > 2) { 167 if (verbose > 2) {
168 pr_info("Histogram for thread filter\n"); 168 pr_info("Histogram for thread filter\n");
169 print_hists_out(hists); 169 print_hists_out(hists);
170 } 170 }
171 171
172 /* normal stats should be invariant */ 172 /* normal stats should be invariant */
173 TEST_ASSERT_VAL("Invalid nr samples", 173 TEST_ASSERT_VAL("Invalid nr samples",
174 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10); 174 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10);
175 TEST_ASSERT_VAL("Invalid nr hist entries", 175 TEST_ASSERT_VAL("Invalid nr hist entries",
176 hists->nr_entries == 9); 176 hists->nr_entries == 9);
177 TEST_ASSERT_VAL("Invalid total period", 177 TEST_ASSERT_VAL("Invalid total period",
178 hists->stats.total_period == 1000); 178 hists->stats.total_period == 1000);
179 179
180 /* but filter stats are changed */ 180 /* but filter stats are changed */
181 TEST_ASSERT_VAL("Unmatched nr samples for thread filter", 181 TEST_ASSERT_VAL("Unmatched nr samples for thread filter",
182 hists->stats.nr_non_filtered_samples == 4); 182 hists->stats.nr_non_filtered_samples == 4);
183 TEST_ASSERT_VAL("Unmatched nr hist entries for thread filter", 183 TEST_ASSERT_VAL("Unmatched nr hist entries for thread filter",
184 hists->nr_non_filtered_entries == 4); 184 hists->nr_non_filtered_entries == 4);
185 TEST_ASSERT_VAL("Unmatched total period for thread filter", 185 TEST_ASSERT_VAL("Unmatched total period for thread filter",
186 hists->stats.total_non_filtered_period == 400); 186 hists->stats.total_non_filtered_period == 400);
187 187
188 /* remove thread filter first */ 188 /* remove thread filter first */
189 hists->thread_filter = NULL; 189 hists->thread_filter = NULL;
190 hists__filter_by_thread(hists); 190 hists__filter_by_thread(hists);
191 191
192 /* now applying dso filter for 'kernel' */ 192 /* now applying dso filter for 'kernel' */
193 hists->dso_filter = fake_samples[0].map->dso; 193 hists->dso_filter = fake_samples[0].map->dso;
194 hists__filter_by_dso(hists); 194 hists__filter_by_dso(hists);
195 195
196 if (verbose > 2) { 196 if (verbose > 2) {
197 pr_info("Histogram for dso filter\n"); 197 pr_info("Histogram for dso filter\n");
198 print_hists_out(hists); 198 print_hists_out(hists);
199 } 199 }
200 200
201 /* normal stats should be invariant */ 201 /* normal stats should be invariant */
202 TEST_ASSERT_VAL("Invalid nr samples", 202 TEST_ASSERT_VAL("Invalid nr samples",
203 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10); 203 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10);
204 TEST_ASSERT_VAL("Invalid nr hist entries", 204 TEST_ASSERT_VAL("Invalid nr hist entries",
205 hists->nr_entries == 9); 205 hists->nr_entries == 9);
206 TEST_ASSERT_VAL("Invalid total period", 206 TEST_ASSERT_VAL("Invalid total period",
207 hists->stats.total_period == 1000); 207 hists->stats.total_period == 1000);
208 208
209 /* but filter stats are changed */ 209 /* but filter stats are changed */
210 TEST_ASSERT_VAL("Unmatched nr samples for dso filter", 210 TEST_ASSERT_VAL("Unmatched nr samples for dso filter",
211 hists->stats.nr_non_filtered_samples == 3); 211 hists->stats.nr_non_filtered_samples == 3);
212 TEST_ASSERT_VAL("Unmatched nr hist entries for dso filter", 212 TEST_ASSERT_VAL("Unmatched nr hist entries for dso filter",
213 hists->nr_non_filtered_entries == 3); 213 hists->nr_non_filtered_entries == 3);
214 TEST_ASSERT_VAL("Unmatched total period for dso filter", 214 TEST_ASSERT_VAL("Unmatched total period for dso filter",
215 hists->stats.total_non_filtered_period == 300); 215 hists->stats.total_non_filtered_period == 300);
216 216
217 /* remove dso filter first */ 217 /* remove dso filter first */
218 hists->dso_filter = NULL; 218 hists->dso_filter = NULL;
219 hists__filter_by_dso(hists); 219 hists__filter_by_dso(hists);
220 220
221 /* 221 /*
222 * now applying symbol filter for 'main'. Also note that 222 * now applying symbol filter for 'main'. Also note that
223 * there's 3 samples that have 'main' symbol but the 4th 223 * there's 3 samples that have 'main' symbol but the 4th
224 * entry of fake_samples was collapsed already so it won't 224 * entry of fake_samples was collapsed already so it won't
225 * be counted as a separate entry but the sample count and 225 * be counted as a separate entry but the sample count and
226 * total period will be remained. 226 * total period will be remained.
227 */ 227 */
228 hists->symbol_filter_str = "main"; 228 hists->symbol_filter_str = "main";
229 hists__filter_by_symbol(hists); 229 hists__filter_by_symbol(hists);
230 230
231 if (verbose > 2) { 231 if (verbose > 2) {
232 pr_info("Histogram for symbol filter\n"); 232 pr_info("Histogram for symbol filter\n");
233 print_hists_out(hists); 233 print_hists_out(hists);
234 } 234 }
235 235
236 /* normal stats should be invariant */ 236 /* normal stats should be invariant */
237 TEST_ASSERT_VAL("Invalid nr samples", 237 TEST_ASSERT_VAL("Invalid nr samples",
238 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10); 238 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10);
239 TEST_ASSERT_VAL("Invalid nr hist entries", 239 TEST_ASSERT_VAL("Invalid nr hist entries",
240 hists->nr_entries == 9); 240 hists->nr_entries == 9);
241 TEST_ASSERT_VAL("Invalid total period", 241 TEST_ASSERT_VAL("Invalid total period",
242 hists->stats.total_period == 1000); 242 hists->stats.total_period == 1000);
243 243
244 /* but filter stats are changed */ 244 /* but filter stats are changed */
245 TEST_ASSERT_VAL("Unmatched nr samples for symbol filter", 245 TEST_ASSERT_VAL("Unmatched nr samples for symbol filter",
246 hists->stats.nr_non_filtered_samples == 3); 246 hists->stats.nr_non_filtered_samples == 3);
247 TEST_ASSERT_VAL("Unmatched nr hist entries for symbol filter", 247 TEST_ASSERT_VAL("Unmatched nr hist entries for symbol filter",
248 hists->nr_non_filtered_entries == 2); 248 hists->nr_non_filtered_entries == 2);
249 TEST_ASSERT_VAL("Unmatched total period for symbol filter", 249 TEST_ASSERT_VAL("Unmatched total period for symbol filter",
250 hists->stats.total_non_filtered_period == 300); 250 hists->stats.total_non_filtered_period == 300);
251 251
252 /* now applying all filters at once. */ 252 /* now applying all filters at once. */
253 hists->thread_filter = fake_samples[1].thread; 253 hists->thread_filter = fake_samples[1].thread;
254 hists->dso_filter = fake_samples[1].map->dso; 254 hists->dso_filter = fake_samples[1].map->dso;
255 hists__filter_by_thread(hists); 255 hists__filter_by_thread(hists);
256 hists__filter_by_dso(hists); 256 hists__filter_by_dso(hists);
257 257
258 if (verbose > 2) { 258 if (verbose > 2) {
259 pr_info("Histogram for all filters\n"); 259 pr_info("Histogram for all filters\n");
260 print_hists_out(hists); 260 print_hists_out(hists);
261 } 261 }
262 262
263 /* normal stats should be invariant */ 263 /* normal stats should be invariant */
264 TEST_ASSERT_VAL("Invalid nr samples", 264 TEST_ASSERT_VAL("Invalid nr samples",
265 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10); 265 hists->stats.nr_events[PERF_RECORD_SAMPLE] == 10);
266 TEST_ASSERT_VAL("Invalid nr hist entries", 266 TEST_ASSERT_VAL("Invalid nr hist entries",
267 hists->nr_entries == 9); 267 hists->nr_entries == 9);
268 TEST_ASSERT_VAL("Invalid total period", 268 TEST_ASSERT_VAL("Invalid total period",
269 hists->stats.total_period == 1000); 269 hists->stats.total_period == 1000);
270 270
271 /* but filter stats are changed */ 271 /* but filter stats are changed */
272 TEST_ASSERT_VAL("Unmatched nr samples for all filter", 272 TEST_ASSERT_VAL("Unmatched nr samples for all filter",
273 hists->stats.nr_non_filtered_samples == 2); 273 hists->stats.nr_non_filtered_samples == 2);
274 TEST_ASSERT_VAL("Unmatched nr hist entries for all filter", 274 TEST_ASSERT_VAL("Unmatched nr hist entries for all filter",
275 hists->nr_non_filtered_entries == 1); 275 hists->nr_non_filtered_entries == 1);
276 TEST_ASSERT_VAL("Unmatched total period for all filter", 276 TEST_ASSERT_VAL("Unmatched total period for all filter",
277 hists->stats.total_non_filtered_period == 200); 277 hists->stats.total_non_filtered_period == 200);
278 } 278 }
279 279
280 280
281 err = TEST_OK; 281 err = TEST_OK;
282 282
283 out: 283 out:
284 /* tear down everything */ 284 /* tear down everything */
285 perf_evlist__delete(evlist); 285 perf_evlist__delete(evlist);
286 reset_output_field(); 286 reset_output_field();
287 machines__exit(&machines); 287 machines__exit(&machines);
288 288
289 return err; 289 return err;
290 } 290 }
291 291
tools/perf/tests/hists_output.c
Diff comments   View file @ ddb321a
1 #include "perf.h" 1 #include "perf.h"
2 #include "util/debug.h" 2 #include "util/debug.h"
3 #include "util/symbol.h" 3 #include "util/symbol.h"
4 #include "util/sort.h" 4 #include "util/sort.h"
5 #include "util/evsel.h" 5 #include "util/evsel.h"
6 #include "util/evlist.h" 6 #include "util/evlist.h"
7 #include "util/machine.h" 7 #include "util/machine.h"
8 #include "util/thread.h" 8 #include "util/thread.h"
9 #include "util/parse-events.h" 9 #include "util/parse-events.h"
10 #include "tests/tests.h" 10 #include "tests/tests.h"
11 #include "tests/hists_common.h" 11 #include "tests/hists_common.h"
12 12
13 struct sample { 13 struct sample {
14 u32 cpu; 14 u32 cpu;
15 u32 pid; 15 u32 pid;
16 u64 ip; 16 u64 ip;
17 struct thread *thread; 17 struct thread *thread;
18 struct map *map; 18 struct map *map;
19 struct symbol *sym; 19 struct symbol *sym;
20 }; 20 };
21 21
22 /* For the numbers, see hists_common.c */ 22 /* For the numbers, see hists_common.c */
23 static struct sample fake_samples[] = { 23 static struct sample fake_samples[] = {
24 /* perf [kernel] schedule() */ 24 /* perf [kernel] schedule() */
25 { .cpu = 0, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_KERNEL_SCHEDULE, }, 25 { .cpu = 0, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_KERNEL_SCHEDULE, },
26 /* perf [perf] main() */ 26 /* perf [perf] main() */
27 { .cpu = 1, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_MAIN, }, 27 { .cpu = 1, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_MAIN, },
28 /* perf [perf] cmd_record() */ 28 /* perf [perf] cmd_record() */
29 { .cpu = 1, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_CMD_RECORD, }, 29 { .cpu = 1, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_PERF_CMD_RECORD, },
30 /* perf [libc] malloc() */ 30 /* perf [libc] malloc() */
31 { .cpu = 1, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_MALLOC, }, 31 { .cpu = 1, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_MALLOC, },
32 /* perf [libc] free() */ 32 /* perf [libc] free() */
33 { .cpu = 2, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_FREE, }, 33 { .cpu = 2, .pid = FAKE_PID_PERF1, .ip = FAKE_IP_LIBC_FREE, },
34 /* perf [perf] main() */ 34 /* perf [perf] main() */
35 { .cpu = 2, .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_MAIN, }, 35 { .cpu = 2, .pid = FAKE_PID_PERF2, .ip = FAKE_IP_PERF_MAIN, },
36 /* perf [kernel] page_fault() */ 36 /* perf [kernel] page_fault() */
37 { .cpu = 2, .pid = FAKE_PID_PERF2, .ip = FAKE_IP_KERNEL_PAGE_FAULT, }, 37 { .cpu = 2, .pid = FAKE_PID_PERF2, .ip = FAKE_IP_KERNEL_PAGE_FAULT, },
38 /* bash [bash] main() */ 38 /* bash [bash] main() */
39 { .cpu = 3, .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_MAIN, }, 39 { .cpu = 3, .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_MAIN, },
40 /* bash [bash] xmalloc() */ 40 /* bash [bash] xmalloc() */
41 { .cpu = 0, .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_XMALLOC, }, 41 { .cpu = 0, .pid = FAKE_PID_BASH, .ip = FAKE_IP_BASH_XMALLOC, },
42 /* bash [kernel] page_fault() */ 42 /* bash [kernel] page_fault() */
43 { .cpu = 1, .pid = FAKE_PID_BASH, .ip = FAKE_IP_KERNEL_PAGE_FAULT, }, 43 { .cpu = 1, .pid = FAKE_PID_BASH, .ip = FAKE_IP_KERNEL_PAGE_FAULT, },
44 }; 44 };
45 45
46 static int add_hist_entries(struct hists *hists, struct machine *machine) 46 static int add_hist_entries(struct hists *hists, struct machine *machine)
47 { 47 {
48 struct addr_location al; 48 struct addr_location al;
49 struct perf_evsel *evsel = hists_to_evsel(hists); 49 struct perf_evsel *evsel = hists_to_evsel(hists);
50 struct perf_sample sample = { .period = 100, }; 50 struct perf_sample sample = { .period = 100, };
51 size_t i; 51 size_t i;
52 52
53 for (i = 0; i < ARRAY_SIZE(fake_samples); i++) { 53 for (i = 0; i < ARRAY_SIZE(fake_samples); i++) {
54 const union perf_event event = { 54 const union perf_event event = {
55 .header = { 55 .header = {
56 .misc = PERF_RECORD_MISC_USER, 56 .misc = PERF_RECORD_MISC_USER,
57 }, 57 },
58 }; 58 };
59 struct hist_entry_iter iter = { 59 struct hist_entry_iter iter = {
60 .ops = &hist_iter_normal, 60 .ops = &hist_iter_normal,
61 .hide_unresolved = false, 61 .hide_unresolved = false,
62 }; 62 };
63 63
64 sample.cpu = fake_samples[i].cpu; 64 sample.cpu = fake_samples[i].cpu;
65 sample.pid = fake_samples[i].pid; 65 sample.pid = fake_samples[i].pid;
66 sample.tid = fake_samples[i].pid; 66 sample.tid = fake_samples[i].pid;
67 sample.ip = fake_samples[i].ip; 67 sample.ip = fake_samples[i].ip;
68 68
69 if (perf_event__preprocess_sample(&event, machine, &al, 69 if (perf_event__preprocess_sample(&event, machine, &al,
70 &sample) < 0) 70 &sample) < 0)
71 goto out; 71 goto out;
72 72
73 if (hist_entry_iter__add(&iter, &al, evsel, &sample, 73 if (hist_entry_iter__add(&iter, &al, evsel, &sample,
74 PERF_MAX_STACK_DEPTH, NULL) < 0) 74 PERF_MAX_STACK_DEPTH, NULL) < 0)
75 goto out; 75 goto out;
76 76
77 fake_samples[i].thread = al.thread; 77 fake_samples[i].thread = al.thread;
78 fake_samples[i].map = al.map; 78 fake_samples[i].map = al.map;
79 fake_samples[i].sym = al.sym; 79 fake_samples[i].sym = al.sym;
80 } 80 }
81 81
82 return TEST_OK; 82 return TEST_OK;
83 83
84 out: 84 out:
85 pr_debug("Not enough memory for adding a hist entry\n"); 85 pr_debug("Not enough memory for adding a hist entry\n");
86 return TEST_FAIL; 86 return TEST_FAIL;
87 } 87 }
88 88
89 static void del_hist_entries(struct hists *hists) 89 static void del_hist_entries(struct hists *hists)
90 { 90 {
91 struct hist_entry *he; 91 struct hist_entry *he;
92 struct rb_root *root_in; 92 struct rb_root *root_in;
93 struct rb_root *root_out; 93 struct rb_root *root_out;
94 struct rb_node *node; 94 struct rb_node *node;
95 95
96 if (sort__need_collapse) 96 if (sort__need_collapse)
97 root_in = &hists->entries_collapsed; 97 root_in = &hists->entries_collapsed;
98 else 98 else
99 root_in = hists->entries_in; 99 root_in = hists->entries_in;
100 100
101 root_out = &hists->entries; 101 root_out = &hists->entries;
102 102
103 while (!RB_EMPTY_ROOT(root_out)) { 103 while (!RB_EMPTY_ROOT(root_out)) {
104 node = rb_first(root_out); 104 node = rb_first(root_out);
105 105
106 he = rb_entry(node, struct hist_entry, rb_node); 106 he = rb_entry(node, struct hist_entry, rb_node);
107 rb_erase(node, root_out); 107 rb_erase(node, root_out);
108 rb_erase(&he->rb_node_in, root_in); 108 rb_erase(&he->rb_node_in, root_in);
109 hist_entry__free(he); 109 hist_entry__free(he);
110 } 110 }
111 } 111 }
112 112
113 typedef int (*test_fn_t)(struct perf_evsel *, struct machine *); 113 typedef int (*test_fn_t)(struct perf_evsel *, struct machine *);
114 114
115 #define COMM(he) (thread__comm_str(he->thread)) 115 #define COMM(he) (thread__comm_str(he->thread))
116 #define DSO(he) (he->ms.map->dso->short_name) 116 #define DSO(he) (he->ms.map->dso->short_name)
117 #define SYM(he) (he->ms.sym->name) 117 #define SYM(he) (he->ms.sym->name)
118 #define CPU(he) (he->cpu) 118 #define CPU(he) (he->cpu)
119 #define PID(he) (he->thread->tid) 119 #define PID(he) (he->thread->tid)
120 120
121 /* default sort keys (no field) */ 121 /* default sort keys (no field) */
122 static int test1(struct perf_evsel *evsel, struct machine *machine) 122 static int test1(struct perf_evsel *evsel, struct machine *machine)
123 { 123 {
124 int err; 124 int err;
125 struct hists *hists = evsel__hists(evsel); 125 struct hists *hists = evsel__hists(evsel);
126 struct hist_entry *he; 126 struct hist_entry *he;
127 struct rb_root *root; 127 struct rb_root *root;
128 struct rb_node *node; 128 struct rb_node *node;
129 129
130 field_order = NULL; 130 field_order = NULL;
131 sort_order = NULL; /* equivalent to sort_order = "comm,dso,sym" */ 131 sort_order = NULL; /* equivalent to sort_order = "comm,dso,sym" */
132 132
133 setup_sorting(); 133 setup_sorting();
134 134
135 /* 135 /*
136 * expected output: 136 * expected output:
137 * 137 *
138 * Overhead Command Shared Object Symbol 138 * Overhead Command Shared Object Symbol
139 * ======== ======= ============= ============== 139 * ======== ======= ============= ==============
140 * 20.00% perf perf [.] main 140 * 20.00% perf perf [.] main
141 * 10.00% bash [kernel] [k] page_fault 141 * 10.00% bash [kernel] [k] page_fault
142 * 10.00% bash bash [.] main 142 * 10.00% bash bash [.] main
143 * 10.00% bash bash [.] xmalloc 143 * 10.00% bash bash [.] xmalloc
144 * 10.00% perf [kernel] [k] page_fault 144 * 10.00% perf [kernel] [k] page_fault
145 * 10.00% perf [kernel] [k] schedule 145 * 10.00% perf [kernel] [k] schedule
146 * 10.00% perf libc [.] free 146 * 10.00% perf libc [.] free
147 * 10.00% perf libc [.] malloc 147 * 10.00% perf libc [.] malloc
148 * 10.00% perf perf [.] cmd_record 148 * 10.00% perf perf [.] cmd_record
149 */ 149 */
150 err = add_hist_entries(hists, machine); 150 err = add_hist_entries(hists, machine);
151 if (err < 0) 151 if (err < 0)
152 goto out; 152 goto out;
153 153
154 hists__collapse_resort(hists, NULL); 154 hists__collapse_resort(hists, NULL);
155 hists__output_resort(hists); 155 hists__output_resort(hists, NULL);
156 156
157 if (verbose > 2) { 157 if (verbose > 2) {
158 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order); 158 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order);
159 print_hists_out(hists); 159 print_hists_out(hists);
160 } 160 }
161 161
162 root = &hists->entries; 162 root = &hists->entries;
163 node = rb_first(root); 163 node = rb_first(root);
164 he = rb_entry(node, struct hist_entry, rb_node); 164 he = rb_entry(node, struct hist_entry, rb_node);
165 TEST_ASSERT_VAL("Invalid hist entry", 165 TEST_ASSERT_VAL("Invalid hist entry",
166 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") && 166 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") &&
167 !strcmp(SYM(he), "main") && he->stat.period == 200); 167 !strcmp(SYM(he), "main") && he->stat.period == 200);
168 168
169 node = rb_next(node); 169 node = rb_next(node);
170 he = rb_entry(node, struct hist_entry, rb_node); 170 he = rb_entry(node, struct hist_entry, rb_node);
171 TEST_ASSERT_VAL("Invalid hist entry", 171 TEST_ASSERT_VAL("Invalid hist entry",
172 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "[kernel]") && 172 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "[kernel]") &&
173 !strcmp(SYM(he), "page_fault") && he->stat.period == 100); 173 !strcmp(SYM(he), "page_fault") && he->stat.period == 100);
174 174
175 node = rb_next(node); 175 node = rb_next(node);
176 he = rb_entry(node, struct hist_entry, rb_node); 176 he = rb_entry(node, struct hist_entry, rb_node);
177 TEST_ASSERT_VAL("Invalid hist entry", 177 TEST_ASSERT_VAL("Invalid hist entry",
178 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") && 178 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") &&
179 !strcmp(SYM(he), "main") && he->stat.period == 100); 179 !strcmp(SYM(he), "main") && he->stat.period == 100);
180 180
181 node = rb_next(node); 181 node = rb_next(node);
182 he = rb_entry(node, struct hist_entry, rb_node); 182 he = rb_entry(node, struct hist_entry, rb_node);
183 TEST_ASSERT_VAL("Invalid hist entry", 183 TEST_ASSERT_VAL("Invalid hist entry",
184 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") && 184 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") &&
185 !strcmp(SYM(he), "xmalloc") && he->stat.period == 100); 185 !strcmp(SYM(he), "xmalloc") && he->stat.period == 100);
186 186
187 node = rb_next(node); 187 node = rb_next(node);
188 he = rb_entry(node, struct hist_entry, rb_node); 188 he = rb_entry(node, struct hist_entry, rb_node);
189 TEST_ASSERT_VAL("Invalid hist entry", 189 TEST_ASSERT_VAL("Invalid hist entry",
190 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") && 190 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") &&
191 !strcmp(SYM(he), "page_fault") && he->stat.period == 100); 191 !strcmp(SYM(he), "page_fault") && he->stat.period == 100);
192 192
193 node = rb_next(node); 193 node = rb_next(node);
194 he = rb_entry(node, struct hist_entry, rb_node); 194 he = rb_entry(node, struct hist_entry, rb_node);
195 TEST_ASSERT_VAL("Invalid hist entry", 195 TEST_ASSERT_VAL("Invalid hist entry",
196 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") && 196 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") &&
197 !strcmp(SYM(he), "schedule") && he->stat.period == 100); 197 !strcmp(SYM(he), "schedule") && he->stat.period == 100);
198 198
199 node = rb_next(node); 199 node = rb_next(node);
200 he = rb_entry(node, struct hist_entry, rb_node); 200 he = rb_entry(node, struct hist_entry, rb_node);
201 TEST_ASSERT_VAL("Invalid hist entry", 201 TEST_ASSERT_VAL("Invalid hist entry",
202 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") && 202 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") &&
203 !strcmp(SYM(he), "free") && he->stat.period == 100); 203 !strcmp(SYM(he), "free") && he->stat.period == 100);
204 204
205 node = rb_next(node); 205 node = rb_next(node);
206 he = rb_entry(node, struct hist_entry, rb_node); 206 he = rb_entry(node, struct hist_entry, rb_node);
207 TEST_ASSERT_VAL("Invalid hist entry", 207 TEST_ASSERT_VAL("Invalid hist entry",
208 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") && 208 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") &&
209 !strcmp(SYM(he), "malloc") && he->stat.period == 100); 209 !strcmp(SYM(he), "malloc") && he->stat.period == 100);
210 210
211 node = rb_next(node); 211 node = rb_next(node);
212 he = rb_entry(node, struct hist_entry, rb_node); 212 he = rb_entry(node, struct hist_entry, rb_node);
213 TEST_ASSERT_VAL("Invalid hist entry", 213 TEST_ASSERT_VAL("Invalid hist entry",
214 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") && 214 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") &&
215 !strcmp(SYM(he), "cmd_record") && he->stat.period == 100); 215 !strcmp(SYM(he), "cmd_record") && he->stat.period == 100);
216 216
217 out: 217 out:
218 del_hist_entries(hists); 218 del_hist_entries(hists);
219 reset_output_field(); 219 reset_output_field();
220 return err; 220 return err;
221 } 221 }
222 222
223 /* mixed fields and sort keys */ 223 /* mixed fields and sort keys */
224 static int test2(struct perf_evsel *evsel, struct machine *machine) 224 static int test2(struct perf_evsel *evsel, struct machine *machine)
225 { 225 {
226 int err; 226 int err;
227 struct hists *hists = evsel__hists(evsel); 227 struct hists *hists = evsel__hists(evsel);
228 struct hist_entry *he; 228 struct hist_entry *he;
229 struct rb_root *root; 229 struct rb_root *root;
230 struct rb_node *node; 230 struct rb_node *node;
231 231
232 field_order = "overhead,cpu"; 232 field_order = "overhead,cpu";
233 sort_order = "pid"; 233 sort_order = "pid";
234 234
235 setup_sorting(); 235 setup_sorting();
236 236
237 /* 237 /*
238 * expected output: 238 * expected output:
239 * 239 *
240 * Overhead CPU Command: Pid 240 * Overhead CPU Command: Pid
241 * ======== === ============= 241 * ======== === =============
242 * 30.00% 1 perf : 100 242 * 30.00% 1 perf : 100
243 * 10.00% 0 perf : 100 243 * 10.00% 0 perf : 100
244 * 10.00% 2 perf : 100 244 * 10.00% 2 perf : 100
245 * 20.00% 2 perf : 200 245 * 20.00% 2 perf : 200
246 * 10.00% 0 bash : 300 246 * 10.00% 0 bash : 300
247 * 10.00% 1 bash : 300 247 * 10.00% 1 bash : 300
248 * 10.00% 3 bash : 300 248 * 10.00% 3 bash : 300
249 */ 249 */
250 err = add_hist_entries(hists, machine); 250 err = add_hist_entries(hists, machine);
251 if (err < 0) 251 if (err < 0)
252 goto out; 252 goto out;
253 253
254 hists__collapse_resort(hists, NULL); 254 hists__collapse_resort(hists, NULL);
255 hists__output_resort(hists); 255 hists__output_resort(hists, NULL);
256 256
257 if (verbose > 2) { 257 if (verbose > 2) {
258 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order); 258 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order);
259 print_hists_out(hists); 259 print_hists_out(hists);
260 } 260 }
261 261
262 root = &hists->entries; 262 root = &hists->entries;
263 node = rb_first(root); 263 node = rb_first(root);
264 he = rb_entry(node, struct hist_entry, rb_node); 264 he = rb_entry(node, struct hist_entry, rb_node);
265 TEST_ASSERT_VAL("Invalid hist entry", 265 TEST_ASSERT_VAL("Invalid hist entry",
266 CPU(he) == 1 && PID(he) == 100 && he->stat.period == 300); 266 CPU(he) == 1 && PID(he) == 100 && he->stat.period == 300);
267 267
268 node = rb_next(node); 268 node = rb_next(node);
269 he = rb_entry(node, struct hist_entry, rb_node); 269 he = rb_entry(node, struct hist_entry, rb_node);
270 TEST_ASSERT_VAL("Invalid hist entry", 270 TEST_ASSERT_VAL("Invalid hist entry",
271 CPU(he) == 0 && PID(he) == 100 && he->stat.period == 100); 271 CPU(he) == 0 && PID(he) == 100 && he->stat.period == 100);
272 272
273 out: 273 out:
274 del_hist_entries(hists); 274 del_hist_entries(hists);
275 reset_output_field(); 275 reset_output_field();
276 return err; 276 return err;
277 } 277 }
278 278
279 /* fields only (no sort key) */ 279 /* fields only (no sort key) */
280 static int test3(struct perf_evsel *evsel, struct machine *machine) 280 static int test3(struct perf_evsel *evsel, struct machine *machine)
281 { 281 {
282 int err; 282 int err;
283 struct hists *hists = evsel__hists(evsel); 283 struct hists *hists = evsel__hists(evsel);
284 struct hist_entry *he; 284 struct hist_entry *he;
285 struct rb_root *root; 285 struct rb_root *root;
286 struct rb_node *node; 286 struct rb_node *node;
287 287
288 field_order = "comm,overhead,dso"; 288 field_order = "comm,overhead,dso";
289 sort_order = NULL; 289 sort_order = NULL;
290 290
291 setup_sorting(); 291 setup_sorting();
292 292
293 /* 293 /*
294 * expected output: 294 * expected output:
295 * 295 *
296 * Command Overhead Shared Object 296 * Command Overhead Shared Object
297 * ======= ======== ============= 297 * ======= ======== =============
298 * bash 20.00% bash 298 * bash 20.00% bash
299 * bash 10.00% [kernel] 299 * bash 10.00% [kernel]
300 * perf 30.00% perf 300 * perf 30.00% perf
301 * perf 20.00% [kernel] 301 * perf 20.00% [kernel]
302 * perf 20.00% libc 302 * perf 20.00% libc
303 */ 303 */
304 err = add_hist_entries(hists, machine); 304 err = add_hist_entries(hists, machine);
305 if (err < 0) 305 if (err < 0)
306 goto out; 306 goto out;
307 307
308 hists__collapse_resort(hists, NULL); 308 hists__collapse_resort(hists, NULL);
309 hists__output_resort(hists); 309 hists__output_resort(hists, NULL);
310 310
311 if (verbose > 2) { 311 if (verbose > 2) {
312 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order); 312 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order);
313 print_hists_out(hists); 313 print_hists_out(hists);
314 } 314 }
315 315
316 root = &hists->entries; 316 root = &hists->entries;
317 node = rb_first(root); 317 node = rb_first(root);
318 he = rb_entry(node, struct hist_entry, rb_node); 318 he = rb_entry(node, struct hist_entry, rb_node);
319 TEST_ASSERT_VAL("Invalid hist entry", 319 TEST_ASSERT_VAL("Invalid hist entry",
320 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") && 320 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") &&
321 he->stat.period == 200); 321 he->stat.period == 200);
322 322
323 node = rb_next(node); 323 node = rb_next(node);
324 he = rb_entry(node, struct hist_entry, rb_node); 324 he = rb_entry(node, struct hist_entry, rb_node);
325 TEST_ASSERT_VAL("Invalid hist entry", 325 TEST_ASSERT_VAL("Invalid hist entry",
326 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "[kernel]") && 326 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "[kernel]") &&
327 he->stat.period == 100); 327 he->stat.period == 100);
328 328
329 node = rb_next(node); 329 node = rb_next(node);
330 he = rb_entry(node, struct hist_entry, rb_node); 330 he = rb_entry(node, struct hist_entry, rb_node);
331 TEST_ASSERT_VAL("Invalid hist entry", 331 TEST_ASSERT_VAL("Invalid hist entry",
332 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") && 332 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") &&
333 he->stat.period == 300); 333 he->stat.period == 300);
334 334
335 node = rb_next(node); 335 node = rb_next(node);
336 he = rb_entry(node, struct hist_entry, rb_node); 336 he = rb_entry(node, struct hist_entry, rb_node);
337 TEST_ASSERT_VAL("Invalid hist entry", 337 TEST_ASSERT_VAL("Invalid hist entry",
338 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") && 338 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") &&
339 he->stat.period == 200); 339 he->stat.period == 200);
340 340
341 node = rb_next(node); 341 node = rb_next(node);
342 he = rb_entry(node, struct hist_entry, rb_node); 342 he = rb_entry(node, struct hist_entry, rb_node);
343 TEST_ASSERT_VAL("Invalid hist entry", 343 TEST_ASSERT_VAL("Invalid hist entry",
344 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") && 344 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") &&
345 he->stat.period == 200); 345 he->stat.period == 200);
346 346
347 out: 347 out:
348 del_hist_entries(hists); 348 del_hist_entries(hists);
349 reset_output_field(); 349 reset_output_field();
350 return err; 350 return err;
351 } 351 }
352 352
353 /* handle duplicate 'dso' field */ 353 /* handle duplicate 'dso' field */
354 static int test4(struct perf_evsel *evsel, struct machine *machine) 354 static int test4(struct perf_evsel *evsel, struct machine *machine)
355 { 355 {
356 int err; 356 int err;
357 struct hists *hists = evsel__hists(evsel); 357 struct hists *hists = evsel__hists(evsel);
358 struct hist_entry *he; 358 struct hist_entry *he;
359 struct rb_root *root; 359 struct rb_root *root;
360 struct rb_node *node; 360 struct rb_node *node;
361 361
362 field_order = "dso,sym,comm,overhead,dso"; 362 field_order = "dso,sym,comm,overhead,dso";
363 sort_order = "sym"; 363 sort_order = "sym";
364 364
365 setup_sorting(); 365 setup_sorting();
366 366
367 /* 367 /*
368 * expected output: 368 * expected output:
369 * 369 *
370 * Shared Object Symbol Command Overhead 370 * Shared Object Symbol Command Overhead
371 * ============= ============== ======= ======== 371 * ============= ============== ======= ========
372 * perf [.] cmd_record perf 10.00% 372 * perf [.] cmd_record perf 10.00%
373 * libc [.] free perf 10.00% 373 * libc [.] free perf 10.00%
374 * bash [.] main bash 10.00% 374 * bash [.] main bash 10.00%
375 * perf [.] main perf 20.00% 375 * perf [.] main perf 20.00%
376 * libc [.] malloc perf 10.00% 376 * libc [.] malloc perf 10.00%
377 * [kernel] [k] page_fault bash 10.00% 377 * [kernel] [k] page_fault bash 10.00%
378 * [kernel] [k] page_fault perf 10.00% 378 * [kernel] [k] page_fault perf 10.00%
379 * [kernel] [k] schedule perf 10.00% 379 * [kernel] [k] schedule perf 10.00%
380 * bash [.] xmalloc bash 10.00% 380 * bash [.] xmalloc bash 10.00%
381 */ 381 */
382 err = add_hist_entries(hists, machine); 382 err = add_hist_entries(hists, machine);
383 if (err < 0) 383 if (err < 0)
384 goto out; 384 goto out;
385 385
386 hists__collapse_resort(hists, NULL); 386 hists__collapse_resort(hists, NULL);
387 hists__output_resort(hists); 387 hists__output_resort(hists, NULL);
388 388
389 if (verbose > 2) { 389 if (verbose > 2) {
390 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order); 390 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order);
391 print_hists_out(hists); 391 print_hists_out(hists);
392 } 392 }
393 393
394 root = &hists->entries; 394 root = &hists->entries;
395 node = rb_first(root); 395 node = rb_first(root);
396 he = rb_entry(node, struct hist_entry, rb_node); 396 he = rb_entry(node, struct hist_entry, rb_node);
397 TEST_ASSERT_VAL("Invalid hist entry", 397 TEST_ASSERT_VAL("Invalid hist entry",
398 !strcmp(DSO(he), "perf") && !strcmp(SYM(he), "cmd_record") && 398 !strcmp(DSO(he), "perf") && !strcmp(SYM(he), "cmd_record") &&
399 !strcmp(COMM(he), "perf") && he->stat.period == 100); 399 !strcmp(COMM(he), "perf") && he->stat.period == 100);
400 400
401 node = rb_next(node); 401 node = rb_next(node);
402 he = rb_entry(node, struct hist_entry, rb_node); 402 he = rb_entry(node, struct hist_entry, rb_node);
403 TEST_ASSERT_VAL("Invalid hist entry", 403 TEST_ASSERT_VAL("Invalid hist entry",
404 !strcmp(DSO(he), "libc") && !strcmp(SYM(he), "free") && 404 !strcmp(DSO(he), "libc") && !strcmp(SYM(he), "free") &&
405 !strcmp(COMM(he), "perf") && he->stat.period == 100); 405 !strcmp(COMM(he), "perf") && he->stat.period == 100);
406 406
407 node = rb_next(node); 407 node = rb_next(node);
408 he = rb_entry(node, struct hist_entry, rb_node); 408 he = rb_entry(node, struct hist_entry, rb_node);
409 TEST_ASSERT_VAL("Invalid hist entry", 409 TEST_ASSERT_VAL("Invalid hist entry",
410 !strcmp(DSO(he), "bash") && !strcmp(SYM(he), "main") && 410 !strcmp(DSO(he), "bash") && !strcmp(SYM(he), "main") &&
411 !strcmp(COMM(he), "bash") && he->stat.period == 100); 411 !strcmp(COMM(he), "bash") && he->stat.period == 100);
412 412
413 node = rb_next(node); 413 node = rb_next(node);
414 he = rb_entry(node, struct hist_entry, rb_node); 414 he = rb_entry(node, struct hist_entry, rb_node);
415 TEST_ASSERT_VAL("Invalid hist entry", 415 TEST_ASSERT_VAL("Invalid hist entry",
416 !strcmp(DSO(he), "perf") && !strcmp(SYM(he), "main") && 416 !strcmp(DSO(he), "perf") && !strcmp(SYM(he), "main") &&
417 !strcmp(COMM(he), "perf") && he->stat.period == 200); 417 !strcmp(COMM(he), "perf") && he->stat.period == 200);
418 418
419 node = rb_next(node); 419 node = rb_next(node);
420 he = rb_entry(node, struct hist_entry, rb_node); 420 he = rb_entry(node, struct hist_entry, rb_node);
421 TEST_ASSERT_VAL("Invalid hist entry", 421 TEST_ASSERT_VAL("Invalid hist entry",
422 !strcmp(DSO(he), "libc") && !strcmp(SYM(he), "malloc") && 422 !strcmp(DSO(he), "libc") && !strcmp(SYM(he), "malloc") &&
423 !strcmp(COMM(he), "perf") && he->stat.period == 100); 423 !strcmp(COMM(he), "perf") && he->stat.period == 100);
424 424
425 node = rb_next(node); 425 node = rb_next(node);
426 he = rb_entry(node, struct hist_entry, rb_node); 426 he = rb_entry(node, struct hist_entry, rb_node);
427 TEST_ASSERT_VAL("Invalid hist entry", 427 TEST_ASSERT_VAL("Invalid hist entry",
428 !strcmp(DSO(he), "[kernel]") && !strcmp(SYM(he), "page_fault") && 428 !strcmp(DSO(he), "[kernel]") && !strcmp(SYM(he), "page_fault") &&
429 !strcmp(COMM(he), "bash") && he->stat.period == 100); 429 !strcmp(COMM(he), "bash") && he->stat.period == 100);
430 430
431 node = rb_next(node); 431 node = rb_next(node);
432 he = rb_entry(node, struct hist_entry, rb_node); 432 he = rb_entry(node, struct hist_entry, rb_node);
433 TEST_ASSERT_VAL("Invalid hist entry", 433 TEST_ASSERT_VAL("Invalid hist entry",
434 !strcmp(DSO(he), "[kernel]") && !strcmp(SYM(he), "page_fault") && 434 !strcmp(DSO(he), "[kernel]") && !strcmp(SYM(he), "page_fault") &&
435 !strcmp(COMM(he), "perf") && he->stat.period == 100); 435 !strcmp(COMM(he), "perf") && he->stat.period == 100);
436 436
437 node = rb_next(node); 437 node = rb_next(node);
438 he = rb_entry(node, struct hist_entry, rb_node); 438 he = rb_entry(node, struct hist_entry, rb_node);
439 TEST_ASSERT_VAL("Invalid hist entry", 439 TEST_ASSERT_VAL("Invalid hist entry",
440 !strcmp(DSO(he), "[kernel]") && !strcmp(SYM(he), "schedule") && 440 !strcmp(DSO(he), "[kernel]") && !strcmp(SYM(he), "schedule") &&
441 !strcmp(COMM(he), "perf") && he->stat.period == 100); 441 !strcmp(COMM(he), "perf") && he->stat.period == 100);
442 442
443 node = rb_next(node); 443 node = rb_next(node);
444 he = rb_entry(node, struct hist_entry, rb_node); 444 he = rb_entry(node, struct hist_entry, rb_node);
445 TEST_ASSERT_VAL("Invalid hist entry", 445 TEST_ASSERT_VAL("Invalid hist entry",
446 !strcmp(DSO(he), "bash") && !strcmp(SYM(he), "xmalloc") && 446 !strcmp(DSO(he), "bash") && !strcmp(SYM(he), "xmalloc") &&
447 !strcmp(COMM(he), "bash") && he->stat.period == 100); 447 !strcmp(COMM(he), "bash") && he->stat.period == 100);
448 448
449 out: 449 out:
450 del_hist_entries(hists); 450 del_hist_entries(hists);
451 reset_output_field(); 451 reset_output_field();
452 return err; 452 return err;
453 } 453 }
454 454
455 /* full sort keys w/o overhead field */ 455 /* full sort keys w/o overhead field */
456 static int test5(struct perf_evsel *evsel, struct machine *machine) 456 static int test5(struct perf_evsel *evsel, struct machine *machine)
457 { 457 {
458 int err; 458 int err;
459 struct hists *hists = evsel__hists(evsel); 459 struct hists *hists = evsel__hists(evsel);
460 struct hist_entry *he; 460 struct hist_entry *he;
461 struct rb_root *root; 461 struct rb_root *root;
462 struct rb_node *node; 462 struct rb_node *node;
463 463
464 field_order = "cpu,pid,comm,dso,sym"; 464 field_order = "cpu,pid,comm,dso,sym";
465 sort_order = "dso,pid"; 465 sort_order = "dso,pid";
466 466
467 setup_sorting(); 467 setup_sorting();
468 468
469 /* 469 /*
470 * expected output: 470 * expected output:
471 * 471 *
472 * CPU Command: Pid Command Shared Object Symbol 472 * CPU Command: Pid Command Shared Object Symbol
473 * === ============= ======= ============= ============== 473 * === ============= ======= ============= ==============
474 * 0 perf: 100 perf [kernel] [k] schedule 474 * 0 perf: 100 perf [kernel] [k] schedule
475 * 2 perf: 200 perf [kernel] [k] page_fault 475 * 2 perf: 200 perf [kernel] [k] page_fault
476 * 1 bash: 300 bash [kernel] [k] page_fault 476 * 1 bash: 300 bash [kernel] [k] page_fault
477 * 0 bash: 300 bash bash [.] xmalloc 477 * 0 bash: 300 bash bash [.] xmalloc
478 * 3 bash: 300 bash bash [.] main 478 * 3 bash: 300 bash bash [.] main
479 * 1 perf: 100 perf libc [.] malloc 479 * 1 perf: 100 perf libc [.] malloc
480 * 2 perf: 100 perf libc [.] free 480 * 2 perf: 100 perf libc [.] free
481 * 1 perf: 100 perf perf [.] cmd_record 481 * 1 perf: 100 perf perf [.] cmd_record
482 * 1 perf: 100 perf perf [.] main 482 * 1 perf: 100 perf perf [.] main
483 * 2 perf: 200 perf perf [.] main 483 * 2 perf: 200 perf perf [.] main
484 */ 484 */
485 err = add_hist_entries(hists, machine); 485 err = add_hist_entries(hists, machine);
486 if (err < 0) 486 if (err < 0)
487 goto out; 487 goto out;
488 488
489 hists__collapse_resort(hists, NULL); 489 hists__collapse_resort(hists, NULL);
490 hists__output_resort(hists); 490 hists__output_resort(hists, NULL);
491 491
492 if (verbose > 2) { 492 if (verbose > 2) {
493 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order); 493 pr_info("[fields = %s, sort = %s]\n", field_order, sort_order);
494 print_hists_out(hists); 494 print_hists_out(hists);
495 } 495 }
496 496
497 root = &hists->entries; 497 root = &hists->entries;
498 node = rb_first(root); 498 node = rb_first(root);
499 he = rb_entry(node, struct hist_entry, rb_node); 499 he = rb_entry(node, struct hist_entry, rb_node);
500 500
501 TEST_ASSERT_VAL("Invalid hist entry", 501 TEST_ASSERT_VAL("Invalid hist entry",
502 CPU(he) == 0 && PID(he) == 100 && 502 CPU(he) == 0 && PID(he) == 100 &&
503 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") && 503 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") &&
504 !strcmp(SYM(he), "schedule") && he->stat.period == 100); 504 !strcmp(SYM(he), "schedule") && he->stat.period == 100);
505 505
506 node = rb_next(node); 506 node = rb_next(node);
507 he = rb_entry(node, struct hist_entry, rb_node); 507 he = rb_entry(node, struct hist_entry, rb_node);
508 TEST_ASSERT_VAL("Invalid hist entry", 508 TEST_ASSERT_VAL("Invalid hist entry",
509 CPU(he) == 2 && PID(he) == 200 && 509 CPU(he) == 2 && PID(he) == 200 &&
510 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") && 510 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "[kernel]") &&
511 !strcmp(SYM(he), "page_fault") && he->stat.period == 100); 511 !strcmp(SYM(he), "page_fault") && he->stat.period == 100);
512 512
513 node = rb_next(node); 513 node = rb_next(node);
514 he = rb_entry(node, struct hist_entry, rb_node); 514 he = rb_entry(node, struct hist_entry, rb_node);
515 TEST_ASSERT_VAL("Invalid hist entry", 515 TEST_ASSERT_VAL("Invalid hist entry",
516 CPU(he) == 1 && PID(he) == 300 && 516 CPU(he) == 1 && PID(he) == 300 &&
517 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "[kernel]") && 517 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "[kernel]") &&
518 !strcmp(SYM(he), "page_fault") && he->stat.period == 100); 518 !strcmp(SYM(he), "page_fault") && he->stat.period == 100);
519 519
520 node = rb_next(node); 520 node = rb_next(node);
521 he = rb_entry(node, struct hist_entry, rb_node); 521 he = rb_entry(node, struct hist_entry, rb_node);
522 TEST_ASSERT_VAL("Invalid hist entry", 522 TEST_ASSERT_VAL("Invalid hist entry",
523 CPU(he) == 0 && PID(he) == 300 && 523 CPU(he) == 0 && PID(he) == 300 &&
524 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") && 524 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") &&
525 !strcmp(SYM(he), "xmalloc") && he->stat.period == 100); 525 !strcmp(SYM(he), "xmalloc") && he->stat.period == 100);
526 526
527 node = rb_next(node); 527 node = rb_next(node);
528 he = rb_entry(node, struct hist_entry, rb_node); 528 he = rb_entry(node, struct hist_entry, rb_node);
529 TEST_ASSERT_VAL("Invalid hist entry", 529 TEST_ASSERT_VAL("Invalid hist entry",
530 CPU(he) == 3 && PID(he) == 300 && 530 CPU(he) == 3 && PID(he) == 300 &&
531 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") && 531 !strcmp(COMM(he), "bash") && !strcmp(DSO(he), "bash") &&
532 !strcmp(SYM(he), "main") && he->stat.period == 100); 532 !strcmp(SYM(he), "main") && he->stat.period == 100);
533 533
534 node = rb_next(node); 534 node = rb_next(node);
535 he = rb_entry(node, struct hist_entry, rb_node); 535 he = rb_entry(node, struct hist_entry, rb_node);
536 TEST_ASSERT_VAL("Invalid hist entry", 536 TEST_ASSERT_VAL("Invalid hist entry",
537 CPU(he) == 1 && PID(he) == 100 && 537 CPU(he) == 1 && PID(he) == 100 &&
538 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") && 538 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") &&
539 !strcmp(SYM(he), "malloc") && he->stat.period == 100); 539 !strcmp(SYM(he), "malloc") && he->stat.period == 100);
540 540
541 node = rb_next(node); 541 node = rb_next(node);
542 he = rb_entry(node, struct hist_entry, rb_node); 542 he = rb_entry(node, struct hist_entry, rb_node);
543 TEST_ASSERT_VAL("Invalid hist entry", 543 TEST_ASSERT_VAL("Invalid hist entry",
544 CPU(he) == 2 && PID(he) == 100 && 544 CPU(he) == 2 && PID(he) == 100 &&
545 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") && 545 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "libc") &&
546 !strcmp(SYM(he), "free") && he->stat.period == 100); 546 !strcmp(SYM(he), "free") && he->stat.period == 100);
547 547
548 node = rb_next(node); 548 node = rb_next(node);
549 he = rb_entry(node, struct hist_entry, rb_node); 549 he = rb_entry(node, struct hist_entry, rb_node);
550 TEST_ASSERT_VAL("Invalid hist entry", 550 TEST_ASSERT_VAL("Invalid hist entry",
551 CPU(he) == 1 && PID(he) == 100 && 551 CPU(he) == 1 && PID(he) == 100 &&
552 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") && 552 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") &&
553 !strcmp(SYM(he), "cmd_record") && he->stat.period == 100); 553 !strcmp(SYM(he), "cmd_record") && he->stat.period == 100);
554 554
555 node = rb_next(node); 555 node = rb_next(node);
556 he = rb_entry(node, struct hist_entry, rb_node); 556 he = rb_entry(node, struct hist_entry, rb_node);
557 TEST_ASSERT_VAL("Invalid hist entry", 557 TEST_ASSERT_VAL("Invalid hist entry",
558 CPU(he) == 1 && PID(he) == 100 && 558 CPU(he) == 1 && PID(he) == 100 &&
559 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") && 559 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") &&
560 !strcmp(SYM(he), "main") && he->stat.period == 100); 560 !strcmp(SYM(he), "main") && he->stat.period == 100);
561 561
562 node = rb_next(node); 562 node = rb_next(node);
563 he = rb_entry(node, struct hist_entry, rb_node); 563 he = rb_entry(node, struct hist_entry, rb_node);
564 TEST_ASSERT_VAL("Invalid hist entry", 564 TEST_ASSERT_VAL("Invalid hist entry",
565 CPU(he) == 2 && PID(he) == 200 && 565 CPU(he) == 2 && PID(he) == 200 &&
566 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") && 566 !strcmp(COMM(he), "perf") && !strcmp(DSO(he), "perf") &&
567 !strcmp(SYM(he), "main") && he->stat.period == 100); 567 !strcmp(SYM(he), "main") && he->stat.period == 100);
568 568
569 out: 569 out:
570 del_hist_entries(hists); 570 del_hist_entries(hists);
571 reset_output_field(); 571 reset_output_field();
572 return err; 572 return err;
573 } 573 }
574 574
575 int test__hists_output(void) 575 int test__hists_output(void)
576 { 576 {
577 int err = TEST_FAIL; 577 int err = TEST_FAIL;
578 struct machines machines; 578 struct machines machines;
579 struct machine *machine; 579 struct machine *machine;
580 struct perf_evsel *evsel; 580 struct perf_evsel *evsel;
581 struct perf_evlist *evlist = perf_evlist__new(); 581 struct perf_evlist *evlist = perf_evlist__new();
582 size_t i; 582 size_t i;
583 test_fn_t testcases[] = { 583 test_fn_t testcases[] = {
584 test1, 584 test1,
585 test2, 585 test2,
586 test3, 586 test3,
587 test4, 587 test4,
588 test5, 588 test5,
589 }; 589 };
590 590
591 TEST_ASSERT_VAL("No memory", evlist); 591 TEST_ASSERT_VAL("No memory", evlist);
592 592
593 err = parse_events(evlist, "cpu-clock"); 593 err = parse_events(evlist, "cpu-clock");
594 if (err) 594 if (err)
595 goto out; 595 goto out;
596 596
597 machines__init(&machines); 597 machines__init(&machines);
598 598
599 /* setup threads/dso/map/symbols also */ 599 /* setup threads/dso/map/symbols also */
600 machine = setup_fake_machine(&machines); 600 machine = setup_fake_machine(&machines);
601 if (!machine) 601 if (!machine)
602 goto out; 602 goto out;
603 603
604 if (verbose > 1) 604 if (verbose > 1)
605 machine__fprintf(machine, stderr); 605 machine__fprintf(machine, stderr);
606 606
607 evsel = perf_evlist__first(evlist); 607 evsel = perf_evlist__first(evlist);
608 608
609 for (i = 0; i < ARRAY_SIZE(testcases); i++) { 609 for (i = 0; i < ARRAY_SIZE(testcases); i++) {
610 err = testcases[i](evsel, machine); 610 err = testcases[i](evsel, machine);
611 if (err < 0) 611 if (err < 0)
612 break; 612 break;
613 } 613 }
614 614
615 out: 615 out:
616 /* tear down everything */ 616 /* tear down everything */
617 perf_evlist__delete(evlist); 617 perf_evlist__delete(evlist);
618 machines__exit(&machines); 618 machines__exit(&machines);
619 619
620 return err; 620 return err;
621 } 621 }
622 622
tools/perf/ui/browsers/hists.c
Diff comments   View file @ ddb321a
1 #include <stdio.h> 1 #include <stdio.h>
2 #include "../libslang.h" 2 #include "../libslang.h"
3 #include <stdlib.h> 3 #include <stdlib.h>
4 #include <string.h> 4 #include <string.h>
5 #include <linux/rbtree.h> 5 #include <linux/rbtree.h>
6 6
7 #include "../../util/evsel.h" 7 #include "../../util/evsel.h"
8 #include "../../util/evlist.h" 8 #include "../../util/evlist.h"
9 #include "../../util/hist.h" 9 #include "../../util/hist.h"
10 #include "../../util/pstack.h" 10 #include "../../util/pstack.h"
11 #include "../../util/sort.h" 11 #include "../../util/sort.h"
12 #include "../../util/util.h" 12 #include "../../util/util.h"
13 #include "../../util/top.h" 13 #include "../../util/top.h"
14 #include "../../arch/common.h" 14 #include "../../arch/common.h"
15 15
16 #include "../browser.h" 16 #include "../browser.h"
17 #include "../helpline.h" 17 #include "../helpline.h"
18 #include "../util.h" 18 #include "../util.h"
19 #include "../ui.h" 19 #include "../ui.h"
20 #include "map.h" 20 #include "map.h"
21 #include "annotate.h" 21 #include "annotate.h"
22 22
23 struct hist_browser { 23 struct hist_browser {
24 struct ui_browser b; 24 struct ui_browser b;
25 struct hists *hists; 25 struct hists *hists;
26 struct hist_entry *he_selection; 26 struct hist_entry *he_selection;
27 struct map_symbol *selection; 27 struct map_symbol *selection;
28 int print_seq; 28 int print_seq;
29 bool show_dso; 29 bool show_dso;
30 bool show_headers; 30 bool show_headers;
31 float min_pcnt; 31 float min_pcnt;
32 u64 nr_non_filtered_entries; 32 u64 nr_non_filtered_entries;
33 u64 nr_callchain_rows; 33 u64 nr_callchain_rows;
34 }; 34 };
35 35
36 extern void hist_browser__init_hpp(void); 36 extern void hist_browser__init_hpp(void);
37 37
38 static int hists__browser_title(struct hists *hists, 38 static int hists__browser_title(struct hists *hists,
39 struct hist_browser_timer *hbt, 39 struct hist_browser_timer *hbt,
40 char *bf, size_t size); 40 char *bf, size_t size);
41 static void hist_browser__update_nr_entries(struct hist_browser *hb); 41 static void hist_browser__update_nr_entries(struct hist_browser *hb);
42 42
43 static struct rb_node *hists__filter_entries(struct rb_node *nd, 43 static struct rb_node *hists__filter_entries(struct rb_node *nd,
44 float min_pcnt); 44 float min_pcnt);
45 45
46 static bool hist_browser__has_filter(struct hist_browser *hb) 46 static bool hist_browser__has_filter(struct hist_browser *hb)
47 { 47 {
48 return hists__has_filter(hb->hists) || hb->min_pcnt; 48 return hists__has_filter(hb->hists) || hb->min_pcnt;
49 } 49 }
50 50
51 static u32 hist_browser__nr_entries(struct hist_browser *hb) 51 static u32 hist_browser__nr_entries(struct hist_browser *hb)
52 { 52 {
53 u32 nr_entries; 53 u32 nr_entries;
54 54
55 if (hist_browser__has_filter(hb)) 55 if (hist_browser__has_filter(hb))
56 nr_entries = hb->nr_non_filtered_entries; 56 nr_entries = hb->nr_non_filtered_entries;
57 else 57 else
58 nr_entries = hb->hists->nr_entries; 58 nr_entries = hb->hists->nr_entries;
59 59
60 return nr_entries + hb->nr_callchain_rows; 60 return nr_entries + hb->nr_callchain_rows;
61 } 61 }
62 62
63 static void hist_browser__update_rows(struct hist_browser *hb) 63 static void hist_browser__update_rows(struct hist_browser *hb)
64 { 64 {
65 struct ui_browser *browser = &hb->b; 65 struct ui_browser *browser = &hb->b;
66 u16 header_offset = hb->show_headers ? 1 : 0, index_row; 66 u16 header_offset = hb->show_headers ? 1 : 0, index_row;
67 67
68 browser->rows = browser->height - header_offset; 68 browser->rows = browser->height - header_offset;
69 /* 69 /*
70 * Verify if we were at the last line and that line isn't 70 * Verify if we were at the last line and that line isn't
71 * visibe because we now show the header line(s). 71 * visibe because we now show the header line(s).
72 */ 72 */
73 index_row = browser->index - browser->top_idx; 73 index_row = browser->index - browser->top_idx;
74 if (index_row >= browser->rows) 74 if (index_row >= browser->rows)
75 browser->index -= index_row - browser->rows + 1; 75 browser->index -= index_row - browser->rows + 1;
76 } 76 }
77 77
78 static void hist_browser__refresh_dimensions(struct ui_browser *browser) 78 static void hist_browser__refresh_dimensions(struct ui_browser *browser)
79 { 79 {
80 struct hist_browser *hb = container_of(browser, struct hist_browser, b); 80 struct hist_browser *hb = container_of(browser, struct hist_browser, b);
81 81
82 /* 3 == +/- toggle symbol before actual hist_entry rendering */ 82 /* 3 == +/- toggle symbol before actual hist_entry rendering */
83 browser->width = 3 + (hists__sort_list_width(hb->hists) + sizeof("[k]")); 83 browser->width = 3 + (hists__sort_list_width(hb->hists) + sizeof("[k]"));
84 /* 84 /*
85 * FIXME: Just keeping existing behaviour, but this really should be 85 * FIXME: Just keeping existing behaviour, but this really should be
86 * before updating browser->width, as it will invalidate the 86 * before updating browser->width, as it will invalidate the
87 * calculation above. Fix this and the fallout in another 87 * calculation above. Fix this and the fallout in another
88 * changeset. 88 * changeset.
89 */ 89 */
90 ui_browser__refresh_dimensions(browser); 90 ui_browser__refresh_dimensions(browser);
91 hist_browser__update_rows(hb); 91 hist_browser__update_rows(hb);
92 } 92 }
93 93
94 static void hist_browser__gotorc(struct hist_browser *browser, int row, int column) 94 static void hist_browser__gotorc(struct hist_browser *browser, int row, int column)
95 { 95 {
96 u16 header_offset = browser->show_headers ? 1 : 0; 96 u16 header_offset = browser->show_headers ? 1 : 0;
97 97
98 ui_browser__gotorc(&browser->b, row + header_offset, column); 98 ui_browser__gotorc(&browser->b, row + header_offset, column);
99 } 99 }
100 100
101 static void hist_browser__reset(struct hist_browser *browser) 101 static void hist_browser__reset(struct hist_browser *browser)
102 { 102 {
103 /* 103 /*
104 * The hists__remove_entry_filter() already folds non-filtered 104 * The hists__remove_entry_filter() already folds non-filtered
105 * entries so we can assume it has 0 callchain rows. 105 * entries so we can assume it has 0 callchain rows.
106 */ 106 */
107 browser->nr_callchain_rows = 0; 107 browser->nr_callchain_rows = 0;
108 108
109 hist_browser__update_nr_entries(browser); 109 hist_browser__update_nr_entries(browser);
110 browser->b.nr_entries = hist_browser__nr_entries(browser); 110 browser->b.nr_entries = hist_browser__nr_entries(browser);
111 hist_browser__refresh_dimensions(&browser->b); 111 hist_browser__refresh_dimensions(&browser->b);
112 ui_browser__reset_index(&browser->b); 112 ui_browser__reset_index(&browser->b);
113 } 113 }
114 114
115 static char tree__folded_sign(bool unfolded) 115 static char tree__folded_sign(bool unfolded)
116 { 116 {
117 return unfolded ? '-' : '+'; 117 return unfolded ? '-' : '+';
118 } 118 }
119 119
120 static char map_symbol__folded(const struct map_symbol *ms) 120 static char map_symbol__folded(const struct map_symbol *ms)
121 { 121 {
122 return ms->has_children ? tree__folded_sign(ms->unfolded) : ' '; 122 return ms->has_children ? tree__folded_sign(ms->unfolded) : ' ';
123 } 123 }
124 124
125 static char hist_entry__folded(const struct hist_entry *he) 125 static char hist_entry__folded(const struct hist_entry *he)
126 { 126 {
127 return map_symbol__folded(&he->ms); 127 return map_symbol__folded(&he->ms);
128 } 128 }
129 129
130 static char callchain_list__folded(const struct callchain_list *cl) 130 static char callchain_list__folded(const struct callchain_list *cl)
131 { 131 {
132 return map_symbol__folded(&cl->ms); 132 return map_symbol__folded(&cl->ms);
133 } 133 }
134 134
135 static void map_symbol__set_folding(struct map_symbol *ms, bool unfold) 135 static void map_symbol__set_folding(struct map_symbol *ms, bool unfold)
136 { 136 {
137 ms->unfolded = unfold ? ms->has_children : false; 137 ms->unfolded = unfold ? ms->has_children : false;
138 } 138 }
139 139
140 static int callchain_node__count_rows_rb_tree(struct callchain_node *node) 140 static int callchain_node__count_rows_rb_tree(struct callchain_node *node)
141 { 141 {
142 int n = 0; 142 int n = 0;
143 struct rb_node *nd; 143 struct rb_node *nd;
144 144
145 for (nd = rb_first(&node->rb_root); nd; nd = rb_next(nd)) { 145 for (nd = rb_first(&node->rb_root); nd; nd = rb_next(nd)) {
146 struct callchain_node *child = rb_entry(nd, struct callchain_node, rb_node); 146 struct callchain_node *child = rb_entry(nd, struct callchain_node, rb_node);
147 struct callchain_list *chain; 147 struct callchain_list *chain;
148 char folded_sign = ' '; /* No children */ 148 char folded_sign = ' '; /* No children */
149 149
150 list_for_each_entry(chain, &child->val, list) { 150 list_for_each_entry(chain, &child->val, list) {
151 ++n; 151 ++n;
152 /* We need this because we may not have children */ 152 /* We need this because we may not have children */
153 folded_sign = callchain_list__folded(chain); 153 folded_sign = callchain_list__folded(chain);
154 if (folded_sign == '+') 154 if (folded_sign == '+')
155 break; 155 break;
156 } 156 }
157 157
158 if (folded_sign == '-') /* Have children and they're unfolded */ 158 if (folded_sign == '-') /* Have children and they're unfolded */
159 n += callchain_node__count_rows_rb_tree(child); 159 n += callchain_node__count_rows_rb_tree(child);
160 } 160 }
161 161
162 return n; 162 return n;
163 } 163 }
164 164
165 static int callchain_node__count_rows(struct callchain_node *node) 165 static int callchain_node__count_rows(struct callchain_node *node)
166 { 166 {
167 struct callchain_list *chain; 167 struct callchain_list *chain;
168 bool unfolded = false; 168 bool unfolded = false;
169 int n = 0; 169 int n = 0;
170 170
171 list_for_each_entry(chain, &node->val, list) { 171 list_for_each_entry(chain, &node->val, list) {
172 ++n; 172 ++n;
173 unfolded = chain->ms.unfolded; 173 unfolded = chain->ms.unfolded;
174 } 174 }
175 175
176 if (unfolded) 176 if (unfolded)
177 n += callchain_node__count_rows_rb_tree(node); 177 n += callchain_node__count_rows_rb_tree(node);
178 178
179 return n; 179 return n;
180 } 180 }
181 181
182 static int callchain__count_rows(struct rb_root *chain) 182 static int callchain__count_rows(struct rb_root *chain)
183 { 183 {
184 struct rb_node *nd; 184 struct rb_node *nd;
185 int n = 0; 185 int n = 0;
186 186
187 for (nd = rb_first(chain); nd; nd = rb_next(nd)) { 187 for (nd = rb_first(chain); nd; nd = rb_next(nd)) {
188 struct callchain_node *node = rb_entry(nd, struct callchain_node, rb_node); 188 struct callchain_node *node = rb_entry(nd, struct callchain_node, rb_node);
189 n += callchain_node__count_rows(node); 189 n += callchain_node__count_rows(node);
190 } 190 }
191 191
192 return n; 192 return n;
193 } 193 }
194 194
195 static bool map_symbol__toggle_fold(struct map_symbol *ms) 195 static bool map_symbol__toggle_fold(struct map_symbol *ms)
196 { 196 {
197 if (!ms) 197 if (!ms)
198 return false; 198 return false;
199 199
200 if (!ms->has_children) 200 if (!ms->has_children)
201 return false; 201 return false;
202 202
203 ms->unfolded = !ms->unfolded; 203 ms->unfolded = !ms->unfolded;
204 return true; 204 return true;
205 } 205 }
206 206
207 static void callchain_node__init_have_children_rb_tree(struct callchain_node *node) 207 static void callchain_node__init_have_children_rb_tree(struct callchain_node *node)
208 { 208 {
209 struct rb_node *nd = rb_first(&node->rb_root); 209 struct rb_node *nd = rb_first(&node->rb_root);
210 210
211 for (nd = rb_first(&node->rb_root); nd; nd = rb_next(nd)) { 211 for (nd = rb_first(&node->rb_root); nd; nd = rb_next(nd)) {
212 struct callchain_node *child = rb_entry(nd, struct callchain_node, rb_node); 212 struct callchain_node *child = rb_entry(nd, struct callchain_node, rb_node);
213 struct callchain_list *chain; 213 struct callchain_list *chain;
214 bool first = true; 214 bool first = true;
215 215
216 list_for_each_entry(chain, &child->val, list) { 216 list_for_each_entry(chain, &child->val, list) {
217 if (first) { 217 if (first) {
218 first = false; 218 first = false;
219 chain->ms.has_children = chain->list.next != &child->val || 219 chain->ms.has_children = chain->list.next != &child->val ||
220 !RB_EMPTY_ROOT(&child->rb_root); 220 !RB_EMPTY_ROOT(&child->rb_root);
221 } else 221 } else
222 chain->ms.has_children = chain->list.next == &child->val && 222 chain->ms.has_children = chain->list.next == &child->val &&
223 !RB_EMPTY_ROOT(&child->rb_root); 223 !RB_EMPTY_ROOT(&child->rb_root);
224 } 224 }
225 225
226 callchain_node__init_have_children_rb_tree(child); 226 callchain_node__init_have_children_rb_tree(child);
227 } 227 }
228 } 228 }
229 229
230 static void callchain_node__init_have_children(struct callchain_node *node, 230 static void callchain_node__init_have_children(struct callchain_node *node,
231 bool has_sibling) 231 bool has_sibling)
232 { 232 {
233 struct callchain_list *chain; 233 struct callchain_list *chain;
234 234
235 chain = list_entry(node->val.next, struct callchain_list, list); 235 chain = list_entry(node->val.next, struct callchain_list, list);
236 chain->ms.has_children = has_sibling; 236 chain->ms.has_children = has_sibling;
237 237
238 if (!list_empty(&node->val)) { 238 if (!list_empty(&node->val)) {
239 chain = list_entry(node->val.prev, struct callchain_list, list); 239 chain = list_entry(node->val.prev, struct callchain_list, list);
240 chain->ms.has_children = !RB_EMPTY_ROOT(&node->rb_root); 240 chain->ms.has_children = !RB_EMPTY_ROOT(&node->rb_root);
241 } 241 }
242 242
243 callchain_node__init_have_children_rb_tree(node); 243 callchain_node__init_have_children_rb_tree(node);
244 } 244 }
245 245
246 static void callchain__init_have_children(struct rb_root *root) 246 static void callchain__init_have_children(struct rb_root *root)
247 { 247 {
248 struct rb_node *nd = rb_first(root); 248 struct rb_node *nd = rb_first(root);
249 bool has_sibling = nd && rb_next(nd); 249 bool has_sibling = nd && rb_next(nd);
250 250
251 for (nd = rb_first(root); nd; nd = rb_next(nd)) { 251 for (nd = rb_first(root); nd; nd = rb_next(nd)) {
252 struct callchain_node *node = rb_entry(nd, struct callchain_node, rb_node); 252 struct callchain_node *node = rb_entry(nd, struct callchain_node, rb_node);
253 callchain_node__init_have_children(node, has_sibling); 253 callchain_node__init_have_children(node, has_sibling);
254 } 254 }
255 } 255 }
256 256
257 static void hist_entry__init_have_children(struct hist_entry *he) 257 static void hist_entry__init_have_children(struct hist_entry *he)
258 { 258 {
259 if (!he->init_have_children) { 259 if (!he->init_have_children) {
260 he->ms.has_children = !RB_EMPTY_ROOT(&he->sorted_chain); 260 he->ms.has_children = !RB_EMPTY_ROOT(&he->sorted_chain);
261 callchain__init_have_children(&he->sorted_chain); 261 callchain__init_have_children(&he->sorted_chain);
262 he->init_have_children = true; 262 he->init_have_children = true;
263 } 263 }
264 } 264 }
265 265
266 static bool hist_browser__toggle_fold(struct hist_browser *browser) 266 static bool hist_browser__toggle_fold(struct hist_browser *browser)
267 { 267 {
268 if (map_symbol__toggle_fold(browser->selection)) { 268 if (map_symbol__toggle_fold(browser->selection)) {
269 struct hist_entry *he = browser->he_selection; 269 struct hist_entry *he = browser->he_selection;
270 270
271 hist_entry__init_have_children(he); 271 hist_entry__init_have_children(he);
272 browser->b.nr_entries -= he->nr_rows; 272 browser->b.nr_entries -= he->nr_rows;
273 browser->nr_callchain_rows -= he->nr_rows; 273 browser->nr_callchain_rows -= he->nr_rows;
274 274
275 if (he->ms.unfolded) 275 if (he->ms.unfolded)
276 he->nr_rows = callchain__count_rows(&he->sorted_chain); 276 he->nr_rows = callchain__count_rows(&he->sorted_chain);
277 else 277 else
278 he->nr_rows = 0; 278 he->nr_rows = 0;
279 279
280 browser->b.nr_entries += he->nr_rows; 280 browser->b.nr_entries += he->nr_rows;
281 browser->nr_callchain_rows += he->nr_rows; 281 browser->nr_callchain_rows += he->nr_rows;
282 282
283 return true; 283 return true;
284 } 284 }
285 285
286 /* If it doesn't have children, no toggling performed */ 286 /* If it doesn't have children, no toggling performed */
287 return false; 287 return false;
288 } 288 }
289 289
290 static int callchain_node__set_folding_rb_tree(struct callchain_node *node, bool unfold) 290 static int callchain_node__set_folding_rb_tree(struct callchain_node *node, bool unfold)
291 { 291 {
292 int n = 0; 292 int n = 0;
293 struct rb_node *nd; 293 struct rb_node *nd;
294 294
295 for (nd = rb_first(&node->rb_root); nd; nd = rb_next(nd)) { 295 for (nd = rb_first(&node->rb_root); nd; nd = rb_next(nd)) {
296 struct callchain_node *child = rb_entry(nd, struct callchain_node, rb_node); 296 struct callchain_node *child = rb_entry(nd, struct callchain_node, rb_node);
297 struct callchain_list *chain; 297 struct callchain_list *chain;
298 bool has_children = false; 298 bool has_children = false;
299 299
300 list_for_each_entry(chain, &child->val, list) { 300 list_for_each_entry(chain, &child->val, list) {
301 ++n; 301 ++n;
302 map_symbol__set_folding(&chain->ms, unfold); 302 map_symbol__set_folding(&chain->ms, unfold);
303 has_children = chain->ms.has_children; 303 has_children = chain->ms.has_children;
304 } 304 }
305 305
306 if (has_children) 306 if (has_children)
307 n += callchain_node__set_folding_rb_tree(child, unfold); 307 n += callchain_node__set_folding_rb_tree(child, unfold);
308 } 308 }
309 309
310 return n; 310 return n;
311 } 311 }
312 312
313 static int callchain_node__set_folding(struct callchain_node *node, bool unfold) 313 static int callchain_node__set_folding(struct callchain_node *node, bool unfold)
314 { 314 {
315 struct callchain_list *chain; 315 struct callchain_list *chain;
316 bool has_children = false; 316 bool has_children = false;
317 int n = 0; 317 int n = 0;
318 318
319 list_for_each_entry(chain, &node->val, list) { 319 list_for_each_entry(chain, &node->val, list) {
320 ++n; 320 ++n;
321 map_symbol__set_folding(&chain->ms, unfold); 321 map_symbol__set_folding(&chain->ms, unfold);
322 has_children = chain->ms.has_children; 322 has_children = chain->ms.has_children;
323 } 323 }
324 324
325 if (has_children) 325 if (has_children)
326 n += callchain_node__set_folding_rb_tree(node, unfold); 326 n += callchain_node__set_folding_rb_tree(node, unfold);
327 327
328 return n; 328 return n;
329 } 329 }
330 330
331 static int callchain__set_folding(struct rb_root *chain, bool unfold) 331 static int callchain__set_folding(struct rb_root *chain, bool unfold)
332 { 332 {
333 struct rb_node *nd; 333 struct rb_node *nd;
334 int n = 0; 334 int n = 0;
335 335
336 for (nd = rb_first(chain); nd; nd = rb_next(nd)) { 336 for (nd = rb_first(chain); nd; nd = rb_next(nd)) {
337 struct callchain_node *node = rb_entry(nd, struct callchain_node, rb_node); 337 struct callchain_node *node = rb_entry(nd, struct callchain_node, rb_node);
338 n += callchain_node__set_folding(node, unfold); 338 n += callchain_node__set_folding(node, unfold);
339 } 339 }
340 340
341 return n; 341 return n;
342 } 342 }
343 343
344 static void hist_entry__set_folding(struct hist_entry *he, bool unfold) 344 static void hist_entry__set_folding(struct hist_entry *he, bool unfold)
345 { 345 {
346 hist_entry__init_have_children(he); 346 hist_entry__init_have_children(he);
347 map_symbol__set_folding(&he->ms, unfold); 347 map_symbol__set_folding(&he->ms, unfold);
348 348
349 if (he->ms.has_children) { 349 if (he->ms.has_children) {
350 int n = callchain__set_folding(&he->sorted_chain, unfold); 350 int n = callchain__set_folding(&he->sorted_chain, unfold);
351 he->nr_rows = unfold ? n : 0; 351 he->nr_rows = unfold ? n : 0;
352 } else 352 } else
353 he->nr_rows = 0; 353 he->nr_rows = 0;
354 } 354 }
355 355
356 static void 356 static void
357 __hist_browser__set_folding(struct hist_browser *browser, bool unfold) 357 __hist_browser__set_folding(struct hist_browser *browser, bool unfold)
358 { 358 {
359 struct rb_node *nd; 359 struct rb_node *nd;
360 struct hists *hists = browser->hists; 360 struct hists *hists = browser->hists;
361 361
362 for (nd = rb_first(&hists->entries); 362 for (nd = rb_first(&hists->entries);
363 (nd = hists__filter_entries(nd, browser->min_pcnt)) != NULL; 363 (nd = hists__filter_entries(nd, browser->min_pcnt)) != NULL;
364 nd = rb_next(nd)) { 364 nd = rb_next(nd)) {
365 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); 365 struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node);
366 hist_entry__set_folding(he, unfold); 366 hist_entry__set_folding(he, unfold);
367 browser->nr_callchain_rows += he->nr_rows; 367 browser->nr_callchain_rows += he->nr_rows;
368 } 368 }
369 } 369 }
370 370
371 static void hist_browser__set_folding(struct hist_browser *browser, bool unfold) 371 static void hist_browser__set_folding(struct hist_browser *browser, bool unfold)
372 { 372 {
373 browser->nr_callchain_rows = 0; 373 browser->nr_callchain_rows = 0;
374 __hist_browser__set_folding(browser, unfold); 374 __hist_browser__set_folding(browser, unfold);
375 375
376 browser->b.nr_entries = hist_browser__nr_entries(browser); 376 browser->b.nr_entries = hist_browser__nr_entries(browser);
377 /* Go to the start, we may be way after valid entries after a collapse */ 377 /* Go to the start, we may be way after valid entries after a collapse */
378 ui_browser__reset_index(&browser->b); 378 ui_browser__reset_index(&browser->b);
379 } 379 }
380 380
381 static void ui_browser__warn_lost_events(struct ui_browser *browser) 381 static void ui_browser__warn_lost_events(struct ui_browser *browser)
382 { 382 {
383 ui_browser__warning(browser, 4, 383 ui_browser__warning(browser, 4,
384 "Events are being lost, check IO/CPU overload!\n\n" 384 "Events are being lost, check IO/CPU overload!\n\n"
385 "You may want to run 'perf' using a RT scheduler policy:\n\n" 385 "You may want to run 'perf' using a RT scheduler policy:\n\n"
386 " perf top -r 80\n\n" 386 " perf top -r 80\n\n"
387 "Or reduce the sampling frequency."); 387 "Or reduce the sampling frequency.");
388 } 388 }
389 389
390 static int hist_browser__run(struct hist_browser *browser, 390 static int hist_browser__run(struct hist_browser *browser,
391 struct hist_browser_timer *hbt) 391 struct hist_browser_timer *hbt)
392 { 392 {
393 int key; 393 int key;
394 char title[160]; 394 char title[160];
395 int delay_secs = hbt ? hbt->refresh : 0; 395 int delay_secs = hbt ? hbt->refresh : 0;
396 396
397 browser->b.entries = &browser->hists->entries; 397 browser->b.entries = &browser->hists->entries;
398 browser->b.nr_entries = hist_browser__nr_entries(browser); 398 browser->b.nr_entries = hist_browser__nr_entries(browser);
399 399
400 hists__browser_title(browser->hists, hbt, title, sizeof(title)); 400 hists__browser_title(browser->hists, hbt, title, sizeof(title));
401 401
402 if (ui_browser__show(&browser->b, title, 402 if (ui_browser__show(&browser->b, title,
403 "Press '?' for help on key bindings") < 0) 403 "Press '?' for help on key bindings") < 0)
404 return -1; 404 return -1;
405 405
406 while (1) { 406 while (1) {
407 key = ui_browser__run(&browser->b, delay_secs); 407 key = ui_browser__run(&browser->b, delay_secs);
408 408
409 switch (key) { 409 switch (key) {
410 case K_TIMER: { 410 case K_TIMER: {
411 u64 nr_entries; 411 u64 nr_entries;
412 hbt->timer(hbt->arg); 412 hbt->timer(hbt->arg);
413 413
414 if (hist_browser__has_filter(browser)) 414 if (hist_browser__has_filter(browser))
415 hist_browser__update_nr_entries(browser); 415 hist_browser__update_nr_entries(browser);
416 416
417 nr_entries = hist_browser__nr_entries(browser); 417 nr_entries = hist_browser__nr_entries(browser);
418 ui_browser__update_nr_entries(&browser->b, nr_entries); 418 ui_browser__update_nr_entries(&browser->b, nr_entries);
419 419
420 if (browser->hists->stats.nr_lost_warned != 420 if (browser->hists->stats.nr_lost_warned !=
421 browser->hists->stats.nr_events[PERF_RECORD_LOST]) { 421 browser->hists->stats.nr_events[PERF_RECORD_LOST]) {
422 browser->hists->stats.nr_lost_warned = 422 browser->hists->stats.nr_lost_warned =
423 browser->hists->stats.nr_events[PERF_RECORD_LOST]; 423 browser->hists->stats.nr_events[PERF_RECORD_LOST];
424 ui_browser__warn_lost_events(&browser->b); 424 ui_browser__warn_lost_events(&browser->b);
425 } 425 }
426 426
427 hists__browser_title(browser->hists, 427 hists__browser_title(browser->hists,
428 hbt, title, sizeof(title)); 428 hbt, title, sizeof(title));
429 ui_browser__show_title(&browser->b, title); 429 ui_browser__show_title(&browser->b, title);
430 continue; 430 continue;
431 } 431 }
432 case 'D': { /* Debug */ 432 case 'D': { /* Debug */
433 static int seq; 433 static int seq;
434 struct hist_entry *h = rb_entry(browser->b.top, 434 struct hist_entry *h = rb_entry(browser->b.top,
435 struct hist_entry, rb_node); 435 struct hist_entry, rb_node);
436 ui_helpline__pop(); 436 ui_helpline__pop();
437 ui_helpline__fpush("%d: nr_ent=(%d,%d), rows=%d, idx=%d, fve: idx=%d, row_off=%d, nrows=%d", 437 ui_helpline__fpush("%d: nr_ent=(%d,%d), rows=%d, idx=%d, fve: idx=%d, row_off=%d, nrows=%d",
438 seq++, browser->b.nr_entries, 438 seq++, browser->b.nr_entries,
439 browser->hists->nr_entries, 439 browser->hists->nr_entries,
440 browser->b.rows, 440 browser->b.rows,
441 browser->b.index, 441 browser->b.index,
442 browser->b.top_idx, 442 browser->b.top_idx,
443 h->row_offset, h->nr_rows); 443 h->row_offset, h->nr_rows);
444 } 444 }
445 break; 445 break;
446 case 'C': 446 case 'C':
447 /* Collapse the whole world. */ 447 /* Collapse the whole world. */
448 hist_browser__set_folding(browser, false); 448 hist_browser__set_folding(browser, false);
449 break; 449 break;
450 case 'E': 450 case 'E':
451 /* Expand the whole world. */ 451 /* Expand the whole world. */
452 hist_browser__set_folding(browser, true); 452 hist_browser__set_folding(browser, true);
453 break; 453 break;
454 case 'H': 454 case 'H':
455 browser->show_headers = !browser->show_headers; 455 browser->show_headers = !browser->show_headers;
456 hist_browser__update_rows(browser); 456 hist_browser__update_rows(browser);
457 break; 457 break;
458 case K_ENTER: 458 case K_ENTER:
459 if (hist_browser__toggle_fold(browser)) 459 if (hist_browser__toggle_fold(browser))
460 break; 460 break;
461 /* fall thru */ 461 /* fall thru */
462 default: 462 default:
463 goto out; 463 goto out;
464 } 464 }
465 } 465 }
466 out: 466 out:
467 ui_browser__hide(&browser->b); 467 ui_browser__hide(&browser->b);
468 return key; 468 return key;
469 } 469 }
470 470
471 struct callchain_print_arg { 471 struct callchain_print_arg {
472 /* for hists browser */ 472 /* for hists browser */
473 off_t row_offset; 473 off_t row_offset;
474 bool is_current_entry; 474 bool is_current_entry;
475 475
476 /* for file dump */ 476 /* for file dump */
477 FILE *fp; 477 FILE *fp;
478 int printed; 478 int printed;
479 }; 479 };
480 480
481 typedef void (*print_callchain_entry_fn)(struct hist_browser *browser, 481 typedef void (*print_callchain_entry_fn)(struct hist_browser *browser,
482 struct callchain_list *chain, 482 struct callchain_list *chain,
483 const char *str, int offset, 483 const char *str, int offset,
484 unsigned short row, 484 unsigned short row,
485 struct callchain_print_arg *arg); 485 struct callchain_print_arg *arg);
486 486
487 static void hist_browser__show_callchain_entry(struct hist_browser *browser, 487 static void hist_browser__show_callchain_entry(struct hist_browser *browser,
488 struct callchain_list *chain, 488 struct callchain_list *chain,
489 const char *str, int offset, 489 const char *str, int offset,
490 unsigned short row, 490 unsigned short row,
491 struct callchain_print_arg *arg) 491 struct callchain_print_arg *arg)
492 { 492 {
493 int color, width; 493 int color, width;
494 char folded_sign = callchain_list__folded(chain); 494 char folded_sign = callchain_list__folded(chain);
495 495
496 color = HE_COLORSET_NORMAL; 496 color = HE_COLORSET_NORMAL;
497 width = browser->b.width - (offset + 2); 497 width = browser->b.width - (offset + 2);
498 if (ui_browser__is_current_entry(&browser->b, row)) { 498 if (ui_browser__is_current_entry(&browser->b, row)) {
499 browser->selection = &chain->ms; 499 browser->selection = &chain->ms;
500 color = HE_COLORSET_SELECTED; 500 color = HE_COLORSET_SELECTED;
501 arg->is_current_entry = true; 501 arg->is_current_entry = true;
502 } 502 }
503 503
504 ui_browser__set_color(&browser->b, color); 504 ui_browser__set_color(&browser->b, color);
505 hist_browser__gotorc(browser, row, 0); 505 hist_browser__gotorc(browser, row, 0);
506 slsmg_write_nstring(" ", offset); 506 slsmg_write_nstring(" ", offset);
507 slsmg_printf("%c ", folded_sign); 507 slsmg_printf("%c ", folded_sign);
508 slsmg_write_nstring(str, width); 508 slsmg_write_nstring(str, width);
509 } 509 }
510 510
511 static void hist_browser__fprintf_callchain_entry(struct hist_browser *b __maybe_unused, 511 static void hist_browser__fprintf_callchain_entry(struct hist_browser *b __maybe_unused,
512 struct callchain_list *chain, 512 struct callchain_list *chain,
513 const char *str, int offset, 513 const char *str, int offset,
514 unsigned short row __maybe_unused, 514 unsigned short row __maybe_unused,
515 struct callchain_print_arg *arg) 515 struct callchain_print_arg *arg)
516 { 516 {
517 char folded_sign = callchain_list__folded(chain); 517 char folded_sign = callchain_list__folded(chain);
518 518
519 arg->printed += fprintf(arg->fp, "%*s%c %s\n", offset, " ", 519 arg->printed += fprintf(arg->fp, "%*s%c %s\n", offset, " ",
520 folded_sign, str); 520 folded_sign, str);
521 } 521 }
522 522
523 typedef bool (*check_output_full_fn)(struct hist_browser *browser, 523 typedef bool (*check_output_full_fn)(struct hist_browser *browser,
524 unsigned short row); 524 unsigned short row);
525 525
526 static bool hist_browser__check_output_full(struct hist_browser *browser, 526 static bool hist_browser__check_output_full(struct hist_browser *browser,
527 unsigned short row) 527 unsigned short row)
528 { 528 {
529 return browser->b.rows == row; 529 return browser->b.rows == row;
530 } 530 }
531 531
532 static bool hist_browser__check_dump_full(struct hist_browser *browser __maybe_unused, 532 static bool hist_browser__check_dump_full(struct hist_browser *browser __maybe_unused,
533 unsigned short row __maybe_unused) 533 unsigned short row __maybe_unused)
534 { 534 {
535 return false; 535 return false;
536 } 536 }
537 537
538 #define LEVEL_OFFSET_STEP 3 538 #define LEVEL_OFFSET_STEP 3
539 539
540 static int hist_browser__show_callchain(struct hist_browser *browser, 540 static int hist_browser__show_callchain(struct hist_browser *browser,
541 struct rb_root *root, int level, 541 struct rb_root *root, int level,
542 unsigned short row, u64 total, 542 unsigned short row, u64 total,
543 print_callchain_entry_fn print, 543 print_callchain_entry_fn print,
544 struct callchain_print_arg *arg, 544 struct callchain_print_arg *arg,
545 check_output_full_fn is_output_full) 545 check_output_full_fn is_output_full)
546 { 546 {
547 struct rb_node *node; 547 struct rb_node *node;
548 int first_row = row, offset = level * LEVEL_OFFSET_STEP; 548 int first_row = row, offset = level * LEVEL_OFFSET_STEP;
549 u64 new_total; 549 u64 new_total;
550 bool need_percent; 550 bool need_percent;
551 551
552 node = rb_first(root); 552 node = rb_first(root);
553 need_percent = !!rb_next(node); 553 need_percent = node && rb_next(node);
554 554
555 while (node) { 555 while (node) {
556 struct callchain_node *child = rb_entry(node, struct callchain_node, rb_node); 556 struct callchain_node *child = rb_entry(node, struct callchain_node, rb_node);
557 struct rb_node *next = rb_next(node); 557 struct rb_node *next = rb_next(node);
558 u64 cumul = callchain_cumul_hits(child); 558 u64 cumul = callchain_cumul_hits(child);
559 struct callchain_list *chain; 559 struct callchain_list *chain;
560 char folded_sign = ' '; 560 char folded_sign = ' ';
561 int first = true; 561 int first = true;
562 int extra_offset = 0; 562 int extra_offset = 0;
563 563
564 list_for_each_entry(chain, &child->val, list) { 564 list_for_each_entry(chain, &child->val, list) {
565 char bf[1024], *alloc_str; 565 char bf[1024], *alloc_str;
566 const char *str; 566 const char *str;
567 bool was_first = first; 567 bool was_first = first;
568 568
569 if (first) 569 if (first)
570 first = false; 570 first = false;
571 else if (need_percent) 571 else if (need_percent)
572 extra_offset = LEVEL_OFFSET_STEP; 572 extra_offset = LEVEL_OFFSET_STEP;
573 573
574 folded_sign = callchain_list__folded(chain); 574 folded_sign = callchain_list__folded(chain);
575 if (arg->row_offset != 0) { 575 if (arg->row_offset != 0) {
576 arg->row_offset--; 576 arg->row_offset--;
577 goto do_next; 577 goto do_next;
578 } 578 }
579 579
580 alloc_str = NULL; 580 alloc_str = NULL;
581 str = callchain_list__sym_name(chain, bf, sizeof(bf), 581 str = callchain_list__sym_name(chain, bf, sizeof(bf),
582 browser->show_dso); 582 browser->show_dso);
583 583
584 if (was_first && need_percent) { 584 if (was_first && need_percent) {
585 double percent = cumul * 100.0 / total; 585 double percent = cumul * 100.0 / total;
586 586
587 if (asprintf(&alloc_str, "%2.2f%% %s", percent, str) < 0) 587 if (asprintf(&alloc_str, "%2.2f%% %s", percent, str) < 0)
588 str = "Not enough memory!"; 588 str = "Not enough memory!";
589 else 589 else
590 str = alloc_str; 590 str = alloc_str;
591 } 591 }
592 592
593 print(browser, chain, str, offset + extra_offset, row, arg); 593 print(browser, chain, str, offset + extra_offset, row, arg);
594 594
595 free(alloc_str); 595 free(alloc_str);
596 596
597 if (is_output_full(browser, ++row)) 597 if (is_output_full(browser, ++row))
598 goto out; 598 goto out;
599 do_next: 599 do_next:
600 if (folded_sign == '+') 600 if (folded_sign == '+')
601 break; 601 break;
602 } 602 }
603 603
604 if (folded_sign == '-') { 604 if (folded_sign == '-') {
605 const int new_level = level + (extra_offset ? 2 : 1); 605 const int new_level = level + (extra_offset ? 2 : 1);
606 606
607 if (callchain_param.mode == CHAIN_GRAPH_REL) 607 if (callchain_param.mode == CHAIN_GRAPH_REL)
608 new_total = child->children_hit; 608 new_total = child->children_hit;
609 else 609 else
610 new_total = total; 610 new_total = total;
611 611
612 row += hist_browser__show_callchain(browser, &child->rb_root, 612 row += hist_browser__show_callchain(browser, &child->rb_root,
613 new_level, row, new_total, 613 new_level, row, new_total,
614 print, arg, is_output_full); 614 print, arg, is_output_full);
615 } 615 }
616 if (is_output_full(browser, row)) 616 if (is_output_full(browser, row))
617 break; 617 break;
618 node = next; 618 node = next;
619 } 619 }
620 out: 620 out:
621 return row - first_row; 621 return row - first_row;
622 } 622 }
623 623
624 struct hpp_arg { 624 struct hpp_arg {
625 struct ui_browser *b; 625 struct ui_browser *b;
626 char folded_sign; 626 char folded_sign;
627 bool current_entry; 627 bool current_entry;
628 }; 628 };
629 629
630 static int __hpp__slsmg_color_printf(struct perf_hpp *hpp, const char *fmt, ...) 630 static int __hpp__slsmg_color_printf(struct perf_hpp *hpp, const char *fmt, ...)
631 { 631 {
632 struct hpp_arg *arg = hpp->ptr; 632 struct hpp_arg *arg = hpp->ptr;
633 int ret, len; 633 int ret, len;
634 va_list args; 634 va_list args;
635 double percent; 635 double percent;
636 636
637 va_start(args, fmt); 637 va_start(args, fmt);
638 len = va_arg(args, int); 638 len = va_arg(args, int);
639 percent = va_arg(args, double); 639 percent = va_arg(args, double);
640 va_end(args); 640 va_end(args);
641 641
642 ui_browser__set_percent_color(arg->b, percent, arg->current_entry); 642 ui_browser__set_percent_color(arg->b, percent, arg->current_entry);
643 643
644 ret = scnprintf(hpp->buf, hpp->size, fmt, len, percent); 644 ret = scnprintf(hpp->buf, hpp->size, fmt, len, percent);
645 slsmg_printf("%s", hpp->buf); 645 slsmg_printf("%s", hpp->buf);
646 646
647 advance_hpp(hpp, ret); 647 advance_hpp(hpp, ret);
648 return ret; 648 return ret;
649 } 649 }
650 650
651 #define __HPP_COLOR_PERCENT_FN(_type, _field) \ 651 #define __HPP_COLOR_PERCENT_FN(_type, _field) \
652 static u64 __hpp_get_##_field(struct hist_entry *he) \ 652 static u64 __hpp_get_##_field(struct hist_entry *he) \
653 { \ 653 { \
654 return he->stat._field; \ 654 return he->stat._field; \
655 } \ 655 } \
656 \ 656 \
657 static int \ 657 static int \
658 hist_browser__hpp_color_##_type(struct perf_hpp_fmt *fmt, \ 658 hist_browser__hpp_color_##_type(struct perf_hpp_fmt *fmt, \
659 struct perf_hpp *hpp, \ 659 struct perf_hpp *hpp, \
660 struct hist_entry *he) \ 660 struct hist_entry *he) \
661 { \ 661 { \
662 return hpp__fmt(fmt, hpp, he, __hpp_get_##_field, " %*.2f%%", \ 662 return hpp__fmt(fmt, hpp, he, __hpp_get_##_field, " %*.2f%%", \
663 __hpp__slsmg_color_printf, true); \ 663 __hpp__slsmg_color_printf, true); \
664 } 664 }
665 665
666 #define __HPP_COLOR_ACC_PERCENT_FN(_type, _field) \ 666 #define __HPP_COLOR_ACC_PERCENT_FN(_type, _field) \
667 static u64 __hpp_get_acc_##_field(struct hist_entry *he) \ 667 static u64 __hpp_get_acc_##_field(struct hist_entry *he) \
668 { \ 668 { \
669 return he->stat_acc->_field; \ 669 return he->stat_acc->_field; \
670 } \ 670 } \
671 \ 671 \
672 static int \ 672 static int \
673 hist_browser__hpp_color_##_type(struct perf_hpp_fmt *fmt, \ 673 hist_browser__hpp_color_##_type(struct perf_hpp_fmt *fmt, \
674 struct perf_hpp *hpp, \ 674 struct perf_hpp *hpp, \
675 struct hist_entry *he) \ 675 struct hist_entry *he) \
676 { \ 676 { \
677 if (!symbol_conf.cumulate_callchain) { \ 677 if (!symbol_conf.cumulate_callchain) { \
678 int len = fmt->user_len ?: fmt->len; \ 678 int len = fmt->user_len ?: fmt->len; \
679 int ret = scnprintf(hpp->buf, hpp->size, \ 679 int ret = scnprintf(hpp->buf, hpp->size, \
680 "%*s", len, "N/A"); \ 680 "%*s", len, "N/A"); \
681 slsmg_printf("%s", hpp->buf); \ 681 slsmg_printf("%s", hpp->buf); \
682 \ 682 \
683 return ret; \ 683 return ret; \
684 } \ 684 } \
685 return hpp__fmt(fmt, hpp, he, __hpp_get_acc_##_field, \ 685 return hpp__fmt(fmt, hpp, he, __hpp_get_acc_##_field, \
686 " %*.2f%%", __hpp__slsmg_color_printf, true); \ 686 " %*.2f%%", __hpp__slsmg_color_printf, true); \
687 } 687 }
688 688
689 __HPP_COLOR_PERCENT_FN(overhead, period) 689 __HPP_COLOR_PERCENT_FN(overhead, period)
690 __HPP_COLOR_PERCENT_FN(overhead_sys, period_sys) 690 __HPP_COLOR_PERCENT_FN(overhead_sys, period_sys)
691 __HPP_COLOR_PERCENT_FN(overhead_us, period_us) 691 __HPP_COLOR_PERCENT_FN(overhead_us, period_us)
692 __HPP_COLOR_PERCENT_FN(overhead_guest_sys, period_guest_sys) 692 __HPP_COLOR_PERCENT_FN(overhead_guest_sys, period_guest_sys)
693 __HPP_COLOR_PERCENT_FN(overhead_guest_us, period_guest_us) 693 __HPP_COLOR_PERCENT_FN(overhead_guest_us, period_guest_us)
694 __HPP_COLOR_ACC_PERCENT_FN(overhead_acc, period) 694 __HPP_COLOR_ACC_PERCENT_FN(overhead_acc, period)
695 695
696 #undef __HPP_COLOR_PERCENT_FN 696 #undef __HPP_COLOR_PERCENT_FN
697 #undef __HPP_COLOR_ACC_PERCENT_FN 697 #undef __HPP_COLOR_ACC_PERCENT_FN
698 698
699 void hist_browser__init_hpp(void) 699 void hist_browser__init_hpp(void)
700 { 700 {
701 perf_hpp__format[PERF_HPP__OVERHEAD].color = 701 perf_hpp__format[PERF_HPP__OVERHEAD].color =
702 hist_browser__hpp_color_overhead; 702 hist_browser__hpp_color_overhead;
703 perf_hpp__format[PERF_HPP__OVERHEAD_SYS].color = 703 perf_hpp__format[PERF_HPP__OVERHEAD_SYS].color =
704 hist_browser__hpp_color_overhead_sys; 704 hist_browser__hpp_color_overhead_sys;
705 perf_hpp__format[PERF_HPP__OVERHEAD_US].color = 705 perf_hpp__format[PERF_HPP__OVERHEAD_US].color =
706 hist_browser__hpp_color_overhead_us; 706 hist_browser__hpp_color_overhead_us;
707 perf_hpp__format[PERF_HPP__OVERHEAD_GUEST_SYS].color = 707 perf_hpp__format[PERF_HPP__OVERHEAD_GUEST_SYS].color =
708 hist_browser__hpp_color_overhead_guest_sys; 708 hist_browser__hpp_color_overhead_guest_sys;
709 perf_hpp__format[PERF_HPP__OVERHEAD_GUEST_US].color = 709 perf_hpp__format[PERF_HPP__OVERHEAD_GUEST_US].color =
710 hist_browser__hpp_color_overhead_guest_us; 710 hist_browser__hpp_color_overhead_guest_us;
711 perf_hpp__format[PERF_HPP__OVERHEAD_ACC].color = 711 perf_hpp__format[PERF_HPP__OVERHEAD_ACC].color =
712 hist_browser__hpp_color_overhead_acc; 712 hist_browser__hpp_color_overhead_acc;
713 } 713 }
714 714
715 static int hist_browser__show_entry(struct hist_browser *browser, 715 static int hist_browser__show_entry(struct hist_browser *browser,
716 struct hist_entry *entry, 716 struct hist_entry *entry,
717 unsigned short row) 717 unsigned short row)
718 { 718 {
719 char s[256]; 719 char s[256];
720 int printed = 0; 720 int printed = 0;
721 int width = browser->b.width; 721 int width = browser->b.width;
722 char folded_sign = ' '; 722 char folded_sign = ' ';
723 bool current_entry = ui_browser__is_current_entry(&browser->b, row); 723 bool current_entry = ui_browser__is_current_entry(&browser->b, row);
724 off_t row_offset = entry->row_offset; 724 off_t row_offset = entry->row_offset;
725 bool first = true; 725 bool first = true;
726 struct perf_hpp_fmt *fmt; 726 struct perf_hpp_fmt *fmt;
727 727
728 if (current_entry) { 728 if (current_entry) {
729 browser->he_selection = entry; 729 browser->he_selection = entry;
730 browser->selection = &entry->ms; 730 browser->selection = &entry->ms;
731 } 731 }
732 732
733 if (symbol_conf.use_callchain) { 733 if (symbol_conf.use_callchain) {
734 hist_entry__init_have_children(entry); 734 hist_entry__init_have_children(entry);
735 folded_sign = hist_entry__folded(entry); 735 folded_sign = hist_entry__folded(entry);
736 } 736 }
737 737
738 if (row_offset == 0) { 738 if (row_offset == 0) {
739 struct hpp_arg arg = { 739 struct hpp_arg arg = {
740 .b = &browser->b, 740 .b = &browser->b,
741 .folded_sign = folded_sign, 741 .folded_sign = folded_sign,
742 .current_entry = current_entry, 742 .current_entry = current_entry,
743 }; 743 };
744 struct perf_hpp hpp = { 744 struct perf_hpp hpp = {
745 .buf = s, 745 .buf = s,
746 .size = sizeof(s), 746 .size = sizeof(s),
747 .ptr = &arg, 747 .ptr = &arg,
748 }; 748 };
749 749
750 hist_browser__gotorc(browser, row, 0); 750 hist_browser__gotorc(browser, row, 0);
751 751
752 perf_hpp__for_each_format(fmt) { 752 perf_hpp__for_each_format(fmt) {
753 if (perf_hpp__should_skip(fmt)) 753 if (perf_hpp__should_skip(fmt))
754 continue; 754 continue;
755 755
756 if (current_entry && browser->b.navkeypressed) { 756 if (current_entry && browser->b.navkeypressed) {
757 ui_browser__set_color(&browser->b, 757 ui_browser__set_color(&browser->b,
758 HE_COLORSET_SELECTED); 758 HE_COLORSET_SELECTED);
759 } else { 759 } else {
760 ui_browser__set_color(&browser->b, 760 ui_browser__set_color(&browser->b,
761 HE_COLORSET_NORMAL); 761 HE_COLORSET_NORMAL);
762 } 762 }
763 763
764 if (first) { 764 if (first) {
765 if (symbol_conf.use_callchain) { 765 if (symbol_conf.use_callchain) {
766 slsmg_printf("%c ", folded_sign); 766 slsmg_printf("%c ", folded_sign);
767 width -= 2; 767 width -= 2;
768 } 768 }
769 first = false; 769 first = false;
770 } else { 770 } else {
771 slsmg_printf(" "); 771 slsmg_printf(" ");
772 width -= 2; 772 width -= 2;
773 } 773 }
774 774
775 if (fmt->color) { 775 if (fmt->color) {
776 width -= fmt->color(fmt, &hpp, entry); 776 width -= fmt->color(fmt, &hpp, entry);
777 } else { 777 } else {
778 width -= fmt->entry(fmt, &hpp, entry); 778 width -= fmt->entry(fmt, &hpp, entry);
779 slsmg_printf("%s", s); 779 slsmg_printf("%s", s);
780 } 780 }
781 } 781 }
782 782
783 /* The scroll bar isn't being used */ 783 /* The scroll bar isn't being used */
784 if (!browser->b.navkeypressed) 784 if (!browser->b.navkeypressed)
785 width += 1; 785 width += 1;
786 786
787 slsmg_write_nstring("", width); 787 slsmg_write_nstring("", width);
788 788
789 ++row; 789 ++row;
790 ++printed; 790 ++printed;
791 } else 791 } else
792 --row_offset; 792 --row_offset;
793 793
794 if (folded_sign == '-' && row != browser->b.rows) { 794 if (folded_sign == '-' && row != browser->b.rows) {
795 u64 total = hists__total_period(entry->hists); 795 u64 total = hists__total_period(entry->hists);
796 struct callchain_print_arg arg = { 796 struct callchain_print_arg arg = {
797 .row_offset = row_offset, 797 .row_offset = row_offset,
798 .is_current_entry = current_entry, 798 .is_current_entry = current_entry,
799 }; 799 };
800 800
801 if (callchain_param.mode == CHAIN_GRAPH_REL) { 801 if (callchain_param.mode == CHAIN_GRAPH_REL) {
802 if (symbol_conf.cumulate_callchain) 802 if (symbol_conf.cumulate_callchain)
803 total = entry->stat_acc->period; 803 total = entry->stat_acc->period;
804 else 804 else
805 total = entry->stat.period; 805 total = entry->stat.period;
806 } 806 }
807 807
808 printed += hist_browser__show_callchain(browser, 808 printed += hist_browser__show_callchain(browser,
809 &entry->sorted_chain, 1, row, total, 809 &entry->sorted_chain, 1, row, total,
810 hist_browser__show_callchain_entry, &arg, 810 hist_browser__show_callchain_entry, &arg,
811 hist_browser__check_output_full); 811 hist_browser__check_output_full);
812 812
813 if (arg.is_current_entry) 813 if (arg.is_current_entry)
814 browser->he_selection = entry; 814 browser->he_selection = entry;
815 } 815 }
816 816
817 return printed; 817 return printed;
818 } 818 }
819 819
820 static int advance_hpp_check(struct perf_hpp *hpp, int inc) 820 static int advance_hpp_check(struct perf_hpp *hpp, int inc)
821 { 821 {
822 advance_hpp(hpp, inc); 822 advance_hpp(hpp, inc);
823 return hpp->size <= 0; 823 return hpp->size <= 0;
824 } 824 }
825 825
826 static int hists__scnprintf_headers(char *buf, size_t size, struct hists *hists) 826 static int hists__scnprintf_headers(char *buf, size_t size, struct hists *hists)
827 { 827 {
828 struct perf_hpp dummy_hpp = { 828 struct perf_hpp dummy_hpp = {
829 .buf = buf, 829 .buf = buf,
830 .size = size, 830 .size = size,
831 }; 831 };
832 struct perf_hpp_fmt *fmt; 832 struct perf_hpp_fmt *fmt;
833 size_t ret = 0; 833 size_t ret = 0;
834 834
835 if (symbol_conf.use_callchain) { 835 if (symbol_conf.use_callchain) {
836 ret = scnprintf(buf, size, " "); 836 ret = scnprintf(buf, size, " ");
837 if (advance_hpp_check(&dummy_hpp, ret)) 837 if (advance_hpp_check(&dummy_hpp, ret))
838 return ret; 838 return ret;
839 } 839 }
840 840
841 perf_hpp__for_each_format(fmt) { 841 perf_hpp__for_each_format(fmt) {
842 if (perf_hpp__should_skip(fmt)) 842 if (perf_hpp__should_skip(fmt))
843 continue; 843 continue;
844 844
845 ret = fmt->header(fmt, &dummy_hpp, hists_to_evsel(hists)); 845 ret = fmt->header(fmt, &dummy_hpp, hists_to_evsel(hists));
846 if (advance_hpp_check(&dummy_hpp, ret)) 846 if (advance_hpp_check(&dummy_hpp, ret))
847 break; 847 break;
848 848
849 ret = scnprintf(dummy_hpp.buf, dummy_hpp.size, " "); 849 ret = scnprintf(dummy_hpp.buf, dummy_hpp.size, " ");
850 if (advance_hpp_check(&dummy_hpp, ret)) 850 if (advance_hpp_check(&dummy_hpp, ret))
851 break; 851 break;
852 } 852 }
853 853
854 return ret; 854 return ret;
855 } 855 }
856 856
857 static void hist_browser__show_headers(struct hist_browser *browser) 857 static void hist_browser__show_headers(struct hist_browser *browser)
858 { 858 {
859 char headers[1024]; 859 char headers[1024];
860 860
861 hists__scnprintf_headers(headers, sizeof(headers), browser->hists); 861 hists__scnprintf_headers(headers, sizeof(headers), browser->hists);
862 ui_browser__gotorc(&browser->b, 0, 0); 862 ui_browser__gotorc(&browser->b, 0, 0);
863 ui_browser__set_color(&browser->b, HE_COLORSET_ROOT); 863 ui_browser__set_color(&browser->b, HE_COLORSET_ROOT);
864 slsmg_write_nstring(headers, browser->b.width + 1); 864 slsmg_write_nstring(headers, browser->b.width + 1);
865 } 865 }
866 866
867 static void ui_browser__hists_init_top(struct ui_browser *browser) 867 static void ui_browser__hists_init_top(struct ui_browser *browser)
868 { 868 {
869 if (browser->top == NULL) { 869 if (browser->top == NULL) {
870 struct hist_browser *hb; 870 struct hist_browser *hb;
871 871
872 hb = container_of(browser, struct hist_browser, b); 872 hb = container_of(browser, struct hist_browser, b);
873 browser->top = rb_first(&hb->hists->entries); 873 browser->top = rb_first(&hb->hists->entries);
874 } 874 }
875 } 875 }
876 876
877 static unsigned int hist_browser__refresh(struct ui_browser *browser) 877 static unsigned int hist_browser__refresh(struct ui_browser *browser)
878 { 878 {
879 unsigned row = 0; 879 unsigned row = 0;
880 u16 header_offset = 0; 880 u16 header_offset = 0;
881 struct rb_node *nd; 881 struct rb_node *nd;
882 struct hist_browser *hb = container_of(browser, struct hist_browser, b); 882 struct hist_browser *hb = container_of(browser, struct hist_browser, b);
883 883
884 if (hb->show_headers) { 884 if (hb->show_headers) {
885 hist_browser__show_headers(hb); 885 hist_browser__show_headers(hb);
886 header_offset = 1; 886 header_offset = 1;
887 } 887 }
888 888
889 ui_browser__hists_init_top(browser); 889 ui_browser__hists_init_top(browser);
890 890
891 for (nd = browser->top; nd; nd = rb_next(nd)) { 891 for (nd = browser->top; nd; nd = rb_next(nd)) {
892 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 892 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
893 float percent; 893 float percent;
894 894
895 if (h->filtered) 895 if (h->filtered)
896 continue; 896 continue;
897 897
898 percent = hist_entry__get_percent_limit(h); 898 percent = hist_entry__get_percent_limit(h);
899 if (percent < hb->min_pcnt) 899 if (percent < hb->min_pcnt)
900 continue; 900 continue;
901 901
902 row += hist_browser__show_entry(hb, h, row); 902 row += hist_browser__show_entry(hb, h, row);
903 if (row == browser->rows) 903 if (row == browser->rows)
904 break; 904 break;
905 } 905 }
906 906
907 return row + header_offset; 907 return row + header_offset;
908 } 908 }
909 909
910 static struct rb_node *hists__filter_entries(struct rb_node *nd, 910 static struct rb_node *hists__filter_entries(struct rb_node *nd,
911 float min_pcnt) 911 float min_pcnt)
912 { 912 {
913 while (nd != NULL) { 913 while (nd != NULL) {
914 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 914 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
915 float percent = hist_entry__get_percent_limit(h); 915 float percent = hist_entry__get_percent_limit(h);
916 916
917 if (!h->filtered && percent >= min_pcnt) 917 if (!h->filtered && percent >= min_pcnt)
918 return nd; 918 return nd;
919 919
920 nd = rb_next(nd); 920 nd = rb_next(nd);
921 } 921 }
922 922
923 return NULL; 923 return NULL;
924 } 924 }
925 925
926 static struct rb_node *hists__filter_prev_entries(struct rb_node *nd, 926 static struct rb_node *hists__filter_prev_entries(struct rb_node *nd,
927 float min_pcnt) 927 float min_pcnt)
928 { 928 {
929 while (nd != NULL) { 929 while (nd != NULL) {
930 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 930 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
931 float percent = hist_entry__get_percent_limit(h); 931 float percent = hist_entry__get_percent_limit(h);
932 932
933 if (!h->filtered && percent >= min_pcnt) 933 if (!h->filtered && percent >= min_pcnt)
934 return nd; 934 return nd;
935 935
936 nd = rb_prev(nd); 936 nd = rb_prev(nd);
937 } 937 }
938 938
939 return NULL; 939 return NULL;
940 } 940 }
941 941
942 static void ui_browser__hists_seek(struct ui_browser *browser, 942 static void ui_browser__hists_seek(struct ui_browser *browser,
943 off_t offset, int whence) 943 off_t offset, int whence)
944 { 944 {
945 struct hist_entry *h; 945 struct hist_entry *h;
946 struct rb_node *nd; 946 struct rb_node *nd;
947 bool first = true; 947 bool first = true;
948 struct hist_browser *hb; 948 struct hist_browser *hb;
949 949
950 hb = container_of(browser, struct hist_browser, b); 950 hb = container_of(browser, struct hist_browser, b);
951 951
952 if (browser->nr_entries == 0) 952 if (browser->nr_entries == 0)
953 return; 953 return;
954 954
955 ui_browser__hists_init_top(browser); 955 ui_browser__hists_init_top(browser);
956 956
957 switch (whence) { 957 switch (whence) {
958 case SEEK_SET: 958 case SEEK_SET:
959 nd = hists__filter_entries(rb_first(browser->entries), 959 nd = hists__filter_entries(rb_first(browser->entries),
960 hb->min_pcnt); 960 hb->min_pcnt);
961 break; 961 break;
962 case SEEK_CUR: 962 case SEEK_CUR:
963 nd = browser->top; 963 nd = browser->top;
964 goto do_offset; 964 goto do_offset;
965 case SEEK_END: 965 case SEEK_END:
966 nd = hists__filter_prev_entries(rb_last(browser->entries), 966 nd = hists__filter_prev_entries(rb_last(browser->entries),
967 hb->min_pcnt); 967 hb->min_pcnt);
968 first = false; 968 first = false;
969 break; 969 break;
970 default: 970 default:
971 return; 971 return;
972 } 972 }
973 973
974 /* 974 /*
975 * Moves not relative to the first visible entry invalidates its 975 * Moves not relative to the first visible entry invalidates its
976 * row_offset: 976 * row_offset:
977 */ 977 */
978 h = rb_entry(browser->top, struct hist_entry, rb_node); 978 h = rb_entry(browser->top, struct hist_entry, rb_node);
979 h->row_offset = 0; 979 h->row_offset = 0;
980 980
981 /* 981 /*
982 * Here we have to check if nd is expanded (+), if it is we can't go 982 * Here we have to check if nd is expanded (+), if it is we can't go
983 * the next top level hist_entry, instead we must compute an offset of 983 * the next top level hist_entry, instead we must compute an offset of
984 * what _not_ to show and not change the first visible entry. 984 * what _not_ to show and not change the first visible entry.
985 * 985 *
986 * This offset increments when we are going from top to bottom and 986 * This offset increments when we are going from top to bottom and
987 * decreases when we're going from bottom to top. 987 * decreases when we're going from bottom to top.
988 * 988 *
989 * As we don't have backpointers to the top level in the callchains 989 * As we don't have backpointers to the top level in the callchains
990 * structure, we need to always print the whole hist_entry callchain, 990 * structure, we need to always print the whole hist_entry callchain,
991 * skipping the first ones that are before the first visible entry 991 * skipping the first ones that are before the first visible entry
992 * and stop when we printed enough lines to fill the screen. 992 * and stop when we printed enough lines to fill the screen.
993 */ 993 */
994 do_offset: 994 do_offset:
995 if (offset > 0) { 995 if (offset > 0) {
996 do { 996 do {
997 h = rb_entry(nd, struct hist_entry, rb_node); 997 h = rb_entry(nd, struct hist_entry, rb_node);
998 if (h->ms.unfolded) { 998 if (h->ms.unfolded) {
999 u16 remaining = h->nr_rows - h->row_offset; 999 u16 remaining = h->nr_rows - h->row_offset;
1000 if (offset > remaining) { 1000 if (offset > remaining) {
1001 offset -= remaining; 1001 offset -= remaining;
1002 h->row_offset = 0; 1002 h->row_offset = 0;
1003 } else { 1003 } else {
1004 h->row_offset += offset; 1004 h->row_offset += offset;
1005 offset = 0; 1005 offset = 0;
1006 browser->top = nd; 1006 browser->top = nd;
1007 break; 1007 break;
1008 } 1008 }
1009 } 1009 }
1010 nd = hists__filter_entries(rb_next(nd), hb->min_pcnt); 1010 nd = hists__filter_entries(rb_next(nd), hb->min_pcnt);
1011 if (nd == NULL) 1011 if (nd == NULL)
1012 break; 1012 break;
1013 --offset; 1013 --offset;
1014 browser->top = nd; 1014 browser->top = nd;
1015 } while (offset != 0); 1015 } while (offset != 0);
1016 } else if (offset < 0) { 1016 } else if (offset < 0) {
1017 while (1) { 1017 while (1) {
1018 h = rb_entry(nd, struct hist_entry, rb_node); 1018 h = rb_entry(nd, struct hist_entry, rb_node);
1019 if (h->ms.unfolded) { 1019 if (h->ms.unfolded) {
1020 if (first) { 1020 if (first) {
1021 if (-offset > h->row_offset) { 1021 if (-offset > h->row_offset) {
1022 offset += h->row_offset; 1022 offset += h->row_offset;
1023 h->row_offset = 0; 1023 h->row_offset = 0;
1024 } else { 1024 } else {
1025 h->row_offset += offset; 1025 h->row_offset += offset;
1026 offset = 0; 1026 offset = 0;
1027 browser->top = nd; 1027 browser->top = nd;
1028 break; 1028 break;
1029 } 1029 }
1030 } else { 1030 } else {
1031 if (-offset > h->nr_rows) { 1031 if (-offset > h->nr_rows) {
1032 offset += h->nr_rows; 1032 offset += h->nr_rows;
1033 h->row_offset = 0; 1033 h->row_offset = 0;
1034 } else { 1034 } else {
1035 h->row_offset = h->nr_rows + offset; 1035 h->row_offset = h->nr_rows + offset;
1036 offset = 0; 1036 offset = 0;
1037 browser->top = nd; 1037 browser->top = nd;
1038 break; 1038 break;
1039 } 1039 }
1040 } 1040 }
1041 } 1041 }
1042 1042
1043 nd = hists__filter_prev_entries(rb_prev(nd), 1043 nd = hists__filter_prev_entries(rb_prev(nd),
1044 hb->min_pcnt); 1044 hb->min_pcnt);
1045 if (nd == NULL) 1045 if (nd == NULL)
1046 break; 1046 break;
1047 ++offset; 1047 ++offset;
1048 browser->top = nd; 1048 browser->top = nd;
1049 if (offset == 0) { 1049 if (offset == 0) {
1050 /* 1050 /*
1051 * Last unfiltered hist_entry, check if it is 1051 * Last unfiltered hist_entry, check if it is
1052 * unfolded, if it is then we should have 1052 * unfolded, if it is then we should have
1053 * row_offset at its last entry. 1053 * row_offset at its last entry.
1054 */ 1054 */
1055 h = rb_entry(nd, struct hist_entry, rb_node); 1055 h = rb_entry(nd, struct hist_entry, rb_node);
1056 if (h->ms.unfolded) 1056 if (h->ms.unfolded)
1057 h->row_offset = h->nr_rows; 1057 h->row_offset = h->nr_rows;
1058 break; 1058 break;
1059 } 1059 }
1060 first = false; 1060 first = false;
1061 } 1061 }
1062 } else { 1062 } else {
1063 browser->top = nd; 1063 browser->top = nd;
1064 h = rb_entry(nd, struct hist_entry, rb_node); 1064 h = rb_entry(nd, struct hist_entry, rb_node);
1065 h->row_offset = 0; 1065 h->row_offset = 0;
1066 } 1066 }
1067 } 1067 }
1068 1068
1069 static int hist_browser__fprintf_callchain(struct hist_browser *browser, 1069 static int hist_browser__fprintf_callchain(struct hist_browser *browser,
1070 struct hist_entry *he, FILE *fp) 1070 struct hist_entry *he, FILE *fp)
1071 { 1071 {
1072 u64 total = hists__total_period(he->hists); 1072 u64 total = hists__total_period(he->hists);
1073 struct callchain_print_arg arg = { 1073 struct callchain_print_arg arg = {
1074 .fp = fp, 1074 .fp = fp,
1075 }; 1075 };
1076 1076
1077 if (symbol_conf.cumulate_callchain) 1077 if (symbol_conf.cumulate_callchain)
1078 total = he->stat_acc->period; 1078 total = he->stat_acc->period;
1079 1079
1080 hist_browser__show_callchain(browser, &he->sorted_chain, 1, 0, total, 1080 hist_browser__show_callchain(browser, &he->sorted_chain, 1, 0, total,
1081 hist_browser__fprintf_callchain_entry, &arg, 1081 hist_browser__fprintf_callchain_entry, &arg,
1082 hist_browser__check_dump_full); 1082 hist_browser__check_dump_full);
1083 return arg.printed; 1083 return arg.printed;
1084 } 1084 }
1085 1085
1086 static int hist_browser__fprintf_entry(struct hist_browser *browser, 1086 static int hist_browser__fprintf_entry(struct hist_browser *browser,
1087 struct hist_entry *he, FILE *fp) 1087 struct hist_entry *he, FILE *fp)
1088 { 1088 {
1089 char s[8192]; 1089 char s[8192];
1090 int printed = 0; 1090 int printed = 0;
1091 char folded_sign = ' '; 1091 char folded_sign = ' ';
1092 struct perf_hpp hpp = { 1092 struct perf_hpp hpp = {
1093 .buf = s, 1093 .buf = s,
1094 .size = sizeof(s), 1094 .size = sizeof(s),
1095 }; 1095 };
1096 struct perf_hpp_fmt *fmt; 1096 struct perf_hpp_fmt *fmt;
1097 bool first = true; 1097 bool first = true;
1098 int ret; 1098 int ret;
1099 1099
1100 if (symbol_conf.use_callchain) 1100 if (symbol_conf.use_callchain)
1101 folded_sign = hist_entry__folded(he); 1101 folded_sign = hist_entry__folded(he);
1102 1102
1103 if (symbol_conf.use_callchain) 1103 if (symbol_conf.use_callchain)
1104 printed += fprintf(fp, "%c ", folded_sign); 1104 printed += fprintf(fp, "%c ", folded_sign);
1105 1105
1106 perf_hpp__for_each_format(fmt) { 1106 perf_hpp__for_each_format(fmt) {
1107 if (perf_hpp__should_skip(fmt)) 1107 if (perf_hpp__should_skip(fmt))
1108 continue; 1108 continue;
1109 1109
1110 if (!first) { 1110 if (!first) {
1111 ret = scnprintf(hpp.buf, hpp.size, " "); 1111 ret = scnprintf(hpp.buf, hpp.size, " ");
1112 advance_hpp(&hpp, ret); 1112 advance_hpp(&hpp, ret);
1113 } else 1113 } else
1114 first = false; 1114 first = false;
1115 1115
1116 ret = fmt->entry(fmt, &hpp, he); 1116 ret = fmt->entry(fmt, &hpp, he);
1117 advance_hpp(&hpp, ret); 1117 advance_hpp(&hpp, ret);
1118 } 1118 }
1119 printed += fprintf(fp, "%s\n", rtrim(s)); 1119 printed += fprintf(fp, "%s\n", rtrim(s));
1120 1120
1121 if (folded_sign == '-') 1121 if (folded_sign == '-')
1122 printed += hist_browser__fprintf_callchain(browser, he, fp); 1122 printed += hist_browser__fprintf_callchain(browser, he, fp);
1123 1123
1124 return printed; 1124 return printed;
1125 } 1125 }
1126 1126
1127 static int hist_browser__fprintf(struct hist_browser *browser, FILE *fp) 1127 static int hist_browser__fprintf(struct hist_browser *browser, FILE *fp)
1128 { 1128 {
1129 struct rb_node *nd = hists__filter_entries(rb_first(browser->b.entries), 1129 struct rb_node *nd = hists__filter_entries(rb_first(browser->b.entries),
1130 browser->min_pcnt); 1130 browser->min_pcnt);
1131 int printed = 0; 1131 int printed = 0;
1132 1132
1133 while (nd) { 1133 while (nd) {
1134 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 1134 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
1135 1135
1136 printed += hist_browser__fprintf_entry(browser, h, fp); 1136 printed += hist_browser__fprintf_entry(browser, h, fp);
1137 nd = hists__filter_entries(rb_next(nd), browser->min_pcnt); 1137 nd = hists__filter_entries(rb_next(nd), browser->min_pcnt);
1138 } 1138 }
1139 1139
1140 return printed; 1140 return printed;
1141 } 1141 }
1142 1142
1143 static int hist_browser__dump(struct hist_browser *browser) 1143 static int hist_browser__dump(struct hist_browser *browser)
1144 { 1144 {
1145 char filename[64]; 1145 char filename[64];
1146 FILE *fp; 1146 FILE *fp;
1147 1147
1148 while (1) { 1148 while (1) {
1149 scnprintf(filename, sizeof(filename), "perf.hist.%d", browser->print_seq); 1149 scnprintf(filename, sizeof(filename), "perf.hist.%d", browser->print_seq);
1150 if (access(filename, F_OK)) 1150 if (access(filename, F_OK))
1151 break; 1151 break;
1152 /* 1152 /*
1153 * XXX: Just an arbitrary lazy upper limit 1153 * XXX: Just an arbitrary lazy upper limit
1154 */ 1154 */
1155 if (++browser->print_seq == 8192) { 1155 if (++browser->print_seq == 8192) {
1156 ui_helpline__fpush("Too many perf.hist.N files, nothing written!"); 1156 ui_helpline__fpush("Too many perf.hist.N files, nothing written!");
1157 return -1; 1157 return -1;
1158 } 1158 }
1159 } 1159 }
1160 1160
1161 fp = fopen(filename, "w"); 1161 fp = fopen(filename, "w");
1162 if (fp == NULL) { 1162 if (fp == NULL) {
1163 char bf[64]; 1163 char bf[64];
1164 const char *err = strerror_r(errno, bf, sizeof(bf)); 1164 const char *err = strerror_r(errno, bf, sizeof(bf));
1165 ui_helpline__fpush("Couldn't write to %s: %s", filename, err); 1165 ui_helpline__fpush("Couldn't write to %s: %s", filename, err);
1166 return -1; 1166 return -1;
1167 } 1167 }
1168 1168
1169 ++browser->print_seq; 1169 ++browser->print_seq;
1170 hist_browser__fprintf(browser, fp); 1170 hist_browser__fprintf(browser, fp);
1171 fclose(fp); 1171 fclose(fp);
1172 ui_helpline__fpush("%s written!", filename); 1172 ui_helpline__fpush("%s written!", filename);
1173 1173
1174 return 0; 1174 return 0;
1175 } 1175 }
1176 1176
1177 static struct hist_browser *hist_browser__new(struct hists *hists) 1177 static struct hist_browser *hist_browser__new(struct hists *hists)
1178 { 1178 {
1179 struct hist_browser *browser = zalloc(sizeof(*browser)); 1179 struct hist_browser *browser = zalloc(sizeof(*browser));
1180 1180
1181 if (browser) { 1181 if (browser) {
1182 browser->hists = hists; 1182 browser->hists = hists;
1183 browser->b.refresh = hist_browser__refresh; 1183 browser->b.refresh = hist_browser__refresh;
1184 browser->b.refresh_dimensions = hist_browser__refresh_dimensions; 1184 browser->b.refresh_dimensions = hist_browser__refresh_dimensions;
1185 browser->b.seek = ui_browser__hists_seek; 1185 browser->b.seek = ui_browser__hists_seek;
1186 browser->b.use_navkeypressed = true; 1186 browser->b.use_navkeypressed = true;
1187 browser->show_headers = symbol_conf.show_hist_headers; 1187 browser->show_headers = symbol_conf.show_hist_headers;
1188 } 1188 }
1189 1189
1190 return browser; 1190 return browser;
1191 } 1191 }
1192 1192
1193 static void hist_browser__delete(struct hist_browser *browser) 1193 static void hist_browser__delete(struct hist_browser *browser)
1194 { 1194 {
1195 free(browser); 1195 free(browser);
1196 } 1196 }
1197 1197
1198 static struct hist_entry *hist_browser__selected_entry(struct hist_browser *browser) 1198 static struct hist_entry *hist_browser__selected_entry(struct hist_browser *browser)
1199 { 1199 {
1200 return browser->he_selection; 1200 return browser->he_selection;
1201 } 1201 }
1202 1202
1203 static struct thread *hist_browser__selected_thread(struct hist_browser *browser) 1203 static struct thread *hist_browser__selected_thread(struct hist_browser *browser)
1204 { 1204 {
1205 return browser->he_selection->thread; 1205 return browser->he_selection->thread;
1206 } 1206 }
1207 1207
1208 /* Check whether the browser is for 'top' or 'report' */ 1208 /* Check whether the browser is for 'top' or 'report' */
1209 static inline bool is_report_browser(void *timer) 1209 static inline bool is_report_browser(void *timer)
1210 { 1210 {
1211 return timer == NULL; 1211 return timer == NULL;
1212 } 1212 }
1213 1213
1214 static int hists__browser_title(struct hists *hists, 1214 static int hists__browser_title(struct hists *hists,
1215 struct hist_browser_timer *hbt, 1215 struct hist_browser_timer *hbt,
1216 char *bf, size_t size) 1216 char *bf, size_t size)
1217 { 1217 {
1218 char unit; 1218 char unit;
1219 int printed; 1219 int printed;
1220 const struct dso *dso = hists->dso_filter; 1220 const struct dso *dso = hists->dso_filter;
1221 const struct thread *thread = hists->thread_filter; 1221 const struct thread *thread = hists->thread_filter;
1222 unsigned long nr_samples = hists->stats.nr_events[PERF_RECORD_SAMPLE]; 1222 unsigned long nr_samples = hists->stats.nr_events[PERF_RECORD_SAMPLE];
1223 u64 nr_events = hists->stats.total_period; 1223 u64 nr_events = hists->stats.total_period;
1224 struct perf_evsel *evsel = hists_to_evsel(hists); 1224 struct perf_evsel *evsel = hists_to_evsel(hists);
1225 const char *ev_name = perf_evsel__name(evsel); 1225 const char *ev_name = perf_evsel__name(evsel);
1226 char buf[512]; 1226 char buf[512];
1227 size_t buflen = sizeof(buf); 1227 size_t buflen = sizeof(buf);
1228 1228
1229 if (symbol_conf.filter_relative) { 1229 if (symbol_conf.filter_relative) {
1230 nr_samples = hists->stats.nr_non_filtered_samples; 1230 nr_samples = hists->stats.nr_non_filtered_samples;
1231 nr_events = hists->stats.total_non_filtered_period; 1231 nr_events = hists->stats.total_non_filtered_period;
1232 } 1232 }
1233 1233
1234 if (perf_evsel__is_group_event(evsel)) { 1234 if (perf_evsel__is_group_event(evsel)) {
1235 struct perf_evsel *pos; 1235 struct perf_evsel *pos;
1236 1236
1237 perf_evsel__group_desc(evsel, buf, buflen); 1237 perf_evsel__group_desc(evsel, buf, buflen);
1238 ev_name = buf; 1238 ev_name = buf;
1239 1239
1240 for_each_group_member(pos, evsel) { 1240 for_each_group_member(pos, evsel) {
1241 struct hists *pos_hists = evsel__hists(pos); 1241 struct hists *pos_hists = evsel__hists(pos);
1242 1242
1243 if (symbol_conf.filter_relative) { 1243 if (symbol_conf.filter_relative) {
1244 nr_samples += pos_hists->stats.nr_non_filtered_samples; 1244 nr_samples += pos_hists->stats.nr_non_filtered_samples;
1245 nr_events += pos_hists->stats.total_non_filtered_period; 1245 nr_events += pos_hists->stats.total_non_filtered_period;
1246 } else { 1246 } else {
1247 nr_samples += pos_hists->stats.nr_events[PERF_RECORD_SAMPLE]; 1247 nr_samples += pos_hists->stats.nr_events[PERF_RECORD_SAMPLE];
1248 nr_events += pos_hists->stats.total_period; 1248 nr_events += pos_hists->stats.total_period;
1249 } 1249 }
1250 } 1250 }
1251 } 1251 }
1252 1252
1253 nr_samples = convert_unit(nr_samples, &unit); 1253 nr_samples = convert_unit(nr_samples, &unit);
1254 printed = scnprintf(bf, size, 1254 printed = scnprintf(bf, size,
1255 "Samples: %lu%c of event '%s', Event count (approx.): %" PRIu64, 1255 "Samples: %lu%c of event '%s', Event count (approx.): %" PRIu64,
1256 nr_samples, unit, ev_name, nr_events); 1256 nr_samples, unit, ev_name, nr_events);
1257 1257
1258 1258
1259 if (hists->uid_filter_str) 1259 if (hists->uid_filter_str)
1260 printed += snprintf(bf + printed, size - printed, 1260 printed += snprintf(bf + printed, size - printed,
1261 ", UID: %s", hists->uid_filter_str); 1261 ", UID: %s", hists->uid_filter_str);
1262 if (thread) 1262 if (thread)
1263 printed += scnprintf(bf + printed, size - printed, 1263 printed += scnprintf(bf + printed, size - printed,
1264 ", Thread: %s(%d)", 1264 ", Thread: %s(%d)",
1265 (thread->comm_set ? thread__comm_str(thread) : ""), 1265 (thread->comm_set ? thread__comm_str(thread) : ""),
1266 thread->tid); 1266 thread->tid);
1267 if (dso) 1267 if (dso)
1268 printed += scnprintf(bf + printed, size - printed, 1268 printed += scnprintf(bf + printed, size - printed,
1269 ", DSO: %s", dso->short_name); 1269 ", DSO: %s", dso->short_name);
1270 if (!is_report_browser(hbt)) { 1270 if (!is_report_browser(hbt)) {
1271 struct perf_top *top = hbt->arg; 1271 struct perf_top *top = hbt->arg;
1272 1272
1273 if (top->zero) 1273 if (top->zero)
1274 printed += scnprintf(bf + printed, size - printed, " [z]"); 1274 printed += scnprintf(bf + printed, size - printed, " [z]");
1275 } 1275 }
1276 1276
1277 return printed; 1277 return printed;
1278 } 1278 }
1279 1279
1280 static inline void free_popup_options(char **options, int n) 1280 static inline void free_popup_options(char **options, int n)
1281 { 1281 {
1282 int i; 1282 int i;
1283 1283
1284 for (i = 0; i < n; ++i) 1284 for (i = 0; i < n; ++i)
1285 zfree(&options[i]); 1285 zfree(&options[i]);
1286 } 1286 }
1287 1287
1288 /* 1288 /*
1289 * Only runtime switching of perf data file will make "input_name" point 1289 * Only runtime switching of perf data file will make "input_name" point
1290 * to a malloced buffer. So add "is_input_name_malloced" flag to decide 1290 * to a malloced buffer. So add "is_input_name_malloced" flag to decide
1291 * whether we need to call free() for current "input_name" during the switch. 1291 * whether we need to call free() for current "input_name" during the switch.
1292 */ 1292 */
1293 static bool is_input_name_malloced = false; 1293 static bool is_input_name_malloced = false;
1294 1294
1295 static int switch_data_file(void) 1295 static int switch_data_file(void)
1296 { 1296 {
1297 char *pwd, *options[32], *abs_path[32], *tmp; 1297 char *pwd, *options[32], *abs_path[32], *tmp;
1298 DIR *pwd_dir; 1298 DIR *pwd_dir;
1299 int nr_options = 0, choice = -1, ret = -1; 1299 int nr_options = 0, choice = -1, ret = -1;
1300 struct dirent *dent; 1300 struct dirent *dent;
1301 1301
1302 pwd = getenv("PWD"); 1302 pwd = getenv("PWD");
1303 if (!pwd) 1303 if (!pwd)
1304 return ret; 1304 return ret;
1305 1305
1306 pwd_dir = opendir(pwd); 1306 pwd_dir = opendir(pwd);
1307 if (!pwd_dir) 1307 if (!pwd_dir)
1308 return ret; 1308 return ret;
1309 1309
1310 memset(options, 0, sizeof(options)); 1310 memset(options, 0, sizeof(options));
1311 memset(options, 0, sizeof(abs_path)); 1311 memset(options, 0, sizeof(abs_path));
1312 1312
1313 while ((dent = readdir(pwd_dir))) { 1313 while ((dent = readdir(pwd_dir))) {
1314 char path[PATH_MAX]; 1314 char path[PATH_MAX];
1315 u64 magic; 1315 u64 magic;
1316 char *name = dent->d_name; 1316 char *name = dent->d_name;
1317 FILE *file; 1317 FILE *file;
1318 1318
1319 if (!(dent->d_type == DT_REG)) 1319 if (!(dent->d_type == DT_REG))
1320 continue; 1320 continue;
1321 1321
1322 snprintf(path, sizeof(path), "%s/%s", pwd, name); 1322 snprintf(path, sizeof(path), "%s/%s", pwd, name);
1323 1323
1324 file = fopen(path, "r"); 1324 file = fopen(path, "r");
1325 if (!file) 1325 if (!file)
1326 continue; 1326 continue;
1327 1327
1328 if (fread(&magic, 1, 8, file) < 8) 1328 if (fread(&magic, 1, 8, file) < 8)
1329 goto close_file_and_continue; 1329 goto close_file_and_continue;
1330 1330
1331 if (is_perf_magic(magic)) { 1331 if (is_perf_magic(magic)) {
1332 options[nr_options] = strdup(name); 1332 options[nr_options] = strdup(name);
1333 if (!options[nr_options]) 1333 if (!options[nr_options])
1334 goto close_file_and_continue; 1334 goto close_file_and_continue;
1335 1335
1336 abs_path[nr_options] = strdup(path); 1336 abs_path[nr_options] = strdup(path);
1337 if (!abs_path[nr_options]) { 1337 if (!abs_path[nr_options]) {
1338 zfree(&options[nr_options]); 1338 zfree(&options[nr_options]);
1339 ui__warning("Can't search all data files due to memory shortage.\n"); 1339 ui__warning("Can't search all data files due to memory shortage.\n");
1340 fclose(file); 1340 fclose(file);
1341 break; 1341 break;
1342 } 1342 }
1343 1343
1344 nr_options++; 1344 nr_options++;
1345 } 1345 }
1346 1346
1347 close_file_and_continue: 1347 close_file_and_continue:
1348 fclose(file); 1348 fclose(file);
1349 if (nr_options >= 32) { 1349 if (nr_options >= 32) {
1350 ui__warning("Too many perf data files in PWD!\n" 1350 ui__warning("Too many perf data files in PWD!\n"
1351 "Only the first 32 files will be listed.\n"); 1351 "Only the first 32 files will be listed.\n");
1352 break; 1352 break;
1353 } 1353 }
1354 } 1354 }
1355 closedir(pwd_dir); 1355 closedir(pwd_dir);
1356 1356
1357 if (nr_options) { 1357 if (nr_options) {
1358 choice = ui__popup_menu(nr_options, options); 1358 choice = ui__popup_menu(nr_options, options);
1359 if (choice < nr_options && choice >= 0) { 1359 if (choice < nr_options && choice >= 0) {
1360 tmp = strdup(abs_path[choice]); 1360 tmp = strdup(abs_path[choice]);
1361 if (tmp) { 1361 if (tmp) {
1362 if (is_input_name_malloced) 1362 if (is_input_name_malloced)
1363 free((void *)input_name); 1363 free((void *)input_name);
1364 input_name = tmp; 1364 input_name = tmp;
1365 is_input_name_malloced = true; 1365 is_input_name_malloced = true;
1366 ret = 0; 1366 ret = 0;
1367 } else 1367 } else
1368 ui__warning("Data switch failed due to memory shortage!\n"); 1368 ui__warning("Data switch failed due to memory shortage!\n");
1369 } 1369 }
1370 } 1370 }
1371 1371
1372 free_popup_options(options, nr_options); 1372 free_popup_options(options, nr_options);
1373 free_popup_options(abs_path, nr_options); 1373 free_popup_options(abs_path, nr_options);
1374 return ret; 1374 return ret;
1375 } 1375 }
1376 1376
1377 static void hist_browser__update_nr_entries(struct hist_browser *hb) 1377 static void hist_browser__update_nr_entries(struct hist_browser *hb)
1378 { 1378 {
1379 u64 nr_entries = 0; 1379 u64 nr_entries = 0;
1380 struct rb_node *nd = rb_first(&hb->hists->entries); 1380 struct rb_node *nd = rb_first(&hb->hists->entries);
1381 1381
1382 if (hb->min_pcnt == 0) { 1382 if (hb->min_pcnt == 0) {
1383 hb->nr_non_filtered_entries = hb->hists->nr_non_filtered_entries; 1383 hb->nr_non_filtered_entries = hb->hists->nr_non_filtered_entries;
1384 return; 1384 return;
1385 } 1385 }
1386 1386
1387 while ((nd = hists__filter_entries(nd, hb->min_pcnt)) != NULL) { 1387 while ((nd = hists__filter_entries(nd, hb->min_pcnt)) != NULL) {
1388 nr_entries++; 1388 nr_entries++;
1389 nd = rb_next(nd); 1389 nd = rb_next(nd);
1390 } 1390 }
1391 1391
1392 hb->nr_non_filtered_entries = nr_entries; 1392 hb->nr_non_filtered_entries = nr_entries;
1393 } 1393 }
1394 1394
1395 static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events, 1395 static int perf_evsel__hists_browse(struct perf_evsel *evsel, int nr_events,
1396 const char *helpline, 1396 const char *helpline,
1397 bool left_exits, 1397 bool left_exits,
1398 struct hist_browser_timer *hbt, 1398 struct hist_browser_timer *hbt,
1399 float min_pcnt, 1399 float min_pcnt,
1400 struct perf_session_env *env) 1400 struct perf_session_env *env)
1401 { 1401 {
1402 struct hists *hists = evsel__hists(evsel); 1402 struct hists *hists = evsel__hists(evsel);
1403 struct hist_browser *browser = hist_browser__new(hists); 1403 struct hist_browser *browser = hist_browser__new(hists);
1404 struct branch_info *bi; 1404 struct branch_info *bi;
1405 struct pstack *fstack; 1405 struct pstack *fstack;
1406 char *options[16]; 1406 char *options[16];
1407 int nr_options = 0; 1407 int nr_options = 0;
1408 int key = -1; 1408 int key = -1;
1409 char buf[64]; 1409 char buf[64];
1410 char script_opt[64]; 1410 char script_opt[64];
1411 int delay_secs = hbt ? hbt->refresh : 0; 1411 int delay_secs = hbt ? hbt->refresh : 0;
1412 struct perf_hpp_fmt *fmt; 1412 struct perf_hpp_fmt *fmt;
1413 1413
1414 #define HIST_BROWSER_HELP_COMMON \ 1414 #define HIST_BROWSER_HELP_COMMON \
1415 "h/?/F1 Show this window\n" \ 1415 "h/?/F1 Show this window\n" \
1416 "UP/DOWN/PGUP\n" \ 1416 "UP/DOWN/PGUP\n" \
1417 "PGDN/SPACE Navigate\n" \ 1417 "PGDN/SPACE Navigate\n" \
1418 "q/ESC/CTRL+C Exit browser\n\n" \ 1418 "q/ESC/CTRL+C Exit browser\n\n" \
1419 "For multiple event sessions:\n\n" \ 1419 "For multiple event sessions:\n\n" \
1420 "TAB/UNTAB Switch events\n\n" \ 1420 "TAB/UNTAB Switch events\n\n" \
1421 "For symbolic views (--sort has sym):\n\n" \ 1421 "For symbolic views (--sort has sym):\n\n" \
1422 "-> Zoom into DSO/Threads & Annotate current symbol\n" \ 1422 "-> Zoom into DSO/Threads & Annotate current symbol\n" \
1423 "<- Zoom out\n" \ 1423 "<- Zoom out\n" \
1424 "a Annotate current symbol\n" \ 1424 "a Annotate current symbol\n" \
1425 "C Collapse all callchains\n" \ 1425 "C Collapse all callchains\n" \
1426 "d Zoom into current DSO\n" \ 1426 "d Zoom into current DSO\n" \
1427 "E Expand all callchains\n" \ 1427 "E Expand all callchains\n" \
1428 "F Toggle percentage of filtered entries\n" \ 1428 "F Toggle percentage of filtered entries\n" \
1429 "H Display column headers\n" \ 1429 "H Display column headers\n" \
1430 1430
1431 /* help messages are sorted by lexical order of the hotkey */ 1431 /* help messages are sorted by lexical order of the hotkey */
1432 const char report_help[] = HIST_BROWSER_HELP_COMMON 1432 const char report_help[] = HIST_BROWSER_HELP_COMMON
1433 "i Show header information\n" 1433 "i Show header information\n"
1434 "P Print histograms to perf.hist.N\n" 1434 "P Print histograms to perf.hist.N\n"
1435 "r Run available scripts\n" 1435 "r Run available scripts\n"
1436 "s Switch to another data file in PWD\n" 1436 "s Switch to another data file in PWD\n"
1437 "t Zoom into current Thread\n" 1437 "t Zoom into current Thread\n"
1438 "V Verbose (DSO names in callchains, etc)\n" 1438 "V Verbose (DSO names in callchains, etc)\n"
1439 "/ Filter symbol by name"; 1439 "/ Filter symbol by name";
1440 const char top_help[] = HIST_BROWSER_HELP_COMMON 1440 const char top_help[] = HIST_BROWSER_HELP_COMMON
1441 "P Print histograms to perf.hist.N\n" 1441 "P Print histograms to perf.hist.N\n"
1442 "t Zoom into current Thread\n" 1442 "t Zoom into current Thread\n"
1443 "V Verbose (DSO names in callchains, etc)\n" 1443 "V Verbose (DSO names in callchains, etc)\n"
1444 "z Toggle zeroing of samples\n" 1444 "z Toggle zeroing of samples\n"
1445 "/ Filter symbol by name"; 1445 "/ Filter symbol by name";
1446 1446
1447 if (browser == NULL) 1447 if (browser == NULL)
1448 return -1; 1448 return -1;
1449 1449
1450 if (min_pcnt) { 1450 if (min_pcnt) {
1451 browser->min_pcnt = min_pcnt; 1451 browser->min_pcnt = min_pcnt;
1452 hist_browser__update_nr_entries(browser); 1452 hist_browser__update_nr_entries(browser);
1453 } 1453 }
1454 1454
1455 fstack = pstack__new(2); 1455 fstack = pstack__new(2);
1456 if (fstack == NULL) 1456 if (fstack == NULL)
1457 goto out; 1457 goto out;
1458 1458
1459 ui_helpline__push(helpline); 1459 ui_helpline__push(helpline);
1460 1460
1461 memset(options, 0, sizeof(options)); 1461 memset(options, 0, sizeof(options));
1462 1462
1463 perf_hpp__for_each_format(fmt) 1463 perf_hpp__for_each_format(fmt)
1464 perf_hpp__reset_width(fmt, hists); 1464 perf_hpp__reset_width(fmt, hists);
1465 1465
1466 if (symbol_conf.col_width_list_str) 1466 if (symbol_conf.col_width_list_str)
1467 perf_hpp__set_user_width(symbol_conf.col_width_list_str); 1467 perf_hpp__set_user_width(symbol_conf.col_width_list_str);
1468 1468
1469 while (1) { 1469 while (1) {
1470 const struct thread *thread = NULL; 1470 const struct thread *thread = NULL;
1471 const struct dso *dso = NULL; 1471 const struct dso *dso = NULL;
1472 int choice = 0, 1472 int choice = 0,
1473 annotate = -2, zoom_dso = -2, zoom_thread = -2, 1473 annotate = -2, zoom_dso = -2, zoom_thread = -2,
1474 annotate_f = -2, annotate_t = -2, browse_map = -2; 1474 annotate_f = -2, annotate_t = -2, browse_map = -2;
1475 int scripts_comm = -2, scripts_symbol = -2, 1475 int scripts_comm = -2, scripts_symbol = -2,
1476 scripts_all = -2, switch_data = -2; 1476 scripts_all = -2, switch_data = -2;
1477 1477
1478 nr_options = 0; 1478 nr_options = 0;
1479 1479
1480 key = hist_browser__run(browser, hbt); 1480 key = hist_browser__run(browser, hbt);
1481 1481
1482 if (browser->he_selection != NULL) { 1482 if (browser->he_selection != NULL) {
1483 thread = hist_browser__selected_thread(browser); 1483 thread = hist_browser__selected_thread(browser);
1484 dso = browser->selection->map ? browser->selection->map->dso : NULL; 1484 dso = browser->selection->map ? browser->selection->map->dso : NULL;
1485 } 1485 }
1486 switch (key) { 1486 switch (key) {
1487 case K_TAB: 1487 case K_TAB:
1488 case K_UNTAB: 1488 case K_UNTAB:
1489 if (nr_events == 1) 1489 if (nr_events == 1)
1490 continue; 1490 continue;
1491 /* 1491 /*
1492 * Exit the browser, let hists__browser_tree 1492 * Exit the browser, let hists__browser_tree
1493 * go to the next or previous 1493 * go to the next or previous
1494 */ 1494 */
1495 goto out_free_stack; 1495 goto out_free_stack;
1496 case 'a': 1496 case 'a':
1497 if (!sort__has_sym) { 1497 if (!sort__has_sym) {
1498 ui_browser__warning(&browser->b, delay_secs * 2, 1498 ui_browser__warning(&browser->b, delay_secs * 2,
1499 "Annotation is only available for symbolic views, " 1499 "Annotation is only available for symbolic views, "
1500 "include \"sym*\" in --sort to use it."); 1500 "include \"sym*\" in --sort to use it.");
1501 continue; 1501 continue;
1502 } 1502 }
1503 1503
1504 if (browser->selection == NULL || 1504 if (browser->selection == NULL ||
1505 browser->selection->sym == NULL || 1505 browser->selection->sym == NULL ||
1506 browser->selection->map->dso->annotate_warned) 1506 browser->selection->map->dso->annotate_warned)
1507 continue; 1507 continue;
1508 goto do_annotate; 1508 goto do_annotate;
1509 case 'P': 1509 case 'P':
1510 hist_browser__dump(browser); 1510 hist_browser__dump(browser);
1511 continue; 1511 continue;
1512 case 'd': 1512 case 'd':
1513 goto zoom_dso; 1513 goto zoom_dso;
1514 case 'V': 1514 case 'V':
1515 browser->show_dso = !browser->show_dso; 1515 browser->show_dso = !browser->show_dso;
1516 continue; 1516 continue;
1517 case 't': 1517 case 't':
1518 goto zoom_thread; 1518 goto zoom_thread;
1519 case '/': 1519 case '/':
1520 if (ui_browser__input_window("Symbol to show", 1520 if (ui_browser__input_window("Symbol to show",
1521 "Please enter the name of symbol you want to see", 1521 "Please enter the name of symbol you want to see",
1522 buf, "ENTER: OK, ESC: Cancel", 1522 buf, "ENTER: OK, ESC: Cancel",
1523 delay_secs * 2) == K_ENTER) { 1523 delay_secs * 2) == K_ENTER) {
1524 hists->symbol_filter_str = *buf ? buf : NULL; 1524 hists->symbol_filter_str = *buf ? buf : NULL;
1525 hists__filter_by_symbol(hists); 1525 hists__filter_by_symbol(hists);
1526 hist_browser__reset(browser); 1526 hist_browser__reset(browser);
1527 } 1527 }
1528 continue; 1528 continue;
1529 case 'r': 1529 case 'r':
1530 if (is_report_browser(hbt)) 1530 if (is_report_browser(hbt))
1531 goto do_scripts; 1531 goto do_scripts;
1532 continue; 1532 continue;
1533 case 's': 1533 case 's':
1534 if (is_report_browser(hbt)) 1534 if (is_report_browser(hbt))
1535 goto do_data_switch; 1535 goto do_data_switch;
1536 continue; 1536 continue;
1537 case 'i': 1537 case 'i':
1538 /* env->arch is NULL for live-mode (i.e. perf top) */ 1538 /* env->arch is NULL for live-mode (i.e. perf top) */
1539 if (env->arch) 1539 if (env->arch)
1540 tui__header_window(env); 1540 tui__header_window(env);
1541 continue; 1541 continue;
1542 case 'F': 1542 case 'F':
1543 symbol_conf.filter_relative ^= 1; 1543 symbol_conf.filter_relative ^= 1;
1544 continue; 1544 continue;
1545 case 'z': 1545 case 'z':
1546 if (!is_report_browser(hbt)) { 1546 if (!is_report_browser(hbt)) {
1547 struct perf_top *top = hbt->arg; 1547 struct perf_top *top = hbt->arg;
1548 1548
1549 top->zero = !top->zero; 1549 top->zero = !top->zero;
1550 } 1550 }
1551 continue; 1551 continue;
1552 case K_F1: 1552 case K_F1:
1553 case 'h': 1553 case 'h':
1554 case '?': 1554 case '?':
1555 ui_browser__help_window(&browser->b, 1555 ui_browser__help_window(&browser->b,
1556 is_report_browser(hbt) ? report_help : top_help); 1556 is_report_browser(hbt) ? report_help : top_help);
1557 continue; 1557 continue;
1558 case K_ENTER: 1558 case K_ENTER:
1559 case K_RIGHT: 1559 case K_RIGHT:
1560 /* menu */ 1560 /* menu */
1561 break; 1561 break;
1562 case K_LEFT: { 1562 case K_LEFT: {
1563 const void *top; 1563 const void *top;
1564 1564
1565 if (pstack__empty(fstack)) { 1565 if (pstack__empty(fstack)) {
1566 /* 1566 /*
1567 * Go back to the perf_evsel_menu__run or other user 1567 * Go back to the perf_evsel_menu__run or other user
1568 */ 1568 */
1569 if (left_exits) 1569 if (left_exits)
1570 goto out_free_stack; 1570 goto out_free_stack;
1571 continue; 1571 continue;
1572 } 1572 }
1573 top = pstack__pop(fstack); 1573 top = pstack__pop(fstack);
1574 if (top == &browser->hists->dso_filter) 1574 if (top == &browser->hists->dso_filter)
1575 goto zoom_out_dso; 1575 goto zoom_out_dso;
1576 if (top == &browser->hists->thread_filter) 1576 if (top == &browser->hists->thread_filter)
1577 goto zoom_out_thread; 1577 goto zoom_out_thread;
1578 continue; 1578 continue;
1579 } 1579 }
1580 case K_ESC: 1580 case K_ESC:
1581 if (!left_exits && 1581 if (!left_exits &&
1582 !ui_browser__dialog_yesno(&browser->b, 1582 !ui_browser__dialog_yesno(&browser->b,
1583 "Do you really want to exit?")) 1583 "Do you really want to exit?"))
1584 continue; 1584 continue;
1585 /* Fall thru */ 1585 /* Fall thru */
1586 case 'q': 1586 case 'q':
1587 case CTRL('c'): 1587 case CTRL('c'):
1588 goto out_free_stack; 1588 goto out_free_stack;
1589 default: 1589 default:
1590 continue; 1590 continue;
1591 } 1591 }
1592 1592
1593 if (!sort__has_sym) 1593 if (!sort__has_sym)
1594 goto add_exit_option; 1594 goto add_exit_option;
1595 1595
1596 if (sort__mode == SORT_MODE__BRANCH) { 1596 if (sort__mode == SORT_MODE__BRANCH) {
1597 bi = browser->he_selection->branch_info; 1597 bi = browser->he_selection->branch_info;
1598 if (browser->selection != NULL && 1598 if (browser->selection != NULL &&
1599 bi && 1599 bi &&
1600 bi->from.sym != NULL && 1600 bi->from.sym != NULL &&
1601 !bi->from.map->dso->annotate_warned && 1601 !bi->from.map->dso->annotate_warned &&
1602 asprintf(&options[nr_options], "Annotate %s", 1602 asprintf(&options[nr_options], "Annotate %s",
1603 bi->from.sym->name) > 0) 1603 bi->from.sym->name) > 0)
1604 annotate_f = nr_options++; 1604 annotate_f = nr_options++;
1605 1605
1606 if (browser->selection != NULL && 1606 if (browser->selection != NULL &&
1607 bi && 1607 bi &&
1608 bi->to.sym != NULL && 1608 bi->to.sym != NULL &&
1609 !bi->to.map->dso->annotate_warned && 1609 !bi->to.map->dso->annotate_warned &&
1610 (bi->to.sym != bi->from.sym || 1610 (bi->to.sym != bi->from.sym ||
1611 bi->to.map->dso != bi->from.map->dso) && 1611 bi->to.map->dso != bi->from.map->dso) &&
1612 asprintf(&options[nr_options], "Annotate %s", 1612 asprintf(&options[nr_options], "Annotate %s",
1613 bi->to.sym->name) > 0) 1613 bi->to.sym->name) > 0)
1614 annotate_t = nr_options++; 1614 annotate_t = nr_options++;
1615 } else { 1615 } else {
1616 if (browser->selection != NULL && 1616 if (browser->selection != NULL &&
1617 browser->selection->sym != NULL && 1617 browser->selection->sym != NULL &&
1618 !browser->selection->map->dso->annotate_warned) { 1618 !browser->selection->map->dso->annotate_warned) {
1619 struct annotation *notes; 1619 struct annotation *notes;
1620 1620
1621 notes = symbol__annotation(browser->selection->sym); 1621 notes = symbol__annotation(browser->selection->sym);
1622 1622
1623 if (notes->src && 1623 if (notes->src &&
1624 asprintf(&options[nr_options], "Annotate %s", 1624 asprintf(&options[nr_options], "Annotate %s",
1625 browser->selection->sym->name) > 0) 1625 browser->selection->sym->name) > 0)
1626 annotate = nr_options++; 1626 annotate = nr_options++;
1627 } 1627 }
1628 } 1628 }
1629 1629
1630 if (thread != NULL && 1630 if (thread != NULL &&
1631 asprintf(&options[nr_options], "Zoom %s %s(%d) thread", 1631 asprintf(&options[nr_options], "Zoom %s %s(%d) thread",
1632 (browser->hists->thread_filter ? "out of" : "into"), 1632 (browser->hists->thread_filter ? "out of" : "into"),
1633 (thread->comm_set ? thread__comm_str(thread) : ""), 1633 (thread->comm_set ? thread__comm_str(thread) : ""),
1634 thread->tid) > 0) 1634 thread->tid) > 0)
1635 zoom_thread = nr_options++; 1635 zoom_thread = nr_options++;
1636 1636
1637 if (dso != NULL && 1637 if (dso != NULL &&
1638 asprintf(&options[nr_options], "Zoom %s %s DSO", 1638 asprintf(&options[nr_options], "Zoom %s %s DSO",
1639 (browser->hists->dso_filter ? "out of" : "into"), 1639 (browser->hists->dso_filter ? "out of" : "into"),
1640 (dso->kernel ? "the Kernel" : dso->short_name)) > 0) 1640 (dso->kernel ? "the Kernel" : dso->short_name)) > 0)
1641 zoom_dso = nr_options++; 1641 zoom_dso = nr_options++;
1642 1642
1643 if (browser->selection != NULL && 1643 if (browser->selection != NULL &&
1644 browser->selection->map != NULL && 1644 browser->selection->map != NULL &&
1645 asprintf(&options[nr_options], "Browse map details") > 0) 1645 asprintf(&options[nr_options], "Browse map details") > 0)
1646 browse_map = nr_options++; 1646 browse_map = nr_options++;
1647 1647
1648 /* perf script support */ 1648 /* perf script support */
1649 if (browser->he_selection) { 1649 if (browser->he_selection) {
1650 struct symbol *sym; 1650 struct symbol *sym;
1651 1651
1652 if (asprintf(&options[nr_options], "Run scripts for samples of thread [%s]", 1652 if (asprintf(&options[nr_options], "Run scripts for samples of thread [%s]",
1653 thread__comm_str(browser->he_selection->thread)) > 0) 1653 thread__comm_str(browser->he_selection->thread)) > 0)
1654 scripts_comm = nr_options++; 1654 scripts_comm = nr_options++;
1655 1655
1656 sym = browser->he_selection->ms.sym; 1656 sym = browser->he_selection->ms.sym;
1657 if (sym && sym->namelen && 1657 if (sym && sym->namelen &&
1658 asprintf(&options[nr_options], "Run scripts for samples of symbol [%s]", 1658 asprintf(&options[nr_options], "Run scripts for samples of symbol [%s]",
1659 sym->name) > 0) 1659 sym->name) > 0)
1660 scripts_symbol = nr_options++; 1660 scripts_symbol = nr_options++;
1661 } 1661 }
1662 1662
1663 if (asprintf(&options[nr_options], "Run scripts for all samples") > 0) 1663 if (asprintf(&options[nr_options], "Run scripts for all samples") > 0)
1664 scripts_all = nr_options++; 1664 scripts_all = nr_options++;
1665 1665
1666 if (is_report_browser(hbt) && asprintf(&options[nr_options], 1666 if (is_report_browser(hbt) && asprintf(&options[nr_options],
1667 "Switch to another data file in PWD") > 0) 1667 "Switch to another data file in PWD") > 0)
1668 switch_data = nr_options++; 1668 switch_data = nr_options++;
1669 add_exit_option: 1669 add_exit_option:
1670 options[nr_options++] = (char *)"Exit"; 1670 options[nr_options++] = (char *)"Exit";
1671 retry_popup_menu: 1671 retry_popup_menu:
1672 choice = ui__popup_menu(nr_options, options); 1672 choice = ui__popup_menu(nr_options, options);
1673 1673
1674 if (choice == nr_options - 1) 1674 if (choice == nr_options - 1)
1675 break; 1675 break;
1676 1676
1677 if (choice == -1) { 1677 if (choice == -1) {
1678 free_popup_options(options, nr_options - 1); 1678 free_popup_options(options, nr_options - 1);
1679 continue; 1679 continue;
1680 } 1680 }
1681 1681
1682 if (choice == annotate || choice == annotate_t || choice == annotate_f) { 1682 if (choice == annotate || choice == annotate_t || choice == annotate_f) {
1683 struct hist_entry *he; 1683 struct hist_entry *he;
1684 struct annotation *notes; 1684 struct annotation *notes;
1685 int err; 1685 int err;
1686 do_annotate: 1686 do_annotate:
1687 if (!objdump_path && perf_session_env__lookup_objdump(env)) 1687 if (!objdump_path && perf_session_env__lookup_objdump(env))
1688 continue; 1688 continue;
1689 1689
1690 he = hist_browser__selected_entry(browser); 1690 he = hist_browser__selected_entry(browser);
1691 if (he == NULL) 1691 if (he == NULL)
1692 continue; 1692 continue;
1693 1693
1694 /* 1694 /*
1695 * we stash the branch_info symbol + map into the 1695 * we stash the branch_info symbol + map into the
1696 * the ms so we don't have to rewrite all the annotation 1696 * the ms so we don't have to rewrite all the annotation
1697 * code to use branch_info. 1697 * code to use branch_info.
1698 * in branch mode, the ms struct is not used 1698 * in branch mode, the ms struct is not used
1699 */ 1699 */
1700 if (choice == annotate_f) { 1700 if (choice == annotate_f) {
1701 he->ms.sym = he->branch_info->from.sym; 1701 he->ms.sym = he->branch_info->from.sym;
1702 he->ms.map = he->branch_info->from.map; 1702 he->ms.map = he->branch_info->from.map;
1703 } else if (choice == annotate_t) { 1703 } else if (choice == annotate_t) {
1704 he->ms.sym = he->branch_info->to.sym; 1704 he->ms.sym = he->branch_info->to.sym;
1705 he->ms.map = he->branch_info->to.map; 1705 he->ms.map = he->branch_info->to.map;
1706 } 1706 }
1707 1707
1708 notes = symbol__annotation(he->ms.sym); 1708 notes = symbol__annotation(he->ms.sym);
1709 if (!notes->src) 1709 if (!notes->src)
1710 continue; 1710 continue;
1711 1711
1712 /* 1712 /*
1713 * Don't let this be freed, say, by hists__decay_entry. 1713 * Don't let this be freed, say, by hists__decay_entry.
1714 */ 1714 */
1715 he->used = true; 1715 he->used = true;
1716 err = hist_entry__tui_annotate(he, evsel, hbt); 1716 err = hist_entry__tui_annotate(he, evsel, hbt);
1717 he->used = false; 1717 he->used = false;
1718 /* 1718 /*
1719 * offer option to annotate the other branch source or target 1719 * offer option to annotate the other branch source or target
1720 * (if they exists) when returning from annotate 1720 * (if they exists) when returning from annotate
1721 */ 1721 */
1722 if ((err == 'q' || err == CTRL('c')) 1722 if ((err == 'q' || err == CTRL('c'))
1723 && annotate_t != -2 && annotate_f != -2) 1723 && annotate_t != -2 && annotate_f != -2)
1724 goto retry_popup_menu; 1724 goto retry_popup_menu;
1725 1725
1726 ui_browser__update_nr_entries(&browser->b, browser->hists->nr_entries); 1726 ui_browser__update_nr_entries(&browser->b, browser->hists->nr_entries);
1727 if (err) 1727 if (err)
1728 ui_browser__handle_resize(&browser->b); 1728 ui_browser__handle_resize(&browser->b);
1729 1729
1730 } else if (choice == browse_map) 1730 } else if (choice == browse_map)
1731 map__browse(browser->selection->map); 1731 map__browse(browser->selection->map);
1732 else if (choice == zoom_dso) { 1732 else if (choice == zoom_dso) {
1733 zoom_dso: 1733 zoom_dso:
1734 if (browser->hists->dso_filter) { 1734 if (browser->hists->dso_filter) {
1735 pstack__remove(fstack, &browser->hists->dso_filter); 1735 pstack__remove(fstack, &browser->hists->dso_filter);
1736 zoom_out_dso: 1736 zoom_out_dso:
1737 ui_helpline__pop(); 1737 ui_helpline__pop();
1738 browser->hists->dso_filter = NULL; 1738 browser->hists->dso_filter = NULL;
1739 perf_hpp__set_elide(HISTC_DSO, false); 1739 perf_hpp__set_elide(HISTC_DSO, false);
1740 } else { 1740 } else {
1741 if (dso == NULL) 1741 if (dso == NULL)
1742 continue; 1742 continue;
1743 ui_helpline__fpush("To zoom out press <- or -> + \"Zoom out of %s DSO\"", 1743 ui_helpline__fpush("To zoom out press <- or -> + \"Zoom out of %s DSO\"",
1744 dso->kernel ? "the Kernel" : dso->short_name); 1744 dso->kernel ? "the Kernel" : dso->short_name);
1745 browser->hists->dso_filter = dso; 1745 browser->hists->dso_filter = dso;
1746 perf_hpp__set_elide(HISTC_DSO, true); 1746 perf_hpp__set_elide(HISTC_DSO, true);
1747 pstack__push(fstack, &browser->hists->dso_filter); 1747 pstack__push(fstack, &browser->hists->dso_filter);
1748 } 1748 }
1749 hists__filter_by_dso(hists); 1749 hists__filter_by_dso(hists);
1750 hist_browser__reset(browser); 1750 hist_browser__reset(browser);
1751 } else if (choice == zoom_thread) { 1751 } else if (choice == zoom_thread) {
1752 zoom_thread: 1752 zoom_thread:
1753 if (browser->hists->thread_filter) { 1753 if (browser->hists->thread_filter) {
1754 pstack__remove(fstack, &browser->hists->thread_filter); 1754 pstack__remove(fstack, &browser->hists->thread_filter);
1755 zoom_out_thread: 1755 zoom_out_thread:
1756 ui_helpline__pop(); 1756 ui_helpline__pop();
1757 browser->hists->thread_filter = NULL; 1757 browser->hists->thread_filter = NULL;
1758 perf_hpp__set_elide(HISTC_THREAD, false); 1758 perf_hpp__set_elide(HISTC_THREAD, false);
1759 } else { 1759 } else {
1760 ui_helpline__fpush("To zoom out press <- or -> + \"Zoom out of %s(%d) thread\"", 1760 ui_helpline__fpush("To zoom out press <- or -> + \"Zoom out of %s(%d) thread\"",
1761 thread->comm_set ? thread__comm_str(thread) : "", 1761 thread->comm_set ? thread__comm_str(thread) : "",
1762 thread->tid); 1762 thread->tid);
1763 browser->hists->thread_filter = thread; 1763 browser->hists->thread_filter = thread;
1764 perf_hpp__set_elide(HISTC_THREAD, false); 1764 perf_hpp__set_elide(HISTC_THREAD, false);
1765 pstack__push(fstack, &browser->hists->thread_filter); 1765 pstack__push(fstack, &browser->hists->thread_filter);
1766 } 1766 }
1767 hists__filter_by_thread(hists); 1767 hists__filter_by_thread(hists);
1768 hist_browser__reset(browser); 1768 hist_browser__reset(browser);
1769 } 1769 }
1770 /* perf scripts support */ 1770 /* perf scripts support */
1771 else if (choice == scripts_all || choice == scripts_comm || 1771 else if (choice == scripts_all || choice == scripts_comm ||
1772 choice == scripts_symbol) { 1772 choice == scripts_symbol) {
1773 do_scripts: 1773 do_scripts:
1774 memset(script_opt, 0, 64); 1774 memset(script_opt, 0, 64);
1775 1775
1776 if (choice == scripts_comm) 1776 if (choice == scripts_comm)
1777 sprintf(script_opt, " -c %s ", thread__comm_str(browser->he_selection->thread)); 1777 sprintf(script_opt, " -c %s ", thread__comm_str(browser->he_selection->thread));
1778 1778
1779 if (choice == scripts_symbol) 1779 if (choice == scripts_symbol)
1780 sprintf(script_opt, " -S %s ", browser->he_selection->ms.sym->name); 1780 sprintf(script_opt, " -S %s ", browser->he_selection->ms.sym->name);
1781 1781
1782 script_browse(script_opt); 1782 script_browse(script_opt);
1783 } 1783 }
1784 /* Switch to another data file */ 1784 /* Switch to another data file */
1785 else if (choice == switch_data) { 1785 else if (choice == switch_data) {
1786 do_data_switch: 1786 do_data_switch:
1787 if (!switch_data_file()) { 1787 if (!switch_data_file()) {
1788 key = K_SWITCH_INPUT_DATA; 1788 key = K_SWITCH_INPUT_DATA;
1789 break; 1789 break;
1790 } else 1790 } else
1791 ui__warning("Won't switch the data files due to\n" 1791 ui__warning("Won't switch the data files due to\n"
1792 "no valid data file get selected!\n"); 1792 "no valid data file get selected!\n");
1793 } 1793 }
1794 } 1794 }
1795 out_free_stack: 1795 out_free_stack:
1796 pstack__delete(fstack); 1796 pstack__delete(fstack);
1797 out: 1797 out:
1798 hist_browser__delete(browser); 1798 hist_browser__delete(browser);
1799 free_popup_options(options, nr_options - 1); 1799 free_popup_options(options, nr_options - 1);
1800 return key; 1800 return key;
1801 } 1801 }
1802 1802
1803 struct perf_evsel_menu { 1803 struct perf_evsel_menu {
1804 struct ui_browser b; 1804 struct ui_browser b;
1805 struct perf_evsel *selection; 1805 struct perf_evsel *selection;
1806 bool lost_events, lost_events_warned; 1806 bool lost_events, lost_events_warned;
1807 float min_pcnt; 1807 float min_pcnt;
1808 struct perf_session_env *env; 1808 struct perf_session_env *env;
1809 }; 1809 };
1810 1810
1811 static void perf_evsel_menu__write(struct ui_browser *browser, 1811 static void perf_evsel_menu__write(struct ui_browser *browser,
1812 void *entry, int row) 1812 void *entry, int row)
1813 { 1813 {
1814 struct perf_evsel_menu *menu = container_of(browser, 1814 struct perf_evsel_menu *menu = container_of(browser,
1815 struct perf_evsel_menu, b); 1815 struct perf_evsel_menu, b);
1816 struct perf_evsel *evsel = list_entry(entry, struct perf_evsel, node); 1816 struct perf_evsel *evsel = list_entry(entry, struct perf_evsel, node);
1817 struct hists *hists = evsel__hists(evsel); 1817 struct hists *hists = evsel__hists(evsel);
1818 bool current_entry = ui_browser__is_current_entry(browser, row); 1818 bool current_entry = ui_browser__is_current_entry(browser, row);
1819 unsigned long nr_events = hists->stats.nr_events[PERF_RECORD_SAMPLE]; 1819 unsigned long nr_events = hists->stats.nr_events[PERF_RECORD_SAMPLE];
1820 const char *ev_name = perf_evsel__name(evsel); 1820 const char *ev_name = perf_evsel__name(evsel);
1821 char bf[256], unit; 1821 char bf[256], unit;
1822 const char *warn = " "; 1822 const char *warn = " ";
1823 size_t printed; 1823 size_t printed;
1824 1824
1825 ui_browser__set_color(browser, current_entry ? HE_COLORSET_SELECTED : 1825 ui_browser__set_color(browser, current_entry ? HE_COLORSET_SELECTED :
1826 HE_COLORSET_NORMAL); 1826 HE_COLORSET_NORMAL);
1827 1827
1828 if (perf_evsel__is_group_event(evsel)) { 1828 if (perf_evsel__is_group_event(evsel)) {
1829 struct perf_evsel *pos; 1829 struct perf_evsel *pos;
1830 1830
1831 ev_name = perf_evsel__group_name(evsel); 1831 ev_name = perf_evsel__group_name(evsel);
1832 1832
1833 for_each_group_member(pos, evsel) { 1833 for_each_group_member(pos, evsel) {
1834 struct hists *pos_hists = evsel__hists(pos); 1834 struct hists *pos_hists = evsel__hists(pos);
1835 nr_events += pos_hists->stats.nr_events[PERF_RECORD_SAMPLE]; 1835 nr_events += pos_hists->stats.nr_events[PERF_RECORD_SAMPLE];
1836 } 1836 }
1837 } 1837 }
1838 1838
1839 nr_events = convert_unit(nr_events, &unit); 1839 nr_events = convert_unit(nr_events, &unit);
1840 printed = scnprintf(bf, sizeof(bf), "%lu%c%s%s", nr_events, 1840 printed = scnprintf(bf, sizeof(bf), "%lu%c%s%s", nr_events,
1841 unit, unit == ' ' ? "" : " ", ev_name); 1841 unit, unit == ' ' ? "" : " ", ev_name);
1842 slsmg_printf("%s", bf); 1842 slsmg_printf("%s", bf);
1843 1843
1844 nr_events = hists->stats.nr_events[PERF_RECORD_LOST]; 1844 nr_events = hists->stats.nr_events[PERF_RECORD_LOST];
1845 if (nr_events != 0) { 1845 if (nr_events != 0) {
1846 menu->lost_events = true; 1846 menu->lost_events = true;
1847 if (!current_entry) 1847 if (!current_entry)
1848 ui_browser__set_color(browser, HE_COLORSET_TOP); 1848 ui_browser__set_color(browser, HE_COLORSET_TOP);
1849 nr_events = convert_unit(nr_events, &unit); 1849 nr_events = convert_unit(nr_events, &unit);
1850 printed += scnprintf(bf, sizeof(bf), ": %ld%c%schunks LOST!", 1850 printed += scnprintf(bf, sizeof(bf), ": %ld%c%schunks LOST!",
1851 nr_events, unit, unit == ' ' ? "" : " "); 1851 nr_events, unit, unit == ' ' ? "" : " ");
1852 warn = bf; 1852 warn = bf;
1853 } 1853 }
1854 1854
1855 slsmg_write_nstring(warn, browser->width - printed); 1855 slsmg_write_nstring(warn, browser->width - printed);
1856 1856
1857 if (current_entry) 1857 if (current_entry)
1858 menu->selection = evsel; 1858 menu->selection = evsel;
1859 } 1859 }
1860 1860
1861 static int perf_evsel_menu__run(struct perf_evsel_menu *menu, 1861 static int perf_evsel_menu__run(struct perf_evsel_menu *menu,
1862 int nr_events, const char *help, 1862 int nr_events, const char *help,
1863 struct hist_browser_timer *hbt) 1863 struct hist_browser_timer *hbt)
1864 { 1864 {
1865 struct perf_evlist *evlist = menu->b.priv; 1865 struct perf_evlist *evlist = menu->b.priv;
1866 struct perf_evsel *pos; 1866 struct perf_evsel *pos;
1867 const char *title = "Available samples"; 1867 const char *title = "Available samples";
1868 int delay_secs = hbt ? hbt->refresh : 0; 1868 int delay_secs = hbt ? hbt->refresh : 0;
1869 int key; 1869 int key;
1870 1870
1871 if (ui_browser__show(&menu->b, title, 1871 if (ui_browser__show(&menu->b, title,
1872 "ESC: exit, ENTER|->: Browse histograms") < 0) 1872 "ESC: exit, ENTER|->: Browse histograms") < 0)
1873 return -1; 1873 return -1;
1874 1874
1875 while (1) { 1875 while (1) {
1876 key = ui_browser__run(&menu->b, delay_secs); 1876 key = ui_browser__run(&menu->b, delay_secs);
1877 1877
1878 switch (key) { 1878 switch (key) {
1879 case K_TIMER: 1879 case K_TIMER:
1880 hbt->timer(hbt->arg); 1880 hbt->timer(hbt->arg);
1881 1881
1882 if (!menu->lost_events_warned && menu->lost_events) { 1882 if (!menu->lost_events_warned && menu->lost_events) {
1883 ui_browser__warn_lost_events(&menu->b); 1883 ui_browser__warn_lost_events(&menu->b);
1884 menu->lost_events_warned = true; 1884 menu->lost_events_warned = true;
1885 } 1885 }
1886 continue; 1886 continue;
1887 case K_RIGHT: 1887 case K_RIGHT:
1888 case K_ENTER: 1888 case K_ENTER:
1889 if (!menu->selection) 1889 if (!menu->selection)
1890 continue; 1890 continue;
1891 pos = menu->selection; 1891 pos = menu->selection;
1892 browse_hists: 1892 browse_hists:
1893 perf_evlist__set_selected(evlist, pos); 1893 perf_evlist__set_selected(evlist, pos);
1894 /* 1894 /*
1895 * Give the calling tool a chance to populate the non 1895 * Give the calling tool a chance to populate the non
1896 * default evsel resorted hists tree. 1896 * default evsel resorted hists tree.
1897 */ 1897 */
1898 if (hbt) 1898 if (hbt)
1899 hbt->timer(hbt->arg); 1899 hbt->timer(hbt->arg);
1900 key = perf_evsel__hists_browse(pos, nr_events, help, 1900 key = perf_evsel__hists_browse(pos, nr_events, help,
1901 true, hbt, 1901 true, hbt,
1902 menu->min_pcnt, 1902 menu->min_pcnt,
1903 menu->env); 1903 menu->env);
1904 ui_browser__show_title(&menu->b, title); 1904 ui_browser__show_title(&menu->b, title);
1905 switch (key) { 1905 switch (key) {
1906 case K_TAB: 1906 case K_TAB:
1907 if (pos->node.next == &evlist->entries) 1907 if (pos->node.next == &evlist->entries)
1908 pos = perf_evlist__first(evlist); 1908 pos = perf_evlist__first(evlist);
1909 else 1909 else
1910 pos = perf_evsel__next(pos); 1910 pos = perf_evsel__next(pos);
1911 goto browse_hists; 1911 goto browse_hists;
1912 case K_UNTAB: 1912 case K_UNTAB:
1913 if (pos->node.prev == &evlist->entries) 1913 if (pos->node.prev == &evlist->entries)
1914 pos = perf_evlist__last(evlist); 1914 pos = perf_evlist__last(evlist);
1915 else 1915 else
1916 pos = perf_evsel__prev(pos); 1916 pos = perf_evsel__prev(pos);
1917 goto browse_hists; 1917 goto browse_hists;
1918 case K_ESC: 1918 case K_ESC:
1919 if (!ui_browser__dialog_yesno(&menu->b, 1919 if (!ui_browser__dialog_yesno(&menu->b,
1920 "Do you really want to exit?")) 1920 "Do you really want to exit?"))
1921 continue; 1921 continue;
1922 /* Fall thru */ 1922 /* Fall thru */
1923 case K_SWITCH_INPUT_DATA: 1923 case K_SWITCH_INPUT_DATA:
1924 case 'q': 1924 case 'q':
1925 case CTRL('c'): 1925 case CTRL('c'):
1926 goto out; 1926 goto out;
1927 default: 1927 default:
1928 continue; 1928 continue;
1929 } 1929 }
1930 case K_LEFT: 1930 case K_LEFT:
1931 continue; 1931 continue;
1932 case K_ESC: 1932 case K_ESC:
1933 if (!ui_browser__dialog_yesno(&menu->b, 1933 if (!ui_browser__dialog_yesno(&menu->b,
1934 "Do you really want to exit?")) 1934 "Do you really want to exit?"))
1935 continue; 1935 continue;
1936 /* Fall thru */ 1936 /* Fall thru */
1937 case 'q': 1937 case 'q':
1938 case CTRL('c'): 1938 case CTRL('c'):
1939 goto out; 1939 goto out;
1940 default: 1940 default:
1941 continue; 1941 continue;
1942 } 1942 }
1943 } 1943 }
1944 1944
1945 out: 1945 out:
1946 ui_browser__hide(&menu->b); 1946 ui_browser__hide(&menu->b);
1947 return key; 1947 return key;
1948 } 1948 }
1949 1949
1950 static bool filter_group_entries(struct ui_browser *browser __maybe_unused, 1950 static bool filter_group_entries(struct ui_browser *browser __maybe_unused,
1951 void *entry) 1951 void *entry)
1952 { 1952 {
1953 struct perf_evsel *evsel = list_entry(entry, struct perf_evsel, node); 1953 struct perf_evsel *evsel = list_entry(entry, struct perf_evsel, node);
1954 1954
1955 if (symbol_conf.event_group && !perf_evsel__is_group_leader(evsel)) 1955 if (symbol_conf.event_group && !perf_evsel__is_group_leader(evsel))
1956 return true; 1956 return true;
1957 1957
1958 return false; 1958 return false;
1959 } 1959 }
1960 1960
1961 static int __perf_evlist__tui_browse_hists(struct perf_evlist *evlist, 1961 static int __perf_evlist__tui_browse_hists(struct perf_evlist *evlist,
1962 int nr_entries, const char *help, 1962 int nr_entries, const char *help,
1963 struct hist_browser_timer *hbt, 1963 struct hist_browser_timer *hbt,
1964 float min_pcnt, 1964 float min_pcnt,
1965 struct perf_session_env *env) 1965 struct perf_session_env *env)
1966 { 1966 {
1967 struct perf_evsel *pos; 1967 struct perf_evsel *pos;
1968 struct perf_evsel_menu menu = { 1968 struct perf_evsel_menu menu = {
1969 .b = { 1969 .b = {
1970 .entries = &evlist->entries, 1970 .entries = &evlist->entries,
1971 .refresh = ui_browser__list_head_refresh, 1971 .refresh = ui_browser__list_head_refresh,
1972 .seek = ui_browser__list_head_seek, 1972 .seek = ui_browser__list_head_seek,
1973 .write = perf_evsel_menu__write, 1973 .write = perf_evsel_menu__write,
1974 .filter = filter_group_entries, 1974 .filter = filter_group_entries,
1975 .nr_entries = nr_entries, 1975 .nr_entries = nr_entries,
1976 .priv = evlist, 1976 .priv = evlist,
1977 }, 1977 },
1978 .min_pcnt = min_pcnt, 1978 .min_pcnt = min_pcnt,
1979 .env = env, 1979 .env = env,
1980 }; 1980 };
1981 1981
1982 ui_helpline__push("Press ESC to exit"); 1982 ui_helpline__push("Press ESC to exit");
1983 1983
1984 evlist__for_each(evlist, pos) { 1984 evlist__for_each(evlist, pos) {
1985 const char *ev_name = perf_evsel__name(pos); 1985 const char *ev_name = perf_evsel__name(pos);
1986 size_t line_len = strlen(ev_name) + 7; 1986 size_t line_len = strlen(ev_name) + 7;
1987 1987
1988 if (menu.b.width < line_len) 1988 if (menu.b.width < line_len)
1989 menu.b.width = line_len; 1989 menu.b.width = line_len;
1990 } 1990 }
1991 1991
1992 return perf_evsel_menu__run(&menu, nr_entries, help, hbt); 1992 return perf_evsel_menu__run(&menu, nr_entries, help, hbt);
1993 } 1993 }
1994 1994
1995 int perf_evlist__tui_browse_hists(struct perf_evlist *evlist, const char *help, 1995 int perf_evlist__tui_browse_hists(struct perf_evlist *evlist, const char *help,
1996 struct hist_browser_timer *hbt, 1996 struct hist_browser_timer *hbt,
1997 float min_pcnt, 1997 float min_pcnt,
1998 struct perf_session_env *env) 1998 struct perf_session_env *env)
1999 { 1999 {
2000 int nr_entries = evlist->nr_entries; 2000 int nr_entries = evlist->nr_entries;
2001 2001
2002 single_entry: 2002 single_entry:
2003 if (nr_entries == 1) { 2003 if (nr_entries == 1) {
2004 struct perf_evsel *first = perf_evlist__first(evlist); 2004 struct perf_evsel *first = perf_evlist__first(evlist);
2005 2005
2006 return perf_evsel__hists_browse(first, nr_entries, help, 2006 return perf_evsel__hists_browse(first, nr_entries, help,
2007 false, hbt, min_pcnt, 2007 false, hbt, min_pcnt,
2008 env); 2008 env);
2009 } 2009 }
2010 2010
2011 if (symbol_conf.event_group) { 2011 if (symbol_conf.event_group) {
2012 struct perf_evsel *pos; 2012 struct perf_evsel *pos;
2013 2013
2014 nr_entries = 0; 2014 nr_entries = 0;
2015 evlist__for_each(evlist, pos) { 2015 evlist__for_each(evlist, pos) {
2016 if (perf_evsel__is_group_leader(pos)) 2016 if (perf_evsel__is_group_leader(pos))
2017 nr_entries++; 2017 nr_entries++;
2018 } 2018 }
2019 2019
2020 if (nr_entries == 1) 2020 if (nr_entries == 1)
2021 goto single_entry; 2021 goto single_entry;
2022 } 2022 }
2023 2023
2024 return __perf_evlist__tui_browse_hists(evlist, nr_entries, help, 2024 return __perf_evlist__tui_browse_hists(evlist, nr_entries, help,
2025 hbt, min_pcnt, env); 2025 hbt, min_pcnt, env);
2026 } 2026 }
2027 2027
tools/perf/ui/hist.c
Diff comments   View file @ ddb321a
1 #include <math.h> 1 #include <math.h>
2 #include <linux/compiler.h> 2 #include <linux/compiler.h>
3 3
4 #include "../util/hist.h" 4 #include "../util/hist.h"
5 #include "../util/util.h" 5 #include "../util/util.h"
6 #include "../util/sort.h" 6 #include "../util/sort.h"
7 #include "../util/evsel.h" 7 #include "../util/evsel.h"
8 8
9 /* hist period print (hpp) functions */ 9 /* hist period print (hpp) functions */
10 10
11 #define hpp__call_print_fn(hpp, fn, fmt, ...) \ 11 #define hpp__call_print_fn(hpp, fn, fmt, ...) \
12 ({ \ 12 ({ \
13 int __ret = fn(hpp, fmt, ##__VA_ARGS__); \ 13 int __ret = fn(hpp, fmt, ##__VA_ARGS__); \
14 advance_hpp(hpp, __ret); \ 14 advance_hpp(hpp, __ret); \
15 __ret; \ 15 __ret; \
16 }) 16 })
17 17
18 static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he, 18 static int __hpp__fmt(struct perf_hpp *hpp, struct hist_entry *he,
19 hpp_field_fn get_field, const char *fmt, int len, 19 hpp_field_fn get_field, const char *fmt, int len,
20 hpp_snprint_fn print_fn, bool fmt_percent) 20 hpp_snprint_fn print_fn, bool fmt_percent)
21 { 21 {
22 int ret; 22 int ret;
23 struct hists *hists = he->hists; 23 struct hists *hists = he->hists;
24 struct perf_evsel *evsel = hists_to_evsel(hists); 24 struct perf_evsel *evsel = hists_to_evsel(hists);
25 char *buf = hpp->buf; 25 char *buf = hpp->buf;
26 size_t size = hpp->size; 26 size_t size = hpp->size;
27 27
28 if (fmt_percent) { 28 if (fmt_percent) {
29 double percent = 0.0; 29 double percent = 0.0;
30 u64 total = hists__total_period(hists); 30 u64 total = hists__total_period(hists);
31 31
32 if (total) 32 if (total)
33 percent = 100.0 * get_field(he) / total; 33 percent = 100.0 * get_field(he) / total;
34 34
35 ret = hpp__call_print_fn(hpp, print_fn, fmt, len, percent); 35 ret = hpp__call_print_fn(hpp, print_fn, fmt, len, percent);
36 } else 36 } else
37 ret = hpp__call_print_fn(hpp, print_fn, fmt, len, get_field(he)); 37 ret = hpp__call_print_fn(hpp, print_fn, fmt, len, get_field(he));
38 38
39 if (perf_evsel__is_group_event(evsel)) { 39 if (perf_evsel__is_group_event(evsel)) {
40 int prev_idx, idx_delta; 40 int prev_idx, idx_delta;
41 struct hist_entry *pair; 41 struct hist_entry *pair;
42 int nr_members = evsel->nr_members; 42 int nr_members = evsel->nr_members;
43 43
44 prev_idx = perf_evsel__group_idx(evsel); 44 prev_idx = perf_evsel__group_idx(evsel);
45 45
46 list_for_each_entry(pair, &he->pairs.head, pairs.node) { 46 list_for_each_entry(pair, &he->pairs.head, pairs.node) {
47 u64 period = get_field(pair); 47 u64 period = get_field(pair);
48 u64 total = hists__total_period(pair->hists); 48 u64 total = hists__total_period(pair->hists);
49 49
50 if (!total) 50 if (!total)
51 continue; 51 continue;
52 52
53 evsel = hists_to_evsel(pair->hists); 53 evsel = hists_to_evsel(pair->hists);
54 idx_delta = perf_evsel__group_idx(evsel) - prev_idx - 1; 54 idx_delta = perf_evsel__group_idx(evsel) - prev_idx - 1;
55 55
56 while (idx_delta--) { 56 while (idx_delta--) {
57 /* 57 /*
58 * zero-fill group members in the middle which 58 * zero-fill group members in the middle which
59 * have no sample 59 * have no sample
60 */ 60 */
61 if (fmt_percent) { 61 if (fmt_percent) {
62 ret += hpp__call_print_fn(hpp, print_fn, 62 ret += hpp__call_print_fn(hpp, print_fn,
63 fmt, len, 0.0); 63 fmt, len, 0.0);
64 } else { 64 } else {
65 ret += hpp__call_print_fn(hpp, print_fn, 65 ret += hpp__call_print_fn(hpp, print_fn,
66 fmt, len, 0ULL); 66 fmt, len, 0ULL);
67 } 67 }
68 } 68 }
69 69
70 if (fmt_percent) { 70 if (fmt_percent) {
71 ret += hpp__call_print_fn(hpp, print_fn, fmt, len, 71 ret += hpp__call_print_fn(hpp, print_fn, fmt, len,
72 100.0 * period / total); 72 100.0 * period / total);
73 } else { 73 } else {
74 ret += hpp__call_print_fn(hpp, print_fn, fmt, 74 ret += hpp__call_print_fn(hpp, print_fn, fmt,
75 len, period); 75 len, period);
76 } 76 }
77 77
78 prev_idx = perf_evsel__group_idx(evsel); 78 prev_idx = perf_evsel__group_idx(evsel);
79 } 79 }
80 80
81 idx_delta = nr_members - prev_idx - 1; 81 idx_delta = nr_members - prev_idx - 1;
82 82
83 while (idx_delta--) { 83 while (idx_delta--) {
84 /* 84 /*
85 * zero-fill group members at last which have no sample 85 * zero-fill group members at last which have no sample
86 */ 86 */
87 if (fmt_percent) { 87 if (fmt_percent) {
88 ret += hpp__call_print_fn(hpp, print_fn, 88 ret += hpp__call_print_fn(hpp, print_fn,
89 fmt, len, 0.0); 89 fmt, len, 0.0);
90 } else { 90 } else {
91 ret += hpp__call_print_fn(hpp, print_fn, 91 ret += hpp__call_print_fn(hpp, print_fn,
92 fmt, len, 0ULL); 92 fmt, len, 0ULL);
93 } 93 }
94 } 94 }
95 } 95 }
96 96
97 /* 97 /*
98 * Restore original buf and size as it's where caller expects 98 * Restore original buf and size as it's where caller expects
99 * the result will be saved. 99 * the result will be saved.
100 */ 100 */
101 hpp->buf = buf; 101 hpp->buf = buf;
102 hpp->size = size; 102 hpp->size = size;
103 103
104 return ret; 104 return ret;
105 } 105 }
106 106
107 int hpp__fmt(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 107 int hpp__fmt(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
108 struct hist_entry *he, hpp_field_fn get_field, 108 struct hist_entry *he, hpp_field_fn get_field,
109 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent) 109 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent)
110 { 110 {
111 int len = fmt->user_len ?: fmt->len; 111 int len = fmt->user_len ?: fmt->len;
112 112
113 if (symbol_conf.field_sep) { 113 if (symbol_conf.field_sep) {
114 return __hpp__fmt(hpp, he, get_field, fmtstr, 1, 114 return __hpp__fmt(hpp, he, get_field, fmtstr, 1,
115 print_fn, fmt_percent); 115 print_fn, fmt_percent);
116 } 116 }
117 117
118 if (fmt_percent) 118 if (fmt_percent)
119 len -= 2; /* 2 for a space and a % sign */ 119 len -= 2; /* 2 for a space and a % sign */
120 else 120 else
121 len -= 1; 121 len -= 1;
122 122
123 return __hpp__fmt(hpp, he, get_field, fmtstr, len, print_fn, fmt_percent); 123 return __hpp__fmt(hpp, he, get_field, fmtstr, len, print_fn, fmt_percent);
124 } 124 }
125 125
126 int hpp__fmt_acc(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 126 int hpp__fmt_acc(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
127 struct hist_entry *he, hpp_field_fn get_field, 127 struct hist_entry *he, hpp_field_fn get_field,
128 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent) 128 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent)
129 { 129 {
130 if (!symbol_conf.cumulate_callchain) { 130 if (!symbol_conf.cumulate_callchain) {
131 int len = fmt->user_len ?: fmt->len; 131 int len = fmt->user_len ?: fmt->len;
132 return snprintf(hpp->buf, hpp->size, " %*s", len - 1, "N/A"); 132 return snprintf(hpp->buf, hpp->size, " %*s", len - 1, "N/A");
133 } 133 }
134 134
135 return hpp__fmt(fmt, hpp, he, get_field, fmtstr, print_fn, fmt_percent); 135 return hpp__fmt(fmt, hpp, he, get_field, fmtstr, print_fn, fmt_percent);
136 } 136 }
137 137
138 static int field_cmp(u64 field_a, u64 field_b) 138 static int field_cmp(u64 field_a, u64 field_b)
139 { 139 {
140 if (field_a > field_b) 140 if (field_a > field_b)
141 return 1; 141 return 1;
142 if (field_a < field_b) 142 if (field_a < field_b)
143 return -1; 143 return -1;
144 return 0; 144 return 0;
145 } 145 }
146 146
147 static int __hpp__sort(struct hist_entry *a, struct hist_entry *b, 147 static int __hpp__sort(struct hist_entry *a, struct hist_entry *b,
148 hpp_field_fn get_field) 148 hpp_field_fn get_field)
149 { 149 {
150 s64 ret; 150 s64 ret;
151 int i, nr_members; 151 int i, nr_members;
152 struct perf_evsel *evsel; 152 struct perf_evsel *evsel;
153 struct hist_entry *pair; 153 struct hist_entry *pair;
154 u64 *fields_a, *fields_b; 154 u64 *fields_a, *fields_b;
155 155
156 ret = field_cmp(get_field(a), get_field(b)); 156 ret = field_cmp(get_field(a), get_field(b));
157 if (ret || !symbol_conf.event_group) 157 if (ret || !symbol_conf.event_group)
158 return ret; 158 return ret;
159 159
160 evsel = hists_to_evsel(a->hists); 160 evsel = hists_to_evsel(a->hists);
161 if (!perf_evsel__is_group_event(evsel)) 161 if (!perf_evsel__is_group_event(evsel))
162 return ret; 162 return ret;
163 163
164 nr_members = evsel->nr_members; 164 nr_members = evsel->nr_members;
165 fields_a = calloc(nr_members, sizeof(*fields_a)); 165 fields_a = calloc(nr_members, sizeof(*fields_a));
166 fields_b = calloc(nr_members, sizeof(*fields_b)); 166 fields_b = calloc(nr_members, sizeof(*fields_b));
167 167
168 if (!fields_a || !fields_b) 168 if (!fields_a || !fields_b)
169 goto out; 169 goto out;
170 170
171 list_for_each_entry(pair, &a->pairs.head, pairs.node) { 171 list_for_each_entry(pair, &a->pairs.head, pairs.node) {
172 evsel = hists_to_evsel(pair->hists); 172 evsel = hists_to_evsel(pair->hists);
173 fields_a[perf_evsel__group_idx(evsel)] = get_field(pair); 173 fields_a[perf_evsel__group_idx(evsel)] = get_field(pair);
174 } 174 }
175 175
176 list_for_each_entry(pair, &b->pairs.head, pairs.node) { 176 list_for_each_entry(pair, &b->pairs.head, pairs.node) {
177 evsel = hists_to_evsel(pair->hists); 177 evsel = hists_to_evsel(pair->hists);
178 fields_b[perf_evsel__group_idx(evsel)] = get_field(pair); 178 fields_b[perf_evsel__group_idx(evsel)] = get_field(pair);
179 } 179 }
180 180
181 for (i = 1; i < nr_members; i++) { 181 for (i = 1; i < nr_members; i++) {
182 ret = field_cmp(fields_a[i], fields_b[i]); 182 ret = field_cmp(fields_a[i], fields_b[i]);
183 if (ret) 183 if (ret)
184 break; 184 break;
185 } 185 }
186 186
187 out: 187 out:
188 free(fields_a); 188 free(fields_a);
189 free(fields_b); 189 free(fields_b);
190 190
191 return ret; 191 return ret;
192 } 192 }
193 193
194 static int __hpp__sort_acc(struct hist_entry *a, struct hist_entry *b, 194 static int __hpp__sort_acc(struct hist_entry *a, struct hist_entry *b,
195 hpp_field_fn get_field) 195 hpp_field_fn get_field)
196 { 196 {
197 s64 ret = 0; 197 s64 ret = 0;
198 198
199 if (symbol_conf.cumulate_callchain) { 199 if (symbol_conf.cumulate_callchain) {
200 /* 200 /*
201 * Put caller above callee when they have equal period. 201 * Put caller above callee when they have equal period.
202 */ 202 */
203 ret = field_cmp(get_field(a), get_field(b)); 203 ret = field_cmp(get_field(a), get_field(b));
204 if (ret) 204 if (ret)
205 return ret; 205 return ret;
206 206
207 if (a->thread != b->thread || !symbol_conf.use_callchain)
208 return 0;
209
207 ret = b->callchain->max_depth - a->callchain->max_depth; 210 ret = b->callchain->max_depth - a->callchain->max_depth;
208 } 211 }
209 return ret; 212 return ret;
210 } 213 }
211 214
212 static int hpp__width_fn(struct perf_hpp_fmt *fmt, 215 static int hpp__width_fn(struct perf_hpp_fmt *fmt,
213 struct perf_hpp *hpp __maybe_unused, 216 struct perf_hpp *hpp __maybe_unused,
214 struct perf_evsel *evsel) 217 struct perf_evsel *evsel)
215 { 218 {
216 int len = fmt->user_len ?: fmt->len; 219 int len = fmt->user_len ?: fmt->len;
217 220
218 if (symbol_conf.event_group) 221 if (symbol_conf.event_group)
219 len = max(len, evsel->nr_members * fmt->len); 222 len = max(len, evsel->nr_members * fmt->len);
220 223
221 if (len < (int)strlen(fmt->name)) 224 if (len < (int)strlen(fmt->name))
222 len = strlen(fmt->name); 225 len = strlen(fmt->name);
223 226
224 return len; 227 return len;
225 } 228 }
226 229
227 static int hpp__header_fn(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 230 static int hpp__header_fn(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
228 struct perf_evsel *evsel) 231 struct perf_evsel *evsel)
229 { 232 {
230 int len = hpp__width_fn(fmt, hpp, evsel); 233 int len = hpp__width_fn(fmt, hpp, evsel);
231 return scnprintf(hpp->buf, hpp->size, "%*s", len, fmt->name); 234 return scnprintf(hpp->buf, hpp->size, "%*s", len, fmt->name);
232 } 235 }
233 236
234 static int hpp_color_scnprintf(struct perf_hpp *hpp, const char *fmt, ...) 237 static int hpp_color_scnprintf(struct perf_hpp *hpp, const char *fmt, ...)
235 { 238 {
236 va_list args; 239 va_list args;
237 ssize_t ssize = hpp->size; 240 ssize_t ssize = hpp->size;
238 double percent; 241 double percent;
239 int ret, len; 242 int ret, len;
240 243
241 va_start(args, fmt); 244 va_start(args, fmt);
242 len = va_arg(args, int); 245 len = va_arg(args, int);
243 percent = va_arg(args, double); 246 percent = va_arg(args, double);
244 ret = percent_color_len_snprintf(hpp->buf, hpp->size, fmt, len, percent); 247 ret = percent_color_len_snprintf(hpp->buf, hpp->size, fmt, len, percent);
245 va_end(args); 248 va_end(args);
246 249
247 return (ret >= ssize) ? (ssize - 1) : ret; 250 return (ret >= ssize) ? (ssize - 1) : ret;
248 } 251 }
249 252
250 static int hpp_entry_scnprintf(struct perf_hpp *hpp, const char *fmt, ...) 253 static int hpp_entry_scnprintf(struct perf_hpp *hpp, const char *fmt, ...)
251 { 254 {
252 va_list args; 255 va_list args;
253 ssize_t ssize = hpp->size; 256 ssize_t ssize = hpp->size;
254 int ret; 257 int ret;
255 258
256 va_start(args, fmt); 259 va_start(args, fmt);
257 ret = vsnprintf(hpp->buf, hpp->size, fmt, args); 260 ret = vsnprintf(hpp->buf, hpp->size, fmt, args);
258 va_end(args); 261 va_end(args);
259 262
260 return (ret >= ssize) ? (ssize - 1) : ret; 263 return (ret >= ssize) ? (ssize - 1) : ret;
261 } 264 }
262 265
263 #define __HPP_COLOR_PERCENT_FN(_type, _field) \ 266 #define __HPP_COLOR_PERCENT_FN(_type, _field) \
264 static u64 he_get_##_field(struct hist_entry *he) \ 267 static u64 he_get_##_field(struct hist_entry *he) \
265 { \ 268 { \
266 return he->stat._field; \ 269 return he->stat._field; \
267 } \ 270 } \
268 \ 271 \
269 static int hpp__color_##_type(struct perf_hpp_fmt *fmt, \ 272 static int hpp__color_##_type(struct perf_hpp_fmt *fmt, \
270 struct perf_hpp *hpp, struct hist_entry *he) \ 273 struct perf_hpp *hpp, struct hist_entry *he) \
271 { \ 274 { \
272 return hpp__fmt(fmt, hpp, he, he_get_##_field, " %*.2f%%", \ 275 return hpp__fmt(fmt, hpp, he, he_get_##_field, " %*.2f%%", \
273 hpp_color_scnprintf, true); \ 276 hpp_color_scnprintf, true); \
274 } 277 }
275 278
276 #define __HPP_ENTRY_PERCENT_FN(_type, _field) \ 279 #define __HPP_ENTRY_PERCENT_FN(_type, _field) \
277 static int hpp__entry_##_type(struct perf_hpp_fmt *fmt, \ 280 static int hpp__entry_##_type(struct perf_hpp_fmt *fmt, \
278 struct perf_hpp *hpp, struct hist_entry *he) \ 281 struct perf_hpp *hpp, struct hist_entry *he) \
279 { \ 282 { \
280 return hpp__fmt(fmt, hpp, he, he_get_##_field, " %*.2f%%", \ 283 return hpp__fmt(fmt, hpp, he, he_get_##_field, " %*.2f%%", \
281 hpp_entry_scnprintf, true); \ 284 hpp_entry_scnprintf, true); \
282 } 285 }
283 286
284 #define __HPP_SORT_FN(_type, _field) \ 287 #define __HPP_SORT_FN(_type, _field) \
285 static int64_t hpp__sort_##_type(struct hist_entry *a, struct hist_entry *b) \ 288 static int64_t hpp__sort_##_type(struct hist_entry *a, struct hist_entry *b) \
286 { \ 289 { \
287 return __hpp__sort(a, b, he_get_##_field); \ 290 return __hpp__sort(a, b, he_get_##_field); \
288 } 291 }
289 292
290 #define __HPP_COLOR_ACC_PERCENT_FN(_type, _field) \ 293 #define __HPP_COLOR_ACC_PERCENT_FN(_type, _field) \
291 static u64 he_get_acc_##_field(struct hist_entry *he) \ 294 static u64 he_get_acc_##_field(struct hist_entry *he) \
292 { \ 295 { \
293 return he->stat_acc->_field; \ 296 return he->stat_acc->_field; \
294 } \ 297 } \
295 \ 298 \
296 static int hpp__color_##_type(struct perf_hpp_fmt *fmt, \ 299 static int hpp__color_##_type(struct perf_hpp_fmt *fmt, \
297 struct perf_hpp *hpp, struct hist_entry *he) \ 300 struct perf_hpp *hpp, struct hist_entry *he) \
298 { \ 301 { \
299 return hpp__fmt_acc(fmt, hpp, he, he_get_acc_##_field, " %*.2f%%", \ 302 return hpp__fmt_acc(fmt, hpp, he, he_get_acc_##_field, " %*.2f%%", \
300 hpp_color_scnprintf, true); \ 303 hpp_color_scnprintf, true); \
301 } 304 }
302 305
303 #define __HPP_ENTRY_ACC_PERCENT_FN(_type, _field) \ 306 #define __HPP_ENTRY_ACC_PERCENT_FN(_type, _field) \
304 static int hpp__entry_##_type(struct perf_hpp_fmt *fmt, \ 307 static int hpp__entry_##_type(struct perf_hpp_fmt *fmt, \
305 struct perf_hpp *hpp, struct hist_entry *he) \ 308 struct perf_hpp *hpp, struct hist_entry *he) \
306 { \ 309 { \
307 return hpp__fmt_acc(fmt, hpp, he, he_get_acc_##_field, " %*.2f%%", \ 310 return hpp__fmt_acc(fmt, hpp, he, he_get_acc_##_field, " %*.2f%%", \
308 hpp_entry_scnprintf, true); \ 311 hpp_entry_scnprintf, true); \
309 } 312 }
310 313
311 #define __HPP_SORT_ACC_FN(_type, _field) \ 314 #define __HPP_SORT_ACC_FN(_type, _field) \
312 static int64_t hpp__sort_##_type(struct hist_entry *a, struct hist_entry *b) \ 315 static int64_t hpp__sort_##_type(struct hist_entry *a, struct hist_entry *b) \
313 { \ 316 { \
314 return __hpp__sort_acc(a, b, he_get_acc_##_field); \ 317 return __hpp__sort_acc(a, b, he_get_acc_##_field); \
315 } 318 }
316 319
317 #define __HPP_ENTRY_RAW_FN(_type, _field) \ 320 #define __HPP_ENTRY_RAW_FN(_type, _field) \
318 static u64 he_get_raw_##_field(struct hist_entry *he) \ 321 static u64 he_get_raw_##_field(struct hist_entry *he) \
319 { \ 322 { \
320 return he->stat._field; \ 323 return he->stat._field; \
321 } \ 324 } \
322 \ 325 \
323 static int hpp__entry_##_type(struct perf_hpp_fmt *fmt, \ 326 static int hpp__entry_##_type(struct perf_hpp_fmt *fmt, \
324 struct perf_hpp *hpp, struct hist_entry *he) \ 327 struct perf_hpp *hpp, struct hist_entry *he) \
325 { \ 328 { \
326 return hpp__fmt(fmt, hpp, he, he_get_raw_##_field, " %*"PRIu64, \ 329 return hpp__fmt(fmt, hpp, he, he_get_raw_##_field, " %*"PRIu64, \
327 hpp_entry_scnprintf, false); \ 330 hpp_entry_scnprintf, false); \
328 } 331 }
329 332
330 #define __HPP_SORT_RAW_FN(_type, _field) \ 333 #define __HPP_SORT_RAW_FN(_type, _field) \
331 static int64_t hpp__sort_##_type(struct hist_entry *a, struct hist_entry *b) \ 334 static int64_t hpp__sort_##_type(struct hist_entry *a, struct hist_entry *b) \
332 { \ 335 { \
333 return __hpp__sort(a, b, he_get_raw_##_field); \ 336 return __hpp__sort(a, b, he_get_raw_##_field); \
334 } 337 }
335 338
336 339
337 #define HPP_PERCENT_FNS(_type, _field) \ 340 #define HPP_PERCENT_FNS(_type, _field) \
338 __HPP_COLOR_PERCENT_FN(_type, _field) \ 341 __HPP_COLOR_PERCENT_FN(_type, _field) \
339 __HPP_ENTRY_PERCENT_FN(_type, _field) \ 342 __HPP_ENTRY_PERCENT_FN(_type, _field) \
340 __HPP_SORT_FN(_type, _field) 343 __HPP_SORT_FN(_type, _field)
341 344
342 #define HPP_PERCENT_ACC_FNS(_type, _field) \ 345 #define HPP_PERCENT_ACC_FNS(_type, _field) \
343 __HPP_COLOR_ACC_PERCENT_FN(_type, _field) \ 346 __HPP_COLOR_ACC_PERCENT_FN(_type, _field) \
344 __HPP_ENTRY_ACC_PERCENT_FN(_type, _field) \ 347 __HPP_ENTRY_ACC_PERCENT_FN(_type, _field) \
345 __HPP_SORT_ACC_FN(_type, _field) 348 __HPP_SORT_ACC_FN(_type, _field)
346 349
347 #define HPP_RAW_FNS(_type, _field) \ 350 #define HPP_RAW_FNS(_type, _field) \
348 __HPP_ENTRY_RAW_FN(_type, _field) \ 351 __HPP_ENTRY_RAW_FN(_type, _field) \
349 __HPP_SORT_RAW_FN(_type, _field) 352 __HPP_SORT_RAW_FN(_type, _field)
350 353
351 HPP_PERCENT_FNS(overhead, period) 354 HPP_PERCENT_FNS(overhead, period)
352 HPP_PERCENT_FNS(overhead_sys, period_sys) 355 HPP_PERCENT_FNS(overhead_sys, period_sys)
353 HPP_PERCENT_FNS(overhead_us, period_us) 356 HPP_PERCENT_FNS(overhead_us, period_us)
354 HPP_PERCENT_FNS(overhead_guest_sys, period_guest_sys) 357 HPP_PERCENT_FNS(overhead_guest_sys, period_guest_sys)
355 HPP_PERCENT_FNS(overhead_guest_us, period_guest_us) 358 HPP_PERCENT_FNS(overhead_guest_us, period_guest_us)
356 HPP_PERCENT_ACC_FNS(overhead_acc, period) 359 HPP_PERCENT_ACC_FNS(overhead_acc, period)
357 360
358 HPP_RAW_FNS(samples, nr_events) 361 HPP_RAW_FNS(samples, nr_events)
359 HPP_RAW_FNS(period, period) 362 HPP_RAW_FNS(period, period)
360 363
361 static int64_t hpp__nop_cmp(struct hist_entry *a __maybe_unused, 364 static int64_t hpp__nop_cmp(struct hist_entry *a __maybe_unused,
362 struct hist_entry *b __maybe_unused) 365 struct hist_entry *b __maybe_unused)
363 { 366 {
364 return 0; 367 return 0;
365 } 368 }
366 369
367 #define HPP__COLOR_PRINT_FNS(_name, _fn) \ 370 #define HPP__COLOR_PRINT_FNS(_name, _fn) \
368 { \ 371 { \
369 .name = _name, \ 372 .name = _name, \
370 .header = hpp__header_fn, \ 373 .header = hpp__header_fn, \
371 .width = hpp__width_fn, \ 374 .width = hpp__width_fn, \
372 .color = hpp__color_ ## _fn, \ 375 .color = hpp__color_ ## _fn, \
373 .entry = hpp__entry_ ## _fn, \ 376 .entry = hpp__entry_ ## _fn, \
374 .cmp = hpp__nop_cmp, \ 377 .cmp = hpp__nop_cmp, \
375 .collapse = hpp__nop_cmp, \ 378 .collapse = hpp__nop_cmp, \
376 .sort = hpp__sort_ ## _fn, \ 379 .sort = hpp__sort_ ## _fn, \
377 } 380 }
378 381
379 #define HPP__COLOR_ACC_PRINT_FNS(_name, _fn) \ 382 #define HPP__COLOR_ACC_PRINT_FNS(_name, _fn) \
380 { \ 383 { \
381 .name = _name, \ 384 .name = _name, \
382 .header = hpp__header_fn, \ 385 .header = hpp__header_fn, \
383 .width = hpp__width_fn, \ 386 .width = hpp__width_fn, \
384 .color = hpp__color_ ## _fn, \ 387 .color = hpp__color_ ## _fn, \
385 .entry = hpp__entry_ ## _fn, \ 388 .entry = hpp__entry_ ## _fn, \
386 .cmp = hpp__nop_cmp, \ 389 .cmp = hpp__nop_cmp, \
387 .collapse = hpp__nop_cmp, \ 390 .collapse = hpp__nop_cmp, \
388 .sort = hpp__sort_ ## _fn, \ 391 .sort = hpp__sort_ ## _fn, \
389 } 392 }
390 393
391 #define HPP__PRINT_FNS(_name, _fn) \ 394 #define HPP__PRINT_FNS(_name, _fn) \
392 { \ 395 { \
393 .name = _name, \ 396 .name = _name, \
394 .header = hpp__header_fn, \ 397 .header = hpp__header_fn, \
395 .width = hpp__width_fn, \ 398 .width = hpp__width_fn, \
396 .entry = hpp__entry_ ## _fn, \ 399 .entry = hpp__entry_ ## _fn, \
397 .cmp = hpp__nop_cmp, \ 400 .cmp = hpp__nop_cmp, \
398 .collapse = hpp__nop_cmp, \ 401 .collapse = hpp__nop_cmp, \
399 .sort = hpp__sort_ ## _fn, \ 402 .sort = hpp__sort_ ## _fn, \
400 } 403 }
401 404
402 struct perf_hpp_fmt perf_hpp__format[] = { 405 struct perf_hpp_fmt perf_hpp__format[] = {
403 HPP__COLOR_PRINT_FNS("Overhead", overhead), 406 HPP__COLOR_PRINT_FNS("Overhead", overhead),
404 HPP__COLOR_PRINT_FNS("sys", overhead_sys), 407 HPP__COLOR_PRINT_FNS("sys", overhead_sys),
405 HPP__COLOR_PRINT_FNS("usr", overhead_us), 408 HPP__COLOR_PRINT_FNS("usr", overhead_us),
406 HPP__COLOR_PRINT_FNS("guest sys", overhead_guest_sys), 409 HPP__COLOR_PRINT_FNS("guest sys", overhead_guest_sys),
407 HPP__COLOR_PRINT_FNS("guest usr", overhead_guest_us), 410 HPP__COLOR_PRINT_FNS("guest usr", overhead_guest_us),
408 HPP__COLOR_ACC_PRINT_FNS("Children", overhead_acc), 411 HPP__COLOR_ACC_PRINT_FNS("Children", overhead_acc),
409 HPP__PRINT_FNS("Samples", samples), 412 HPP__PRINT_FNS("Samples", samples),
410 HPP__PRINT_FNS("Period", period) 413 HPP__PRINT_FNS("Period", period)
411 }; 414 };
412 415
413 LIST_HEAD(perf_hpp__list); 416 LIST_HEAD(perf_hpp__list);
414 LIST_HEAD(perf_hpp__sort_list); 417 LIST_HEAD(perf_hpp__sort_list);
415 418
416 419
417 #undef HPP__COLOR_PRINT_FNS 420 #undef HPP__COLOR_PRINT_FNS
418 #undef HPP__COLOR_ACC_PRINT_FNS 421 #undef HPP__COLOR_ACC_PRINT_FNS
419 #undef HPP__PRINT_FNS 422 #undef HPP__PRINT_FNS
420 423
421 #undef HPP_PERCENT_FNS 424 #undef HPP_PERCENT_FNS
422 #undef HPP_PERCENT_ACC_FNS 425 #undef HPP_PERCENT_ACC_FNS
423 #undef HPP_RAW_FNS 426 #undef HPP_RAW_FNS
424 427
425 #undef __HPP_HEADER_FN 428 #undef __HPP_HEADER_FN
426 #undef __HPP_WIDTH_FN 429 #undef __HPP_WIDTH_FN
427 #undef __HPP_COLOR_PERCENT_FN 430 #undef __HPP_COLOR_PERCENT_FN
428 #undef __HPP_ENTRY_PERCENT_FN 431 #undef __HPP_ENTRY_PERCENT_FN
429 #undef __HPP_COLOR_ACC_PERCENT_FN 432 #undef __HPP_COLOR_ACC_PERCENT_FN
430 #undef __HPP_ENTRY_ACC_PERCENT_FN 433 #undef __HPP_ENTRY_ACC_PERCENT_FN
431 #undef __HPP_ENTRY_RAW_FN 434 #undef __HPP_ENTRY_RAW_FN
432 #undef __HPP_SORT_FN 435 #undef __HPP_SORT_FN
433 #undef __HPP_SORT_ACC_FN 436 #undef __HPP_SORT_ACC_FN
434 #undef __HPP_SORT_RAW_FN 437 #undef __HPP_SORT_RAW_FN
435 438
436 439
437 void perf_hpp__init(void) 440 void perf_hpp__init(void)
438 { 441 {
439 struct list_head *list; 442 struct list_head *list;
440 int i; 443 int i;
441 444
442 for (i = 0; i < PERF_HPP__MAX_INDEX; i++) { 445 for (i = 0; i < PERF_HPP__MAX_INDEX; i++) {
443 struct perf_hpp_fmt *fmt = &perf_hpp__format[i]; 446 struct perf_hpp_fmt *fmt = &perf_hpp__format[i];
444 447
445 INIT_LIST_HEAD(&fmt->list); 448 INIT_LIST_HEAD(&fmt->list);
446 449
447 /* sort_list may be linked by setup_sorting() */ 450 /* sort_list may be linked by setup_sorting() */
448 if (fmt->sort_list.next == NULL) 451 if (fmt->sort_list.next == NULL)
449 INIT_LIST_HEAD(&fmt->sort_list); 452 INIT_LIST_HEAD(&fmt->sort_list);
450 } 453 }
451 454
452 /* 455 /*
453 * If user specified field order, no need to setup default fields. 456 * If user specified field order, no need to setup default fields.
454 */ 457 */
455 if (is_strict_order(field_order)) 458 if (is_strict_order(field_order))
456 return; 459 return;
457 460
458 if (symbol_conf.cumulate_callchain) { 461 if (symbol_conf.cumulate_callchain) {
459 perf_hpp__column_enable(PERF_HPP__OVERHEAD_ACC); 462 perf_hpp__column_enable(PERF_HPP__OVERHEAD_ACC);
460 perf_hpp__format[PERF_HPP__OVERHEAD].name = "Self"; 463 perf_hpp__format[PERF_HPP__OVERHEAD].name = "Self";
461 } 464 }
462 465
463 perf_hpp__column_enable(PERF_HPP__OVERHEAD); 466 perf_hpp__column_enable(PERF_HPP__OVERHEAD);
464 467
465 if (symbol_conf.show_cpu_utilization) { 468 if (symbol_conf.show_cpu_utilization) {
466 perf_hpp__column_enable(PERF_HPP__OVERHEAD_SYS); 469 perf_hpp__column_enable(PERF_HPP__OVERHEAD_SYS);
467 perf_hpp__column_enable(PERF_HPP__OVERHEAD_US); 470 perf_hpp__column_enable(PERF_HPP__OVERHEAD_US);
468 471
469 if (perf_guest) { 472 if (perf_guest) {
470 perf_hpp__column_enable(PERF_HPP__OVERHEAD_GUEST_SYS); 473 perf_hpp__column_enable(PERF_HPP__OVERHEAD_GUEST_SYS);
471 perf_hpp__column_enable(PERF_HPP__OVERHEAD_GUEST_US); 474 perf_hpp__column_enable(PERF_HPP__OVERHEAD_GUEST_US);
472 } 475 }
473 } 476 }
474 477
475 if (symbol_conf.show_nr_samples) 478 if (symbol_conf.show_nr_samples)
476 perf_hpp__column_enable(PERF_HPP__SAMPLES); 479 perf_hpp__column_enable(PERF_HPP__SAMPLES);
477 480
478 if (symbol_conf.show_total_period) 481 if (symbol_conf.show_total_period)
479 perf_hpp__column_enable(PERF_HPP__PERIOD); 482 perf_hpp__column_enable(PERF_HPP__PERIOD);
480 483
481 /* prepend overhead field for backward compatiblity. */ 484 /* prepend overhead field for backward compatiblity. */
482 list = &perf_hpp__format[PERF_HPP__OVERHEAD].sort_list; 485 list = &perf_hpp__format[PERF_HPP__OVERHEAD].sort_list;
483 if (list_empty(list)) 486 if (list_empty(list))
484 list_add(list, &perf_hpp__sort_list); 487 list_add(list, &perf_hpp__sort_list);
485 488
486 if (symbol_conf.cumulate_callchain) { 489 if (symbol_conf.cumulate_callchain) {
487 list = &perf_hpp__format[PERF_HPP__OVERHEAD_ACC].sort_list; 490 list = &perf_hpp__format[PERF_HPP__OVERHEAD_ACC].sort_list;
488 if (list_empty(list)) 491 if (list_empty(list))
489 list_add(list, &perf_hpp__sort_list); 492 list_add(list, &perf_hpp__sort_list);
490 } 493 }
491 } 494 }
492 495
493 void perf_hpp__column_register(struct perf_hpp_fmt *format) 496 void perf_hpp__column_register(struct perf_hpp_fmt *format)
494 { 497 {
495 list_add_tail(&format->list, &perf_hpp__list); 498 list_add_tail(&format->list, &perf_hpp__list);
496 } 499 }
497 500
498 void perf_hpp__column_unregister(struct perf_hpp_fmt *format) 501 void perf_hpp__column_unregister(struct perf_hpp_fmt *format)
499 { 502 {
500 list_del(&format->list); 503 list_del(&format->list);
501 } 504 }
502 505
503 void perf_hpp__register_sort_field(struct perf_hpp_fmt *format) 506 void perf_hpp__register_sort_field(struct perf_hpp_fmt *format)
504 { 507 {
505 list_add_tail(&format->sort_list, &perf_hpp__sort_list); 508 list_add_tail(&format->sort_list, &perf_hpp__sort_list);
506 } 509 }
507 510
508 void perf_hpp__column_enable(unsigned col) 511 void perf_hpp__column_enable(unsigned col)
509 { 512 {
510 BUG_ON(col >= PERF_HPP__MAX_INDEX); 513 BUG_ON(col >= PERF_HPP__MAX_INDEX);
511 perf_hpp__column_register(&perf_hpp__format[col]); 514 perf_hpp__column_register(&perf_hpp__format[col]);
512 } 515 }
513 516
514 void perf_hpp__column_disable(unsigned col) 517 void perf_hpp__column_disable(unsigned col)
515 { 518 {
516 BUG_ON(col >= PERF_HPP__MAX_INDEX); 519 BUG_ON(col >= PERF_HPP__MAX_INDEX);
517 perf_hpp__column_unregister(&perf_hpp__format[col]); 520 perf_hpp__column_unregister(&perf_hpp__format[col]);
518 } 521 }
519 522
520 void perf_hpp__cancel_cumulate(void) 523 void perf_hpp__cancel_cumulate(void)
521 { 524 {
522 if (is_strict_order(field_order)) 525 if (is_strict_order(field_order))
523 return; 526 return;
524 527
525 perf_hpp__column_disable(PERF_HPP__OVERHEAD_ACC); 528 perf_hpp__column_disable(PERF_HPP__OVERHEAD_ACC);
526 perf_hpp__format[PERF_HPP__OVERHEAD].name = "Overhead"; 529 perf_hpp__format[PERF_HPP__OVERHEAD].name = "Overhead";
527 } 530 }
528 531
529 void perf_hpp__setup_output_field(void) 532 void perf_hpp__setup_output_field(void)
530 { 533 {
531 struct perf_hpp_fmt *fmt; 534 struct perf_hpp_fmt *fmt;
532 535
533 /* append sort keys to output field */ 536 /* append sort keys to output field */
534 perf_hpp__for_each_sort_list(fmt) { 537 perf_hpp__for_each_sort_list(fmt) {
535 if (!list_empty(&fmt->list)) 538 if (!list_empty(&fmt->list))
536 continue; 539 continue;
537 540
538 /* 541 /*
539 * sort entry fields are dynamically created, 542 * sort entry fields are dynamically created,
540 * so they can share a same sort key even though 543 * so they can share a same sort key even though
541 * the list is empty. 544 * the list is empty.
542 */ 545 */
543 if (perf_hpp__is_sort_entry(fmt)) { 546 if (perf_hpp__is_sort_entry(fmt)) {
544 struct perf_hpp_fmt *pos; 547 struct perf_hpp_fmt *pos;
545 548
546 perf_hpp__for_each_format(pos) { 549 perf_hpp__for_each_format(pos) {
547 if (perf_hpp__same_sort_entry(pos, fmt)) 550 if (perf_hpp__same_sort_entry(pos, fmt))
548 goto next; 551 goto next;
549 } 552 }
550 } 553 }
551 554
552 perf_hpp__column_register(fmt); 555 perf_hpp__column_register(fmt);
553 next: 556 next:
554 continue; 557 continue;
555 } 558 }
556 } 559 }
557 560
558 void perf_hpp__append_sort_keys(void) 561 void perf_hpp__append_sort_keys(void)
559 { 562 {
560 struct perf_hpp_fmt *fmt; 563 struct perf_hpp_fmt *fmt;
561 564
562 /* append output fields to sort keys */ 565 /* append output fields to sort keys */
563 perf_hpp__for_each_format(fmt) { 566 perf_hpp__for_each_format(fmt) {
564 if (!list_empty(&fmt->sort_list)) 567 if (!list_empty(&fmt->sort_list))
565 continue; 568 continue;
566 569
567 /* 570 /*
568 * sort entry fields are dynamically created, 571 * sort entry fields are dynamically created,
569 * so they can share a same sort key even though 572 * so they can share a same sort key even though
570 * the list is empty. 573 * the list is empty.
571 */ 574 */
572 if (perf_hpp__is_sort_entry(fmt)) { 575 if (perf_hpp__is_sort_entry(fmt)) {
573 struct perf_hpp_fmt *pos; 576 struct perf_hpp_fmt *pos;
574 577
575 perf_hpp__for_each_sort_list(pos) { 578 perf_hpp__for_each_sort_list(pos) {
576 if (perf_hpp__same_sort_entry(pos, fmt)) 579 if (perf_hpp__same_sort_entry(pos, fmt))
577 goto next; 580 goto next;
578 } 581 }
579 } 582 }
580 583
581 perf_hpp__register_sort_field(fmt); 584 perf_hpp__register_sort_field(fmt);
582 next: 585 next:
583 continue; 586 continue;
584 } 587 }
585 } 588 }
586 589
587 void perf_hpp__reset_output_field(void) 590 void perf_hpp__reset_output_field(void)
588 { 591 {
589 struct perf_hpp_fmt *fmt, *tmp; 592 struct perf_hpp_fmt *fmt, *tmp;
590 593
591 /* reset output fields */ 594 /* reset output fields */
592 perf_hpp__for_each_format_safe(fmt, tmp) { 595 perf_hpp__for_each_format_safe(fmt, tmp) {
593 list_del_init(&fmt->list); 596 list_del_init(&fmt->list);
594 list_del_init(&fmt->sort_list); 597 list_del_init(&fmt->sort_list);
595 } 598 }
596 599
597 /* reset sort keys */ 600 /* reset sort keys */
598 perf_hpp__for_each_sort_list_safe(fmt, tmp) { 601 perf_hpp__for_each_sort_list_safe(fmt, tmp) {
599 list_del_init(&fmt->list); 602 list_del_init(&fmt->list);
600 list_del_init(&fmt->sort_list); 603 list_del_init(&fmt->sort_list);
601 } 604 }
602 } 605 }
603 606
604 /* 607 /*
605 * See hists__fprintf to match the column widths 608 * See hists__fprintf to match the column widths
606 */ 609 */
607 unsigned int hists__sort_list_width(struct hists *hists) 610 unsigned int hists__sort_list_width(struct hists *hists)
608 { 611 {
609 struct perf_hpp_fmt *fmt; 612 struct perf_hpp_fmt *fmt;
610 int ret = 0; 613 int ret = 0;
611 bool first = true; 614 bool first = true;
612 struct perf_hpp dummy_hpp; 615 struct perf_hpp dummy_hpp;
613 616
614 perf_hpp__for_each_format(fmt) { 617 perf_hpp__for_each_format(fmt) {
615 if (perf_hpp__should_skip(fmt)) 618 if (perf_hpp__should_skip(fmt))
616 continue; 619 continue;
617 620
618 if (first) 621 if (first)
619 first = false; 622 first = false;
620 else 623 else
621 ret += 2; 624 ret += 2;
622 625
623 ret += fmt->width(fmt, &dummy_hpp, hists_to_evsel(hists)); 626 ret += fmt->width(fmt, &dummy_hpp, hists_to_evsel(hists));
624 } 627 }
625 628
626 if (verbose && sort__has_sym) /* Addr + origin */ 629 if (verbose && sort__has_sym) /* Addr + origin */
627 ret += 3 + BITS_PER_LONG / 4; 630 ret += 3 + BITS_PER_LONG / 4;
628 631
629 return ret; 632 return ret;
630 } 633 }
631 634
632 void perf_hpp__reset_width(struct perf_hpp_fmt *fmt, struct hists *hists) 635 void perf_hpp__reset_width(struct perf_hpp_fmt *fmt, struct hists *hists)
633 { 636 {
634 int idx; 637 int idx;
635 638
636 if (perf_hpp__is_sort_entry(fmt)) 639 if (perf_hpp__is_sort_entry(fmt))
637 return perf_hpp__reset_sort_width(fmt, hists); 640 return perf_hpp__reset_sort_width(fmt, hists);
638 641
639 for (idx = 0; idx < PERF_HPP__MAX_INDEX; idx++) { 642 for (idx = 0; idx < PERF_HPP__MAX_INDEX; idx++) {
640 if (fmt == &perf_hpp__format[idx]) 643 if (fmt == &perf_hpp__format[idx])
641 break; 644 break;
642 } 645 }
643 646
644 if (idx == PERF_HPP__MAX_INDEX) 647 if (idx == PERF_HPP__MAX_INDEX)
645 return; 648 return;
646 649
647 switch (idx) { 650 switch (idx) {
648 case PERF_HPP__OVERHEAD: 651 case PERF_HPP__OVERHEAD:
649 case PERF_HPP__OVERHEAD_SYS: 652 case PERF_HPP__OVERHEAD_SYS:
650 case PERF_HPP__OVERHEAD_US: 653 case PERF_HPP__OVERHEAD_US:
651 case PERF_HPP__OVERHEAD_ACC: 654 case PERF_HPP__OVERHEAD_ACC:
652 fmt->len = 8; 655 fmt->len = 8;
653 break; 656 break;
654 657
655 case PERF_HPP__OVERHEAD_GUEST_SYS: 658 case PERF_HPP__OVERHEAD_GUEST_SYS:
656 case PERF_HPP__OVERHEAD_GUEST_US: 659 case PERF_HPP__OVERHEAD_GUEST_US:
657 fmt->len = 9; 660 fmt->len = 9;
658 break; 661 break;
659 662
660 case PERF_HPP__SAMPLES: 663 case PERF_HPP__SAMPLES:
661 case PERF_HPP__PERIOD: 664 case PERF_HPP__PERIOD:
662 fmt->len = 12; 665 fmt->len = 12;
663 break; 666 break;
664 667
665 default: 668 default:
666 break; 669 break;
667 } 670 }
668 } 671 }
669 672
670 void perf_hpp__set_user_width(const char *width_list_str) 673 void perf_hpp__set_user_width(const char *width_list_str)
671 { 674 {
672 struct perf_hpp_fmt *fmt; 675 struct perf_hpp_fmt *fmt;
673 const char *ptr = width_list_str; 676 const char *ptr = width_list_str;
674 677
675 perf_hpp__for_each_format(fmt) { 678 perf_hpp__for_each_format(fmt) {
676 char *p; 679 char *p;
677 680
678 int len = strtol(ptr, &p, 10); 681 int len = strtol(ptr, &p, 10);
679 fmt->user_len = len; 682 fmt->user_len = len;
680 683
681 if (*p == ',') 684 if (*p == ',')
682 ptr = p + 1; 685 ptr = p + 1;
683 else 686 else
684 break; 687 break;
685 } 688 }
686 } 689 }
687 690
tools/perf/ui/tui/setup.c
Diff comments   View file @ ddb321a
1 #include <signal.h> 1 #include <signal.h>
2 #include <stdbool.h> 2 #include <stdbool.h>
3 #ifdef HAVE_BACKTRACE_SUPPORT
4 #include <execinfo.h>
5 #endif
3 6
4 #include "../../util/cache.h" 7 #include "../../util/cache.h"
5 #include "../../util/debug.h" 8 #include "../../util/debug.h"
6 #include "../browser.h" 9 #include "../browser.h"
7 #include "../helpline.h" 10 #include "../helpline.h"
8 #include "../ui.h" 11 #include "../ui.h"
9 #include "../util.h" 12 #include "../util.h"
10 #include "../libslang.h" 13 #include "../libslang.h"
11 #include "../keysyms.h" 14 #include "../keysyms.h"
12 #include "tui.h" 15 #include "tui.h"
13 16
14 static volatile int ui__need_resize; 17 static volatile int ui__need_resize;
15 18
16 extern struct perf_error_ops perf_tui_eops; 19 extern struct perf_error_ops perf_tui_eops;
17 20
18 extern void hist_browser__init_hpp(void); 21 extern void hist_browser__init_hpp(void);
19 22
20 void ui__refresh_dimensions(bool force) 23 void ui__refresh_dimensions(bool force)
21 { 24 {
22 if (force || ui__need_resize) { 25 if (force || ui__need_resize) {
23 ui__need_resize = 0; 26 ui__need_resize = 0;
24 pthread_mutex_lock(&ui__lock); 27 pthread_mutex_lock(&ui__lock);
25 SLtt_get_screen_size(); 28 SLtt_get_screen_size();
26 SLsmg_reinit_smg(); 29 SLsmg_reinit_smg();
27 pthread_mutex_unlock(&ui__lock); 30 pthread_mutex_unlock(&ui__lock);
28 } 31 }
29 } 32 }
30 33
31 static void ui__sigwinch(int sig __maybe_unused) 34 static void ui__sigwinch(int sig __maybe_unused)
32 { 35 {
33 ui__need_resize = 1; 36 ui__need_resize = 1;
34 } 37 }
35 38
36 static void ui__setup_sigwinch(void) 39 static void ui__setup_sigwinch(void)
37 { 40 {
38 static bool done; 41 static bool done;
39 42
40 if (done) 43 if (done)
41 return; 44 return;
42 45
43 done = true; 46 done = true;
44 pthread__unblock_sigwinch(); 47 pthread__unblock_sigwinch();
45 signal(SIGWINCH, ui__sigwinch); 48 signal(SIGWINCH, ui__sigwinch);
46 } 49 }
47 50
48 int ui__getch(int delay_secs) 51 int ui__getch(int delay_secs)
49 { 52 {
50 struct timeval timeout, *ptimeout = delay_secs ? &timeout : NULL; 53 struct timeval timeout, *ptimeout = delay_secs ? &timeout : NULL;
51 fd_set read_set; 54 fd_set read_set;
52 int err, key; 55 int err, key;
53 56
54 ui__setup_sigwinch(); 57 ui__setup_sigwinch();
55 58
56 FD_ZERO(&read_set); 59 FD_ZERO(&read_set);
57 FD_SET(0, &read_set); 60 FD_SET(0, &read_set);
58 61
59 if (delay_secs) { 62 if (delay_secs) {
60 timeout.tv_sec = delay_secs; 63 timeout.tv_sec = delay_secs;
61 timeout.tv_usec = 0; 64 timeout.tv_usec = 0;
62 } 65 }
63 66
64 err = select(1, &read_set, NULL, NULL, ptimeout); 67 err = select(1, &read_set, NULL, NULL, ptimeout);
65 68
66 if (err == 0) 69 if (err == 0)
67 return K_TIMER; 70 return K_TIMER;
68 71
69 if (err == -1) { 72 if (err == -1) {
70 if (errno == EINTR) 73 if (errno == EINTR)
71 return K_RESIZE; 74 return K_RESIZE;
72 return K_ERROR; 75 return K_ERROR;
73 } 76 }
74 77
75 key = SLang_getkey(); 78 key = SLang_getkey();
76 if (key != K_ESC) 79 if (key != K_ESC)
77 return key; 80 return key;
78 81
79 FD_ZERO(&read_set); 82 FD_ZERO(&read_set);
80 FD_SET(0, &read_set); 83 FD_SET(0, &read_set);
81 timeout.tv_sec = 0; 84 timeout.tv_sec = 0;
82 timeout.tv_usec = 20; 85 timeout.tv_usec = 20;
83 err = select(1, &read_set, NULL, NULL, &timeout); 86 err = select(1, &read_set, NULL, NULL, &timeout);
84 if (err == 0) 87 if (err == 0)
85 return K_ESC; 88 return K_ESC;
86 89
87 SLang_ungetkey(key); 90 SLang_ungetkey(key);
88 return SLkp_getkey(); 91 return SLkp_getkey();
89 } 92 }
90 93
94 #ifdef HAVE_BACKTRACE_SUPPORT
95 static void ui__signal_backtrace(int sig)
96 {
97 void *stackdump[32];
98 size_t size;
99
100 ui__exit(false);
101 psignal(sig, "perf");
102
103 printf("-------- backtrace --------\n");
104 size = backtrace(stackdump, ARRAY_SIZE(stackdump));
105 backtrace_symbols_fd(stackdump, size, STDOUT_FILENO);
106
107 exit(0);
108 }
109 #else
110 # define ui__signal_backtrace ui__signal
111 #endif
112
91 static void ui__signal(int sig) 113 static void ui__signal(int sig)
92 { 114 {
93 ui__exit(false); 115 ui__exit(false);
94 psignal(sig, "perf"); 116 psignal(sig, "perf");
95 exit(0); 117 exit(0);
96 } 118 }
97 119
98 int ui__init(void) 120 int ui__init(void)
99 { 121 {
100 int err; 122 int err;
101 123
102 SLutf8_enable(-1); 124 SLutf8_enable(-1);
103 SLtt_get_terminfo(); 125 SLtt_get_terminfo();
104 SLtt_get_screen_size(); 126 SLtt_get_screen_size();
105 127
106 err = SLsmg_init_smg(); 128 err = SLsmg_init_smg();
107 if (err < 0) 129 if (err < 0)
108 goto out; 130 goto out;
109 err = SLang_init_tty(0, 0, 0); 131 err = SLang_init_tty(0, 0, 0);
110 if (err < 0) 132 if (err < 0)
111 goto out; 133 goto out;
112 134
113 err = SLkp_init(); 135 err = SLkp_init();
114 if (err < 0) { 136 if (err < 0) {
115 pr_err("TUI initialization failed.\n"); 137 pr_err("TUI initialization failed.\n");
116 goto out; 138 goto out;
117 } 139 }
118 140
119 SLkp_define_keysym((char *)"^(kB)", SL_KEY_UNTAB); 141 SLkp_define_keysym((char *)"^(kB)", SL_KEY_UNTAB);
120 142
121 ui_helpline__init(); 143 ui_helpline__init();
122 ui_browser__init(); 144 ui_browser__init();
123 tui_progress__init(); 145 tui_progress__init();
124 146
125 signal(SIGSEGV, ui__signal); 147 signal(SIGSEGV, ui__signal_backtrace);
126 signal(SIGFPE, ui__signal); 148 signal(SIGFPE, ui__signal_backtrace);
127 signal(SIGINT, ui__signal); 149 signal(SIGINT, ui__signal);
128 signal(SIGQUIT, ui__signal); 150 signal(SIGQUIT, ui__signal);
129 signal(SIGTERM, ui__signal); 151 signal(SIGTERM, ui__signal);
130 152
131 perf_error__register(&perf_tui_eops); 153 perf_error__register(&perf_tui_eops);
132 154
133 hist_browser__init_hpp(); 155 hist_browser__init_hpp();
134 out: 156 out:
135 return err; 157 return err;
136 } 158 }
137 159
138 void ui__exit(bool wait_for_ok) 160 void ui__exit(bool wait_for_ok)
139 { 161 {
140 if (wait_for_ok) 162 if (wait_for_ok)
141 ui__question_window("Fatal Error", 163 ui__question_window("Fatal Error",
142 ui_helpline__last_msg, 164 ui_helpline__last_msg,
143 "Press any key...", 0); 165 "Press any key...", 0);
144 166
145 SLtt_set_cursor_visibility(1); 167 SLtt_set_cursor_visibility(1);
146 SLsmg_refresh(); 168 SLsmg_refresh();
147 SLsmg_reset_smg(); 169 SLsmg_reset_smg();
148 SLang_reset_tty(); 170 SLang_reset_tty();
149 171
150 perf_error__unregister(&perf_tui_eops); 172 perf_error__unregister(&perf_tui_eops);
151 } 173 }
152 174
tools/perf/util/callchain.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * Copyright (C) 2009-2011, Frederic Weisbecker <fweisbec@gmail.com> 2 * Copyright (C) 2009-2011, Frederic Weisbecker <fweisbec@gmail.com>
3 * 3 *
4 * Handle the callchains from the stream in an ad-hoc radix tree and then 4 * Handle the callchains from the stream in an ad-hoc radix tree and then
5 * sort them in an rbtree. 5 * sort them in an rbtree.
6 * 6 *
7 * Using a radix for code path provides a fast retrieval and factorizes 7 * Using a radix for code path provides a fast retrieval and factorizes
8 * memory use. Also that lets us use the paths in a hierarchical graph view. 8 * memory use. Also that lets us use the paths in a hierarchical graph view.
9 * 9 *
10 */ 10 */
11 11
12 #include <stdlib.h> 12 #include <stdlib.h>
13 #include <stdio.h> 13 #include <stdio.h>
14 #include <stdbool.h> 14 #include <stdbool.h>
15 #include <errno.h> 15 #include <errno.h>
16 #include <math.h> 16 #include <math.h>
17 17
18 #include "asm/bug.h" 18 #include "asm/bug.h"
19 19
20 #include "hist.h" 20 #include "hist.h"
21 #include "util.h" 21 #include "util.h"
22 #include "sort.h" 22 #include "sort.h"
23 #include "machine.h" 23 #include "machine.h"
24 #include "callchain.h" 24 #include "callchain.h"
25 25
26 __thread struct callchain_cursor callchain_cursor; 26 __thread struct callchain_cursor callchain_cursor;
27 27
28 #ifdef HAVE_DWARF_UNWIND_SUPPORT 28 #ifdef HAVE_DWARF_UNWIND_SUPPORT
29 static int get_stack_size(const char *str, unsigned long *_size) 29 static int get_stack_size(const char *str, unsigned long *_size)
30 { 30 {
31 char *endptr; 31 char *endptr;
32 unsigned long size; 32 unsigned long size;
33 unsigned long max_size = round_down(USHRT_MAX, sizeof(u64)); 33 unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
34 34
35 size = strtoul(str, &endptr, 0); 35 size = strtoul(str, &endptr, 0);
36 36
37 do { 37 do {
38 if (*endptr) 38 if (*endptr)
39 break; 39 break;
40 40
41 size = round_up(size, sizeof(u64)); 41 size = round_up(size, sizeof(u64));
42 if (!size || size > max_size) 42 if (!size || size > max_size)
43 break; 43 break;
44 44
45 *_size = size; 45 *_size = size;
46 return 0; 46 return 0;
47 47
48 } while (0); 48 } while (0);
49 49
50 pr_err("callchain: Incorrect stack dump size (max %ld): %s\n", 50 pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
51 max_size, str); 51 max_size, str);
52 return -1; 52 return -1;
53 } 53 }
54 #endif /* HAVE_DWARF_UNWIND_SUPPORT */ 54 #endif /* HAVE_DWARF_UNWIND_SUPPORT */
55 55
56 int parse_callchain_record_opt(const char *arg) 56 int parse_callchain_record_opt(const char *arg)
57 { 57 {
58 char *tok, *name, *saveptr = NULL; 58 char *tok, *name, *saveptr = NULL;
59 char *buf; 59 char *buf;
60 int ret = -1; 60 int ret = -1;
61 61
62 /* We need buffer that we know we can write to. */ 62 /* We need buffer that we know we can write to. */
63 buf = malloc(strlen(arg) + 1); 63 buf = malloc(strlen(arg) + 1);
64 if (!buf) 64 if (!buf)
65 return -ENOMEM; 65 return -ENOMEM;
66 66
67 strcpy(buf, arg); 67 strcpy(buf, arg);
68 68
69 tok = strtok_r((char *)buf, ",", &saveptr); 69 tok = strtok_r((char *)buf, ",", &saveptr);
70 name = tok ? : (char *)buf; 70 name = tok ? : (char *)buf;
71 71
72 do { 72 do {
73 /* Framepointer style */ 73 /* Framepointer style */
74 if (!strncmp(name, "fp", sizeof("fp"))) { 74 if (!strncmp(name, "fp", sizeof("fp"))) {
75 if (!strtok_r(NULL, ",", &saveptr)) { 75 if (!strtok_r(NULL, ",", &saveptr)) {
76 callchain_param.record_mode = CALLCHAIN_FP; 76 callchain_param.record_mode = CALLCHAIN_FP;
77 ret = 0; 77 ret = 0;
78 } else 78 } else
79 pr_err("callchain: No more arguments " 79 pr_err("callchain: No more arguments "
80 "needed for --call-graph fp\n"); 80 "needed for --call-graph fp\n");
81 break; 81 break;
82 82
83 #ifdef HAVE_DWARF_UNWIND_SUPPORT 83 #ifdef HAVE_DWARF_UNWIND_SUPPORT
84 /* Dwarf style */ 84 /* Dwarf style */
85 } else if (!strncmp(name, "dwarf", sizeof("dwarf"))) { 85 } else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
86 const unsigned long default_stack_dump_size = 8192; 86 const unsigned long default_stack_dump_size = 8192;
87 87
88 ret = 0; 88 ret = 0;
89 callchain_param.record_mode = CALLCHAIN_DWARF; 89 callchain_param.record_mode = CALLCHAIN_DWARF;
90 callchain_param.dump_size = default_stack_dump_size; 90 callchain_param.dump_size = default_stack_dump_size;
91 91
92 tok = strtok_r(NULL, ",", &saveptr); 92 tok = strtok_r(NULL, ",", &saveptr);
93 if (tok) { 93 if (tok) {
94 unsigned long size = 0; 94 unsigned long size = 0;
95 95
96 ret = get_stack_size(tok, &size); 96 ret = get_stack_size(tok, &size);
97 callchain_param.dump_size = size; 97 callchain_param.dump_size = size;
98 } 98 }
99 #endif /* HAVE_DWARF_UNWIND_SUPPORT */ 99 #endif /* HAVE_DWARF_UNWIND_SUPPORT */
100 } else { 100 } else {
101 pr_err("callchain: Unknown --call-graph option " 101 pr_err("callchain: Unknown --call-graph option "
102 "value: %s\n", arg); 102 "value: %s\n", arg);
103 break; 103 break;
104 } 104 }
105 105
106 } while (0); 106 } while (0);
107 107
108 free(buf); 108 free(buf);
109 return ret; 109 return ret;
110 } 110 }
111 111
112 static int parse_callchain_mode(const char *value) 112 static int parse_callchain_mode(const char *value)
113 { 113 {
114 if (!strncmp(value, "graph", strlen(value))) { 114 if (!strncmp(value, "graph", strlen(value))) {
115 callchain_param.mode = CHAIN_GRAPH_ABS; 115 callchain_param.mode = CHAIN_GRAPH_ABS;
116 return 0; 116 return 0;
117 } 117 }
118 if (!strncmp(value, "flat", strlen(value))) { 118 if (!strncmp(value, "flat", strlen(value))) {
119 callchain_param.mode = CHAIN_FLAT; 119 callchain_param.mode = CHAIN_FLAT;
120 return 0; 120 return 0;
121 } 121 }
122 if (!strncmp(value, "fractal", strlen(value))) { 122 if (!strncmp(value, "fractal", strlen(value))) {
123 callchain_param.mode = CHAIN_GRAPH_REL; 123 callchain_param.mode = CHAIN_GRAPH_REL;
124 return 0; 124 return 0;
125 } 125 }
126 return -1; 126 return -1;
127 } 127 }
128 128
129 static int parse_callchain_order(const char *value) 129 static int parse_callchain_order(const char *value)
130 { 130 {
131 if (!strncmp(value, "caller", strlen(value))) { 131 if (!strncmp(value, "caller", strlen(value))) {
132 callchain_param.order = ORDER_CALLER; 132 callchain_param.order = ORDER_CALLER;
133 return 0; 133 return 0;
134 } 134 }
135 if (!strncmp(value, "callee", strlen(value))) { 135 if (!strncmp(value, "callee", strlen(value))) {
136 callchain_param.order = ORDER_CALLEE; 136 callchain_param.order = ORDER_CALLEE;
137 return 0; 137 return 0;
138 } 138 }
139 return -1; 139 return -1;
140 } 140 }
141 141
142 static int parse_callchain_sort_key(const char *value) 142 static int parse_callchain_sort_key(const char *value)
143 { 143 {
144 if (!strncmp(value, "function", strlen(value))) { 144 if (!strncmp(value, "function", strlen(value))) {
145 callchain_param.key = CCKEY_FUNCTION; 145 callchain_param.key = CCKEY_FUNCTION;
146 return 0; 146 return 0;
147 } 147 }
148 if (!strncmp(value, "address", strlen(value))) { 148 if (!strncmp(value, "address", strlen(value))) {
149 callchain_param.key = CCKEY_ADDRESS; 149 callchain_param.key = CCKEY_ADDRESS;
150 return 0; 150 return 0;
151 } 151 }
152 if (!strncmp(value, "branch", strlen(value))) { 152 if (!strncmp(value, "branch", strlen(value))) {
153 callchain_param.branch_callstack = 1; 153 callchain_param.branch_callstack = 1;
154 return 0; 154 return 0;
155 } 155 }
156 return -1; 156 return -1;
157 } 157 }
158 158
159 int 159 int
160 parse_callchain_report_opt(const char *arg) 160 parse_callchain_report_opt(const char *arg)
161 { 161 {
162 char *tok; 162 char *tok;
163 char *endptr; 163 char *endptr;
164 bool minpcnt_set = false; 164 bool minpcnt_set = false;
165 165
166 symbol_conf.use_callchain = true; 166 symbol_conf.use_callchain = true;
167 167
168 if (!arg) 168 if (!arg)
169 return 0; 169 return 0;
170 170
171 while ((tok = strtok((char *)arg, ",")) != NULL) { 171 while ((tok = strtok((char *)arg, ",")) != NULL) {
172 if (!strncmp(tok, "none", strlen(tok))) { 172 if (!strncmp(tok, "none", strlen(tok))) {
173 callchain_param.mode = CHAIN_NONE; 173 callchain_param.mode = CHAIN_NONE;
174 symbol_conf.use_callchain = false; 174 symbol_conf.use_callchain = false;
175 return 0; 175 return 0;
176 } 176 }
177 177
178 if (!parse_callchain_mode(tok) || 178 if (!parse_callchain_mode(tok) ||
179 !parse_callchain_order(tok) || 179 !parse_callchain_order(tok) ||
180 !parse_callchain_sort_key(tok)) { 180 !parse_callchain_sort_key(tok)) {
181 /* parsing ok - move on to the next */ 181 /* parsing ok - move on to the next */
182 } else if (!minpcnt_set) { 182 } else if (!minpcnt_set) {
183 /* try to get the min percent */ 183 /* try to get the min percent */
184 callchain_param.min_percent = strtod(tok, &endptr); 184 callchain_param.min_percent = strtod(tok, &endptr);
185 if (tok == endptr) 185 if (tok == endptr)
186 return -1; 186 return -1;
187 minpcnt_set = true; 187 minpcnt_set = true;
188 } else { 188 } else {
189 /* try print limit at last */ 189 /* try print limit at last */
190 callchain_param.print_limit = strtoul(tok, &endptr, 0); 190 callchain_param.print_limit = strtoul(tok, &endptr, 0);
191 if (tok == endptr) 191 if (tok == endptr)
192 return -1; 192 return -1;
193 } 193 }
194 194
195 arg = NULL; 195 arg = NULL;
196 } 196 }
197 197
198 if (callchain_register_param(&callchain_param) < 0) { 198 if (callchain_register_param(&callchain_param) < 0) {
199 pr_err("Can't register callchain params\n"); 199 pr_err("Can't register callchain params\n");
200 return -1; 200 return -1;
201 } 201 }
202 return 0; 202 return 0;
203 } 203 }
204 204
205 int perf_callchain_config(const char *var, const char *value) 205 int perf_callchain_config(const char *var, const char *value)
206 { 206 {
207 char *endptr; 207 char *endptr;
208 208
209 if (prefixcmp(var, "call-graph.")) 209 if (prefixcmp(var, "call-graph."))
210 return 0; 210 return 0;
211 var += sizeof("call-graph.") - 1; 211 var += sizeof("call-graph.") - 1;
212 212
213 if (!strcmp(var, "record-mode")) 213 if (!strcmp(var, "record-mode"))
214 return parse_callchain_record_opt(value); 214 return parse_callchain_record_opt(value);
215 #ifdef HAVE_DWARF_UNWIND_SUPPORT 215 #ifdef HAVE_DWARF_UNWIND_SUPPORT
216 if (!strcmp(var, "dump-size")) { 216 if (!strcmp(var, "dump-size")) {
217 unsigned long size = 0; 217 unsigned long size = 0;
218 int ret; 218 int ret;
219 219
220 ret = get_stack_size(value, &size); 220 ret = get_stack_size(value, &size);
221 callchain_param.dump_size = size; 221 callchain_param.dump_size = size;
222 222
223 return ret; 223 return ret;
224 } 224 }
225 #endif 225 #endif
226 if (!strcmp(var, "print-type")) 226 if (!strcmp(var, "print-type"))
227 return parse_callchain_mode(value); 227 return parse_callchain_mode(value);
228 if (!strcmp(var, "order")) 228 if (!strcmp(var, "order"))
229 return parse_callchain_order(value); 229 return parse_callchain_order(value);
230 if (!strcmp(var, "sort-key")) 230 if (!strcmp(var, "sort-key"))
231 return parse_callchain_sort_key(value); 231 return parse_callchain_sort_key(value);
232 if (!strcmp(var, "threshold")) { 232 if (!strcmp(var, "threshold")) {
233 callchain_param.min_percent = strtod(value, &endptr); 233 callchain_param.min_percent = strtod(value, &endptr);
234 if (value == endptr) 234 if (value == endptr)
235 return -1; 235 return -1;
236 } 236 }
237 if (!strcmp(var, "print-limit")) { 237 if (!strcmp(var, "print-limit")) {
238 callchain_param.print_limit = strtod(value, &endptr); 238 callchain_param.print_limit = strtod(value, &endptr);
239 if (value == endptr) 239 if (value == endptr)
240 return -1; 240 return -1;
241 } 241 }
242 242
243 return 0; 243 return 0;
244 } 244 }
245 245
246 static void 246 static void
247 rb_insert_callchain(struct rb_root *root, struct callchain_node *chain, 247 rb_insert_callchain(struct rb_root *root, struct callchain_node *chain,
248 enum chain_mode mode) 248 enum chain_mode mode)
249 { 249 {
250 struct rb_node **p = &root->rb_node; 250 struct rb_node **p = &root->rb_node;
251 struct rb_node *parent = NULL; 251 struct rb_node *parent = NULL;
252 struct callchain_node *rnode; 252 struct callchain_node *rnode;
253 u64 chain_cumul = callchain_cumul_hits(chain); 253 u64 chain_cumul = callchain_cumul_hits(chain);
254 254
255 while (*p) { 255 while (*p) {
256 u64 rnode_cumul; 256 u64 rnode_cumul;
257 257
258 parent = *p; 258 parent = *p;
259 rnode = rb_entry(parent, struct callchain_node, rb_node); 259 rnode = rb_entry(parent, struct callchain_node, rb_node);
260 rnode_cumul = callchain_cumul_hits(rnode); 260 rnode_cumul = callchain_cumul_hits(rnode);
261 261
262 switch (mode) { 262 switch (mode) {
263 case CHAIN_FLAT: 263 case CHAIN_FLAT:
264 if (rnode->hit < chain->hit) 264 if (rnode->hit < chain->hit)
265 p = &(*p)->rb_left; 265 p = &(*p)->rb_left;
266 else 266 else
267 p = &(*p)->rb_right; 267 p = &(*p)->rb_right;
268 break; 268 break;
269 case CHAIN_GRAPH_ABS: /* Falldown */ 269 case CHAIN_GRAPH_ABS: /* Falldown */
270 case CHAIN_GRAPH_REL: 270 case CHAIN_GRAPH_REL:
271 if (rnode_cumul < chain_cumul) 271 if (rnode_cumul < chain_cumul)
272 p = &(*p)->rb_left; 272 p = &(*p)->rb_left;
273 else 273 else
274 p = &(*p)->rb_right; 274 p = &(*p)->rb_right;
275 break; 275 break;
276 case CHAIN_NONE: 276 case CHAIN_NONE:
277 default: 277 default:
278 break; 278 break;
279 } 279 }
280 } 280 }
281 281
282 rb_link_node(&chain->rb_node, parent, p); 282 rb_link_node(&chain->rb_node, parent, p);
283 rb_insert_color(&chain->rb_node, root); 283 rb_insert_color(&chain->rb_node, root);
284 } 284 }
285 285
286 static void 286 static void
287 __sort_chain_flat(struct rb_root *rb_root, struct callchain_node *node, 287 __sort_chain_flat(struct rb_root *rb_root, struct callchain_node *node,
288 u64 min_hit) 288 u64 min_hit)
289 { 289 {
290 struct rb_node *n; 290 struct rb_node *n;
291 struct callchain_node *child; 291 struct callchain_node *child;
292 292
293 n = rb_first(&node->rb_root_in); 293 n = rb_first(&node->rb_root_in);
294 while (n) { 294 while (n) {
295 child = rb_entry(n, struct callchain_node, rb_node_in); 295 child = rb_entry(n, struct callchain_node, rb_node_in);
296 n = rb_next(n); 296 n = rb_next(n);
297 297
298 __sort_chain_flat(rb_root, child, min_hit); 298 __sort_chain_flat(rb_root, child, min_hit);
299 } 299 }
300 300
301 if (node->hit && node->hit >= min_hit) 301 if (node->hit && node->hit >= min_hit)
302 rb_insert_callchain(rb_root, node, CHAIN_FLAT); 302 rb_insert_callchain(rb_root, node, CHAIN_FLAT);
303 } 303 }
304 304
305 /* 305 /*
306 * Once we get every callchains from the stream, we can now 306 * Once we get every callchains from the stream, we can now
307 * sort them by hit 307 * sort them by hit
308 */ 308 */
309 static void 309 static void
310 sort_chain_flat(struct rb_root *rb_root, struct callchain_root *root, 310 sort_chain_flat(struct rb_root *rb_root, struct callchain_root *root,
311 u64 min_hit, struct callchain_param *param __maybe_unused) 311 u64 min_hit, struct callchain_param *param __maybe_unused)
312 { 312 {
313 __sort_chain_flat(rb_root, &root->node, min_hit); 313 __sort_chain_flat(rb_root, &root->node, min_hit);
314 } 314 }
315 315
316 static void __sort_chain_graph_abs(struct callchain_node *node, 316 static void __sort_chain_graph_abs(struct callchain_node *node,
317 u64 min_hit) 317 u64 min_hit)
318 { 318 {
319 struct rb_node *n; 319 struct rb_node *n;
320 struct callchain_node *child; 320 struct callchain_node *child;
321 321
322 node->rb_root = RB_ROOT; 322 node->rb_root = RB_ROOT;
323 n = rb_first(&node->rb_root_in); 323 n = rb_first(&node->rb_root_in);
324 324
325 while (n) { 325 while (n) {
326 child = rb_entry(n, struct callchain_node, rb_node_in); 326 child = rb_entry(n, struct callchain_node, rb_node_in);
327 n = rb_next(n); 327 n = rb_next(n);
328 328
329 __sort_chain_graph_abs(child, min_hit); 329 __sort_chain_graph_abs(child, min_hit);
330 if (callchain_cumul_hits(child) >= min_hit) 330 if (callchain_cumul_hits(child) >= min_hit)
331 rb_insert_callchain(&node->rb_root, child, 331 rb_insert_callchain(&node->rb_root, child,
332 CHAIN_GRAPH_ABS); 332 CHAIN_GRAPH_ABS);
333 } 333 }
334 } 334 }
335 335
336 static void 336 static void
337 sort_chain_graph_abs(struct rb_root *rb_root, struct callchain_root *chain_root, 337 sort_chain_graph_abs(struct rb_root *rb_root, struct callchain_root *chain_root,
338 u64 min_hit, struct callchain_param *param __maybe_unused) 338 u64 min_hit, struct callchain_param *param __maybe_unused)
339 { 339 {
340 __sort_chain_graph_abs(&chain_root->node, min_hit); 340 __sort_chain_graph_abs(&chain_root->node, min_hit);
341 rb_root->rb_node = chain_root->node.rb_root.rb_node; 341 rb_root->rb_node = chain_root->node.rb_root.rb_node;
342 } 342 }
343 343
344 static void __sort_chain_graph_rel(struct callchain_node *node, 344 static void __sort_chain_graph_rel(struct callchain_node *node,
345 double min_percent) 345 double min_percent)
346 { 346 {
347 struct rb_node *n; 347 struct rb_node *n;
348 struct callchain_node *child; 348 struct callchain_node *child;
349 u64 min_hit; 349 u64 min_hit;
350 350
351 node->rb_root = RB_ROOT; 351 node->rb_root = RB_ROOT;
352 min_hit = ceil(node->children_hit * min_percent); 352 min_hit = ceil(node->children_hit * min_percent);
353 353
354 n = rb_first(&node->rb_root_in); 354 n = rb_first(&node->rb_root_in);
355 while (n) { 355 while (n) {
356 child = rb_entry(n, struct callchain_node, rb_node_in); 356 child = rb_entry(n, struct callchain_node, rb_node_in);
357 n = rb_next(n); 357 n = rb_next(n);
358 358
359 __sort_chain_graph_rel(child, min_percent); 359 __sort_chain_graph_rel(child, min_percent);
360 if (callchain_cumul_hits(child) >= min_hit) 360 if (callchain_cumul_hits(child) >= min_hit)
361 rb_insert_callchain(&node->rb_root, child, 361 rb_insert_callchain(&node->rb_root, child,
362 CHAIN_GRAPH_REL); 362 CHAIN_GRAPH_REL);
363 } 363 }
364 } 364 }
365 365
366 static void 366 static void
367 sort_chain_graph_rel(struct rb_root *rb_root, struct callchain_root *chain_root, 367 sort_chain_graph_rel(struct rb_root *rb_root, struct callchain_root *chain_root,
368 u64 min_hit __maybe_unused, struct callchain_param *param) 368 u64 min_hit __maybe_unused, struct callchain_param *param)
369 { 369 {
370 __sort_chain_graph_rel(&chain_root->node, param->min_percent / 100.0); 370 __sort_chain_graph_rel(&chain_root->node, param->min_percent / 100.0);
371 rb_root->rb_node = chain_root->node.rb_root.rb_node; 371 rb_root->rb_node = chain_root->node.rb_root.rb_node;
372 } 372 }
373 373
374 int callchain_register_param(struct callchain_param *param) 374 int callchain_register_param(struct callchain_param *param)
375 { 375 {
376 switch (param->mode) { 376 switch (param->mode) {
377 case CHAIN_GRAPH_ABS: 377 case CHAIN_GRAPH_ABS:
378 param->sort = sort_chain_graph_abs; 378 param->sort = sort_chain_graph_abs;
379 break; 379 break;
380 case CHAIN_GRAPH_REL: 380 case CHAIN_GRAPH_REL:
381 param->sort = sort_chain_graph_rel; 381 param->sort = sort_chain_graph_rel;
382 break; 382 break;
383 case CHAIN_FLAT: 383 case CHAIN_FLAT:
384 param->sort = sort_chain_flat; 384 param->sort = sort_chain_flat;
385 break; 385 break;
386 case CHAIN_NONE: 386 case CHAIN_NONE:
387 default: 387 default:
388 return -1; 388 return -1;
389 } 389 }
390 return 0; 390 return 0;
391 } 391 }
392 392
393 /* 393 /*
394 * Create a child for a parent. If inherit_children, then the new child 394 * Create a child for a parent. If inherit_children, then the new child
395 * will become the new parent of it's parent children 395 * will become the new parent of it's parent children
396 */ 396 */
397 static struct callchain_node * 397 static struct callchain_node *
398 create_child(struct callchain_node *parent, bool inherit_children) 398 create_child(struct callchain_node *parent, bool inherit_children)
399 { 399 {
400 struct callchain_node *new; 400 struct callchain_node *new;
401 401
402 new = zalloc(sizeof(*new)); 402 new = zalloc(sizeof(*new));
403 if (!new) { 403 if (!new) {
404 perror("not enough memory to create child for code path tree"); 404 perror("not enough memory to create child for code path tree");
405 return NULL; 405 return NULL;
406 } 406 }
407 new->parent = parent; 407 new->parent = parent;
408 INIT_LIST_HEAD(&new->val); 408 INIT_LIST_HEAD(&new->val);
409 409
410 if (inherit_children) { 410 if (inherit_children) {
411 struct rb_node *n; 411 struct rb_node *n;
412 struct callchain_node *child; 412 struct callchain_node *child;
413 413
414 new->rb_root_in = parent->rb_root_in; 414 new->rb_root_in = parent->rb_root_in;
415 parent->rb_root_in = RB_ROOT; 415 parent->rb_root_in = RB_ROOT;
416 416
417 n = rb_first(&new->rb_root_in); 417 n = rb_first(&new->rb_root_in);
418 while (n) { 418 while (n) {
419 child = rb_entry(n, struct callchain_node, rb_node_in); 419 child = rb_entry(n, struct callchain_node, rb_node_in);
420 child->parent = new; 420 child->parent = new;
421 n = rb_next(n); 421 n = rb_next(n);
422 } 422 }
423 423
424 /* make it the first child */ 424 /* make it the first child */
425 rb_link_node(&new->rb_node_in, NULL, &parent->rb_root_in.rb_node); 425 rb_link_node(&new->rb_node_in, NULL, &parent->rb_root_in.rb_node);
426 rb_insert_color(&new->rb_node_in, &parent->rb_root_in); 426 rb_insert_color(&new->rb_node_in, &parent->rb_root_in);
427 } 427 }
428 428
429 return new; 429 return new;
430 } 430 }
431 431
432 432
433 /* 433 /*
434 * Fill the node with callchain values 434 * Fill the node with callchain values
435 */ 435 */
436 static void 436 static void
437 fill_node(struct callchain_node *node, struct callchain_cursor *cursor) 437 fill_node(struct callchain_node *node, struct callchain_cursor *cursor)
438 { 438 {
439 struct callchain_cursor_node *cursor_node; 439 struct callchain_cursor_node *cursor_node;
440 440
441 node->val_nr = cursor->nr - cursor->pos; 441 node->val_nr = cursor->nr - cursor->pos;
442 if (!node->val_nr) 442 if (!node->val_nr)
443 pr_warning("Warning: empty node in callchain tree\n"); 443 pr_warning("Warning: empty node in callchain tree\n");
444 444
445 cursor_node = callchain_cursor_current(cursor); 445 cursor_node = callchain_cursor_current(cursor);
446 446
447 while (cursor_node) { 447 while (cursor_node) {
448 struct callchain_list *call; 448 struct callchain_list *call;
449 449
450 call = zalloc(sizeof(*call)); 450 call = zalloc(sizeof(*call));
451 if (!call) { 451 if (!call) {
452 perror("not enough memory for the code path tree"); 452 perror("not enough memory for the code path tree");
453 return; 453 return;
454 } 454 }
455 call->ip = cursor_node->ip; 455 call->ip = cursor_node->ip;
456 call->ms.sym = cursor_node->sym; 456 call->ms.sym = cursor_node->sym;
457 call->ms.map = cursor_node->map; 457 call->ms.map = cursor_node->map;
458 list_add_tail(&call->list, &node->val); 458 list_add_tail(&call->list, &node->val);
459 459
460 callchain_cursor_advance(cursor); 460 callchain_cursor_advance(cursor);
461 cursor_node = callchain_cursor_current(cursor); 461 cursor_node = callchain_cursor_current(cursor);
462 } 462 }
463 } 463 }
464 464
465 static struct callchain_node * 465 static struct callchain_node *
466 add_child(struct callchain_node *parent, 466 add_child(struct callchain_node *parent,
467 struct callchain_cursor *cursor, 467 struct callchain_cursor *cursor,
468 u64 period) 468 u64 period)
469 { 469 {
470 struct callchain_node *new; 470 struct callchain_node *new;
471 471
472 new = create_child(parent, false); 472 new = create_child(parent, false);
473 fill_node(new, cursor); 473 fill_node(new, cursor);
474 474
475 new->children_hit = 0; 475 new->children_hit = 0;
476 new->hit = period; 476 new->hit = period;
477 return new; 477 return new;
478 } 478 }
479 479
480 static s64 match_chain(struct callchain_cursor_node *node, 480 static s64 match_chain(struct callchain_cursor_node *node,
481 struct callchain_list *cnode) 481 struct callchain_list *cnode)
482 { 482 {
483 struct symbol *sym = node->sym; 483 struct symbol *sym = node->sym;
484 484
485 if (cnode->ms.sym && sym && 485 if (cnode->ms.sym && sym &&
486 callchain_param.key == CCKEY_FUNCTION) 486 callchain_param.key == CCKEY_FUNCTION)
487 return cnode->ms.sym->start - sym->start; 487 return cnode->ms.sym->start - sym->start;
488 else 488 else
489 return cnode->ip - node->ip; 489 return cnode->ip - node->ip;
490 } 490 }
491 491
492 /* 492 /*
493 * Split the parent in two parts (a new child is created) and 493 * Split the parent in two parts (a new child is created) and
494 * give a part of its callchain to the created child. 494 * give a part of its callchain to the created child.
495 * Then create another child to host the given callchain of new branch 495 * Then create another child to host the given callchain of new branch
496 */ 496 */
497 static void 497 static void
498 split_add_child(struct callchain_node *parent, 498 split_add_child(struct callchain_node *parent,
499 struct callchain_cursor *cursor, 499 struct callchain_cursor *cursor,
500 struct callchain_list *to_split, 500 struct callchain_list *to_split,
501 u64 idx_parents, u64 idx_local, u64 period) 501 u64 idx_parents, u64 idx_local, u64 period)
502 { 502 {
503 struct callchain_node *new; 503 struct callchain_node *new;
504 struct list_head *old_tail; 504 struct list_head *old_tail;
505 unsigned int idx_total = idx_parents + idx_local; 505 unsigned int idx_total = idx_parents + idx_local;
506 506
507 /* split */ 507 /* split */
508 new = create_child(parent, true); 508 new = create_child(parent, true);
509 509
510 /* split the callchain and move a part to the new child */ 510 /* split the callchain and move a part to the new child */
511 old_tail = parent->val.prev; 511 old_tail = parent->val.prev;
512 list_del_range(&to_split->list, old_tail); 512 list_del_range(&to_split->list, old_tail);
513 new->val.next = &to_split->list; 513 new->val.next = &to_split->list;
514 new->val.prev = old_tail; 514 new->val.prev = old_tail;
515 to_split->list.prev = &new->val; 515 to_split->list.prev = &new->val;
516 old_tail->next = &new->val; 516 old_tail->next = &new->val;
517 517
518 /* split the hits */ 518 /* split the hits */
519 new->hit = parent->hit; 519 new->hit = parent->hit;
520 new->children_hit = parent->children_hit; 520 new->children_hit = parent->children_hit;
521 parent->children_hit = callchain_cumul_hits(new); 521 parent->children_hit = callchain_cumul_hits(new);
522 new->val_nr = parent->val_nr - idx_local; 522 new->val_nr = parent->val_nr - idx_local;
523 parent->val_nr = idx_local; 523 parent->val_nr = idx_local;
524 524
525 /* create a new child for the new branch if any */ 525 /* create a new child for the new branch if any */
526 if (idx_total < cursor->nr) { 526 if (idx_total < cursor->nr) {
527 struct callchain_node *first; 527 struct callchain_node *first;
528 struct callchain_list *cnode; 528 struct callchain_list *cnode;
529 struct callchain_cursor_node *node; 529 struct callchain_cursor_node *node;
530 struct rb_node *p, **pp; 530 struct rb_node *p, **pp;
531 531
532 parent->hit = 0; 532 parent->hit = 0;
533 parent->children_hit += period; 533 parent->children_hit += period;
534 534
535 node = callchain_cursor_current(cursor); 535 node = callchain_cursor_current(cursor);
536 new = add_child(parent, cursor, period); 536 new = add_child(parent, cursor, period);
537 537
538 /* 538 /*
539 * This is second child since we moved parent's children 539 * This is second child since we moved parent's children
540 * to new (first) child above. 540 * to new (first) child above.
541 */ 541 */
542 p = parent->rb_root_in.rb_node; 542 p = parent->rb_root_in.rb_node;
543 first = rb_entry(p, struct callchain_node, rb_node_in); 543 first = rb_entry(p, struct callchain_node, rb_node_in);
544 cnode = list_first_entry(&first->val, struct callchain_list, 544 cnode = list_first_entry(&first->val, struct callchain_list,
545 list); 545 list);
546 546
547 if (match_chain(node, cnode) < 0) 547 if (match_chain(node, cnode) < 0)
548 pp = &p->rb_left; 548 pp = &p->rb_left;
549 else 549 else
550 pp = &p->rb_right; 550 pp = &p->rb_right;
551 551
552 rb_link_node(&new->rb_node_in, p, pp); 552 rb_link_node(&new->rb_node_in, p, pp);
553 rb_insert_color(&new->rb_node_in, &parent->rb_root_in); 553 rb_insert_color(&new->rb_node_in, &parent->rb_root_in);
554 } else { 554 } else {
555 parent->hit = period; 555 parent->hit = period;
556 } 556 }
557 } 557 }
558 558
559 static int 559 static int
560 append_chain(struct callchain_node *root, 560 append_chain(struct callchain_node *root,
561 struct callchain_cursor *cursor, 561 struct callchain_cursor *cursor,
562 u64 period); 562 u64 period);
563 563
564 static void 564 static void
565 append_chain_children(struct callchain_node *root, 565 append_chain_children(struct callchain_node *root,
566 struct callchain_cursor *cursor, 566 struct callchain_cursor *cursor,
567 u64 period) 567 u64 period)
568 { 568 {
569 struct callchain_node *rnode; 569 struct callchain_node *rnode;
570 struct callchain_cursor_node *node; 570 struct callchain_cursor_node *node;
571 struct rb_node **p = &root->rb_root_in.rb_node; 571 struct rb_node **p = &root->rb_root_in.rb_node;
572 struct rb_node *parent = NULL; 572 struct rb_node *parent = NULL;
573 573
574 node = callchain_cursor_current(cursor); 574 node = callchain_cursor_current(cursor);
575 if (!node) 575 if (!node)
576 return; 576 return;
577 577
578 /* lookup in childrens */ 578 /* lookup in childrens */
579 while (*p) { 579 while (*p) {
580 s64 ret; 580 s64 ret;
581 581
582 parent = *p; 582 parent = *p;
583 rnode = rb_entry(parent, struct callchain_node, rb_node_in); 583 rnode = rb_entry(parent, struct callchain_node, rb_node_in);
584 584
585 /* If at least first entry matches, rely to children */ 585 /* If at least first entry matches, rely to children */
586 ret = append_chain(rnode, cursor, period); 586 ret = append_chain(rnode, cursor, period);
587 if (ret == 0) 587 if (ret == 0)
588 goto inc_children_hit; 588 goto inc_children_hit;
589 589
590 if (ret < 0) 590 if (ret < 0)
591 p = &parent->rb_left; 591 p = &parent->rb_left;
592 else 592 else
593 p = &parent->rb_right; 593 p = &parent->rb_right;
594 } 594 }
595 /* nothing in children, add to the current node */ 595 /* nothing in children, add to the current node */
596 rnode = add_child(root, cursor, period); 596 rnode = add_child(root, cursor, period);
597 rb_link_node(&rnode->rb_node_in, parent, p); 597 rb_link_node(&rnode->rb_node_in, parent, p);
598 rb_insert_color(&rnode->rb_node_in, &root->rb_root_in); 598 rb_insert_color(&rnode->rb_node_in, &root->rb_root_in);
599 599
600 inc_children_hit: 600 inc_children_hit:
601 root->children_hit += period; 601 root->children_hit += period;
602 } 602 }
603 603
604 static int 604 static int
605 append_chain(struct callchain_node *root, 605 append_chain(struct callchain_node *root,
606 struct callchain_cursor *cursor, 606 struct callchain_cursor *cursor,
607 u64 period) 607 u64 period)
608 { 608 {
609 struct callchain_list *cnode; 609 struct callchain_list *cnode;
610 u64 start = cursor->pos; 610 u64 start = cursor->pos;
611 bool found = false; 611 bool found = false;
612 u64 matches; 612 u64 matches;
613 int cmp = 0; 613 int cmp = 0;
614 614
615 /* 615 /*
616 * Lookup in the current node 616 * Lookup in the current node
617 * If we have a symbol, then compare the start to match 617 * If we have a symbol, then compare the start to match
618 * anywhere inside a function, unless function 618 * anywhere inside a function, unless function
619 * mode is disabled. 619 * mode is disabled.
620 */ 620 */
621 list_for_each_entry(cnode, &root->val, list) { 621 list_for_each_entry(cnode, &root->val, list) {
622 struct callchain_cursor_node *node; 622 struct callchain_cursor_node *node;
623 623
624 node = callchain_cursor_current(cursor); 624 node = callchain_cursor_current(cursor);
625 if (!node) 625 if (!node)
626 break; 626 break;
627 627
628 cmp = match_chain(node, cnode); 628 cmp = match_chain(node, cnode);
629 if (cmp) 629 if (cmp)
630 break; 630 break;
631 631
632 found = true; 632 found = true;
633 633
634 callchain_cursor_advance(cursor); 634 callchain_cursor_advance(cursor);
635 } 635 }
636 636
637 /* matches not, relay no the parent */ 637 /* matches not, relay no the parent */
638 if (!found) { 638 if (!found) {
639 WARN_ONCE(!cmp, "Chain comparison error\n"); 639 WARN_ONCE(!cmp, "Chain comparison error\n");
640 return cmp; 640 return cmp;
641 } 641 }
642 642
643 matches = cursor->pos - start; 643 matches = cursor->pos - start;
644 644
645 /* we match only a part of the node. Split it and add the new chain */ 645 /* we match only a part of the node. Split it and add the new chain */
646 if (matches < root->val_nr) { 646 if (matches < root->val_nr) {
647 split_add_child(root, cursor, cnode, start, matches, period); 647 split_add_child(root, cursor, cnode, start, matches, period);
648 return 0; 648 return 0;
649 } 649 }
650 650
651 /* we match 100% of the path, increment the hit */ 651 /* we match 100% of the path, increment the hit */
652 if (matches == root->val_nr && cursor->pos == cursor->nr) { 652 if (matches == root->val_nr && cursor->pos == cursor->nr) {
653 root->hit += period; 653 root->hit += period;
654 return 0; 654 return 0;
655 } 655 }
656 656
657 /* We match the node and still have a part remaining */ 657 /* We match the node and still have a part remaining */
658 append_chain_children(root, cursor, period); 658 append_chain_children(root, cursor, period);
659 659
660 return 0; 660 return 0;
661 } 661 }
662 662
663 int callchain_append(struct callchain_root *root, 663 int callchain_append(struct callchain_root *root,
664 struct callchain_cursor *cursor, 664 struct callchain_cursor *cursor,
665 u64 period) 665 u64 period)
666 { 666 {
667 if (!cursor->nr) 667 if (!cursor->nr)
668 return 0; 668 return 0;
669 669
670 callchain_cursor_commit(cursor); 670 callchain_cursor_commit(cursor);
671 671
672 append_chain_children(&root->node, cursor, period); 672 append_chain_children(&root->node, cursor, period);
673 673
674 if (cursor->nr > root->max_depth) 674 if (cursor->nr > root->max_depth)
675 root->max_depth = cursor->nr; 675 root->max_depth = cursor->nr;
676 676
677 return 0; 677 return 0;
678 } 678 }
679 679
680 static int 680 static int
681 merge_chain_branch(struct callchain_cursor *cursor, 681 merge_chain_branch(struct callchain_cursor *cursor,
682 struct callchain_node *dst, struct callchain_node *src) 682 struct callchain_node *dst, struct callchain_node *src)
683 { 683 {
684 struct callchain_cursor_node **old_last = cursor->last; 684 struct callchain_cursor_node **old_last = cursor->last;
685 struct callchain_node *child; 685 struct callchain_node *child;
686 struct callchain_list *list, *next_list; 686 struct callchain_list *list, *next_list;
687 struct rb_node *n; 687 struct rb_node *n;
688 int old_pos = cursor->nr; 688 int old_pos = cursor->nr;
689 int err = 0; 689 int err = 0;
690 690
691 list_for_each_entry_safe(list, next_list, &src->val, list) { 691 list_for_each_entry_safe(list, next_list, &src->val, list) {
692 callchain_cursor_append(cursor, list->ip, 692 callchain_cursor_append(cursor, list->ip,
693 list->ms.map, list->ms.sym); 693 list->ms.map, list->ms.sym);
694 list_del(&list->list); 694 list_del(&list->list);
695 free(list); 695 free(list);
696 } 696 }
697 697
698 if (src->hit) { 698 if (src->hit) {
699 callchain_cursor_commit(cursor); 699 callchain_cursor_commit(cursor);
700 append_chain_children(dst, cursor, src->hit); 700 append_chain_children(dst, cursor, src->hit);
701 } 701 }
702 702
703 n = rb_first(&src->rb_root_in); 703 n = rb_first(&src->rb_root_in);
704 while (n) { 704 while (n) {
705 child = container_of(n, struct callchain_node, rb_node_in); 705 child = container_of(n, struct callchain_node, rb_node_in);
706 n = rb_next(n); 706 n = rb_next(n);
707 rb_erase(&child->rb_node_in, &src->rb_root_in); 707 rb_erase(&child->rb_node_in, &src->rb_root_in);
708 708
709 err = merge_chain_branch(cursor, dst, child); 709 err = merge_chain_branch(cursor, dst, child);
710 if (err) 710 if (err)
711 break; 711 break;
712 712
713 free(child); 713 free(child);
714 } 714 }
715 715
716 cursor->nr = old_pos; 716 cursor->nr = old_pos;
717 cursor->last = old_last; 717 cursor->last = old_last;
718 718
719 return err; 719 return err;
720 } 720 }
721 721
722 int callchain_merge(struct callchain_cursor *cursor, 722 int callchain_merge(struct callchain_cursor *cursor,
723 struct callchain_root *dst, struct callchain_root *src) 723 struct callchain_root *dst, struct callchain_root *src)
724 { 724 {
725 return merge_chain_branch(cursor, &dst->node, &src->node); 725 return merge_chain_branch(cursor, &dst->node, &src->node);
726 } 726 }
727 727
728 int callchain_cursor_append(struct callchain_cursor *cursor, 728 int callchain_cursor_append(struct callchain_cursor *cursor,
729 u64 ip, struct map *map, struct symbol *sym) 729 u64 ip, struct map *map, struct symbol *sym)
730 { 730 {
731 struct callchain_cursor_node *node = *cursor->last; 731 struct callchain_cursor_node *node = *cursor->last;
732 732
733 if (!node) { 733 if (!node) {
734 node = calloc(1, sizeof(*node)); 734 node = calloc(1, sizeof(*node));
735 if (!node) 735 if (!node)
736 return -ENOMEM; 736 return -ENOMEM;
737 737
738 *cursor->last = node; 738 *cursor->last = node;
739 } 739 }
740 740
741 node->ip = ip; 741 node->ip = ip;
742 node->map = map; 742 node->map = map;
743 node->sym = sym; 743 node->sym = sym;
744 744
745 cursor->nr++; 745 cursor->nr++;
746 746
747 cursor->last = &node->next; 747 cursor->last = &node->next;
748 748
749 return 0; 749 return 0;
750 } 750 }
751 751
752 int sample__resolve_callchain(struct perf_sample *sample, struct symbol **parent, 752 int sample__resolve_callchain(struct perf_sample *sample, struct symbol **parent,
753 struct perf_evsel *evsel, struct addr_location *al, 753 struct perf_evsel *evsel, struct addr_location *al,
754 int max_stack) 754 int max_stack)
755 { 755 {
756 if (sample->callchain == NULL) 756 if (sample->callchain == NULL)
757 return 0; 757 return 0;
758 758
759 if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain || 759 if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain ||
760 sort__has_parent) { 760 sort__has_parent) {
761 return thread__resolve_callchain(al->thread, evsel, sample, 761 return thread__resolve_callchain(al->thread, evsel, sample,
762 parent, al, max_stack); 762 parent, al, max_stack);
763 } 763 }
764 return 0; 764 return 0;
765 } 765 }
766 766
767 int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample) 767 int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample)
768 { 768 {
769 if (!symbol_conf.use_callchain || sample->callchain == NULL) 769 if (!symbol_conf.use_callchain || sample->callchain == NULL)
770 return 0; 770 return 0;
771 return callchain_append(he->callchain, &callchain_cursor, sample->period); 771 return callchain_append(he->callchain, &callchain_cursor, sample->period);
772 } 772 }
773 773
774 int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, 774 int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node,
775 bool hide_unresolved) 775 bool hide_unresolved)
776 { 776 {
777 al->map = node->map; 777 al->map = node->map;
778 al->sym = node->sym; 778 al->sym = node->sym;
779 if (node->map) 779 if (node->map)
780 al->addr = node->map->map_ip(node->map, node->ip); 780 al->addr = node->map->map_ip(node->map, node->ip);
781 else 781 else
782 al->addr = node->ip; 782 al->addr = node->ip;
783 783
784 if (al->sym == NULL) { 784 if (al->sym == NULL) {
785 if (hide_unresolved) 785 if (hide_unresolved)
786 return 0; 786 return 0;
787 if (al->map == NULL) 787 if (al->map == NULL)
788 goto out; 788 goto out;
789 } 789 }
790 790
791 if (al->map->groups == &al->machine->kmaps) { 791 if (al->map->groups == &al->machine->kmaps) {
792 if (machine__is_host(al->machine)) { 792 if (machine__is_host(al->machine)) {
793 al->cpumode = PERF_RECORD_MISC_KERNEL; 793 al->cpumode = PERF_RECORD_MISC_KERNEL;
794 al->level = 'k'; 794 al->level = 'k';
795 } else { 795 } else {
796 al->cpumode = PERF_RECORD_MISC_GUEST_KERNEL; 796 al->cpumode = PERF_RECORD_MISC_GUEST_KERNEL;
797 al->level = 'g'; 797 al->level = 'g';
798 } 798 }
799 } else { 799 } else {
800 if (machine__is_host(al->machine)) { 800 if (machine__is_host(al->machine)) {
801 al->cpumode = PERF_RECORD_MISC_USER; 801 al->cpumode = PERF_RECORD_MISC_USER;
802 al->level = '.'; 802 al->level = '.';
803 } else if (perf_guest) { 803 } else if (perf_guest) {
804 al->cpumode = PERF_RECORD_MISC_GUEST_USER; 804 al->cpumode = PERF_RECORD_MISC_GUEST_USER;
805 al->level = 'u'; 805 al->level = 'u';
806 } else { 806 } else {
807 al->cpumode = PERF_RECORD_MISC_HYPERVISOR; 807 al->cpumode = PERF_RECORD_MISC_HYPERVISOR;
808 al->level = 'H'; 808 al->level = 'H';
809 } 809 }
810 } 810 }
811 811
812 out: 812 out:
813 return 1; 813 return 1;
814 } 814 }
815 815
816 char *callchain_list__sym_name(struct callchain_list *cl, 816 char *callchain_list__sym_name(struct callchain_list *cl,
817 char *bf, size_t bfsize, bool show_dso) 817 char *bf, size_t bfsize, bool show_dso)
818 { 818 {
819 int printed; 819 int printed;
820 820
821 if (cl->ms.sym) { 821 if (cl->ms.sym) {
822 if (callchain_param.key == CCKEY_ADDRESS && 822 if (callchain_param.key == CCKEY_ADDRESS &&
823 cl->ms.map && !cl->srcline) 823 cl->ms.map && !cl->srcline)
824 cl->srcline = get_srcline(cl->ms.map->dso, 824 cl->srcline = get_srcline(cl->ms.map->dso,
825 map__rip_2objdump(cl->ms.map, 825 map__rip_2objdump(cl->ms.map,
826 cl->ip), 826 cl->ip),
827 cl->ms.sym, false); 827 cl->ms.sym, false);
828 if (cl->srcline) 828 if (cl->srcline)
829 printed = scnprintf(bf, bfsize, "%s %s", 829 printed = scnprintf(bf, bfsize, "%s %s",
830 cl->ms.sym->name, cl->srcline); 830 cl->ms.sym->name, cl->srcline);
831 else 831 else
832 printed = scnprintf(bf, bfsize, "%s", cl->ms.sym->name); 832 printed = scnprintf(bf, bfsize, "%s", cl->ms.sym->name);
833 } else 833 } else
834 printed = scnprintf(bf, bfsize, "%#" PRIx64, cl->ip); 834 printed = scnprintf(bf, bfsize, "%#" PRIx64, cl->ip);
835 835
836 if (show_dso) 836 if (show_dso)
837 scnprintf(bf + printed, bfsize - printed, " %s", 837 scnprintf(bf + printed, bfsize - printed, " %s",
838 cl->ms.map ? 838 cl->ms.map ?
839 cl->ms.map->dso->short_name : 839 cl->ms.map->dso->short_name :
840 "unknown"); 840 "unknown");
841 841
842 return bf; 842 return bf;
843 } 843 }
844
845 static void free_callchain_node(struct callchain_node *node)
846 {
847 struct callchain_list *list, *tmp;
848 struct callchain_node *child;
849 struct rb_node *n;
850
851 list_for_each_entry_safe(list, tmp, &node->val, list) {
852 list_del(&list->list);
853 free(list);
854 }
855
856 n = rb_first(&node->rb_root_in);
857 while (n) {
858 child = container_of(n, struct callchain_node, rb_node_in);
859 n = rb_next(n);
860 rb_erase(&child->rb_node_in, &node->rb_root_in);
861
862 free_callchain_node(child);
863 free(child);
864 }
865 }
866
867 void free_callchain(struct callchain_root *root)
868 {
869 if (!symbol_conf.use_callchain)
870 return;
871
872 free_callchain_node(&root->node);
873 }
844 874
tools/perf/util/callchain.h
Diff comments   View file @ ddb321a
1 #ifndef __PERF_CALLCHAIN_H 1 #ifndef __PERF_CALLCHAIN_H
2 #define __PERF_CALLCHAIN_H 2 #define __PERF_CALLCHAIN_H
3 3
4 #include "../perf.h" 4 #include "../perf.h"
5 #include <linux/list.h> 5 #include <linux/list.h>
6 #include <linux/rbtree.h> 6 #include <linux/rbtree.h>
7 #include "event.h" 7 #include "event.h"
8 #include "symbol.h" 8 #include "symbol.h"
9 9
10 enum perf_call_graph_mode { 10 enum perf_call_graph_mode {
11 CALLCHAIN_NONE, 11 CALLCHAIN_NONE,
12 CALLCHAIN_FP, 12 CALLCHAIN_FP,
13 CALLCHAIN_DWARF, 13 CALLCHAIN_DWARF,
14 CALLCHAIN_MAX 14 CALLCHAIN_MAX
15 }; 15 };
16 16
17 enum chain_mode { 17 enum chain_mode {
18 CHAIN_NONE, 18 CHAIN_NONE,
19 CHAIN_FLAT, 19 CHAIN_FLAT,
20 CHAIN_GRAPH_ABS, 20 CHAIN_GRAPH_ABS,
21 CHAIN_GRAPH_REL 21 CHAIN_GRAPH_REL
22 }; 22 };
23 23
24 enum chain_order { 24 enum chain_order {
25 ORDER_CALLER, 25 ORDER_CALLER,
26 ORDER_CALLEE 26 ORDER_CALLEE
27 }; 27 };
28 28
29 struct callchain_node { 29 struct callchain_node {
30 struct callchain_node *parent; 30 struct callchain_node *parent;
31 struct list_head val; 31 struct list_head val;
32 struct rb_node rb_node_in; /* to insert nodes in an rbtree */ 32 struct rb_node rb_node_in; /* to insert nodes in an rbtree */
33 struct rb_node rb_node; /* to sort nodes in an output tree */ 33 struct rb_node rb_node; /* to sort nodes in an output tree */
34 struct rb_root rb_root_in; /* input tree of children */ 34 struct rb_root rb_root_in; /* input tree of children */
35 struct rb_root rb_root; /* sorted output tree of children */ 35 struct rb_root rb_root; /* sorted output tree of children */
36 unsigned int val_nr; 36 unsigned int val_nr;
37 u64 hit; 37 u64 hit;
38 u64 children_hit; 38 u64 children_hit;
39 }; 39 };
40 40
41 struct callchain_root { 41 struct callchain_root {
42 u64 max_depth; 42 u64 max_depth;
43 struct callchain_node node; 43 struct callchain_node node;
44 }; 44 };
45 45
46 struct callchain_param; 46 struct callchain_param;
47 47
48 typedef void (*sort_chain_func_t)(struct rb_root *, struct callchain_root *, 48 typedef void (*sort_chain_func_t)(struct rb_root *, struct callchain_root *,
49 u64, struct callchain_param *); 49 u64, struct callchain_param *);
50 50
51 enum chain_key { 51 enum chain_key {
52 CCKEY_FUNCTION, 52 CCKEY_FUNCTION,
53 CCKEY_ADDRESS 53 CCKEY_ADDRESS
54 }; 54 };
55 55
56 struct callchain_param { 56 struct callchain_param {
57 bool enabled; 57 bool enabled;
58 enum perf_call_graph_mode record_mode; 58 enum perf_call_graph_mode record_mode;
59 u32 dump_size; 59 u32 dump_size;
60 enum chain_mode mode; 60 enum chain_mode mode;
61 u32 print_limit; 61 u32 print_limit;
62 double min_percent; 62 double min_percent;
63 sort_chain_func_t sort; 63 sort_chain_func_t sort;
64 enum chain_order order; 64 enum chain_order order;
65 enum chain_key key; 65 enum chain_key key;
66 bool branch_callstack; 66 bool branch_callstack;
67 }; 67 };
68 68
69 extern struct callchain_param callchain_param; 69 extern struct callchain_param callchain_param;
70 70
71 struct callchain_list { 71 struct callchain_list {
72 u64 ip; 72 u64 ip;
73 struct map_symbol ms; 73 struct map_symbol ms;
74 char *srcline; 74 char *srcline;
75 struct list_head list; 75 struct list_head list;
76 }; 76 };
77 77
78 /* 78 /*
79 * A callchain cursor is a single linked list that 79 * A callchain cursor is a single linked list that
80 * let one feed a callchain progressively. 80 * let one feed a callchain progressively.
81 * It keeps persistent allocated entries to minimize 81 * It keeps persistent allocated entries to minimize
82 * allocations. 82 * allocations.
83 */ 83 */
84 struct callchain_cursor_node { 84 struct callchain_cursor_node {
85 u64 ip; 85 u64 ip;
86 struct map *map; 86 struct map *map;
87 struct symbol *sym; 87 struct symbol *sym;
88 struct callchain_cursor_node *next; 88 struct callchain_cursor_node *next;
89 }; 89 };
90 90
91 struct callchain_cursor { 91 struct callchain_cursor {
92 u64 nr; 92 u64 nr;
93 struct callchain_cursor_node *first; 93 struct callchain_cursor_node *first;
94 struct callchain_cursor_node **last; 94 struct callchain_cursor_node **last;
95 u64 pos; 95 u64 pos;
96 struct callchain_cursor_node *curr; 96 struct callchain_cursor_node *curr;
97 }; 97 };
98 98
99 extern __thread struct callchain_cursor callchain_cursor; 99 extern __thread struct callchain_cursor callchain_cursor;
100 100
101 static inline void callchain_init(struct callchain_root *root) 101 static inline void callchain_init(struct callchain_root *root)
102 { 102 {
103 INIT_LIST_HEAD(&root->node.val); 103 INIT_LIST_HEAD(&root->node.val);
104 104
105 root->node.parent = NULL; 105 root->node.parent = NULL;
106 root->node.hit = 0; 106 root->node.hit = 0;
107 root->node.children_hit = 0; 107 root->node.children_hit = 0;
108 root->node.rb_root_in = RB_ROOT; 108 root->node.rb_root_in = RB_ROOT;
109 root->max_depth = 0; 109 root->max_depth = 0;
110 } 110 }
111 111
112 static inline u64 callchain_cumul_hits(struct callchain_node *node) 112 static inline u64 callchain_cumul_hits(struct callchain_node *node)
113 { 113 {
114 return node->hit + node->children_hit; 114 return node->hit + node->children_hit;
115 } 115 }
116 116
117 int callchain_register_param(struct callchain_param *param); 117 int callchain_register_param(struct callchain_param *param);
118 int callchain_append(struct callchain_root *root, 118 int callchain_append(struct callchain_root *root,
119 struct callchain_cursor *cursor, 119 struct callchain_cursor *cursor,
120 u64 period); 120 u64 period);
121 121
122 int callchain_merge(struct callchain_cursor *cursor, 122 int callchain_merge(struct callchain_cursor *cursor,
123 struct callchain_root *dst, struct callchain_root *src); 123 struct callchain_root *dst, struct callchain_root *src);
124 124
125 /* 125 /*
126 * Initialize a cursor before adding entries inside, but keep 126 * Initialize a cursor before adding entries inside, but keep
127 * the previously allocated entries as a cache. 127 * the previously allocated entries as a cache.
128 */ 128 */
129 static inline void callchain_cursor_reset(struct callchain_cursor *cursor) 129 static inline void callchain_cursor_reset(struct callchain_cursor *cursor)
130 { 130 {
131 cursor->nr = 0; 131 cursor->nr = 0;
132 cursor->last = &cursor->first; 132 cursor->last = &cursor->first;
133 } 133 }
134 134
135 int callchain_cursor_append(struct callchain_cursor *cursor, u64 ip, 135 int callchain_cursor_append(struct callchain_cursor *cursor, u64 ip,
136 struct map *map, struct symbol *sym); 136 struct map *map, struct symbol *sym);
137 137
138 /* Close a cursor writing session. Initialize for the reader */ 138 /* Close a cursor writing session. Initialize for the reader */
139 static inline void callchain_cursor_commit(struct callchain_cursor *cursor) 139 static inline void callchain_cursor_commit(struct callchain_cursor *cursor)
140 { 140 {
141 cursor->curr = cursor->first; 141 cursor->curr = cursor->first;
142 cursor->pos = 0; 142 cursor->pos = 0;
143 } 143 }
144 144
145 /* Cursor reading iteration helpers */ 145 /* Cursor reading iteration helpers */
146 static inline struct callchain_cursor_node * 146 static inline struct callchain_cursor_node *
147 callchain_cursor_current(struct callchain_cursor *cursor) 147 callchain_cursor_current(struct callchain_cursor *cursor)
148 { 148 {
149 if (cursor->pos == cursor->nr) 149 if (cursor->pos == cursor->nr)
150 return NULL; 150 return NULL;
151 151
152 return cursor->curr; 152 return cursor->curr;
153 } 153 }
154 154
155 static inline void callchain_cursor_advance(struct callchain_cursor *cursor) 155 static inline void callchain_cursor_advance(struct callchain_cursor *cursor)
156 { 156 {
157 cursor->curr = cursor->curr->next; 157 cursor->curr = cursor->curr->next;
158 cursor->pos++; 158 cursor->pos++;
159 } 159 }
160 160
161 struct option; 161 struct option;
162 struct hist_entry; 162 struct hist_entry;
163 163
164 int record_parse_callchain_opt(const struct option *opt, const char *arg, int unset); 164 int record_parse_callchain_opt(const struct option *opt, const char *arg, int unset);
165 int record_callchain_opt(const struct option *opt, const char *arg, int unset); 165 int record_callchain_opt(const struct option *opt, const char *arg, int unset);
166 166
167 int sample__resolve_callchain(struct perf_sample *sample, struct symbol **parent, 167 int sample__resolve_callchain(struct perf_sample *sample, struct symbol **parent,
168 struct perf_evsel *evsel, struct addr_location *al, 168 struct perf_evsel *evsel, struct addr_location *al,
169 int max_stack); 169 int max_stack);
170 int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample); 170 int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample);
171 int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, 171 int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node,
172 bool hide_unresolved); 172 bool hide_unresolved);
173 173
174 extern const char record_callchain_help[]; 174 extern const char record_callchain_help[];
175 int parse_callchain_record_opt(const char *arg); 175 int parse_callchain_record_opt(const char *arg);
176 int parse_callchain_report_opt(const char *arg); 176 int parse_callchain_report_opt(const char *arg);
177 int perf_callchain_config(const char *var, const char *value); 177 int perf_callchain_config(const char *var, const char *value);
178 178
179 static inline void callchain_cursor_snapshot(struct callchain_cursor *dest, 179 static inline void callchain_cursor_snapshot(struct callchain_cursor *dest,
180 struct callchain_cursor *src) 180 struct callchain_cursor *src)
181 { 181 {
182 *dest = *src; 182 *dest = *src;
183 183
184 dest->first = src->curr; 184 dest->first = src->curr;
185 dest->nr -= src->pos; 185 dest->nr -= src->pos;
186 } 186 }
187 187
188 #ifdef HAVE_SKIP_CALLCHAIN_IDX 188 #ifdef HAVE_SKIP_CALLCHAIN_IDX
189 extern int arch_skip_callchain_idx(struct thread *thread, struct ip_callchain *chain); 189 extern int arch_skip_callchain_idx(struct thread *thread, struct ip_callchain *chain);
190 #else 190 #else
191 static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused, 191 static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused,
192 struct ip_callchain *chain __maybe_unused) 192 struct ip_callchain *chain __maybe_unused)
193 { 193 {
194 return -1; 194 return -1;
195 } 195 }
196 #endif 196 #endif
197 197
198 char *callchain_list__sym_name(struct callchain_list *cl, 198 char *callchain_list__sym_name(struct callchain_list *cl,
199 char *bf, size_t bfsize, bool show_dso); 199 char *bf, size_t bfsize, bool show_dso);
200 200
201 void free_callchain(struct callchain_root *root);
202
201 #endif /* __PERF_CALLCHAIN_H */ 203 #endif /* __PERF_CALLCHAIN_H */
202 204
tools/perf/util/hist.c
Diff comments   View file @ ddb321a
1 #include "util.h" 1 #include "util.h"
2 #include "build-id.h" 2 #include "build-id.h"
3 #include "hist.h" 3 #include "hist.h"
4 #include "session.h" 4 #include "session.h"
5 #include "sort.h" 5 #include "sort.h"
6 #include "evlist.h" 6 #include "evlist.h"
7 #include "evsel.h" 7 #include "evsel.h"
8 #include "annotate.h" 8 #include "annotate.h"
9 #include "ui/progress.h"
9 #include <math.h> 10 #include <math.h>
10 11
11 static bool hists__filter_entry_by_dso(struct hists *hists, 12 static bool hists__filter_entry_by_dso(struct hists *hists,
12 struct hist_entry *he); 13 struct hist_entry *he);
13 static bool hists__filter_entry_by_thread(struct hists *hists, 14 static bool hists__filter_entry_by_thread(struct hists *hists,
14 struct hist_entry *he); 15 struct hist_entry *he);
15 static bool hists__filter_entry_by_symbol(struct hists *hists, 16 static bool hists__filter_entry_by_symbol(struct hists *hists,
16 struct hist_entry *he); 17 struct hist_entry *he);
17 18
18 u16 hists__col_len(struct hists *hists, enum hist_column col) 19 u16 hists__col_len(struct hists *hists, enum hist_column col)
19 { 20 {
20 return hists->col_len[col]; 21 return hists->col_len[col];
21 } 22 }
22 23
23 void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len) 24 void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len)
24 { 25 {
25 hists->col_len[col] = len; 26 hists->col_len[col] = len;
26 } 27 }
27 28
28 bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len) 29 bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len)
29 { 30 {
30 if (len > hists__col_len(hists, col)) { 31 if (len > hists__col_len(hists, col)) {
31 hists__set_col_len(hists, col, len); 32 hists__set_col_len(hists, col, len);
32 return true; 33 return true;
33 } 34 }
34 return false; 35 return false;
35 } 36 }
36 37
37 void hists__reset_col_len(struct hists *hists) 38 void hists__reset_col_len(struct hists *hists)
38 { 39 {
39 enum hist_column col; 40 enum hist_column col;
40 41
41 for (col = 0; col < HISTC_NR_COLS; ++col) 42 for (col = 0; col < HISTC_NR_COLS; ++col)
42 hists__set_col_len(hists, col, 0); 43 hists__set_col_len(hists, col, 0);
43 } 44 }
44 45
45 static void hists__set_unres_dso_col_len(struct hists *hists, int dso) 46 static void hists__set_unres_dso_col_len(struct hists *hists, int dso)
46 { 47 {
47 const unsigned int unresolved_col_width = BITS_PER_LONG / 4; 48 const unsigned int unresolved_col_width = BITS_PER_LONG / 4;
48 49
49 if (hists__col_len(hists, dso) < unresolved_col_width && 50 if (hists__col_len(hists, dso) < unresolved_col_width &&
50 !symbol_conf.col_width_list_str && !symbol_conf.field_sep && 51 !symbol_conf.col_width_list_str && !symbol_conf.field_sep &&
51 !symbol_conf.dso_list) 52 !symbol_conf.dso_list)
52 hists__set_col_len(hists, dso, unresolved_col_width); 53 hists__set_col_len(hists, dso, unresolved_col_width);
53 } 54 }
54 55
55 void hists__calc_col_len(struct hists *hists, struct hist_entry *h) 56 void hists__calc_col_len(struct hists *hists, struct hist_entry *h)
56 { 57 {
57 const unsigned int unresolved_col_width = BITS_PER_LONG / 4; 58 const unsigned int unresolved_col_width = BITS_PER_LONG / 4;
58 int symlen; 59 int symlen;
59 u16 len; 60 u16 len;
60 61
61 /* 62 /*
62 * +4 accounts for '[x] ' priv level info 63 * +4 accounts for '[x] ' priv level info
63 * +2 accounts for 0x prefix on raw addresses 64 * +2 accounts for 0x prefix on raw addresses
64 * +3 accounts for ' y ' symtab origin info 65 * +3 accounts for ' y ' symtab origin info
65 */ 66 */
66 if (h->ms.sym) { 67 if (h->ms.sym) {
67 symlen = h->ms.sym->namelen + 4; 68 symlen = h->ms.sym->namelen + 4;
68 if (verbose) 69 if (verbose)
69 symlen += BITS_PER_LONG / 4 + 2 + 3; 70 symlen += BITS_PER_LONG / 4 + 2 + 3;
70 hists__new_col_len(hists, HISTC_SYMBOL, symlen); 71 hists__new_col_len(hists, HISTC_SYMBOL, symlen);
71 } else { 72 } else {
72 symlen = unresolved_col_width + 4 + 2; 73 symlen = unresolved_col_width + 4 + 2;
73 hists__new_col_len(hists, HISTC_SYMBOL, symlen); 74 hists__new_col_len(hists, HISTC_SYMBOL, symlen);
74 hists__set_unres_dso_col_len(hists, HISTC_DSO); 75 hists__set_unres_dso_col_len(hists, HISTC_DSO);
75 } 76 }
76 77
77 len = thread__comm_len(h->thread); 78 len = thread__comm_len(h->thread);
78 if (hists__new_col_len(hists, HISTC_COMM, len)) 79 if (hists__new_col_len(hists, HISTC_COMM, len))
79 hists__set_col_len(hists, HISTC_THREAD, len + 6); 80 hists__set_col_len(hists, HISTC_THREAD, len + 6);
80 81
81 if (h->ms.map) { 82 if (h->ms.map) {
82 len = dso__name_len(h->ms.map->dso); 83 len = dso__name_len(h->ms.map->dso);
83 hists__new_col_len(hists, HISTC_DSO, len); 84 hists__new_col_len(hists, HISTC_DSO, len);
84 } 85 }
85 86
86 if (h->parent) 87 if (h->parent)
87 hists__new_col_len(hists, HISTC_PARENT, h->parent->namelen); 88 hists__new_col_len(hists, HISTC_PARENT, h->parent->namelen);
88 89
89 if (h->branch_info) { 90 if (h->branch_info) {
90 if (h->branch_info->from.sym) { 91 if (h->branch_info->from.sym) {
91 symlen = (int)h->branch_info->from.sym->namelen + 4; 92 symlen = (int)h->branch_info->from.sym->namelen + 4;
92 if (verbose) 93 if (verbose)
93 symlen += BITS_PER_LONG / 4 + 2 + 3; 94 symlen += BITS_PER_LONG / 4 + 2 + 3;
94 hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen); 95 hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen);
95 96
96 symlen = dso__name_len(h->branch_info->from.map->dso); 97 symlen = dso__name_len(h->branch_info->from.map->dso);
97 hists__new_col_len(hists, HISTC_DSO_FROM, symlen); 98 hists__new_col_len(hists, HISTC_DSO_FROM, symlen);
98 } else { 99 } else {
99 symlen = unresolved_col_width + 4 + 2; 100 symlen = unresolved_col_width + 4 + 2;
100 hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen); 101 hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen);
101 hists__set_unres_dso_col_len(hists, HISTC_DSO_FROM); 102 hists__set_unres_dso_col_len(hists, HISTC_DSO_FROM);
102 } 103 }
103 104
104 if (h->branch_info->to.sym) { 105 if (h->branch_info->to.sym) {
105 symlen = (int)h->branch_info->to.sym->namelen + 4; 106 symlen = (int)h->branch_info->to.sym->namelen + 4;
106 if (verbose) 107 if (verbose)
107 symlen += BITS_PER_LONG / 4 + 2 + 3; 108 symlen += BITS_PER_LONG / 4 + 2 + 3;
108 hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen); 109 hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen);
109 110
110 symlen = dso__name_len(h->branch_info->to.map->dso); 111 symlen = dso__name_len(h->branch_info->to.map->dso);
111 hists__new_col_len(hists, HISTC_DSO_TO, symlen); 112 hists__new_col_len(hists, HISTC_DSO_TO, symlen);
112 } else { 113 } else {
113 symlen = unresolved_col_width + 4 + 2; 114 symlen = unresolved_col_width + 4 + 2;
114 hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen); 115 hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen);
115 hists__set_unres_dso_col_len(hists, HISTC_DSO_TO); 116 hists__set_unres_dso_col_len(hists, HISTC_DSO_TO);
116 } 117 }
117 } 118 }
118 119
119 if (h->mem_info) { 120 if (h->mem_info) {
120 if (h->mem_info->daddr.sym) { 121 if (h->mem_info->daddr.sym) {
121 symlen = (int)h->mem_info->daddr.sym->namelen + 4 122 symlen = (int)h->mem_info->daddr.sym->namelen + 4
122 + unresolved_col_width + 2; 123 + unresolved_col_width + 2;
123 hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, 124 hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL,
124 symlen); 125 symlen);
125 hists__new_col_len(hists, HISTC_MEM_DCACHELINE, 126 hists__new_col_len(hists, HISTC_MEM_DCACHELINE,
126 symlen + 1); 127 symlen + 1);
127 } else { 128 } else {
128 symlen = unresolved_col_width + 4 + 2; 129 symlen = unresolved_col_width + 4 + 2;
129 hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, 130 hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL,
130 symlen); 131 symlen);
131 } 132 }
132 if (h->mem_info->daddr.map) { 133 if (h->mem_info->daddr.map) {
133 symlen = dso__name_len(h->mem_info->daddr.map->dso); 134 symlen = dso__name_len(h->mem_info->daddr.map->dso);
134 hists__new_col_len(hists, HISTC_MEM_DADDR_DSO, 135 hists__new_col_len(hists, HISTC_MEM_DADDR_DSO,
135 symlen); 136 symlen);
136 } else { 137 } else {
137 symlen = unresolved_col_width + 4 + 2; 138 symlen = unresolved_col_width + 4 + 2;
138 hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO); 139 hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO);
139 } 140 }
140 } else { 141 } else {
141 symlen = unresolved_col_width + 4 + 2; 142 symlen = unresolved_col_width + 4 + 2;
142 hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, symlen); 143 hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, symlen);
143 hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO); 144 hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO);
144 } 145 }
145 146
146 hists__new_col_len(hists, HISTC_MEM_LOCKED, 6); 147 hists__new_col_len(hists, HISTC_MEM_LOCKED, 6);
147 hists__new_col_len(hists, HISTC_MEM_TLB, 22); 148 hists__new_col_len(hists, HISTC_MEM_TLB, 22);
148 hists__new_col_len(hists, HISTC_MEM_SNOOP, 12); 149 hists__new_col_len(hists, HISTC_MEM_SNOOP, 12);
149 hists__new_col_len(hists, HISTC_MEM_LVL, 21 + 3); 150 hists__new_col_len(hists, HISTC_MEM_LVL, 21 + 3);
150 hists__new_col_len(hists, HISTC_LOCAL_WEIGHT, 12); 151 hists__new_col_len(hists, HISTC_LOCAL_WEIGHT, 12);
151 hists__new_col_len(hists, HISTC_GLOBAL_WEIGHT, 12); 152 hists__new_col_len(hists, HISTC_GLOBAL_WEIGHT, 12);
152 153
153 if (h->transaction) 154 if (h->transaction)
154 hists__new_col_len(hists, HISTC_TRANSACTION, 155 hists__new_col_len(hists, HISTC_TRANSACTION,
155 hist_entry__transaction_len()); 156 hist_entry__transaction_len());
156 } 157 }
157 158
158 void hists__output_recalc_col_len(struct hists *hists, int max_rows) 159 void hists__output_recalc_col_len(struct hists *hists, int max_rows)
159 { 160 {
160 struct rb_node *next = rb_first(&hists->entries); 161 struct rb_node *next = rb_first(&hists->entries);
161 struct hist_entry *n; 162 struct hist_entry *n;
162 int row = 0; 163 int row = 0;
163 164
164 hists__reset_col_len(hists); 165 hists__reset_col_len(hists);
165 166
166 while (next && row++ < max_rows) { 167 while (next && row++ < max_rows) {
167 n = rb_entry(next, struct hist_entry, rb_node); 168 n = rb_entry(next, struct hist_entry, rb_node);
168 if (!n->filtered) 169 if (!n->filtered)
169 hists__calc_col_len(hists, n); 170 hists__calc_col_len(hists, n);
170 next = rb_next(&n->rb_node); 171 next = rb_next(&n->rb_node);
171 } 172 }
172 } 173 }
173 174
174 static void he_stat__add_cpumode_period(struct he_stat *he_stat, 175 static void he_stat__add_cpumode_period(struct he_stat *he_stat,
175 unsigned int cpumode, u64 period) 176 unsigned int cpumode, u64 period)
176 { 177 {
177 switch (cpumode) { 178 switch (cpumode) {
178 case PERF_RECORD_MISC_KERNEL: 179 case PERF_RECORD_MISC_KERNEL:
179 he_stat->period_sys += period; 180 he_stat->period_sys += period;
180 break; 181 break;
181 case PERF_RECORD_MISC_USER: 182 case PERF_RECORD_MISC_USER:
182 he_stat->period_us += period; 183 he_stat->period_us += period;
183 break; 184 break;
184 case PERF_RECORD_MISC_GUEST_KERNEL: 185 case PERF_RECORD_MISC_GUEST_KERNEL:
185 he_stat->period_guest_sys += period; 186 he_stat->period_guest_sys += period;
186 break; 187 break;
187 case PERF_RECORD_MISC_GUEST_USER: 188 case PERF_RECORD_MISC_GUEST_USER:
188 he_stat->period_guest_us += period; 189 he_stat->period_guest_us += period;
189 break; 190 break;
190 default: 191 default:
191 break; 192 break;
192 } 193 }
193 } 194 }
194 195
195 static void he_stat__add_period(struct he_stat *he_stat, u64 period, 196 static void he_stat__add_period(struct he_stat *he_stat, u64 period,
196 u64 weight) 197 u64 weight)
197 { 198 {
198 199
199 he_stat->period += period; 200 he_stat->period += period;
200 he_stat->weight += weight; 201 he_stat->weight += weight;
201 he_stat->nr_events += 1; 202 he_stat->nr_events += 1;
202 } 203 }
203 204
204 static void he_stat__add_stat(struct he_stat *dest, struct he_stat *src) 205 static void he_stat__add_stat(struct he_stat *dest, struct he_stat *src)
205 { 206 {
206 dest->period += src->period; 207 dest->period += src->period;
207 dest->period_sys += src->period_sys; 208 dest->period_sys += src->period_sys;
208 dest->period_us += src->period_us; 209 dest->period_us += src->period_us;
209 dest->period_guest_sys += src->period_guest_sys; 210 dest->period_guest_sys += src->period_guest_sys;
210 dest->period_guest_us += src->period_guest_us; 211 dest->period_guest_us += src->period_guest_us;
211 dest->nr_events += src->nr_events; 212 dest->nr_events += src->nr_events;
212 dest->weight += src->weight; 213 dest->weight += src->weight;
213 } 214 }
214 215
215 static void he_stat__decay(struct he_stat *he_stat) 216 static void he_stat__decay(struct he_stat *he_stat)
216 { 217 {
217 he_stat->period = (he_stat->period * 7) / 8; 218 he_stat->period = (he_stat->period * 7) / 8;
218 he_stat->nr_events = (he_stat->nr_events * 7) / 8; 219 he_stat->nr_events = (he_stat->nr_events * 7) / 8;
219 /* XXX need decay for weight too? */ 220 /* XXX need decay for weight too? */
220 } 221 }
221 222
222 static bool hists__decay_entry(struct hists *hists, struct hist_entry *he) 223 static bool hists__decay_entry(struct hists *hists, struct hist_entry *he)
223 { 224 {
224 u64 prev_period = he->stat.period; 225 u64 prev_period = he->stat.period;
225 u64 diff; 226 u64 diff;
226 227
227 if (prev_period == 0) 228 if (prev_period == 0)
228 return true; 229 return true;
229 230
230 he_stat__decay(&he->stat); 231 he_stat__decay(&he->stat);
231 if (symbol_conf.cumulate_callchain) 232 if (symbol_conf.cumulate_callchain)
232 he_stat__decay(he->stat_acc); 233 he_stat__decay(he->stat_acc);
233 234
234 diff = prev_period - he->stat.period; 235 diff = prev_period - he->stat.period;
235 236
236 hists->stats.total_period -= diff; 237 hists->stats.total_period -= diff;
237 if (!he->filtered) 238 if (!he->filtered)
238 hists->stats.total_non_filtered_period -= diff; 239 hists->stats.total_non_filtered_period -= diff;
239 240
240 return he->stat.period == 0; 241 return he->stat.period == 0;
241 } 242 }
242 243
243 void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel) 244 void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel)
244 { 245 {
245 struct rb_node *next = rb_first(&hists->entries); 246 struct rb_node *next = rb_first(&hists->entries);
246 struct hist_entry *n; 247 struct hist_entry *n;
247 248
248 while (next) { 249 while (next) {
249 n = rb_entry(next, struct hist_entry, rb_node); 250 n = rb_entry(next, struct hist_entry, rb_node);
250 next = rb_next(&n->rb_node); 251 next = rb_next(&n->rb_node);
251 /* 252 /*
252 * We may be annotating this, for instance, so keep it here in 253 * We may be annotating this, for instance, so keep it here in
253 * case some it gets new samples, we'll eventually free it when 254 * case some it gets new samples, we'll eventually free it when
254 * the user stops browsing and it agains gets fully decayed. 255 * the user stops browsing and it agains gets fully decayed.
255 */ 256 */
256 if (((zap_user && n->level == '.') || 257 if (((zap_user && n->level == '.') ||
257 (zap_kernel && n->level != '.') || 258 (zap_kernel && n->level != '.') ||
258 hists__decay_entry(hists, n)) && 259 hists__decay_entry(hists, n)) &&
259 !n->used) { 260 !n->used) {
260 rb_erase(&n->rb_node, &hists->entries); 261 rb_erase(&n->rb_node, &hists->entries);
261 262
262 if (sort__need_collapse) 263 if (sort__need_collapse)
263 rb_erase(&n->rb_node_in, &hists->entries_collapsed); 264 rb_erase(&n->rb_node_in, &hists->entries_collapsed);
264 265
265 --hists->nr_entries; 266 --hists->nr_entries;
266 if (!n->filtered) 267 if (!n->filtered)
267 --hists->nr_non_filtered_entries; 268 --hists->nr_non_filtered_entries;
268 269
269 hist_entry__free(n); 270 hist_entry__free(n);
270 } 271 }
271 } 272 }
272 } 273 }
273 274
274 void hists__delete_entries(struct hists *hists) 275 void hists__delete_entries(struct hists *hists)
275 { 276 {
276 struct rb_node *next = rb_first(&hists->entries); 277 struct rb_node *next = rb_first(&hists->entries);
277 struct hist_entry *n; 278 struct hist_entry *n;
278 279
279 while (next) { 280 while (next) {
280 n = rb_entry(next, struct hist_entry, rb_node); 281 n = rb_entry(next, struct hist_entry, rb_node);
281 next = rb_next(&n->rb_node); 282 next = rb_next(&n->rb_node);
282 283
283 rb_erase(&n->rb_node, &hists->entries); 284 rb_erase(&n->rb_node, &hists->entries);
284 285
285 if (sort__need_collapse) 286 if (sort__need_collapse)
286 rb_erase(&n->rb_node_in, &hists->entries_collapsed); 287 rb_erase(&n->rb_node_in, &hists->entries_collapsed);
287 288
288 --hists->nr_entries; 289 --hists->nr_entries;
289 if (!n->filtered) 290 if (!n->filtered)
290 --hists->nr_non_filtered_entries; 291 --hists->nr_non_filtered_entries;
291 292
292 hist_entry__free(n); 293 hist_entry__free(n);
293 } 294 }
294 } 295 }
295 296
296 /* 297 /*
297 * histogram, sorted on item, collects periods 298 * histogram, sorted on item, collects periods
298 */ 299 */
299 300
300 static struct hist_entry *hist_entry__new(struct hist_entry *template, 301 static struct hist_entry *hist_entry__new(struct hist_entry *template,
301 bool sample_self) 302 bool sample_self)
302 { 303 {
303 size_t callchain_size = 0; 304 size_t callchain_size = 0;
304 struct hist_entry *he; 305 struct hist_entry *he;
305 306
306 if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain) 307 if (symbol_conf.use_callchain)
307 callchain_size = sizeof(struct callchain_root); 308 callchain_size = sizeof(struct callchain_root);
308 309
309 he = zalloc(sizeof(*he) + callchain_size); 310 he = zalloc(sizeof(*he) + callchain_size);
310 311
311 if (he != NULL) { 312 if (he != NULL) {
312 *he = *template; 313 *he = *template;
313 314
314 if (symbol_conf.cumulate_callchain) { 315 if (symbol_conf.cumulate_callchain) {
315 he->stat_acc = malloc(sizeof(he->stat)); 316 he->stat_acc = malloc(sizeof(he->stat));
316 if (he->stat_acc == NULL) { 317 if (he->stat_acc == NULL) {
317 free(he); 318 free(he);
318 return NULL; 319 return NULL;
319 } 320 }
320 memcpy(he->stat_acc, &he->stat, sizeof(he->stat)); 321 memcpy(he->stat_acc, &he->stat, sizeof(he->stat));
321 if (!sample_self) 322 if (!sample_self)
322 memset(&he->stat, 0, sizeof(he->stat)); 323 memset(&he->stat, 0, sizeof(he->stat));
323 } 324 }
324 325
325 if (he->ms.map) 326 if (he->ms.map)
326 he->ms.map->referenced = true; 327 he->ms.map->referenced = true;
327 328
328 if (he->branch_info) { 329 if (he->branch_info) {
329 /* 330 /*
330 * This branch info is (a part of) allocated from 331 * This branch info is (a part of) allocated from
331 * sample__resolve_bstack() and will be freed after 332 * sample__resolve_bstack() and will be freed after
332 * adding new entries. So we need to save a copy. 333 * adding new entries. So we need to save a copy.
333 */ 334 */
334 he->branch_info = malloc(sizeof(*he->branch_info)); 335 he->branch_info = malloc(sizeof(*he->branch_info));
335 if (he->branch_info == NULL) { 336 if (he->branch_info == NULL) {
336 free(he->stat_acc); 337 free(he->stat_acc);
337 free(he); 338 free(he);
338 return NULL; 339 return NULL;
339 } 340 }
340 341
341 memcpy(he->branch_info, template->branch_info, 342 memcpy(he->branch_info, template->branch_info,
342 sizeof(*he->branch_info)); 343 sizeof(*he->branch_info));
343 344
344 if (he->branch_info->from.map) 345 if (he->branch_info->from.map)
345 he->branch_info->from.map->referenced = true; 346 he->branch_info->from.map->referenced = true;
346 if (he->branch_info->to.map) 347 if (he->branch_info->to.map)
347 he->branch_info->to.map->referenced = true; 348 he->branch_info->to.map->referenced = true;
348 } 349 }
349 350
350 if (he->mem_info) { 351 if (he->mem_info) {
351 if (he->mem_info->iaddr.map) 352 if (he->mem_info->iaddr.map)
352 he->mem_info->iaddr.map->referenced = true; 353 he->mem_info->iaddr.map->referenced = true;
353 if (he->mem_info->daddr.map) 354 if (he->mem_info->daddr.map)
354 he->mem_info->daddr.map->referenced = true; 355 he->mem_info->daddr.map->referenced = true;
355 } 356 }
356 357
357 if (symbol_conf.use_callchain) 358 if (symbol_conf.use_callchain)
358 callchain_init(he->callchain); 359 callchain_init(he->callchain);
359 360
360 INIT_LIST_HEAD(&he->pairs.node); 361 INIT_LIST_HEAD(&he->pairs.node);
361 } 362 }
362 363
363 return he; 364 return he;
364 } 365 }
365 366
366 static u8 symbol__parent_filter(const struct symbol *parent) 367 static u8 symbol__parent_filter(const struct symbol *parent)
367 { 368 {
368 if (symbol_conf.exclude_other && parent == NULL) 369 if (symbol_conf.exclude_other && parent == NULL)
369 return 1 << HIST_FILTER__PARENT; 370 return 1 << HIST_FILTER__PARENT;
370 return 0; 371 return 0;
371 } 372 }
372 373
373 static struct hist_entry *add_hist_entry(struct hists *hists, 374 static struct hist_entry *add_hist_entry(struct hists *hists,
374 struct hist_entry *entry, 375 struct hist_entry *entry,
375 struct addr_location *al, 376 struct addr_location *al,
376 bool sample_self) 377 bool sample_self)
377 { 378 {
378 struct rb_node **p; 379 struct rb_node **p;
379 struct rb_node *parent = NULL; 380 struct rb_node *parent = NULL;
380 struct hist_entry *he; 381 struct hist_entry *he;
381 int64_t cmp; 382 int64_t cmp;
382 u64 period = entry->stat.period; 383 u64 period = entry->stat.period;
383 u64 weight = entry->stat.weight; 384 u64 weight = entry->stat.weight;
384 385
385 p = &hists->entries_in->rb_node; 386 p = &hists->entries_in->rb_node;
386 387
387 while (*p != NULL) { 388 while (*p != NULL) {
388 parent = *p; 389 parent = *p;
389 he = rb_entry(parent, struct hist_entry, rb_node_in); 390 he = rb_entry(parent, struct hist_entry, rb_node_in);
390 391
391 /* 392 /*
392 * Make sure that it receives arguments in a same order as 393 * Make sure that it receives arguments in a same order as
393 * hist_entry__collapse() so that we can use an appropriate 394 * hist_entry__collapse() so that we can use an appropriate
394 * function when searching an entry regardless which sort 395 * function when searching an entry regardless which sort
395 * keys were used. 396 * keys were used.
396 */ 397 */
397 cmp = hist_entry__cmp(he, entry); 398 cmp = hist_entry__cmp(he, entry);
398 399
399 if (!cmp) { 400 if (!cmp) {
400 if (sample_self) 401 if (sample_self)
401 he_stat__add_period(&he->stat, period, weight); 402 he_stat__add_period(&he->stat, period, weight);
402 if (symbol_conf.cumulate_callchain) 403 if (symbol_conf.cumulate_callchain)
403 he_stat__add_period(he->stat_acc, period, weight); 404 he_stat__add_period(he->stat_acc, period, weight);
404 405
405 /* 406 /*
406 * This mem info was allocated from sample__resolve_mem 407 * This mem info was allocated from sample__resolve_mem
407 * and will not be used anymore. 408 * and will not be used anymore.
408 */ 409 */
409 zfree(&entry->mem_info); 410 zfree(&entry->mem_info);
410 411
411 /* If the map of an existing hist_entry has 412 /* If the map of an existing hist_entry has
412 * become out-of-date due to an exec() or 413 * become out-of-date due to an exec() or
413 * similar, update it. Otherwise we will 414 * similar, update it. Otherwise we will
414 * mis-adjust symbol addresses when computing 415 * mis-adjust symbol addresses when computing
415 * the history counter to increment. 416 * the history counter to increment.
416 */ 417 */
417 if (he->ms.map != entry->ms.map) { 418 if (he->ms.map != entry->ms.map) {
418 he->ms.map = entry->ms.map; 419 he->ms.map = entry->ms.map;
419 if (he->ms.map) 420 if (he->ms.map)
420 he->ms.map->referenced = true; 421 he->ms.map->referenced = true;
421 } 422 }
422 goto out; 423 goto out;
423 } 424 }
424 425
425 if (cmp < 0) 426 if (cmp < 0)
426 p = &(*p)->rb_left; 427 p = &(*p)->rb_left;
427 else 428 else
428 p = &(*p)->rb_right; 429 p = &(*p)->rb_right;
429 } 430 }
430 431
431 he = hist_entry__new(entry, sample_self); 432 he = hist_entry__new(entry, sample_self);
432 if (!he) 433 if (!he)
433 return NULL; 434 return NULL;
434 435
435 rb_link_node(&he->rb_node_in, parent, p); 436 rb_link_node(&he->rb_node_in, parent, p);
436 rb_insert_color(&he->rb_node_in, hists->entries_in); 437 rb_insert_color(&he->rb_node_in, hists->entries_in);
437 out: 438 out:
438 if (sample_self) 439 if (sample_self)
439 he_stat__add_cpumode_period(&he->stat, al->cpumode, period); 440 he_stat__add_cpumode_period(&he->stat, al->cpumode, period);
440 if (symbol_conf.cumulate_callchain) 441 if (symbol_conf.cumulate_callchain)
441 he_stat__add_cpumode_period(he->stat_acc, al->cpumode, period); 442 he_stat__add_cpumode_period(he->stat_acc, al->cpumode, period);
442 return he; 443 return he;
443 } 444 }
444 445
445 struct hist_entry *__hists__add_entry(struct hists *hists, 446 struct hist_entry *__hists__add_entry(struct hists *hists,
446 struct addr_location *al, 447 struct addr_location *al,
447 struct symbol *sym_parent, 448 struct symbol *sym_parent,
448 struct branch_info *bi, 449 struct branch_info *bi,
449 struct mem_info *mi, 450 struct mem_info *mi,
450 u64 period, u64 weight, u64 transaction, 451 u64 period, u64 weight, u64 transaction,
451 bool sample_self) 452 bool sample_self)
452 { 453 {
453 struct hist_entry entry = { 454 struct hist_entry entry = {
454 .thread = al->thread, 455 .thread = al->thread,
455 .comm = thread__comm(al->thread), 456 .comm = thread__comm(al->thread),
456 .ms = { 457 .ms = {
457 .map = al->map, 458 .map = al->map,
458 .sym = al->sym, 459 .sym = al->sym,
459 }, 460 },
460 .cpu = al->cpu, 461 .cpu = al->cpu,
461 .cpumode = al->cpumode, 462 .cpumode = al->cpumode,
462 .ip = al->addr, 463 .ip = al->addr,
463 .level = al->level, 464 .level = al->level,
464 .stat = { 465 .stat = {
465 .nr_events = 1, 466 .nr_events = 1,
466 .period = period, 467 .period = period,
467 .weight = weight, 468 .weight = weight,
468 }, 469 },
469 .parent = sym_parent, 470 .parent = sym_parent,
470 .filtered = symbol__parent_filter(sym_parent) | al->filtered, 471 .filtered = symbol__parent_filter(sym_parent) | al->filtered,
471 .hists = hists, 472 .hists = hists,
472 .branch_info = bi, 473 .branch_info = bi,
473 .mem_info = mi, 474 .mem_info = mi,
474 .transaction = transaction, 475 .transaction = transaction,
475 }; 476 };
476 477
477 return add_hist_entry(hists, &entry, al, sample_self); 478 return add_hist_entry(hists, &entry, al, sample_self);
478 } 479 }
479 480
480 static int 481 static int
481 iter_next_nop_entry(struct hist_entry_iter *iter __maybe_unused, 482 iter_next_nop_entry(struct hist_entry_iter *iter __maybe_unused,
482 struct addr_location *al __maybe_unused) 483 struct addr_location *al __maybe_unused)
483 { 484 {
484 return 0; 485 return 0;
485 } 486 }
486 487
487 static int 488 static int
488 iter_add_next_nop_entry(struct hist_entry_iter *iter __maybe_unused, 489 iter_add_next_nop_entry(struct hist_entry_iter *iter __maybe_unused,
489 struct addr_location *al __maybe_unused) 490 struct addr_location *al __maybe_unused)
490 { 491 {
491 return 0; 492 return 0;
492 } 493 }
493 494
494 static int 495 static int
495 iter_prepare_mem_entry(struct hist_entry_iter *iter, struct addr_location *al) 496 iter_prepare_mem_entry(struct hist_entry_iter *iter, struct addr_location *al)
496 { 497 {
497 struct perf_sample *sample = iter->sample; 498 struct perf_sample *sample = iter->sample;
498 struct mem_info *mi; 499 struct mem_info *mi;
499 500
500 mi = sample__resolve_mem(sample, al); 501 mi = sample__resolve_mem(sample, al);
501 if (mi == NULL) 502 if (mi == NULL)
502 return -ENOMEM; 503 return -ENOMEM;
503 504
504 iter->priv = mi; 505 iter->priv = mi;
505 return 0; 506 return 0;
506 } 507 }
507 508
508 static int 509 static int
509 iter_add_single_mem_entry(struct hist_entry_iter *iter, struct addr_location *al) 510 iter_add_single_mem_entry(struct hist_entry_iter *iter, struct addr_location *al)
510 { 511 {
511 u64 cost; 512 u64 cost;
512 struct mem_info *mi = iter->priv; 513 struct mem_info *mi = iter->priv;
513 struct hists *hists = evsel__hists(iter->evsel); 514 struct hists *hists = evsel__hists(iter->evsel);
514 struct hist_entry *he; 515 struct hist_entry *he;
515 516
516 if (mi == NULL) 517 if (mi == NULL)
517 return -EINVAL; 518 return -EINVAL;
518 519
519 cost = iter->sample->weight; 520 cost = iter->sample->weight;
520 if (!cost) 521 if (!cost)
521 cost = 1; 522 cost = 1;
522 523
523 /* 524 /*
524 * must pass period=weight in order to get the correct 525 * must pass period=weight in order to get the correct
525 * sorting from hists__collapse_resort() which is solely 526 * sorting from hists__collapse_resort() which is solely
526 * based on periods. We want sorting be done on nr_events * weight 527 * based on periods. We want sorting be done on nr_events * weight
527 * and this is indirectly achieved by passing period=weight here 528 * and this is indirectly achieved by passing period=weight here
528 * and the he_stat__add_period() function. 529 * and the he_stat__add_period() function.
529 */ 530 */
530 he = __hists__add_entry(hists, al, iter->parent, NULL, mi, 531 he = __hists__add_entry(hists, al, iter->parent, NULL, mi,
531 cost, cost, 0, true); 532 cost, cost, 0, true);
532 if (!he) 533 if (!he)
533 return -ENOMEM; 534 return -ENOMEM;
534 535
535 iter->he = he; 536 iter->he = he;
536 return 0; 537 return 0;
537 } 538 }
538 539
539 static int 540 static int
540 iter_finish_mem_entry(struct hist_entry_iter *iter, 541 iter_finish_mem_entry(struct hist_entry_iter *iter,
541 struct addr_location *al __maybe_unused) 542 struct addr_location *al __maybe_unused)
542 { 543 {
543 struct perf_evsel *evsel = iter->evsel; 544 struct perf_evsel *evsel = iter->evsel;
544 struct hists *hists = evsel__hists(evsel); 545 struct hists *hists = evsel__hists(evsel);
545 struct hist_entry *he = iter->he; 546 struct hist_entry *he = iter->he;
546 int err = -EINVAL; 547 int err = -EINVAL;
547 548
548 if (he == NULL) 549 if (he == NULL)
549 goto out; 550 goto out;
550 551
551 hists__inc_nr_samples(hists, he->filtered); 552 hists__inc_nr_samples(hists, he->filtered);
552 553
553 err = hist_entry__append_callchain(he, iter->sample); 554 err = hist_entry__append_callchain(he, iter->sample);
554 555
555 out: 556 out:
556 /* 557 /*
557 * We don't need to free iter->priv (mem_info) here since 558 * We don't need to free iter->priv (mem_info) here since
558 * the mem info was either already freed in add_hist_entry() or 559 * the mem info was either already freed in add_hist_entry() or
559 * passed to a new hist entry by hist_entry__new(). 560 * passed to a new hist entry by hist_entry__new().
560 */ 561 */
561 iter->priv = NULL; 562 iter->priv = NULL;
562 563
563 iter->he = NULL; 564 iter->he = NULL;
564 return err; 565 return err;
565 } 566 }
566 567
567 static int 568 static int
568 iter_prepare_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) 569 iter_prepare_branch_entry(struct hist_entry_iter *iter, struct addr_location *al)
569 { 570 {
570 struct branch_info *bi; 571 struct branch_info *bi;
571 struct perf_sample *sample = iter->sample; 572 struct perf_sample *sample = iter->sample;
572 573
573 bi = sample__resolve_bstack(sample, al); 574 bi = sample__resolve_bstack(sample, al);
574 if (!bi) 575 if (!bi)
575 return -ENOMEM; 576 return -ENOMEM;
576 577
577 iter->curr = 0; 578 iter->curr = 0;
578 iter->total = sample->branch_stack->nr; 579 iter->total = sample->branch_stack->nr;
579 580
580 iter->priv = bi; 581 iter->priv = bi;
581 return 0; 582 return 0;
582 } 583 }
583 584
584 static int 585 static int
585 iter_add_single_branch_entry(struct hist_entry_iter *iter __maybe_unused, 586 iter_add_single_branch_entry(struct hist_entry_iter *iter __maybe_unused,
586 struct addr_location *al __maybe_unused) 587 struct addr_location *al __maybe_unused)
587 { 588 {
588 /* to avoid calling callback function */ 589 /* to avoid calling callback function */
589 iter->he = NULL; 590 iter->he = NULL;
590 591
591 return 0; 592 return 0;
592 } 593 }
593 594
594 static int 595 static int
595 iter_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) 596 iter_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al)
596 { 597 {
597 struct branch_info *bi = iter->priv; 598 struct branch_info *bi = iter->priv;
598 int i = iter->curr; 599 int i = iter->curr;
599 600
600 if (bi == NULL) 601 if (bi == NULL)
601 return 0; 602 return 0;
602 603
603 if (iter->curr >= iter->total) 604 if (iter->curr >= iter->total)
604 return 0; 605 return 0;
605 606
606 al->map = bi[i].to.map; 607 al->map = bi[i].to.map;
607 al->sym = bi[i].to.sym; 608 al->sym = bi[i].to.sym;
608 al->addr = bi[i].to.addr; 609 al->addr = bi[i].to.addr;
609 return 1; 610 return 1;
610 } 611 }
611 612
612 static int 613 static int
613 iter_add_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) 614 iter_add_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al)
614 { 615 {
615 struct branch_info *bi; 616 struct branch_info *bi;
616 struct perf_evsel *evsel = iter->evsel; 617 struct perf_evsel *evsel = iter->evsel;
617 struct hists *hists = evsel__hists(evsel); 618 struct hists *hists = evsel__hists(evsel);
618 struct hist_entry *he = NULL; 619 struct hist_entry *he = NULL;
619 int i = iter->curr; 620 int i = iter->curr;
620 int err = 0; 621 int err = 0;
621 622
622 bi = iter->priv; 623 bi = iter->priv;
623 624
624 if (iter->hide_unresolved && !(bi[i].from.sym && bi[i].to.sym)) 625 if (iter->hide_unresolved && !(bi[i].from.sym && bi[i].to.sym))
625 goto out; 626 goto out;
626 627
627 /* 628 /*
628 * The report shows the percentage of total branches captured 629 * The report shows the percentage of total branches captured
629 * and not events sampled. Thus we use a pseudo period of 1. 630 * and not events sampled. Thus we use a pseudo period of 1.
630 */ 631 */
631 he = __hists__add_entry(hists, al, iter->parent, &bi[i], NULL, 632 he = __hists__add_entry(hists, al, iter->parent, &bi[i], NULL,
632 1, 1, 0, true); 633 1, 1, 0, true);
633 if (he == NULL) 634 if (he == NULL)
634 return -ENOMEM; 635 return -ENOMEM;
635 636
636 hists__inc_nr_samples(hists, he->filtered); 637 hists__inc_nr_samples(hists, he->filtered);
637 638
638 out: 639 out:
639 iter->he = he; 640 iter->he = he;
640 iter->curr++; 641 iter->curr++;
641 return err; 642 return err;
642 } 643 }
643 644
644 static int 645 static int
645 iter_finish_branch_entry(struct hist_entry_iter *iter, 646 iter_finish_branch_entry(struct hist_entry_iter *iter,
646 struct addr_location *al __maybe_unused) 647 struct addr_location *al __maybe_unused)
647 { 648 {
648 zfree(&iter->priv); 649 zfree(&iter->priv);
649 iter->he = NULL; 650 iter->he = NULL;
650 651
651 return iter->curr >= iter->total ? 0 : -1; 652 return iter->curr >= iter->total ? 0 : -1;
652 } 653 }
653 654
654 static int 655 static int
655 iter_prepare_normal_entry(struct hist_entry_iter *iter __maybe_unused, 656 iter_prepare_normal_entry(struct hist_entry_iter *iter __maybe_unused,
656 struct addr_location *al __maybe_unused) 657 struct addr_location *al __maybe_unused)
657 { 658 {
658 return 0; 659 return 0;
659 } 660 }
660 661
661 static int 662 static int
662 iter_add_single_normal_entry(struct hist_entry_iter *iter, struct addr_location *al) 663 iter_add_single_normal_entry(struct hist_entry_iter *iter, struct addr_location *al)
663 { 664 {
664 struct perf_evsel *evsel = iter->evsel; 665 struct perf_evsel *evsel = iter->evsel;
665 struct perf_sample *sample = iter->sample; 666 struct perf_sample *sample = iter->sample;
666 struct hist_entry *he; 667 struct hist_entry *he;
667 668
668 he = __hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, 669 he = __hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL,
669 sample->period, sample->weight, 670 sample->period, sample->weight,
670 sample->transaction, true); 671 sample->transaction, true);
671 if (he == NULL) 672 if (he == NULL)
672 return -ENOMEM; 673 return -ENOMEM;
673 674
674 iter->he = he; 675 iter->he = he;
675 return 0; 676 return 0;
676 } 677 }
677 678
678 static int 679 static int
679 iter_finish_normal_entry(struct hist_entry_iter *iter, 680 iter_finish_normal_entry(struct hist_entry_iter *iter,
680 struct addr_location *al __maybe_unused) 681 struct addr_location *al __maybe_unused)
681 { 682 {
682 struct hist_entry *he = iter->he; 683 struct hist_entry *he = iter->he;
683 struct perf_evsel *evsel = iter->evsel; 684 struct perf_evsel *evsel = iter->evsel;
684 struct perf_sample *sample = iter->sample; 685 struct perf_sample *sample = iter->sample;
685 686
686 if (he == NULL) 687 if (he == NULL)
687 return 0; 688 return 0;
688 689
689 iter->he = NULL; 690 iter->he = NULL;
690 691
691 hists__inc_nr_samples(evsel__hists(evsel), he->filtered); 692 hists__inc_nr_samples(evsel__hists(evsel), he->filtered);
692 693
693 return hist_entry__append_callchain(he, sample); 694 return hist_entry__append_callchain(he, sample);
694 } 695 }
695 696
696 static int 697 static int
697 iter_prepare_cumulative_entry(struct hist_entry_iter *iter __maybe_unused, 698 iter_prepare_cumulative_entry(struct hist_entry_iter *iter __maybe_unused,
698 struct addr_location *al __maybe_unused) 699 struct addr_location *al __maybe_unused)
699 { 700 {
700 struct hist_entry **he_cache; 701 struct hist_entry **he_cache;
701 702
702 callchain_cursor_commit(&callchain_cursor); 703 callchain_cursor_commit(&callchain_cursor);
703 704
704 /* 705 /*
705 * This is for detecting cycles or recursions so that they're 706 * This is for detecting cycles or recursions so that they're
706 * cumulated only one time to prevent entries more than 100% 707 * cumulated only one time to prevent entries more than 100%
707 * overhead. 708 * overhead.
708 */ 709 */
709 he_cache = malloc(sizeof(*he_cache) * (PERF_MAX_STACK_DEPTH + 1)); 710 he_cache = malloc(sizeof(*he_cache) * (PERF_MAX_STACK_DEPTH + 1));
710 if (he_cache == NULL) 711 if (he_cache == NULL)
711 return -ENOMEM; 712 return -ENOMEM;
712 713
713 iter->priv = he_cache; 714 iter->priv = he_cache;
714 iter->curr = 0; 715 iter->curr = 0;
715 716
716 return 0; 717 return 0;
717 } 718 }
718 719
719 static int 720 static int
720 iter_add_single_cumulative_entry(struct hist_entry_iter *iter, 721 iter_add_single_cumulative_entry(struct hist_entry_iter *iter,
721 struct addr_location *al) 722 struct addr_location *al)
722 { 723 {
723 struct perf_evsel *evsel = iter->evsel; 724 struct perf_evsel *evsel = iter->evsel;
724 struct hists *hists = evsel__hists(evsel); 725 struct hists *hists = evsel__hists(evsel);
725 struct perf_sample *sample = iter->sample; 726 struct perf_sample *sample = iter->sample;
726 struct hist_entry **he_cache = iter->priv; 727 struct hist_entry **he_cache = iter->priv;
727 struct hist_entry *he; 728 struct hist_entry *he;
728 int err = 0; 729 int err = 0;
729 730
730 he = __hists__add_entry(hists, al, iter->parent, NULL, NULL, 731 he = __hists__add_entry(hists, al, iter->parent, NULL, NULL,
731 sample->period, sample->weight, 732 sample->period, sample->weight,
732 sample->transaction, true); 733 sample->transaction, true);
733 if (he == NULL) 734 if (he == NULL)
734 return -ENOMEM; 735 return -ENOMEM;
735 736
736 iter->he = he; 737 iter->he = he;
737 he_cache[iter->curr++] = he; 738 he_cache[iter->curr++] = he;
738 739
739 callchain_append(he->callchain, &callchain_cursor, sample->period); 740 hist_entry__append_callchain(he, sample);
740 741
741 /* 742 /*
742 * We need to re-initialize the cursor since callchain_append() 743 * We need to re-initialize the cursor since callchain_append()
743 * advanced the cursor to the end. 744 * advanced the cursor to the end.
744 */ 745 */
745 callchain_cursor_commit(&callchain_cursor); 746 callchain_cursor_commit(&callchain_cursor);
746 747
747 hists__inc_nr_samples(hists, he->filtered); 748 hists__inc_nr_samples(hists, he->filtered);
748 749
749 return err; 750 return err;
750 } 751 }
751 752
752 static int 753 static int
753 iter_next_cumulative_entry(struct hist_entry_iter *iter, 754 iter_next_cumulative_entry(struct hist_entry_iter *iter,
754 struct addr_location *al) 755 struct addr_location *al)
755 { 756 {
756 struct callchain_cursor_node *node; 757 struct callchain_cursor_node *node;
757 758
758 node = callchain_cursor_current(&callchain_cursor); 759 node = callchain_cursor_current(&callchain_cursor);
759 if (node == NULL) 760 if (node == NULL)
760 return 0; 761 return 0;
761 762
762 return fill_callchain_info(al, node, iter->hide_unresolved); 763 return fill_callchain_info(al, node, iter->hide_unresolved);
763 } 764 }
764 765
765 static int 766 static int
766 iter_add_next_cumulative_entry(struct hist_entry_iter *iter, 767 iter_add_next_cumulative_entry(struct hist_entry_iter *iter,
767 struct addr_location *al) 768 struct addr_location *al)
768 { 769 {
769 struct perf_evsel *evsel = iter->evsel; 770 struct perf_evsel *evsel = iter->evsel;
770 struct perf_sample *sample = iter->sample; 771 struct perf_sample *sample = iter->sample;
771 struct hist_entry **he_cache = iter->priv; 772 struct hist_entry **he_cache = iter->priv;
772 struct hist_entry *he; 773 struct hist_entry *he;
773 struct hist_entry he_tmp = { 774 struct hist_entry he_tmp = {
774 .cpu = al->cpu, 775 .cpu = al->cpu,
775 .thread = al->thread, 776 .thread = al->thread,
776 .comm = thread__comm(al->thread), 777 .comm = thread__comm(al->thread),
777 .ip = al->addr, 778 .ip = al->addr,
778 .ms = { 779 .ms = {
779 .map = al->map, 780 .map = al->map,
780 .sym = al->sym, 781 .sym = al->sym,
781 }, 782 },
782 .parent = iter->parent, 783 .parent = iter->parent,
783 }; 784 };
784 int i; 785 int i;
785 struct callchain_cursor cursor; 786 struct callchain_cursor cursor;
786 787
787 callchain_cursor_snapshot(&cursor, &callchain_cursor); 788 callchain_cursor_snapshot(&cursor, &callchain_cursor);
788 789
789 callchain_cursor_advance(&callchain_cursor); 790 callchain_cursor_advance(&callchain_cursor);
790 791
791 /* 792 /*
792 * Check if there's duplicate entries in the callchain. 793 * Check if there's duplicate entries in the callchain.
793 * It's possible that it has cycles or recursive calls. 794 * It's possible that it has cycles or recursive calls.
794 */ 795 */
795 for (i = 0; i < iter->curr; i++) { 796 for (i = 0; i < iter->curr; i++) {
796 if (hist_entry__cmp(he_cache[i], &he_tmp) == 0) { 797 if (hist_entry__cmp(he_cache[i], &he_tmp) == 0) {
797 /* to avoid calling callback function */ 798 /* to avoid calling callback function */
798 iter->he = NULL; 799 iter->he = NULL;
799 return 0; 800 return 0;
800 } 801 }
801 } 802 }
802 803
803 he = __hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, 804 he = __hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL,
804 sample->period, sample->weight, 805 sample->period, sample->weight,
805 sample->transaction, false); 806 sample->transaction, false);
806 if (he == NULL) 807 if (he == NULL)
807 return -ENOMEM; 808 return -ENOMEM;
808 809
809 iter->he = he; 810 iter->he = he;
810 he_cache[iter->curr++] = he; 811 he_cache[iter->curr++] = he;
811 812
812 callchain_append(he->callchain, &cursor, sample->period); 813 if (symbol_conf.use_callchain)
814 callchain_append(he->callchain, &cursor, sample->period);
813 return 0; 815 return 0;
814 } 816 }
815 817
816 static int 818 static int
817 iter_finish_cumulative_entry(struct hist_entry_iter *iter, 819 iter_finish_cumulative_entry(struct hist_entry_iter *iter,
818 struct addr_location *al __maybe_unused) 820 struct addr_location *al __maybe_unused)
819 { 821 {
820 zfree(&iter->priv); 822 zfree(&iter->priv);
821 iter->he = NULL; 823 iter->he = NULL;
822 824
823 return 0; 825 return 0;
824 } 826 }
825 827
826 const struct hist_iter_ops hist_iter_mem = { 828 const struct hist_iter_ops hist_iter_mem = {
827 .prepare_entry = iter_prepare_mem_entry, 829 .prepare_entry = iter_prepare_mem_entry,
828 .add_single_entry = iter_add_single_mem_entry, 830 .add_single_entry = iter_add_single_mem_entry,
829 .next_entry = iter_next_nop_entry, 831 .next_entry = iter_next_nop_entry,
830 .add_next_entry = iter_add_next_nop_entry, 832 .add_next_entry = iter_add_next_nop_entry,
831 .finish_entry = iter_finish_mem_entry, 833 .finish_entry = iter_finish_mem_entry,
832 }; 834 };
833 835
834 const struct hist_iter_ops hist_iter_branch = { 836 const struct hist_iter_ops hist_iter_branch = {
835 .prepare_entry = iter_prepare_branch_entry, 837 .prepare_entry = iter_prepare_branch_entry,
836 .add_single_entry = iter_add_single_branch_entry, 838 .add_single_entry = iter_add_single_branch_entry,
837 .next_entry = iter_next_branch_entry, 839 .next_entry = iter_next_branch_entry,
838 .add_next_entry = iter_add_next_branch_entry, 840 .add_next_entry = iter_add_next_branch_entry,
839 .finish_entry = iter_finish_branch_entry, 841 .finish_entry = iter_finish_branch_entry,
840 }; 842 };
841 843
842 const struct hist_iter_ops hist_iter_normal = { 844 const struct hist_iter_ops hist_iter_normal = {
843 .prepare_entry = iter_prepare_normal_entry, 845 .prepare_entry = iter_prepare_normal_entry,
844 .add_single_entry = iter_add_single_normal_entry, 846 .add_single_entry = iter_add_single_normal_entry,
845 .next_entry = iter_next_nop_entry, 847 .next_entry = iter_next_nop_entry,
846 .add_next_entry = iter_add_next_nop_entry, 848 .add_next_entry = iter_add_next_nop_entry,
847 .finish_entry = iter_finish_normal_entry, 849 .finish_entry = iter_finish_normal_entry,
848 }; 850 };
849 851
850 const struct hist_iter_ops hist_iter_cumulative = { 852 const struct hist_iter_ops hist_iter_cumulative = {
851 .prepare_entry = iter_prepare_cumulative_entry, 853 .prepare_entry = iter_prepare_cumulative_entry,
852 .add_single_entry = iter_add_single_cumulative_entry, 854 .add_single_entry = iter_add_single_cumulative_entry,
853 .next_entry = iter_next_cumulative_entry, 855 .next_entry = iter_next_cumulative_entry,
854 .add_next_entry = iter_add_next_cumulative_entry, 856 .add_next_entry = iter_add_next_cumulative_entry,
855 .finish_entry = iter_finish_cumulative_entry, 857 .finish_entry = iter_finish_cumulative_entry,
856 }; 858 };
857 859
858 int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, 860 int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al,
859 struct perf_evsel *evsel, struct perf_sample *sample, 861 struct perf_evsel *evsel, struct perf_sample *sample,
860 int max_stack_depth, void *arg) 862 int max_stack_depth, void *arg)
861 { 863 {
862 int err, err2; 864 int err, err2;
863 865
864 err = sample__resolve_callchain(sample, &iter->parent, evsel, al, 866 err = sample__resolve_callchain(sample, &iter->parent, evsel, al,
865 max_stack_depth); 867 max_stack_depth);
866 if (err) 868 if (err)
867 return err; 869 return err;
868 870
869 iter->evsel = evsel; 871 iter->evsel = evsel;
870 iter->sample = sample; 872 iter->sample = sample;
871 873
872 err = iter->ops->prepare_entry(iter, al); 874 err = iter->ops->prepare_entry(iter, al);
873 if (err) 875 if (err)
874 goto out; 876 goto out;
875 877
876 err = iter->ops->add_single_entry(iter, al); 878 err = iter->ops->add_single_entry(iter, al);
877 if (err) 879 if (err)
878 goto out; 880 goto out;
879 881
880 if (iter->he && iter->add_entry_cb) { 882 if (iter->he && iter->add_entry_cb) {
881 err = iter->add_entry_cb(iter, al, true, arg); 883 err = iter->add_entry_cb(iter, al, true, arg);
882 if (err) 884 if (err)
883 goto out; 885 goto out;
884 } 886 }
885 887
886 while (iter->ops->next_entry(iter, al)) { 888 while (iter->ops->next_entry(iter, al)) {
887 err = iter->ops->add_next_entry(iter, al); 889 err = iter->ops->add_next_entry(iter, al);
888 if (err) 890 if (err)
889 break; 891 break;
890 892
891 if (iter->he && iter->add_entry_cb) { 893 if (iter->he && iter->add_entry_cb) {
892 err = iter->add_entry_cb(iter, al, false, arg); 894 err = iter->add_entry_cb(iter, al, false, arg);
893 if (err) 895 if (err)
894 goto out; 896 goto out;
895 } 897 }
896 } 898 }
897 899
898 out: 900 out:
899 err2 = iter->ops->finish_entry(iter, al); 901 err2 = iter->ops->finish_entry(iter, al);
900 if (!err) 902 if (!err)
901 err = err2; 903 err = err2;
902 904
903 return err; 905 return err;
904 } 906 }
905 907
906 int64_t 908 int64_t
907 hist_entry__cmp(struct hist_entry *left, struct hist_entry *right) 909 hist_entry__cmp(struct hist_entry *left, struct hist_entry *right)
908 { 910 {
909 struct perf_hpp_fmt *fmt; 911 struct perf_hpp_fmt *fmt;
910 int64_t cmp = 0; 912 int64_t cmp = 0;
911 913
912 perf_hpp__for_each_sort_list(fmt) { 914 perf_hpp__for_each_sort_list(fmt) {
913 if (perf_hpp__should_skip(fmt)) 915 if (perf_hpp__should_skip(fmt))
914 continue; 916 continue;
915 917
916 cmp = fmt->cmp(left, right); 918 cmp = fmt->cmp(left, right);
917 if (cmp) 919 if (cmp)
918 break; 920 break;
919 } 921 }
920 922
921 return cmp; 923 return cmp;
922 } 924 }
923 925
924 int64_t 926 int64_t
925 hist_entry__collapse(struct hist_entry *left, struct hist_entry *right) 927 hist_entry__collapse(struct hist_entry *left, struct hist_entry *right)
926 { 928 {
927 struct perf_hpp_fmt *fmt; 929 struct perf_hpp_fmt *fmt;
928 int64_t cmp = 0; 930 int64_t cmp = 0;
929 931
930 perf_hpp__for_each_sort_list(fmt) { 932 perf_hpp__for_each_sort_list(fmt) {
931 if (perf_hpp__should_skip(fmt)) 933 if (perf_hpp__should_skip(fmt))
932 continue; 934 continue;
933 935
934 cmp = fmt->collapse(left, right); 936 cmp = fmt->collapse(left, right);
935 if (cmp) 937 if (cmp)
936 break; 938 break;
937 } 939 }
938 940
939 return cmp; 941 return cmp;
940 } 942 }
941 943
942 void hist_entry__free(struct hist_entry *he) 944 void hist_entry__free(struct hist_entry *he)
943 { 945 {
944 zfree(&he->branch_info); 946 zfree(&he->branch_info);
945 zfree(&he->mem_info); 947 zfree(&he->mem_info);
946 zfree(&he->stat_acc); 948 zfree(&he->stat_acc);
947 free_srcline(he->srcline); 949 free_srcline(he->srcline);
950 free_callchain(he->callchain);
948 free(he); 951 free(he);
949 } 952 }
950 953
951 /* 954 /*
952 * collapse the histogram 955 * collapse the histogram
953 */ 956 */
954 957
955 static bool hists__collapse_insert_entry(struct hists *hists __maybe_unused, 958 static bool hists__collapse_insert_entry(struct hists *hists __maybe_unused,
956 struct rb_root *root, 959 struct rb_root *root,
957 struct hist_entry *he) 960 struct hist_entry *he)
958 { 961 {
959 struct rb_node **p = &root->rb_node; 962 struct rb_node **p = &root->rb_node;
960 struct rb_node *parent = NULL; 963 struct rb_node *parent = NULL;
961 struct hist_entry *iter; 964 struct hist_entry *iter;
962 int64_t cmp; 965 int64_t cmp;
963 966
964 while (*p != NULL) { 967 while (*p != NULL) {
965 parent = *p; 968 parent = *p;
966 iter = rb_entry(parent, struct hist_entry, rb_node_in); 969 iter = rb_entry(parent, struct hist_entry, rb_node_in);
967 970
968 cmp = hist_entry__collapse(iter, he); 971 cmp = hist_entry__collapse(iter, he);
969 972
970 if (!cmp) { 973 if (!cmp) {
971 he_stat__add_stat(&iter->stat, &he->stat); 974 he_stat__add_stat(&iter->stat, &he->stat);
972 if (symbol_conf.cumulate_callchain) 975 if (symbol_conf.cumulate_callchain)
973 he_stat__add_stat(iter->stat_acc, he->stat_acc); 976 he_stat__add_stat(iter->stat_acc, he->stat_acc);
974 977
975 if (symbol_conf.use_callchain) { 978 if (symbol_conf.use_callchain) {
976 callchain_cursor_reset(&callchain_cursor); 979 callchain_cursor_reset(&callchain_cursor);
977 callchain_merge(&callchain_cursor, 980 callchain_merge(&callchain_cursor,
978 iter->callchain, 981 iter->callchain,
979 he->callchain); 982 he->callchain);
980 } 983 }
981 hist_entry__free(he); 984 hist_entry__free(he);
982 return false; 985 return false;
983 } 986 }
984 987
985 if (cmp < 0) 988 if (cmp < 0)
986 p = &(*p)->rb_left; 989 p = &(*p)->rb_left;
987 else 990 else
988 p = &(*p)->rb_right; 991 p = &(*p)->rb_right;
989 } 992 }
993 hists->nr_entries++;
990 994
991 rb_link_node(&he->rb_node_in, parent, p); 995 rb_link_node(&he->rb_node_in, parent, p);
992 rb_insert_color(&he->rb_node_in, root); 996 rb_insert_color(&he->rb_node_in, root);
993 return true; 997 return true;
994 } 998 }
995 999
996 static struct rb_root *hists__get_rotate_entries_in(struct hists *hists) 1000 static struct rb_root *hists__get_rotate_entries_in(struct hists *hists)
997 { 1001 {
998 struct rb_root *root; 1002 struct rb_root *root;
999 1003
1000 pthread_mutex_lock(&hists->lock); 1004 pthread_mutex_lock(&hists->lock);
1001 1005
1002 root = hists->entries_in; 1006 root = hists->entries_in;
1003 if (++hists->entries_in > &hists->entries_in_array[1]) 1007 if (++hists->entries_in > &hists->entries_in_array[1])
1004 hists->entries_in = &hists->entries_in_array[0]; 1008 hists->entries_in = &hists->entries_in_array[0];
1005 1009
1006 pthread_mutex_unlock(&hists->lock); 1010 pthread_mutex_unlock(&hists->lock);
1007 1011
1008 return root; 1012 return root;
1009 } 1013 }
1010 1014
1011 static void hists__apply_filters(struct hists *hists, struct hist_entry *he) 1015 static void hists__apply_filters(struct hists *hists, struct hist_entry *he)
1012 { 1016 {
1013 hists__filter_entry_by_dso(hists, he); 1017 hists__filter_entry_by_dso(hists, he);
1014 hists__filter_entry_by_thread(hists, he); 1018 hists__filter_entry_by_thread(hists, he);
1015 hists__filter_entry_by_symbol(hists, he); 1019 hists__filter_entry_by_symbol(hists, he);
1016 } 1020 }
1017 1021
1018 void hists__collapse_resort(struct hists *hists, struct ui_progress *prog) 1022 void hists__collapse_resort(struct hists *hists, struct ui_progress *prog)
1019 { 1023 {
1020 struct rb_root *root; 1024 struct rb_root *root;
1021 struct rb_node *next; 1025 struct rb_node *next;
1022 struct hist_entry *n; 1026 struct hist_entry *n;
1023 1027
1024 if (!sort__need_collapse) 1028 if (!sort__need_collapse)
1025 return; 1029 return;
1026 1030
1031 hists->nr_entries = 0;
1032
1027 root = hists__get_rotate_entries_in(hists); 1033 root = hists__get_rotate_entries_in(hists);
1034
1028 next = rb_first(root); 1035 next = rb_first(root);
1029 1036
1030 while (next) { 1037 while (next) {
1031 if (session_done()) 1038 if (session_done())
1032 break; 1039 break;
1033 n = rb_entry(next, struct hist_entry, rb_node_in); 1040 n = rb_entry(next, struct hist_entry, rb_node_in);
1034 next = rb_next(&n->rb_node_in); 1041 next = rb_next(&n->rb_node_in);
1035 1042
1036 rb_erase(&n->rb_node_in, root); 1043 rb_erase(&n->rb_node_in, root);
1037 if (hists__collapse_insert_entry(hists, &hists->entries_collapsed, n)) { 1044 if (hists__collapse_insert_entry(hists, &hists->entries_collapsed, n)) {
1038 /* 1045 /*
1039 * If it wasn't combined with one of the entries already 1046 * If it wasn't combined with one of the entries already
1040 * collapsed, we need to apply the filters that may have 1047 * collapsed, we need to apply the filters that may have
1041 * been set by, say, the hist_browser. 1048 * been set by, say, the hist_browser.
1042 */ 1049 */
1043 hists__apply_filters(hists, n); 1050 hists__apply_filters(hists, n);
1044 } 1051 }
1045 if (prog) 1052 if (prog)
1046 ui_progress__update(prog, 1); 1053 ui_progress__update(prog, 1);
1047 } 1054 }
1048 } 1055 }
1049 1056
1050 static int hist_entry__sort(struct hist_entry *a, struct hist_entry *b) 1057 static int hist_entry__sort(struct hist_entry *a, struct hist_entry *b)
1051 { 1058 {
1052 struct perf_hpp_fmt *fmt; 1059 struct perf_hpp_fmt *fmt;
1053 int64_t cmp = 0; 1060 int64_t cmp = 0;
1054 1061
1055 perf_hpp__for_each_sort_list(fmt) { 1062 perf_hpp__for_each_sort_list(fmt) {
1056 if (perf_hpp__should_skip(fmt)) 1063 if (perf_hpp__should_skip(fmt))
1057 continue; 1064 continue;
1058 1065
1059 cmp = fmt->sort(a, b); 1066 cmp = fmt->sort(a, b);
1060 if (cmp) 1067 if (cmp)
1061 break; 1068 break;
1062 } 1069 }
1063 1070
1064 return cmp; 1071 return cmp;
1065 } 1072 }
1066 1073
1067 static void hists__reset_filter_stats(struct hists *hists) 1074 static void hists__reset_filter_stats(struct hists *hists)
1068 { 1075 {
1069 hists->nr_non_filtered_entries = 0; 1076 hists->nr_non_filtered_entries = 0;
1070 hists->stats.total_non_filtered_period = 0; 1077 hists->stats.total_non_filtered_period = 0;
1071 } 1078 }
1072 1079
1073 void hists__reset_stats(struct hists *hists) 1080 void hists__reset_stats(struct hists *hists)
1074 { 1081 {
1075 hists->nr_entries = 0; 1082 hists->nr_entries = 0;
1076 hists->stats.total_period = 0; 1083 hists->stats.total_period = 0;
1077 1084
1078 hists__reset_filter_stats(hists); 1085 hists__reset_filter_stats(hists);
1079 } 1086 }
1080 1087
1081 static void hists__inc_filter_stats(struct hists *hists, struct hist_entry *h) 1088 static void hists__inc_filter_stats(struct hists *hists, struct hist_entry *h)
1082 { 1089 {
1083 hists->nr_non_filtered_entries++; 1090 hists->nr_non_filtered_entries++;
1084 hists->stats.total_non_filtered_period += h->stat.period; 1091 hists->stats.total_non_filtered_period += h->stat.period;
1085 } 1092 }
1086 1093
1087 void hists__inc_stats(struct hists *hists, struct hist_entry *h) 1094 void hists__inc_stats(struct hists *hists, struct hist_entry *h)
1088 { 1095 {
1089 if (!h->filtered) 1096 if (!h->filtered)
1090 hists__inc_filter_stats(hists, h); 1097 hists__inc_filter_stats(hists, h);
1091 1098
1092 hists->nr_entries++; 1099 hists->nr_entries++;
1093 hists->stats.total_period += h->stat.period; 1100 hists->stats.total_period += h->stat.period;
1094 } 1101 }
1095 1102
1096 static void __hists__insert_output_entry(struct rb_root *entries, 1103 static void __hists__insert_output_entry(struct rb_root *entries,
1097 struct hist_entry *he, 1104 struct hist_entry *he,
1098 u64 min_callchain_hits) 1105 u64 min_callchain_hits)
1099 { 1106 {
1100 struct rb_node **p = &entries->rb_node; 1107 struct rb_node **p = &entries->rb_node;
1101 struct rb_node *parent = NULL; 1108 struct rb_node *parent = NULL;
1102 struct hist_entry *iter; 1109 struct hist_entry *iter;
1103 1110
1104 if (symbol_conf.use_callchain) 1111 if (symbol_conf.use_callchain)
1105 callchain_param.sort(&he->sorted_chain, he->callchain, 1112 callchain_param.sort(&he->sorted_chain, he->callchain,
1106 min_callchain_hits, &callchain_param); 1113 min_callchain_hits, &callchain_param);
1107 1114
1108 while (*p != NULL) { 1115 while (*p != NULL) {
1109 parent = *p; 1116 parent = *p;
1110 iter = rb_entry(parent, struct hist_entry, rb_node); 1117 iter = rb_entry(parent, struct hist_entry, rb_node);
1111 1118
1112 if (hist_entry__sort(he, iter) > 0) 1119 if (hist_entry__sort(he, iter) > 0)
1113 p = &(*p)->rb_left; 1120 p = &(*p)->rb_left;
1114 else 1121 else
1115 p = &(*p)->rb_right; 1122 p = &(*p)->rb_right;
1116 } 1123 }
1117 1124
1118 rb_link_node(&he->rb_node, parent, p); 1125 rb_link_node(&he->rb_node, parent, p);
1119 rb_insert_color(&he->rb_node, entries); 1126 rb_insert_color(&he->rb_node, entries);
1120 } 1127 }
1121 1128
1122 void hists__output_resort(struct hists *hists) 1129 void hists__output_resort(struct hists *hists, struct ui_progress *prog)
1123 { 1130 {
1124 struct rb_root *root; 1131 struct rb_root *root;
1125 struct rb_node *next; 1132 struct rb_node *next;
1126 struct hist_entry *n; 1133 struct hist_entry *n;
1127 u64 min_callchain_hits; 1134 u64 min_callchain_hits;
1128 1135
1129 min_callchain_hits = hists->stats.total_period * (callchain_param.min_percent / 100); 1136 min_callchain_hits = hists->stats.total_period * (callchain_param.min_percent / 100);
1130 1137
1131 if (sort__need_collapse) 1138 if (sort__need_collapse)
1132 root = &hists->entries_collapsed; 1139 root = &hists->entries_collapsed;
1133 else 1140 else
1134 root = hists->entries_in; 1141 root = hists->entries_in;
1135 1142
1136 next = rb_first(root); 1143 next = rb_first(root);
1137 hists->entries = RB_ROOT; 1144 hists->entries = RB_ROOT;
1138 1145
1139 hists__reset_stats(hists); 1146 hists__reset_stats(hists);
1140 hists__reset_col_len(hists); 1147 hists__reset_col_len(hists);
1141 1148
1142 while (next) { 1149 while (next) {
1143 n = rb_entry(next, struct hist_entry, rb_node_in); 1150 n = rb_entry(next, struct hist_entry, rb_node_in);
1144 next = rb_next(&n->rb_node_in); 1151 next = rb_next(&n->rb_node_in);
1145 1152
1146 __hists__insert_output_entry(&hists->entries, n, min_callchain_hits); 1153 __hists__insert_output_entry(&hists->entries, n, min_callchain_hits);
1147 hists__inc_stats(hists, n); 1154 hists__inc_stats(hists, n);
1148 1155
1149 if (!n->filtered) 1156 if (!n->filtered)
1150 hists__calc_col_len(hists, n); 1157 hists__calc_col_len(hists, n);
1158
1159 if (prog)
1160 ui_progress__update(prog, 1);
1151 } 1161 }
1152 } 1162 }
1153 1163
1154 static void hists__remove_entry_filter(struct hists *hists, struct hist_entry *h, 1164 static void hists__remove_entry_filter(struct hists *hists, struct hist_entry *h,
1155 enum hist_filter filter) 1165 enum hist_filter filter)
1156 { 1166 {
1157 h->filtered &= ~(1 << filter); 1167 h->filtered &= ~(1 << filter);
1158 if (h->filtered) 1168 if (h->filtered)
1159 return; 1169 return;
1160 1170
1161 /* force fold unfiltered entry for simplicity */ 1171 /* force fold unfiltered entry for simplicity */
1162 h->ms.unfolded = false; 1172 h->ms.unfolded = false;
1163 h->row_offset = 0; 1173 h->row_offset = 0;
1164 1174
1165 hists->stats.nr_non_filtered_samples += h->stat.nr_events; 1175 hists->stats.nr_non_filtered_samples += h->stat.nr_events;
1166 1176
1167 hists__inc_filter_stats(hists, h); 1177 hists__inc_filter_stats(hists, h);
1168 hists__calc_col_len(hists, h); 1178 hists__calc_col_len(hists, h);
1169 } 1179 }
1170 1180
1171 1181
1172 static bool hists__filter_entry_by_dso(struct hists *hists, 1182 static bool hists__filter_entry_by_dso(struct hists *hists,
1173 struct hist_entry *he) 1183 struct hist_entry *he)
1174 { 1184 {
1175 if (hists->dso_filter != NULL && 1185 if (hists->dso_filter != NULL &&
1176 (he->ms.map == NULL || he->ms.map->dso != hists->dso_filter)) { 1186 (he->ms.map == NULL || he->ms.map->dso != hists->dso_filter)) {
1177 he->filtered |= (1 << HIST_FILTER__DSO); 1187 he->filtered |= (1 << HIST_FILTER__DSO);
1178 return true; 1188 return true;
1179 } 1189 }
1180 1190
1181 return false; 1191 return false;
1182 } 1192 }
1183 1193
1184 void hists__filter_by_dso(struct hists *hists) 1194 void hists__filter_by_dso(struct hists *hists)
1185 { 1195 {
1186 struct rb_node *nd; 1196 struct rb_node *nd;
1187 1197
1188 hists->stats.nr_non_filtered_samples = 0; 1198 hists->stats.nr_non_filtered_samples = 0;
1189 1199
1190 hists__reset_filter_stats(hists); 1200 hists__reset_filter_stats(hists);
1191 hists__reset_col_len(hists); 1201 hists__reset_col_len(hists);
1192 1202
1193 for (nd = rb_first(&hists->entries); nd; nd = rb_next(nd)) { 1203 for (nd = rb_first(&hists->entries); nd; nd = rb_next(nd)) {
1194 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 1204 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
1195 1205
1196 if (symbol_conf.exclude_other && !h->parent) 1206 if (symbol_conf.exclude_other && !h->parent)
1197 continue; 1207 continue;
1198 1208
1199 if (hists__filter_entry_by_dso(hists, h)) 1209 if (hists__filter_entry_by_dso(hists, h))
1200 continue; 1210 continue;
1201 1211
1202 hists__remove_entry_filter(hists, h, HIST_FILTER__DSO); 1212 hists__remove_entry_filter(hists, h, HIST_FILTER__DSO);
1203 } 1213 }
1204 } 1214 }
1205 1215
1206 static bool hists__filter_entry_by_thread(struct hists *hists, 1216 static bool hists__filter_entry_by_thread(struct hists *hists,
1207 struct hist_entry *he) 1217 struct hist_entry *he)
1208 { 1218 {
1209 if (hists->thread_filter != NULL && 1219 if (hists->thread_filter != NULL &&
1210 he->thread != hists->thread_filter) { 1220 he->thread != hists->thread_filter) {
1211 he->filtered |= (1 << HIST_FILTER__THREAD); 1221 he->filtered |= (1 << HIST_FILTER__THREAD);
1212 return true; 1222 return true;
1213 } 1223 }
1214 1224
1215 return false; 1225 return false;
1216 } 1226 }
1217 1227
1218 void hists__filter_by_thread(struct hists *hists) 1228 void hists__filter_by_thread(struct hists *hists)
1219 { 1229 {
1220 struct rb_node *nd; 1230 struct rb_node *nd;
1221 1231
1222 hists->stats.nr_non_filtered_samples = 0; 1232 hists->stats.nr_non_filtered_samples = 0;
1223 1233
1224 hists__reset_filter_stats(hists); 1234 hists__reset_filter_stats(hists);
1225 hists__reset_col_len(hists); 1235 hists__reset_col_len(hists);
1226 1236
1227 for (nd = rb_first(&hists->entries); nd; nd = rb_next(nd)) { 1237 for (nd = rb_first(&hists->entries); nd; nd = rb_next(nd)) {
1228 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 1238 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
1229 1239
1230 if (hists__filter_entry_by_thread(hists, h)) 1240 if (hists__filter_entry_by_thread(hists, h))
1231 continue; 1241 continue;
1232 1242
1233 hists__remove_entry_filter(hists, h, HIST_FILTER__THREAD); 1243 hists__remove_entry_filter(hists, h, HIST_FILTER__THREAD);
1234 } 1244 }
1235 } 1245 }
1236 1246
1237 static bool hists__filter_entry_by_symbol(struct hists *hists, 1247 static bool hists__filter_entry_by_symbol(struct hists *hists,
1238 struct hist_entry *he) 1248 struct hist_entry *he)
1239 { 1249 {
1240 if (hists->symbol_filter_str != NULL && 1250 if (hists->symbol_filter_str != NULL &&
1241 (!he->ms.sym || strstr(he->ms.sym->name, 1251 (!he->ms.sym || strstr(he->ms.sym->name,
1242 hists->symbol_filter_str) == NULL)) { 1252 hists->symbol_filter_str) == NULL)) {
1243 he->filtered |= (1 << HIST_FILTER__SYMBOL); 1253 he->filtered |= (1 << HIST_FILTER__SYMBOL);
1244 return true; 1254 return true;
1245 } 1255 }
1246 1256
1247 return false; 1257 return false;
1248 } 1258 }
1249 1259
1250 void hists__filter_by_symbol(struct hists *hists) 1260 void hists__filter_by_symbol(struct hists *hists)
1251 { 1261 {
1252 struct rb_node *nd; 1262 struct rb_node *nd;
1253 1263
1254 hists->stats.nr_non_filtered_samples = 0; 1264 hists->stats.nr_non_filtered_samples = 0;
1255 1265
1256 hists__reset_filter_stats(hists); 1266 hists__reset_filter_stats(hists);
1257 hists__reset_col_len(hists); 1267 hists__reset_col_len(hists);
1258 1268
1259 for (nd = rb_first(&hists->entries); nd; nd = rb_next(nd)) { 1269 for (nd = rb_first(&hists->entries); nd; nd = rb_next(nd)) {
1260 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); 1270 struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node);
1261 1271
1262 if (hists__filter_entry_by_symbol(hists, h)) 1272 if (hists__filter_entry_by_symbol(hists, h))
1263 continue; 1273 continue;
1264 1274
1265 hists__remove_entry_filter(hists, h, HIST_FILTER__SYMBOL); 1275 hists__remove_entry_filter(hists, h, HIST_FILTER__SYMBOL);
1266 } 1276 }
1267 } 1277 }
1268 1278
1269 void events_stats__inc(struct events_stats *stats, u32 type) 1279 void events_stats__inc(struct events_stats *stats, u32 type)
1270 { 1280 {
1271 ++stats->nr_events[0]; 1281 ++stats->nr_events[0];
1272 ++stats->nr_events[type]; 1282 ++stats->nr_events[type];
1273 } 1283 }
1274 1284
1275 void hists__inc_nr_events(struct hists *hists, u32 type) 1285 void hists__inc_nr_events(struct hists *hists, u32 type)
1276 { 1286 {
1277 events_stats__inc(&hists->stats, type); 1287 events_stats__inc(&hists->stats, type);
1278 } 1288 }
1279 1289
1280 void hists__inc_nr_samples(struct hists *hists, bool filtered) 1290 void hists__inc_nr_samples(struct hists *hists, bool filtered)
1281 { 1291 {
1282 events_stats__inc(&hists->stats, PERF_RECORD_SAMPLE); 1292 events_stats__inc(&hists->stats, PERF_RECORD_SAMPLE);
1283 if (!filtered) 1293 if (!filtered)
1284 hists->stats.nr_non_filtered_samples++; 1294 hists->stats.nr_non_filtered_samples++;
1285 } 1295 }
1286 1296
1287 static struct hist_entry *hists__add_dummy_entry(struct hists *hists, 1297 static struct hist_entry *hists__add_dummy_entry(struct hists *hists,
1288 struct hist_entry *pair) 1298 struct hist_entry *pair)
1289 { 1299 {
1290 struct rb_root *root; 1300 struct rb_root *root;
1291 struct rb_node **p; 1301 struct rb_node **p;
1292 struct rb_node *parent = NULL; 1302 struct rb_node *parent = NULL;
1293 struct hist_entry *he; 1303 struct hist_entry *he;
1294 int64_t cmp; 1304 int64_t cmp;
1295 1305
1296 if (sort__need_collapse) 1306 if (sort__need_collapse)
1297 root = &hists->entries_collapsed; 1307 root = &hists->entries_collapsed;
1298 else 1308 else
1299 root = hists->entries_in; 1309 root = hists->entries_in;
1300 1310
1301 p = &root->rb_node; 1311 p = &root->rb_node;
1302 1312
1303 while (*p != NULL) { 1313 while (*p != NULL) {
1304 parent = *p; 1314 parent = *p;
1305 he = rb_entry(parent, struct hist_entry, rb_node_in); 1315 he = rb_entry(parent, struct hist_entry, rb_node_in);
1306 1316
1307 cmp = hist_entry__collapse(he, pair); 1317 cmp = hist_entry__collapse(he, pair);
1308 1318
1309 if (!cmp) 1319 if (!cmp)
1310 goto out; 1320 goto out;
1311 1321
1312 if (cmp < 0) 1322 if (cmp < 0)
1313 p = &(*p)->rb_left; 1323 p = &(*p)->rb_left;
1314 else 1324 else
1315 p = &(*p)->rb_right; 1325 p = &(*p)->rb_right;
1316 } 1326 }
1317 1327
1318 he = hist_entry__new(pair, true); 1328 he = hist_entry__new(pair, true);
1319 if (he) { 1329 if (he) {
1320 memset(&he->stat, 0, sizeof(he->stat)); 1330 memset(&he->stat, 0, sizeof(he->stat));
1321 he->hists = hists; 1331 he->hists = hists;
1322 rb_link_node(&he->rb_node_in, parent, p); 1332 rb_link_node(&he->rb_node_in, parent, p);
1323 rb_insert_color(&he->rb_node_in, root); 1333 rb_insert_color(&he->rb_node_in, root);
1324 hists__inc_stats(hists, he); 1334 hists__inc_stats(hists, he);
1325 he->dummy = true; 1335 he->dummy = true;
1326 } 1336 }
1327 out: 1337 out:
1328 return he; 1338 return he;
1329 } 1339 }
1330 1340
1331 static struct hist_entry *hists__find_entry(struct hists *hists, 1341 static struct hist_entry *hists__find_entry(struct hists *hists,
1332 struct hist_entry *he) 1342 struct hist_entry *he)
1333 { 1343 {
1334 struct rb_node *n; 1344 struct rb_node *n;
1335 1345
1336 if (sort__need_collapse) 1346 if (sort__need_collapse)
1337 n = hists->entries_collapsed.rb_node; 1347 n = hists->entries_collapsed.rb_node;
1338 else 1348 else
1339 n = hists->entries_in->rb_node; 1349 n = hists->entries_in->rb_node;
1340 1350
1341 while (n) { 1351 while (n) {
1342 struct hist_entry *iter = rb_entry(n, struct hist_entry, rb_node_in); 1352 struct hist_entry *iter = rb_entry(n, struct hist_entry, rb_node_in);
1343 int64_t cmp = hist_entry__collapse(iter, he); 1353 int64_t cmp = hist_entry__collapse(iter, he);
1344 1354
1345 if (cmp < 0) 1355 if (cmp < 0)
1346 n = n->rb_left; 1356 n = n->rb_left;
1347 else if (cmp > 0) 1357 else if (cmp > 0)
1348 n = n->rb_right; 1358 n = n->rb_right;
1349 else 1359 else
1350 return iter; 1360 return iter;
1351 } 1361 }
1352 1362
1353 return NULL; 1363 return NULL;
1354 } 1364 }
1355 1365
1356 /* 1366 /*
1357 * Look for pairs to link to the leader buckets (hist_entries): 1367 * Look for pairs to link to the leader buckets (hist_entries):
1358 */ 1368 */
1359 void hists__match(struct hists *leader, struct hists *other) 1369 void hists__match(struct hists *leader, struct hists *other)
1360 { 1370 {
1361 struct rb_root *root; 1371 struct rb_root *root;
1362 struct rb_node *nd; 1372 struct rb_node *nd;
1363 struct hist_entry *pos, *pair; 1373 struct hist_entry *pos, *pair;
1364 1374
1365 if (sort__need_collapse) 1375 if (sort__need_collapse)
1366 root = &leader->entries_collapsed; 1376 root = &leader->entries_collapsed;
1367 else 1377 else
1368 root = leader->entries_in; 1378 root = leader->entries_in;
1369 1379
1370 for (nd = rb_first(root); nd; nd = rb_next(nd)) { 1380 for (nd = rb_first(root); nd; nd = rb_next(nd)) {
1371 pos = rb_entry(nd, struct hist_entry, rb_node_in); 1381 pos = rb_entry(nd, struct hist_entry, rb_node_in);
1372 pair = hists__find_entry(other, pos); 1382 pair = hists__find_entry(other, pos);
1373 1383
1374 if (pair) 1384 if (pair)
1375 hist_entry__add_pair(pair, pos); 1385 hist_entry__add_pair(pair, pos);
1376 } 1386 }
1377 } 1387 }
1378 1388
1379 /* 1389 /*
1380 * Look for entries in the other hists that are not present in the leader, if 1390 * Look for entries in the other hists that are not present in the leader, if
1381 * we find them, just add a dummy entry on the leader hists, with period=0, 1391 * we find them, just add a dummy entry on the leader hists, with period=0,
1382 * nr_events=0, to serve as the list header. 1392 * nr_events=0, to serve as the list header.
1383 */ 1393 */
1384 int hists__link(struct hists *leader, struct hists *other) 1394 int hists__link(struct hists *leader, struct hists *other)
1385 { 1395 {
1386 struct rb_root *root; 1396 struct rb_root *root;
1387 struct rb_node *nd; 1397 struct rb_node *nd;
1388 struct hist_entry *pos, *pair; 1398 struct hist_entry *pos, *pair;
1389 1399
1390 if (sort__need_collapse) 1400 if (sort__need_collapse)
1391 root = &other->entries_collapsed; 1401 root = &other->entries_collapsed;
1392 else 1402 else
1393 root = other->entries_in; 1403 root = other->entries_in;
1394 1404
1395 for (nd = rb_first(root); nd; nd = rb_next(nd)) { 1405 for (nd = rb_first(root); nd; nd = rb_next(nd)) {
1396 pos = rb_entry(nd, struct hist_entry, rb_node_in); 1406 pos = rb_entry(nd, struct hist_entry, rb_node_in);
1397 1407
1398 if (!hist_entry__has_pairs(pos)) { 1408 if (!hist_entry__has_pairs(pos)) {
1399 pair = hists__add_dummy_entry(leader, pos); 1409 pair = hists__add_dummy_entry(leader, pos);
1400 if (pair == NULL) 1410 if (pair == NULL)
1401 return -1; 1411 return -1;
1402 hist_entry__add_pair(pos, pair); 1412 hist_entry__add_pair(pos, pair);
1403 } 1413 }
1404 } 1414 }
1405 1415
1406 return 0; 1416 return 0;
1407 } 1417 }
1408 1418
1409 1419
1410 size_t perf_evlist__fprintf_nr_events(struct perf_evlist *evlist, FILE *fp) 1420 size_t perf_evlist__fprintf_nr_events(struct perf_evlist *evlist, FILE *fp)
1411 { 1421 {
1412 struct perf_evsel *pos; 1422 struct perf_evsel *pos;
1413 size_t ret = 0; 1423 size_t ret = 0;
1414 1424
1415 evlist__for_each(evlist, pos) { 1425 evlist__for_each(evlist, pos) {
1416 ret += fprintf(fp, "%s stats:\n", perf_evsel__name(pos)); 1426 ret += fprintf(fp, "%s stats:\n", perf_evsel__name(pos));
1417 ret += events_stats__fprintf(&evsel__hists(pos)->stats, fp); 1427 ret += events_stats__fprintf(&evsel__hists(pos)->stats, fp);
1418 } 1428 }
1419 1429
1420 return ret; 1430 return ret;
1421 } 1431 }
1422 1432
1423 1433
1424 u64 hists__total_period(struct hists *hists) 1434 u64 hists__total_period(struct hists *hists)
1425 { 1435 {
1426 return symbol_conf.filter_relative ? hists->stats.total_non_filtered_period : 1436 return symbol_conf.filter_relative ? hists->stats.total_non_filtered_period :
1427 hists->stats.total_period; 1437 hists->stats.total_period;
1428 } 1438 }
1429 1439
1430 int parse_filter_percentage(const struct option *opt __maybe_unused, 1440 int parse_filter_percentage(const struct option *opt __maybe_unused,
1431 const char *arg, int unset __maybe_unused) 1441 const char *arg, int unset __maybe_unused)
1432 { 1442 {
1433 if (!strcmp(arg, "relative")) 1443 if (!strcmp(arg, "relative"))
1434 symbol_conf.filter_relative = true; 1444 symbol_conf.filter_relative = true;
1435 else if (!strcmp(arg, "absolute")) 1445 else if (!strcmp(arg, "absolute"))
1436 symbol_conf.filter_relative = false; 1446 symbol_conf.filter_relative = false;
1437 else 1447 else
1438 return -1; 1448 return -1;
1439 1449
1440 return 0; 1450 return 0;
1441 } 1451 }
1442 1452
1443 int perf_hist_config(const char *var, const char *value) 1453 int perf_hist_config(const char *var, const char *value)
1444 { 1454 {
1445 if (!strcmp(var, "hist.percentage")) 1455 if (!strcmp(var, "hist.percentage"))
1446 return parse_filter_percentage(NULL, value, 0); 1456 return parse_filter_percentage(NULL, value, 0);
1447 1457
1448 return 0; 1458 return 0;
1449 } 1459 }
1450 1460
1451 static int hists_evsel__init(struct perf_evsel *evsel) 1461 static int hists_evsel__init(struct perf_evsel *evsel)
1452 { 1462 {
1453 struct hists *hists = evsel__hists(evsel); 1463 struct hists *hists = evsel__hists(evsel);
1454 1464
1455 memset(hists, 0, sizeof(*hists)); 1465 memset(hists, 0, sizeof(*hists));
1456 hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT; 1466 hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT;
1457 hists->entries_in = &hists->entries_in_array[0]; 1467 hists->entries_in = &hists->entries_in_array[0];
1458 hists->entries_collapsed = RB_ROOT; 1468 hists->entries_collapsed = RB_ROOT;
1459 hists->entries = RB_ROOT; 1469 hists->entries = RB_ROOT;
1460 pthread_mutex_init(&hists->lock, NULL); 1470 pthread_mutex_init(&hists->lock, NULL);
1461 return 0; 1471 return 0;
1462 } 1472 }
1463 1473
1464 /* 1474 /*
1465 * XXX We probably need a hists_evsel__exit() to free the hist_entries 1475 * XXX We probably need a hists_evsel__exit() to free the hist_entries
1466 * stored in the rbtree... 1476 * stored in the rbtree...
1467 */ 1477 */
1468 1478
1469 int hists__init(void) 1479 int hists__init(void)
1470 { 1480 {
1471 int err = perf_evsel__object_config(sizeof(struct hists_evsel), 1481 int err = perf_evsel__object_config(sizeof(struct hists_evsel),
1472 hists_evsel__init, NULL); 1482 hists_evsel__init, NULL);
1473 if (err) 1483 if (err)
1474 fputs("FATAL ERROR: Couldn't setup hists class\n", stderr); 1484 fputs("FATAL ERROR: Couldn't setup hists class\n", stderr);
1475 1485
1476 return err; 1486 return err;
1477 } 1487 }
1478 1488
tools/perf/util/hist.h
Diff comments   View file @ ddb321a
1 #ifndef __PERF_HIST_H 1 #ifndef __PERF_HIST_H
2 #define __PERF_HIST_H 2 #define __PERF_HIST_H
3 3
4 #include <linux/types.h> 4 #include <linux/types.h>
5 #include <pthread.h> 5 #include <pthread.h>
6 #include "callchain.h" 6 #include "callchain.h"
7 #include "evsel.h" 7 #include "evsel.h"
8 #include "header.h" 8 #include "header.h"
9 #include "color.h" 9 #include "color.h"
10 #include "ui/progress.h" 10 #include "ui/progress.h"
11 11
12 struct hist_entry; 12 struct hist_entry;
13 struct addr_location; 13 struct addr_location;
14 struct symbol; 14 struct symbol;
15 15
16 enum hist_filter { 16 enum hist_filter {
17 HIST_FILTER__DSO, 17 HIST_FILTER__DSO,
18 HIST_FILTER__THREAD, 18 HIST_FILTER__THREAD,
19 HIST_FILTER__PARENT, 19 HIST_FILTER__PARENT,
20 HIST_FILTER__SYMBOL, 20 HIST_FILTER__SYMBOL,
21 HIST_FILTER__GUEST, 21 HIST_FILTER__GUEST,
22 HIST_FILTER__HOST, 22 HIST_FILTER__HOST,
23 }; 23 };
24 24
25 enum hist_column { 25 enum hist_column {
26 HISTC_SYMBOL, 26 HISTC_SYMBOL,
27 HISTC_DSO, 27 HISTC_DSO,
28 HISTC_THREAD, 28 HISTC_THREAD,
29 HISTC_COMM, 29 HISTC_COMM,
30 HISTC_PARENT, 30 HISTC_PARENT,
31 HISTC_CPU, 31 HISTC_CPU,
32 HISTC_SRCLINE, 32 HISTC_SRCLINE,
33 HISTC_MISPREDICT, 33 HISTC_MISPREDICT,
34 HISTC_IN_TX, 34 HISTC_IN_TX,
35 HISTC_ABORT, 35 HISTC_ABORT,
36 HISTC_SYMBOL_FROM, 36 HISTC_SYMBOL_FROM,
37 HISTC_SYMBOL_TO, 37 HISTC_SYMBOL_TO,
38 HISTC_DSO_FROM, 38 HISTC_DSO_FROM,
39 HISTC_DSO_TO, 39 HISTC_DSO_TO,
40 HISTC_LOCAL_WEIGHT, 40 HISTC_LOCAL_WEIGHT,
41 HISTC_GLOBAL_WEIGHT, 41 HISTC_GLOBAL_WEIGHT,
42 HISTC_MEM_DADDR_SYMBOL, 42 HISTC_MEM_DADDR_SYMBOL,
43 HISTC_MEM_DADDR_DSO, 43 HISTC_MEM_DADDR_DSO,
44 HISTC_MEM_LOCKED, 44 HISTC_MEM_LOCKED,
45 HISTC_MEM_TLB, 45 HISTC_MEM_TLB,
46 HISTC_MEM_LVL, 46 HISTC_MEM_LVL,
47 HISTC_MEM_SNOOP, 47 HISTC_MEM_SNOOP,
48 HISTC_MEM_DCACHELINE, 48 HISTC_MEM_DCACHELINE,
49 HISTC_TRANSACTION, 49 HISTC_TRANSACTION,
50 HISTC_NR_COLS, /* Last entry */ 50 HISTC_NR_COLS, /* Last entry */
51 }; 51 };
52 52
53 struct thread; 53 struct thread;
54 struct dso; 54 struct dso;
55 55
56 struct hists { 56 struct hists {
57 struct rb_root entries_in_array[2]; 57 struct rb_root entries_in_array[2];
58 struct rb_root *entries_in; 58 struct rb_root *entries_in;
59 struct rb_root entries; 59 struct rb_root entries;
60 struct rb_root entries_collapsed; 60 struct rb_root entries_collapsed;
61 u64 nr_entries; 61 u64 nr_entries;
62 u64 nr_non_filtered_entries; 62 u64 nr_non_filtered_entries;
63 const struct thread *thread_filter; 63 const struct thread *thread_filter;
64 const struct dso *dso_filter; 64 const struct dso *dso_filter;
65 const char *uid_filter_str; 65 const char *uid_filter_str;
66 const char *symbol_filter_str; 66 const char *symbol_filter_str;
67 pthread_mutex_t lock; 67 pthread_mutex_t lock;
68 struct events_stats stats; 68 struct events_stats stats;
69 u64 event_stream; 69 u64 event_stream;
70 u16 col_len[HISTC_NR_COLS]; 70 u16 col_len[HISTC_NR_COLS];
71 }; 71 };
72 72
73 struct hist_entry_iter; 73 struct hist_entry_iter;
74 74
75 struct hist_iter_ops { 75 struct hist_iter_ops {
76 int (*prepare_entry)(struct hist_entry_iter *, struct addr_location *); 76 int (*prepare_entry)(struct hist_entry_iter *, struct addr_location *);
77 int (*add_single_entry)(struct hist_entry_iter *, struct addr_location *); 77 int (*add_single_entry)(struct hist_entry_iter *, struct addr_location *);
78 int (*next_entry)(struct hist_entry_iter *, struct addr_location *); 78 int (*next_entry)(struct hist_entry_iter *, struct addr_location *);
79 int (*add_next_entry)(struct hist_entry_iter *, struct addr_location *); 79 int (*add_next_entry)(struct hist_entry_iter *, struct addr_location *);
80 int (*finish_entry)(struct hist_entry_iter *, struct addr_location *); 80 int (*finish_entry)(struct hist_entry_iter *, struct addr_location *);
81 }; 81 };
82 82
83 struct hist_entry_iter { 83 struct hist_entry_iter {
84 int total; 84 int total;
85 int curr; 85 int curr;
86 86
87 bool hide_unresolved; 87 bool hide_unresolved;
88 88
89 struct perf_evsel *evsel; 89 struct perf_evsel *evsel;
90 struct perf_sample *sample; 90 struct perf_sample *sample;
91 struct hist_entry *he; 91 struct hist_entry *he;
92 struct symbol *parent; 92 struct symbol *parent;
93 void *priv; 93 void *priv;
94 94
95 const struct hist_iter_ops *ops; 95 const struct hist_iter_ops *ops;
96 /* user-defined callback function (optional) */ 96 /* user-defined callback function (optional) */
97 int (*add_entry_cb)(struct hist_entry_iter *iter, 97 int (*add_entry_cb)(struct hist_entry_iter *iter,
98 struct addr_location *al, bool single, void *arg); 98 struct addr_location *al, bool single, void *arg);
99 }; 99 };
100 100
101 extern const struct hist_iter_ops hist_iter_normal; 101 extern const struct hist_iter_ops hist_iter_normal;
102 extern const struct hist_iter_ops hist_iter_branch; 102 extern const struct hist_iter_ops hist_iter_branch;
103 extern const struct hist_iter_ops hist_iter_mem; 103 extern const struct hist_iter_ops hist_iter_mem;
104 extern const struct hist_iter_ops hist_iter_cumulative; 104 extern const struct hist_iter_ops hist_iter_cumulative;
105 105
106 struct hist_entry *__hists__add_entry(struct hists *hists, 106 struct hist_entry *__hists__add_entry(struct hists *hists,
107 struct addr_location *al, 107 struct addr_location *al,
108 struct symbol *parent, 108 struct symbol *parent,
109 struct branch_info *bi, 109 struct branch_info *bi,
110 struct mem_info *mi, u64 period, 110 struct mem_info *mi, u64 period,
111 u64 weight, u64 transaction, 111 u64 weight, u64 transaction,
112 bool sample_self); 112 bool sample_self);
113 int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, 113 int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al,
114 struct perf_evsel *evsel, struct perf_sample *sample, 114 struct perf_evsel *evsel, struct perf_sample *sample,
115 int max_stack_depth, void *arg); 115 int max_stack_depth, void *arg);
116 116
117 int64_t hist_entry__cmp(struct hist_entry *left, struct hist_entry *right); 117 int64_t hist_entry__cmp(struct hist_entry *left, struct hist_entry *right);
118 int64_t hist_entry__collapse(struct hist_entry *left, struct hist_entry *right); 118 int64_t hist_entry__collapse(struct hist_entry *left, struct hist_entry *right);
119 int hist_entry__transaction_len(void); 119 int hist_entry__transaction_len(void);
120 int hist_entry__sort_snprintf(struct hist_entry *he, char *bf, size_t size, 120 int hist_entry__sort_snprintf(struct hist_entry *he, char *bf, size_t size,
121 struct hists *hists); 121 struct hists *hists);
122 void hist_entry__free(struct hist_entry *); 122 void hist_entry__free(struct hist_entry *);
123 123
124 void hists__output_resort(struct hists *hists); 124 void hists__output_resort(struct hists *hists, struct ui_progress *prog);
125 void hists__collapse_resort(struct hists *hists, struct ui_progress *prog); 125 void hists__collapse_resort(struct hists *hists, struct ui_progress *prog);
126 126
127 void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel); 127 void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel);
128 void hists__delete_entries(struct hists *hists); 128 void hists__delete_entries(struct hists *hists);
129 void hists__output_recalc_col_len(struct hists *hists, int max_rows); 129 void hists__output_recalc_col_len(struct hists *hists, int max_rows);
130 130
131 u64 hists__total_period(struct hists *hists); 131 u64 hists__total_period(struct hists *hists);
132 void hists__reset_stats(struct hists *hists); 132 void hists__reset_stats(struct hists *hists);
133 void hists__inc_stats(struct hists *hists, struct hist_entry *h); 133 void hists__inc_stats(struct hists *hists, struct hist_entry *h);
134 void hists__inc_nr_events(struct hists *hists, u32 type); 134 void hists__inc_nr_events(struct hists *hists, u32 type);
135 void hists__inc_nr_samples(struct hists *hists, bool filtered); 135 void hists__inc_nr_samples(struct hists *hists, bool filtered);
136 void events_stats__inc(struct events_stats *stats, u32 type); 136 void events_stats__inc(struct events_stats *stats, u32 type);
137 size_t events_stats__fprintf(struct events_stats *stats, FILE *fp); 137 size_t events_stats__fprintf(struct events_stats *stats, FILE *fp);
138 138
139 size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows, 139 size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows,
140 int max_cols, float min_pcnt, FILE *fp); 140 int max_cols, float min_pcnt, FILE *fp);
141 size_t perf_evlist__fprintf_nr_events(struct perf_evlist *evlist, FILE *fp); 141 size_t perf_evlist__fprintf_nr_events(struct perf_evlist *evlist, FILE *fp);
142 142
143 void hists__filter_by_dso(struct hists *hists); 143 void hists__filter_by_dso(struct hists *hists);
144 void hists__filter_by_thread(struct hists *hists); 144 void hists__filter_by_thread(struct hists *hists);
145 void hists__filter_by_symbol(struct hists *hists); 145 void hists__filter_by_symbol(struct hists *hists);
146 146
147 static inline bool hists__has_filter(struct hists *hists) 147 static inline bool hists__has_filter(struct hists *hists)
148 { 148 {
149 return hists->thread_filter || hists->dso_filter || 149 return hists->thread_filter || hists->dso_filter ||
150 hists->symbol_filter_str; 150 hists->symbol_filter_str;
151 } 151 }
152 152
153 u16 hists__col_len(struct hists *hists, enum hist_column col); 153 u16 hists__col_len(struct hists *hists, enum hist_column col);
154 void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len); 154 void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len);
155 bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len); 155 bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len);
156 void hists__reset_col_len(struct hists *hists); 156 void hists__reset_col_len(struct hists *hists);
157 void hists__calc_col_len(struct hists *hists, struct hist_entry *he); 157 void hists__calc_col_len(struct hists *hists, struct hist_entry *he);
158 158
159 void hists__match(struct hists *leader, struct hists *other); 159 void hists__match(struct hists *leader, struct hists *other);
160 int hists__link(struct hists *leader, struct hists *other); 160 int hists__link(struct hists *leader, struct hists *other);
161 161
162 struct hists_evsel { 162 struct hists_evsel {
163 struct perf_evsel evsel; 163 struct perf_evsel evsel;
164 struct hists hists; 164 struct hists hists;
165 }; 165 };
166 166
167 static inline struct perf_evsel *hists_to_evsel(struct hists *hists) 167 static inline struct perf_evsel *hists_to_evsel(struct hists *hists)
168 { 168 {
169 struct hists_evsel *hevsel = container_of(hists, struct hists_evsel, hists); 169 struct hists_evsel *hevsel = container_of(hists, struct hists_evsel, hists);
170 return &hevsel->evsel; 170 return &hevsel->evsel;
171 } 171 }
172 172
173 static inline struct hists *evsel__hists(struct perf_evsel *evsel) 173 static inline struct hists *evsel__hists(struct perf_evsel *evsel)
174 { 174 {
175 struct hists_evsel *hevsel = (struct hists_evsel *)evsel; 175 struct hists_evsel *hevsel = (struct hists_evsel *)evsel;
176 return &hevsel->hists; 176 return &hevsel->hists;
177 } 177 }
178 178
179 int hists__init(void); 179 int hists__init(void);
180 180
181 struct perf_hpp { 181 struct perf_hpp {
182 char *buf; 182 char *buf;
183 size_t size; 183 size_t size;
184 const char *sep; 184 const char *sep;
185 void *ptr; 185 void *ptr;
186 }; 186 };
187 187
188 struct perf_hpp_fmt { 188 struct perf_hpp_fmt {
189 const char *name; 189 const char *name;
190 int (*header)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 190 int (*header)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
191 struct perf_evsel *evsel); 191 struct perf_evsel *evsel);
192 int (*width)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 192 int (*width)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
193 struct perf_evsel *evsel); 193 struct perf_evsel *evsel);
194 int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 194 int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
195 struct hist_entry *he); 195 struct hist_entry *he);
196 int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 196 int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
197 struct hist_entry *he); 197 struct hist_entry *he);
198 int64_t (*cmp)(struct hist_entry *a, struct hist_entry *b); 198 int64_t (*cmp)(struct hist_entry *a, struct hist_entry *b);
199 int64_t (*collapse)(struct hist_entry *a, struct hist_entry *b); 199 int64_t (*collapse)(struct hist_entry *a, struct hist_entry *b);
200 int64_t (*sort)(struct hist_entry *a, struct hist_entry *b); 200 int64_t (*sort)(struct hist_entry *a, struct hist_entry *b);
201 201
202 struct list_head list; 202 struct list_head list;
203 struct list_head sort_list; 203 struct list_head sort_list;
204 bool elide; 204 bool elide;
205 int len; 205 int len;
206 int user_len; 206 int user_len;
207 }; 207 };
208 208
209 extern struct list_head perf_hpp__list; 209 extern struct list_head perf_hpp__list;
210 extern struct list_head perf_hpp__sort_list; 210 extern struct list_head perf_hpp__sort_list;
211 211
212 #define perf_hpp__for_each_format(format) \ 212 #define perf_hpp__for_each_format(format) \
213 list_for_each_entry(format, &perf_hpp__list, list) 213 list_for_each_entry(format, &perf_hpp__list, list)
214 214
215 #define perf_hpp__for_each_format_safe(format, tmp) \ 215 #define perf_hpp__for_each_format_safe(format, tmp) \
216 list_for_each_entry_safe(format, tmp, &perf_hpp__list, list) 216 list_for_each_entry_safe(format, tmp, &perf_hpp__list, list)
217 217
218 #define perf_hpp__for_each_sort_list(format) \ 218 #define perf_hpp__for_each_sort_list(format) \
219 list_for_each_entry(format, &perf_hpp__sort_list, sort_list) 219 list_for_each_entry(format, &perf_hpp__sort_list, sort_list)
220 220
221 #define perf_hpp__for_each_sort_list_safe(format, tmp) \ 221 #define perf_hpp__for_each_sort_list_safe(format, tmp) \
222 list_for_each_entry_safe(format, tmp, &perf_hpp__sort_list, sort_list) 222 list_for_each_entry_safe(format, tmp, &perf_hpp__sort_list, sort_list)
223 223
224 extern struct perf_hpp_fmt perf_hpp__format[]; 224 extern struct perf_hpp_fmt perf_hpp__format[];
225 225
226 enum { 226 enum {
227 /* Matches perf_hpp__format array. */ 227 /* Matches perf_hpp__format array. */
228 PERF_HPP__OVERHEAD, 228 PERF_HPP__OVERHEAD,
229 PERF_HPP__OVERHEAD_SYS, 229 PERF_HPP__OVERHEAD_SYS,
230 PERF_HPP__OVERHEAD_US, 230 PERF_HPP__OVERHEAD_US,
231 PERF_HPP__OVERHEAD_GUEST_SYS, 231 PERF_HPP__OVERHEAD_GUEST_SYS,
232 PERF_HPP__OVERHEAD_GUEST_US, 232 PERF_HPP__OVERHEAD_GUEST_US,
233 PERF_HPP__OVERHEAD_ACC, 233 PERF_HPP__OVERHEAD_ACC,
234 PERF_HPP__SAMPLES, 234 PERF_HPP__SAMPLES,
235 PERF_HPP__PERIOD, 235 PERF_HPP__PERIOD,
236 236
237 PERF_HPP__MAX_INDEX 237 PERF_HPP__MAX_INDEX
238 }; 238 };
239 239
240 void perf_hpp__init(void); 240 void perf_hpp__init(void);
241 void perf_hpp__column_register(struct perf_hpp_fmt *format); 241 void perf_hpp__column_register(struct perf_hpp_fmt *format);
242 void perf_hpp__column_unregister(struct perf_hpp_fmt *format); 242 void perf_hpp__column_unregister(struct perf_hpp_fmt *format);
243 void perf_hpp__column_enable(unsigned col); 243 void perf_hpp__column_enable(unsigned col);
244 void perf_hpp__column_disable(unsigned col); 244 void perf_hpp__column_disable(unsigned col);
245 void perf_hpp__cancel_cumulate(void); 245 void perf_hpp__cancel_cumulate(void);
246 246
247 void perf_hpp__register_sort_field(struct perf_hpp_fmt *format); 247 void perf_hpp__register_sort_field(struct perf_hpp_fmt *format);
248 void perf_hpp__setup_output_field(void); 248 void perf_hpp__setup_output_field(void);
249 void perf_hpp__reset_output_field(void); 249 void perf_hpp__reset_output_field(void);
250 void perf_hpp__append_sort_keys(void); 250 void perf_hpp__append_sort_keys(void);
251 251
252 bool perf_hpp__is_sort_entry(struct perf_hpp_fmt *format); 252 bool perf_hpp__is_sort_entry(struct perf_hpp_fmt *format);
253 bool perf_hpp__same_sort_entry(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b); 253 bool perf_hpp__same_sort_entry(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b);
254 254
255 static inline bool perf_hpp__should_skip(struct perf_hpp_fmt *format) 255 static inline bool perf_hpp__should_skip(struct perf_hpp_fmt *format)
256 { 256 {
257 return format->elide; 257 return format->elide;
258 } 258 }
259 259
260 void perf_hpp__reset_width(struct perf_hpp_fmt *fmt, struct hists *hists); 260 void perf_hpp__reset_width(struct perf_hpp_fmt *fmt, struct hists *hists);
261 void perf_hpp__reset_sort_width(struct perf_hpp_fmt *fmt, struct hists *hists); 261 void perf_hpp__reset_sort_width(struct perf_hpp_fmt *fmt, struct hists *hists);
262 void perf_hpp__set_user_width(const char *width_list_str); 262 void perf_hpp__set_user_width(const char *width_list_str);
263 263
264 typedef u64 (*hpp_field_fn)(struct hist_entry *he); 264 typedef u64 (*hpp_field_fn)(struct hist_entry *he);
265 typedef int (*hpp_callback_fn)(struct perf_hpp *hpp, bool front); 265 typedef int (*hpp_callback_fn)(struct perf_hpp *hpp, bool front);
266 typedef int (*hpp_snprint_fn)(struct perf_hpp *hpp, const char *fmt, ...); 266 typedef int (*hpp_snprint_fn)(struct perf_hpp *hpp, const char *fmt, ...);
267 267
268 int hpp__fmt(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 268 int hpp__fmt(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
269 struct hist_entry *he, hpp_field_fn get_field, 269 struct hist_entry *he, hpp_field_fn get_field,
270 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent); 270 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent);
271 int hpp__fmt_acc(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, 271 int hpp__fmt_acc(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp,
272 struct hist_entry *he, hpp_field_fn get_field, 272 struct hist_entry *he, hpp_field_fn get_field,
273 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent); 273 const char *fmtstr, hpp_snprint_fn print_fn, bool fmt_percent);
274 274
275 static inline void advance_hpp(struct perf_hpp *hpp, int inc) 275 static inline void advance_hpp(struct perf_hpp *hpp, int inc)
276 { 276 {
277 hpp->buf += inc; 277 hpp->buf += inc;
278 hpp->size -= inc; 278 hpp->size -= inc;
279 } 279 }
280 280
281 static inline size_t perf_hpp__use_color(void) 281 static inline size_t perf_hpp__use_color(void)
282 { 282 {
283 return !symbol_conf.field_sep; 283 return !symbol_conf.field_sep;
284 } 284 }
285 285
286 static inline size_t perf_hpp__color_overhead(void) 286 static inline size_t perf_hpp__color_overhead(void)
287 { 287 {
288 return perf_hpp__use_color() ? 288 return perf_hpp__use_color() ?
289 (COLOR_MAXLEN + sizeof(PERF_COLOR_RESET)) * PERF_HPP__MAX_INDEX 289 (COLOR_MAXLEN + sizeof(PERF_COLOR_RESET)) * PERF_HPP__MAX_INDEX
290 : 0; 290 : 0;
291 } 291 }
292 292
293 struct perf_evlist; 293 struct perf_evlist;
294 294
295 struct hist_browser_timer { 295 struct hist_browser_timer {
296 void (*timer)(void *arg); 296 void (*timer)(void *arg);
297 void *arg; 297 void *arg;
298 int refresh; 298 int refresh;
299 }; 299 };
300 300
301 #ifdef HAVE_SLANG_SUPPORT 301 #ifdef HAVE_SLANG_SUPPORT
302 #include "../ui/keysyms.h" 302 #include "../ui/keysyms.h"
303 int hist_entry__tui_annotate(struct hist_entry *he, struct perf_evsel *evsel, 303 int hist_entry__tui_annotate(struct hist_entry *he, struct perf_evsel *evsel,
304 struct hist_browser_timer *hbt); 304 struct hist_browser_timer *hbt);
305 305
306 int perf_evlist__tui_browse_hists(struct perf_evlist *evlist, const char *help, 306 int perf_evlist__tui_browse_hists(struct perf_evlist *evlist, const char *help,
307 struct hist_browser_timer *hbt, 307 struct hist_browser_timer *hbt,
308 float min_pcnt, 308 float min_pcnt,
309 struct perf_session_env *env); 309 struct perf_session_env *env);
310 int script_browse(const char *script_opt); 310 int script_browse(const char *script_opt);
311 #else 311 #else
312 static inline 312 static inline
313 int perf_evlist__tui_browse_hists(struct perf_evlist *evlist __maybe_unused, 313 int perf_evlist__tui_browse_hists(struct perf_evlist *evlist __maybe_unused,
314 const char *help __maybe_unused, 314 const char *help __maybe_unused,
315 struct hist_browser_timer *hbt __maybe_unused, 315 struct hist_browser_timer *hbt __maybe_unused,
316 float min_pcnt __maybe_unused, 316 float min_pcnt __maybe_unused,
317 struct perf_session_env *env __maybe_unused) 317 struct perf_session_env *env __maybe_unused)
318 { 318 {
319 return 0; 319 return 0;
320 } 320 }
321 321
322 static inline int hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, 322 static inline int hist_entry__tui_annotate(struct hist_entry *he __maybe_unused,
323 struct perf_evsel *evsel __maybe_unused, 323 struct perf_evsel *evsel __maybe_unused,
324 struct hist_browser_timer *hbt __maybe_unused) 324 struct hist_browser_timer *hbt __maybe_unused)
325 { 325 {
326 return 0; 326 return 0;
327 } 327 }
328 328
329 static inline int script_browse(const char *script_opt __maybe_unused) 329 static inline int script_browse(const char *script_opt __maybe_unused)
330 { 330 {
331 return 0; 331 return 0;
332 } 332 }
333 333
334 #define K_LEFT -1000 334 #define K_LEFT -1000
335 #define K_RIGHT -2000 335 #define K_RIGHT -2000
336 #define K_SWITCH_INPUT_DATA -3000 336 #define K_SWITCH_INPUT_DATA -3000
337 #endif 337 #endif
338 338
339 unsigned int hists__sort_list_width(struct hists *hists); 339 unsigned int hists__sort_list_width(struct hists *hists);
340 340
341 struct option; 341 struct option;
342 int parse_filter_percentage(const struct option *opt __maybe_unused, 342 int parse_filter_percentage(const struct option *opt __maybe_unused,
343 const char *arg, int unset __maybe_unused); 343 const char *arg, int unset __maybe_unused);
344 int perf_hist_config(const char *var, const char *value); 344 int perf_hist_config(const char *var, const char *value);
345 345
346 #endif /* __PERF_HIST_H */ 346 #endif /* __PERF_HIST_H */
347 347
tools/perf/util/probe-event.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * probe-event.c : perf-probe definition to probe_events format converter 2 * probe-event.c : perf-probe definition to probe_events format converter
3 * 3 *
4 * Written by Masami Hiramatsu <mhiramat@redhat.com> 4 * Written by Masami Hiramatsu <mhiramat@redhat.com>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or 8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version. 9 * (at your option) any later version.
10 * 10 *
11 * This program is distributed in the hope that it will be useful, 11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details. 14 * GNU General Public License for more details.
15 * 15 *
16 * You should have received a copy of the GNU General Public License 16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software 17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * 19 *
20 */ 20 */
21 21
22 #include <sys/utsname.h> 22 #include <sys/utsname.h>
23 #include <sys/types.h> 23 #include <sys/types.h>
24 #include <sys/stat.h> 24 #include <sys/stat.h>
25 #include <fcntl.h> 25 #include <fcntl.h>
26 #include <errno.h> 26 #include <errno.h>
27 #include <stdio.h> 27 #include <stdio.h>
28 #include <unistd.h> 28 #include <unistd.h>
29 #include <stdlib.h> 29 #include <stdlib.h>
30 #include <string.h> 30 #include <string.h>
31 #include <stdarg.h> 31 #include <stdarg.h>
32 #include <limits.h> 32 #include <limits.h>
33 #include <elf.h> 33 #include <elf.h>
34 34
35 #include "util.h" 35 #include "util.h"
36 #include "event.h" 36 #include "event.h"
37 #include "strlist.h" 37 #include "strlist.h"
38 #include "debug.h" 38 #include "debug.h"
39 #include "cache.h" 39 #include "cache.h"
40 #include "color.h" 40 #include "color.h"
41 #include "symbol.h" 41 #include "symbol.h"
42 #include "thread.h" 42 #include "thread.h"
43 #include <api/fs/debugfs.h> 43 #include <api/fs/debugfs.h>
44 #include "trace-event.h" /* For __maybe_unused */ 44 #include "trace-event.h" /* For __maybe_unused */
45 #include "probe-event.h" 45 #include "probe-event.h"
46 #include "probe-finder.h" 46 #include "probe-finder.h"
47 #include "session.h" 47 #include "session.h"
48 48
49 #define MAX_CMDLEN 256 49 #define MAX_CMDLEN 256
50 #define PERFPROBE_GROUP "probe" 50 #define PERFPROBE_GROUP "probe"
51 51
52 bool probe_event_dry_run; /* Dry run flag */ 52 bool probe_event_dry_run; /* Dry run flag */
53 53
54 #define semantic_error(msg ...) pr_err("Semantic error :" msg) 54 #define semantic_error(msg ...) pr_err("Semantic error :" msg)
55 55
56 /* If there is no space to write, returns -E2BIG. */ 56 /* If there is no space to write, returns -E2BIG. */
57 static int e_snprintf(char *str, size_t size, const char *format, ...) 57 static int e_snprintf(char *str, size_t size, const char *format, ...)
58 __attribute__((format(printf, 3, 4))); 58 __attribute__((format(printf, 3, 4)));
59 59
60 static int e_snprintf(char *str, size_t size, const char *format, ...) 60 static int e_snprintf(char *str, size_t size, const char *format, ...)
61 { 61 {
62 int ret; 62 int ret;
63 va_list ap; 63 va_list ap;
64 va_start(ap, format); 64 va_start(ap, format);
65 ret = vsnprintf(str, size, format, ap); 65 ret = vsnprintf(str, size, format, ap);
66 va_end(ap); 66 va_end(ap);
67 if (ret >= (int)size) 67 if (ret >= (int)size)
68 ret = -E2BIG; 68 ret = -E2BIG;
69 return ret; 69 return ret;
70 } 70 }
71 71
72 static char *synthesize_perf_probe_point(struct perf_probe_point *pp); 72 static char *synthesize_perf_probe_point(struct perf_probe_point *pp);
73 static void clear_probe_trace_event(struct probe_trace_event *tev); 73 static void clear_probe_trace_event(struct probe_trace_event *tev);
74 static struct machine *host_machine; 74 static struct machine *host_machine;
75 75
76 /* Initialize symbol maps and path of vmlinux/modules */ 76 /* Initialize symbol maps and path of vmlinux/modules */
77 static int init_symbol_maps(bool user_only) 77 static int init_symbol_maps(bool user_only)
78 { 78 {
79 int ret; 79 int ret;
80 80
81 symbol_conf.sort_by_name = true; 81 symbol_conf.sort_by_name = true;
82 ret = symbol__init(NULL); 82 ret = symbol__init(NULL);
83 if (ret < 0) { 83 if (ret < 0) {
84 pr_debug("Failed to init symbol map.\n"); 84 pr_debug("Failed to init symbol map.\n");
85 goto out; 85 goto out;
86 } 86 }
87 87
88 if (host_machine || user_only) /* already initialized */ 88 if (host_machine || user_only) /* already initialized */
89 return 0; 89 return 0;
90 90
91 if (symbol_conf.vmlinux_name) 91 if (symbol_conf.vmlinux_name)
92 pr_debug("Use vmlinux: %s\n", symbol_conf.vmlinux_name); 92 pr_debug("Use vmlinux: %s\n", symbol_conf.vmlinux_name);
93 93
94 host_machine = machine__new_host(); 94 host_machine = machine__new_host();
95 if (!host_machine) { 95 if (!host_machine) {
96 pr_debug("machine__new_host() failed.\n"); 96 pr_debug("machine__new_host() failed.\n");
97 symbol__exit(); 97 symbol__exit();
98 ret = -1; 98 ret = -1;
99 } 99 }
100 out: 100 out:
101 if (ret < 0) 101 if (ret < 0)
102 pr_warning("Failed to init vmlinux path.\n"); 102 pr_warning("Failed to init vmlinux path.\n");
103 return ret; 103 return ret;
104 } 104 }
105 105
106 static void exit_symbol_maps(void) 106 static void exit_symbol_maps(void)
107 { 107 {
108 if (host_machine) { 108 if (host_machine) {
109 machine__delete(host_machine); 109 machine__delete(host_machine);
110 host_machine = NULL; 110 host_machine = NULL;
111 } 111 }
112 symbol__exit(); 112 symbol__exit();
113 } 113 }
114 114
115 static struct symbol *__find_kernel_function_by_name(const char *name, 115 static struct symbol *__find_kernel_function_by_name(const char *name,
116 struct map **mapp) 116 struct map **mapp)
117 { 117 {
118 return machine__find_kernel_function_by_name(host_machine, name, mapp, 118 return machine__find_kernel_function_by_name(host_machine, name, mapp,
119 NULL); 119 NULL);
120 } 120 }
121 121
122 static struct symbol *__find_kernel_function(u64 addr, struct map **mapp) 122 static struct symbol *__find_kernel_function(u64 addr, struct map **mapp)
123 { 123 {
124 return machine__find_kernel_function(host_machine, addr, mapp, NULL); 124 return machine__find_kernel_function(host_machine, addr, mapp, NULL);
125 } 125 }
126 126
127 static struct ref_reloc_sym *kernel_get_ref_reloc_sym(void) 127 static struct ref_reloc_sym *kernel_get_ref_reloc_sym(void)
128 { 128 {
129 /* kmap->ref_reloc_sym should be set if host_machine is initialized */ 129 /* kmap->ref_reloc_sym should be set if host_machine is initialized */
130 struct kmap *kmap; 130 struct kmap *kmap;
131 131
132 if (map__load(host_machine->vmlinux_maps[MAP__FUNCTION], NULL) < 0) 132 if (map__load(host_machine->vmlinux_maps[MAP__FUNCTION], NULL) < 0)
133 return NULL; 133 return NULL;
134 134
135 kmap = map__kmap(host_machine->vmlinux_maps[MAP__FUNCTION]); 135 kmap = map__kmap(host_machine->vmlinux_maps[MAP__FUNCTION]);
136 return kmap->ref_reloc_sym; 136 return kmap->ref_reloc_sym;
137 } 137 }
138 138
139 static u64 kernel_get_symbol_address_by_name(const char *name, bool reloc) 139 static u64 kernel_get_symbol_address_by_name(const char *name, bool reloc)
140 { 140 {
141 struct ref_reloc_sym *reloc_sym; 141 struct ref_reloc_sym *reloc_sym;
142 struct symbol *sym; 142 struct symbol *sym;
143 struct map *map; 143 struct map *map;
144 144
145 /* ref_reloc_sym is just a label. Need a special fix*/ 145 /* ref_reloc_sym is just a label. Need a special fix*/
146 reloc_sym = kernel_get_ref_reloc_sym(); 146 reloc_sym = kernel_get_ref_reloc_sym();
147 if (reloc_sym && strcmp(name, reloc_sym->name) == 0) 147 if (reloc_sym && strcmp(name, reloc_sym->name) == 0)
148 return (reloc) ? reloc_sym->addr : reloc_sym->unrelocated_addr; 148 return (reloc) ? reloc_sym->addr : reloc_sym->unrelocated_addr;
149 else { 149 else {
150 sym = __find_kernel_function_by_name(name, &map); 150 sym = __find_kernel_function_by_name(name, &map);
151 if (sym) 151 if (sym)
152 return map->unmap_ip(map, sym->start) - 152 return map->unmap_ip(map, sym->start) -
153 (reloc) ? 0 : map->reloc; 153 (reloc) ? 0 : map->reloc;
154 } 154 }
155 return 0; 155 return 0;
156 } 156 }
157 157
158 static struct map *kernel_get_module_map(const char *module) 158 static struct map *kernel_get_module_map(const char *module)
159 { 159 {
160 struct rb_node *nd; 160 struct rb_node *nd;
161 struct map_groups *grp = &host_machine->kmaps; 161 struct map_groups *grp = &host_machine->kmaps;
162 162
163 /* A file path -- this is an offline module */ 163 /* A file path -- this is an offline module */
164 if (module && strchr(module, '/')) 164 if (module && strchr(module, '/'))
165 return machine__new_module(host_machine, 0, module); 165 return machine__new_module(host_machine, 0, module);
166 166
167 if (!module) 167 if (!module)
168 module = "kernel"; 168 module = "kernel";
169 169
170 for (nd = rb_first(&grp->maps[MAP__FUNCTION]); nd; nd = rb_next(nd)) { 170 for (nd = rb_first(&grp->maps[MAP__FUNCTION]); nd; nd = rb_next(nd)) {
171 struct map *pos = rb_entry(nd, struct map, rb_node); 171 struct map *pos = rb_entry(nd, struct map, rb_node);
172 if (strncmp(pos->dso->short_name + 1, module, 172 if (strncmp(pos->dso->short_name + 1, module,
173 pos->dso->short_name_len - 2) == 0) { 173 pos->dso->short_name_len - 2) == 0) {
174 return pos; 174 return pos;
175 } 175 }
176 } 176 }
177 return NULL; 177 return NULL;
178 } 178 }
179 179
180 static struct dso *kernel_get_module_dso(const char *module) 180 static struct dso *kernel_get_module_dso(const char *module)
181 { 181 {
182 struct dso *dso; 182 struct dso *dso;
183 struct map *map; 183 struct map *map;
184 const char *vmlinux_name; 184 const char *vmlinux_name;
185 185
186 if (module) { 186 if (module) {
187 list_for_each_entry(dso, &host_machine->kernel_dsos.head, 187 list_for_each_entry(dso, &host_machine->kernel_dsos.head,
188 node) { 188 node) {
189 if (strncmp(dso->short_name + 1, module, 189 if (strncmp(dso->short_name + 1, module,
190 dso->short_name_len - 2) == 0) 190 dso->short_name_len - 2) == 0)
191 goto found; 191 goto found;
192 } 192 }
193 pr_debug("Failed to find module %s.\n", module); 193 pr_debug("Failed to find module %s.\n", module);
194 return NULL; 194 return NULL;
195 } 195 }
196 196
197 map = host_machine->vmlinux_maps[MAP__FUNCTION]; 197 map = host_machine->vmlinux_maps[MAP__FUNCTION];
198 dso = map->dso; 198 dso = map->dso;
199 199
200 vmlinux_name = symbol_conf.vmlinux_name; 200 vmlinux_name = symbol_conf.vmlinux_name;
201 if (vmlinux_name) { 201 if (vmlinux_name) {
202 if (dso__load_vmlinux(dso, map, vmlinux_name, false, NULL) <= 0) 202 if (dso__load_vmlinux(dso, map, vmlinux_name, false, NULL) <= 0)
203 return NULL; 203 return NULL;
204 } else { 204 } else {
205 if (dso__load_vmlinux_path(dso, map, NULL) <= 0) { 205 if (dso__load_vmlinux_path(dso, map, NULL) <= 0) {
206 pr_debug("Failed to load kernel map.\n"); 206 pr_debug("Failed to load kernel map.\n");
207 return NULL; 207 return NULL;
208 } 208 }
209 } 209 }
210 found: 210 found:
211 return dso; 211 return dso;
212 } 212 }
213 213
214 const char *kernel_get_module_path(const char *module) 214 const char *kernel_get_module_path(const char *module)
215 { 215 {
216 struct dso *dso = kernel_get_module_dso(module); 216 struct dso *dso = kernel_get_module_dso(module);
217 return (dso) ? dso->long_name : NULL; 217 return (dso) ? dso->long_name : NULL;
218 } 218 }
219 219
220 static int convert_exec_to_group(const char *exec, char **result) 220 static int convert_exec_to_group(const char *exec, char **result)
221 { 221 {
222 char *ptr1, *ptr2, *exec_copy; 222 char *ptr1, *ptr2, *exec_copy;
223 char buf[64]; 223 char buf[64];
224 int ret; 224 int ret;
225 225
226 exec_copy = strdup(exec); 226 exec_copy = strdup(exec);
227 if (!exec_copy) 227 if (!exec_copy)
228 return -ENOMEM; 228 return -ENOMEM;
229 229
230 ptr1 = basename(exec_copy); 230 ptr1 = basename(exec_copy);
231 if (!ptr1) { 231 if (!ptr1) {
232 ret = -EINVAL; 232 ret = -EINVAL;
233 goto out; 233 goto out;
234 } 234 }
235 235
236 ptr2 = strpbrk(ptr1, "-._"); 236 ptr2 = strpbrk(ptr1, "-._");
237 if (ptr2) 237 if (ptr2)
238 *ptr2 = '\0'; 238 *ptr2 = '\0';
239 ret = e_snprintf(buf, 64, "%s_%s", PERFPROBE_GROUP, ptr1); 239 ret = e_snprintf(buf, 64, "%s_%s", PERFPROBE_GROUP, ptr1);
240 if (ret < 0) 240 if (ret < 0)
241 goto out; 241 goto out;
242 242
243 *result = strdup(buf); 243 *result = strdup(buf);
244 ret = *result ? 0 : -ENOMEM; 244 ret = *result ? 0 : -ENOMEM;
245 245
246 out: 246 out:
247 free(exec_copy); 247 free(exec_copy);
248 return ret; 248 return ret;
249 } 249 }
250 250
251 static void clear_probe_trace_events(struct probe_trace_event *tevs, int ntevs) 251 static void clear_probe_trace_events(struct probe_trace_event *tevs, int ntevs)
252 { 252 {
253 int i; 253 int i;
254 254
255 for (i = 0; i < ntevs; i++) 255 for (i = 0; i < ntevs; i++)
256 clear_probe_trace_event(tevs + i); 256 clear_probe_trace_event(tevs + i);
257 } 257 }
258 258
259 #ifdef HAVE_DWARF_SUPPORT 259 #ifdef HAVE_DWARF_SUPPORT
260 260
261 /* Open new debuginfo of given module */ 261 /* Open new debuginfo of given module */
262 static struct debuginfo *open_debuginfo(const char *module, bool silent) 262 static struct debuginfo *open_debuginfo(const char *module, bool silent)
263 { 263 {
264 const char *path = module; 264 const char *path = module;
265 struct debuginfo *ret; 265 struct debuginfo *ret;
266 266
267 if (!module || !strchr(module, '/')) { 267 if (!module || !strchr(module, '/')) {
268 path = kernel_get_module_path(module); 268 path = kernel_get_module_path(module);
269 if (!path) { 269 if (!path) {
270 if (!silent) 270 if (!silent)
271 pr_err("Failed to find path of %s module.\n", 271 pr_err("Failed to find path of %s module.\n",
272 module ?: "kernel"); 272 module ?: "kernel");
273 return NULL; 273 return NULL;
274 } 274 }
275 } 275 }
276 ret = debuginfo__new(path); 276 ret = debuginfo__new(path);
277 if (!ret && !silent) { 277 if (!ret && !silent) {
278 pr_warning("The %s file has no debug information.\n", path); 278 pr_warning("The %s file has no debug information.\n", path);
279 if (!module || !strtailcmp(path, ".ko")) 279 if (!module || !strtailcmp(path, ".ko"))
280 pr_warning("Rebuild with CONFIG_DEBUG_INFO=y, "); 280 pr_warning("Rebuild with CONFIG_DEBUG_INFO=y, ");
281 else 281 else
282 pr_warning("Rebuild with -g, "); 282 pr_warning("Rebuild with -g, ");
283 pr_warning("or install an appropriate debuginfo package.\n"); 283 pr_warning("or install an appropriate debuginfo package.\n");
284 } 284 }
285 return ret; 285 return ret;
286 } 286 }
287 287
288 288
289 static int get_text_start_address(const char *exec, unsigned long *address) 289 static int get_text_start_address(const char *exec, unsigned long *address)
290 { 290 {
291 Elf *elf; 291 Elf *elf;
292 GElf_Ehdr ehdr; 292 GElf_Ehdr ehdr;
293 GElf_Shdr shdr; 293 GElf_Shdr shdr;
294 int fd, ret = -ENOENT; 294 int fd, ret = -ENOENT;
295 295
296 fd = open(exec, O_RDONLY); 296 fd = open(exec, O_RDONLY);
297 if (fd < 0) 297 if (fd < 0)
298 return -errno; 298 return -errno;
299 299
300 elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL); 300 elf = elf_begin(fd, PERF_ELF_C_READ_MMAP, NULL);
301 if (elf == NULL) 301 if (elf == NULL)
302 return -EINVAL; 302 return -EINVAL;
303 303
304 if (gelf_getehdr(elf, &ehdr) == NULL) 304 if (gelf_getehdr(elf, &ehdr) == NULL)
305 goto out; 305 goto out;
306 306
307 if (!elf_section_by_name(elf, &ehdr, &shdr, ".text", NULL)) 307 if (!elf_section_by_name(elf, &ehdr, &shdr, ".text", NULL))
308 goto out; 308 goto out;
309 309
310 *address = shdr.sh_addr - shdr.sh_offset; 310 *address = shdr.sh_addr - shdr.sh_offset;
311 ret = 0; 311 ret = 0;
312 out: 312 out:
313 elf_end(elf); 313 elf_end(elf);
314 return ret; 314 return ret;
315 } 315 }
316 316
317 /* 317 /*
318 * Convert trace point to probe point with debuginfo 318 * Convert trace point to probe point with debuginfo
319 */ 319 */
320 static int find_perf_probe_point_from_dwarf(struct probe_trace_point *tp, 320 static int find_perf_probe_point_from_dwarf(struct probe_trace_point *tp,
321 struct perf_probe_point *pp, 321 struct perf_probe_point *pp,
322 bool is_kprobe) 322 bool is_kprobe)
323 { 323 {
324 struct debuginfo *dinfo = NULL; 324 struct debuginfo *dinfo = NULL;
325 unsigned long stext = 0; 325 unsigned long stext = 0;
326 u64 addr = tp->address; 326 u64 addr = tp->address;
327 int ret = -ENOENT; 327 int ret = -ENOENT;
328 328
329 /* convert the address to dwarf address */ 329 /* convert the address to dwarf address */
330 if (!is_kprobe) { 330 if (!is_kprobe) {
331 if (!addr) { 331 if (!addr) {
332 ret = -EINVAL; 332 ret = -EINVAL;
333 goto error; 333 goto error;
334 } 334 }
335 ret = get_text_start_address(tp->module, &stext); 335 ret = get_text_start_address(tp->module, &stext);
336 if (ret < 0) 336 if (ret < 0)
337 goto error; 337 goto error;
338 addr += stext; 338 addr += stext;
339 } else { 339 } else {
340 addr = kernel_get_symbol_address_by_name(tp->symbol, false); 340 addr = kernel_get_symbol_address_by_name(tp->symbol, false);
341 if (addr == 0) 341 if (addr == 0)
342 goto error; 342 goto error;
343 addr += tp->offset; 343 addr += tp->offset;
344 } 344 }
345 345
346 pr_debug("try to find information at %" PRIx64 " in %s\n", addr, 346 pr_debug("try to find information at %" PRIx64 " in %s\n", addr,
347 tp->module ? : "kernel"); 347 tp->module ? : "kernel");
348 348
349 dinfo = open_debuginfo(tp->module, verbose == 0); 349 dinfo = open_debuginfo(tp->module, verbose == 0);
350 if (dinfo) { 350 if (dinfo) {
351 ret = debuginfo__find_probe_point(dinfo, 351 ret = debuginfo__find_probe_point(dinfo,
352 (unsigned long)addr, pp); 352 (unsigned long)addr, pp);
353 debuginfo__delete(dinfo); 353 debuginfo__delete(dinfo);
354 } else 354 } else
355 ret = -ENOENT; 355 ret = -ENOENT;
356 356
357 if (ret > 0) { 357 if (ret > 0) {
358 pp->retprobe = tp->retprobe; 358 pp->retprobe = tp->retprobe;
359 return 0; 359 return 0;
360 } 360 }
361 error: 361 error:
362 pr_debug("Failed to find corresponding probes from debuginfo.\n"); 362 pr_debug("Failed to find corresponding probes from debuginfo.\n");
363 return ret ? : -ENOENT; 363 return ret ? : -ENOENT;
364 } 364 }
365 365
366 static int add_exec_to_probe_trace_events(struct probe_trace_event *tevs, 366 static int add_exec_to_probe_trace_events(struct probe_trace_event *tevs,
367 int ntevs, const char *exec) 367 int ntevs, const char *exec)
368 { 368 {
369 int i, ret = 0; 369 int i, ret = 0;
370 unsigned long stext = 0; 370 unsigned long stext = 0;
371 371
372 if (!exec) 372 if (!exec)
373 return 0; 373 return 0;
374 374
375 ret = get_text_start_address(exec, &stext); 375 ret = get_text_start_address(exec, &stext);
376 if (ret < 0) 376 if (ret < 0)
377 return ret; 377 return ret;
378 378
379 for (i = 0; i < ntevs && ret >= 0; i++) { 379 for (i = 0; i < ntevs && ret >= 0; i++) {
380 /* point.address is the addres of point.symbol + point.offset */ 380 /* point.address is the addres of point.symbol + point.offset */
381 tevs[i].point.address -= stext; 381 tevs[i].point.address -= stext;
382 tevs[i].point.module = strdup(exec); 382 tevs[i].point.module = strdup(exec);
383 if (!tevs[i].point.module) { 383 if (!tevs[i].point.module) {
384 ret = -ENOMEM; 384 ret = -ENOMEM;
385 break; 385 break;
386 } 386 }
387 tevs[i].uprobes = true; 387 tevs[i].uprobes = true;
388 } 388 }
389 389
390 return ret; 390 return ret;
391 } 391 }
392 392
393 static int add_module_to_probe_trace_events(struct probe_trace_event *tevs, 393 static int add_module_to_probe_trace_events(struct probe_trace_event *tevs,
394 int ntevs, const char *module) 394 int ntevs, const char *module)
395 { 395 {
396 int i, ret = 0; 396 int i, ret = 0;
397 char *tmp; 397 char *tmp;
398 398
399 if (!module) 399 if (!module)
400 return 0; 400 return 0;
401 401
402 tmp = strrchr(module, '/'); 402 tmp = strrchr(module, '/');
403 if (tmp) { 403 if (tmp) {
404 /* This is a module path -- get the module name */ 404 /* This is a module path -- get the module name */
405 module = strdup(tmp + 1); 405 module = strdup(tmp + 1);
406 if (!module) 406 if (!module)
407 return -ENOMEM; 407 return -ENOMEM;
408 tmp = strchr(module, '.'); 408 tmp = strchr(module, '.');
409 if (tmp) 409 if (tmp)
410 *tmp = '\0'; 410 *tmp = '\0';
411 tmp = (char *)module; /* For free() */ 411 tmp = (char *)module; /* For free() */
412 } 412 }
413 413
414 for (i = 0; i < ntevs; i++) { 414 for (i = 0; i < ntevs; i++) {
415 tevs[i].point.module = strdup(module); 415 tevs[i].point.module = strdup(module);
416 if (!tevs[i].point.module) { 416 if (!tevs[i].point.module) {
417 ret = -ENOMEM; 417 ret = -ENOMEM;
418 break; 418 break;
419 } 419 }
420 } 420 }
421 421
422 free(tmp); 422 free(tmp);
423 return ret; 423 return ret;
424 } 424 }
425 425
426 /* Post processing the probe events */ 426 /* Post processing the probe events */
427 static int post_process_probe_trace_events(struct probe_trace_event *tevs, 427 static int post_process_probe_trace_events(struct probe_trace_event *tevs,
428 int ntevs, const char *module, 428 int ntevs, const char *module,
429 bool uprobe) 429 bool uprobe)
430 { 430 {
431 struct ref_reloc_sym *reloc_sym; 431 struct ref_reloc_sym *reloc_sym;
432 char *tmp; 432 char *tmp;
433 int i; 433 int i;
434 434
435 if (uprobe) 435 if (uprobe)
436 return add_exec_to_probe_trace_events(tevs, ntevs, module); 436 return add_exec_to_probe_trace_events(tevs, ntevs, module);
437 437
438 /* Note that currently ref_reloc_sym based probe is not for drivers */ 438 /* Note that currently ref_reloc_sym based probe is not for drivers */
439 if (module) 439 if (module)
440 return add_module_to_probe_trace_events(tevs, ntevs, module); 440 return add_module_to_probe_trace_events(tevs, ntevs, module);
441 441
442 reloc_sym = kernel_get_ref_reloc_sym(); 442 reloc_sym = kernel_get_ref_reloc_sym();
443 if (!reloc_sym) { 443 if (!reloc_sym) {
444 pr_warning("Relocated base symbol is not found!\n"); 444 pr_warning("Relocated base symbol is not found!\n");
445 return -EINVAL; 445 return -EINVAL;
446 } 446 }
447 447
448 for (i = 0; i < ntevs; i++) { 448 for (i = 0; i < ntevs; i++) {
449 if (tevs[i].point.address) { 449 if (tevs[i].point.address) {
450 tmp = strdup(reloc_sym->name); 450 tmp = strdup(reloc_sym->name);
451 if (!tmp) 451 if (!tmp)
452 return -ENOMEM; 452 return -ENOMEM;
453 free(tevs[i].point.symbol); 453 free(tevs[i].point.symbol);
454 tevs[i].point.symbol = tmp; 454 tevs[i].point.symbol = tmp;
455 tevs[i].point.offset = tevs[i].point.address - 455 tevs[i].point.offset = tevs[i].point.address -
456 reloc_sym->unrelocated_addr; 456 reloc_sym->unrelocated_addr;
457 } 457 }
458 } 458 }
459 return 0; 459 return 0;
460 } 460 }
461 461
462 /* Try to find perf_probe_event with debuginfo */ 462 /* Try to find perf_probe_event with debuginfo */
463 static int try_to_find_probe_trace_events(struct perf_probe_event *pev, 463 static int try_to_find_probe_trace_events(struct perf_probe_event *pev,
464 struct probe_trace_event **tevs, 464 struct probe_trace_event **tevs,
465 int max_tevs, const char *target) 465 int max_tevs, const char *target)
466 { 466 {
467 bool need_dwarf = perf_probe_event_need_dwarf(pev); 467 bool need_dwarf = perf_probe_event_need_dwarf(pev);
468 struct debuginfo *dinfo; 468 struct debuginfo *dinfo;
469 int ntevs, ret = 0; 469 int ntevs, ret = 0;
470 470
471 dinfo = open_debuginfo(target, !need_dwarf); 471 dinfo = open_debuginfo(target, !need_dwarf);
472 472
473 if (!dinfo) { 473 if (!dinfo) {
474 if (need_dwarf) 474 if (need_dwarf)
475 return -ENOENT; 475 return -ENOENT;
476 pr_debug("Could not open debuginfo. Try to use symbols.\n"); 476 pr_debug("Could not open debuginfo. Try to use symbols.\n");
477 return 0; 477 return 0;
478 } 478 }
479 479
480 pr_debug("Try to find probe point from debuginfo.\n"); 480 pr_debug("Try to find probe point from debuginfo.\n");
481 /* Searching trace events corresponding to a probe event */ 481 /* Searching trace events corresponding to a probe event */
482 ntevs = debuginfo__find_trace_events(dinfo, pev, tevs, max_tevs); 482 ntevs = debuginfo__find_trace_events(dinfo, pev, tevs, max_tevs);
483 483
484 debuginfo__delete(dinfo); 484 debuginfo__delete(dinfo);
485 485
486 if (ntevs > 0) { /* Succeeded to find trace events */ 486 if (ntevs > 0) { /* Succeeded to find trace events */
487 pr_debug("Found %d probe_trace_events.\n", ntevs); 487 pr_debug("Found %d probe_trace_events.\n", ntevs);
488 ret = post_process_probe_trace_events(*tevs, ntevs, 488 ret = post_process_probe_trace_events(*tevs, ntevs,
489 target, pev->uprobes); 489 target, pev->uprobes);
490 if (ret < 0) { 490 if (ret < 0) {
491 clear_probe_trace_events(*tevs, ntevs); 491 clear_probe_trace_events(*tevs, ntevs);
492 zfree(tevs); 492 zfree(tevs);
493 } 493 }
494 return ret < 0 ? ret : ntevs; 494 return ret < 0 ? ret : ntevs;
495 } 495 }
496 496
497 if (ntevs == 0) { /* No error but failed to find probe point. */ 497 if (ntevs == 0) { /* No error but failed to find probe point. */
498 pr_warning("Probe point '%s' not found.\n", 498 pr_warning("Probe point '%s' not found in debuginfo.\n",
499 synthesize_perf_probe_point(&pev->point)); 499 synthesize_perf_probe_point(&pev->point));
500 return -ENOENT; 500 if (need_dwarf)
501 return -ENOENT;
502 return 0;
501 } 503 }
502 /* Error path : ntevs < 0 */ 504 /* Error path : ntevs < 0 */
503 pr_debug("An error occurred in debuginfo analysis (%d).\n", ntevs); 505 pr_debug("An error occurred in debuginfo analysis (%d).\n", ntevs);
504 if (ntevs == -EBADF) { 506 if (ntevs == -EBADF) {
505 pr_warning("Warning: No dwarf info found in the vmlinux - " 507 pr_warning("Warning: No dwarf info found in the vmlinux - "
506 "please rebuild kernel with CONFIG_DEBUG_INFO=y.\n"); 508 "please rebuild kernel with CONFIG_DEBUG_INFO=y.\n");
507 if (!need_dwarf) { 509 if (!need_dwarf) {
508 pr_debug("Trying to use symbols.\n"); 510 pr_debug("Trying to use symbols.\n");
509 return 0; 511 return 0;
510 } 512 }
511 } 513 }
512 return ntevs; 514 return ntevs;
513 } 515 }
514 516
515 /* 517 /*
516 * Find a src file from a DWARF tag path. Prepend optional source path prefix 518 * Find a src file from a DWARF tag path. Prepend optional source path prefix
517 * and chop off leading directories that do not exist. Result is passed back as 519 * and chop off leading directories that do not exist. Result is passed back as
518 * a newly allocated path on success. 520 * a newly allocated path on success.
519 * Return 0 if file was found and readable, -errno otherwise. 521 * Return 0 if file was found and readable, -errno otherwise.
520 */ 522 */
521 static int get_real_path(const char *raw_path, const char *comp_dir, 523 static int get_real_path(const char *raw_path, const char *comp_dir,
522 char **new_path) 524 char **new_path)
523 { 525 {
524 const char *prefix = symbol_conf.source_prefix; 526 const char *prefix = symbol_conf.source_prefix;
525 527
526 if (!prefix) { 528 if (!prefix) {
527 if (raw_path[0] != '/' && comp_dir) 529 if (raw_path[0] != '/' && comp_dir)
528 /* If not an absolute path, try to use comp_dir */ 530 /* If not an absolute path, try to use comp_dir */
529 prefix = comp_dir; 531 prefix = comp_dir;
530 else { 532 else {
531 if (access(raw_path, R_OK) == 0) { 533 if (access(raw_path, R_OK) == 0) {
532 *new_path = strdup(raw_path); 534 *new_path = strdup(raw_path);
533 return 0; 535 return 0;
534 } else 536 } else
535 return -errno; 537 return -errno;
536 } 538 }
537 } 539 }
538 540
539 *new_path = malloc((strlen(prefix) + strlen(raw_path) + 2)); 541 *new_path = malloc((strlen(prefix) + strlen(raw_path) + 2));
540 if (!*new_path) 542 if (!*new_path)
541 return -ENOMEM; 543 return -ENOMEM;
542 544
543 for (;;) { 545 for (;;) {
544 sprintf(*new_path, "%s/%s", prefix, raw_path); 546 sprintf(*new_path, "%s/%s", prefix, raw_path);
545 547
546 if (access(*new_path, R_OK) == 0) 548 if (access(*new_path, R_OK) == 0)
547 return 0; 549 return 0;
548 550
549 if (!symbol_conf.source_prefix) 551 if (!symbol_conf.source_prefix)
550 /* In case of searching comp_dir, don't retry */ 552 /* In case of searching comp_dir, don't retry */
551 return -errno; 553 return -errno;
552 554
553 switch (errno) { 555 switch (errno) {
554 case ENAMETOOLONG: 556 case ENAMETOOLONG:
555 case ENOENT: 557 case ENOENT:
556 case EROFS: 558 case EROFS:
557 case EFAULT: 559 case EFAULT:
558 raw_path = strchr(++raw_path, '/'); 560 raw_path = strchr(++raw_path, '/');
559 if (!raw_path) { 561 if (!raw_path) {
560 zfree(new_path); 562 zfree(new_path);
561 return -ENOENT; 563 return -ENOENT;
562 } 564 }
563 continue; 565 continue;
564 566
565 default: 567 default:
566 zfree(new_path); 568 zfree(new_path);
567 return -errno; 569 return -errno;
568 } 570 }
569 } 571 }
570 } 572 }
571 573
572 #define LINEBUF_SIZE 256 574 #define LINEBUF_SIZE 256
573 #define NR_ADDITIONAL_LINES 2 575 #define NR_ADDITIONAL_LINES 2
574 576
575 static int __show_one_line(FILE *fp, int l, bool skip, bool show_num) 577 static int __show_one_line(FILE *fp, int l, bool skip, bool show_num)
576 { 578 {
577 char buf[LINEBUF_SIZE], sbuf[STRERR_BUFSIZE]; 579 char buf[LINEBUF_SIZE], sbuf[STRERR_BUFSIZE];
578 const char *color = show_num ? "" : PERF_COLOR_BLUE; 580 const char *color = show_num ? "" : PERF_COLOR_BLUE;
579 const char *prefix = NULL; 581 const char *prefix = NULL;
580 582
581 do { 583 do {
582 if (fgets(buf, LINEBUF_SIZE, fp) == NULL) 584 if (fgets(buf, LINEBUF_SIZE, fp) == NULL)
583 goto error; 585 goto error;
584 if (skip) 586 if (skip)
585 continue; 587 continue;
586 if (!prefix) { 588 if (!prefix) {
587 prefix = show_num ? "%7d " : " "; 589 prefix = show_num ? "%7d " : " ";
588 color_fprintf(stdout, color, prefix, l); 590 color_fprintf(stdout, color, prefix, l);
589 } 591 }
590 color_fprintf(stdout, color, "%s", buf); 592 color_fprintf(stdout, color, "%s", buf);
591 593
592 } while (strchr(buf, '\n') == NULL); 594 } while (strchr(buf, '\n') == NULL);
593 595
594 return 1; 596 return 1;
595 error: 597 error:
596 if (ferror(fp)) { 598 if (ferror(fp)) {
597 pr_warning("File read error: %s\n", 599 pr_warning("File read error: %s\n",
598 strerror_r(errno, sbuf, sizeof(sbuf))); 600 strerror_r(errno, sbuf, sizeof(sbuf)));
599 return -1; 601 return -1;
600 } 602 }
601 return 0; 603 return 0;
602 } 604 }
603 605
604 static int _show_one_line(FILE *fp, int l, bool skip, bool show_num) 606 static int _show_one_line(FILE *fp, int l, bool skip, bool show_num)
605 { 607 {
606 int rv = __show_one_line(fp, l, skip, show_num); 608 int rv = __show_one_line(fp, l, skip, show_num);
607 if (rv == 0) { 609 if (rv == 0) {
608 pr_warning("Source file is shorter than expected.\n"); 610 pr_warning("Source file is shorter than expected.\n");
609 rv = -1; 611 rv = -1;
610 } 612 }
611 return rv; 613 return rv;
612 } 614 }
613 615
614 #define show_one_line_with_num(f,l) _show_one_line(f,l,false,true) 616 #define show_one_line_with_num(f,l) _show_one_line(f,l,false,true)
615 #define show_one_line(f,l) _show_one_line(f,l,false,false) 617 #define show_one_line(f,l) _show_one_line(f,l,false,false)
616 #define skip_one_line(f,l) _show_one_line(f,l,true,false) 618 #define skip_one_line(f,l) _show_one_line(f,l,true,false)
617 #define show_one_line_or_eof(f,l) __show_one_line(f,l,false,false) 619 #define show_one_line_or_eof(f,l) __show_one_line(f,l,false,false)
618 620
619 /* 621 /*
620 * Show line-range always requires debuginfo to find source file and 622 * Show line-range always requires debuginfo to find source file and
621 * line number. 623 * line number.
622 */ 624 */
623 static int __show_line_range(struct line_range *lr, const char *module) 625 static int __show_line_range(struct line_range *lr, const char *module)
624 { 626 {
625 int l = 1; 627 int l = 1;
626 struct int_node *ln; 628 struct int_node *ln;
627 struct debuginfo *dinfo; 629 struct debuginfo *dinfo;
628 FILE *fp; 630 FILE *fp;
629 int ret; 631 int ret;
630 char *tmp; 632 char *tmp;
631 char sbuf[STRERR_BUFSIZE]; 633 char sbuf[STRERR_BUFSIZE];
632 634
633 /* Search a line range */ 635 /* Search a line range */
634 dinfo = open_debuginfo(module, false); 636 dinfo = open_debuginfo(module, false);
635 if (!dinfo) 637 if (!dinfo)
636 return -ENOENT; 638 return -ENOENT;
637 639
638 ret = debuginfo__find_line_range(dinfo, lr); 640 ret = debuginfo__find_line_range(dinfo, lr);
639 debuginfo__delete(dinfo); 641 debuginfo__delete(dinfo);
640 if (ret == 0 || ret == -ENOENT) { 642 if (ret == 0 || ret == -ENOENT) {
641 pr_warning("Specified source line is not found.\n"); 643 pr_warning("Specified source line is not found.\n");
642 return -ENOENT; 644 return -ENOENT;
643 } else if (ret < 0) { 645 } else if (ret < 0) {
644 pr_warning("Debuginfo analysis failed.\n"); 646 pr_warning("Debuginfo analysis failed.\n");
645 return ret; 647 return ret;
646 } 648 }
647 649
648 /* Convert source file path */ 650 /* Convert source file path */
649 tmp = lr->path; 651 tmp = lr->path;
650 ret = get_real_path(tmp, lr->comp_dir, &lr->path); 652 ret = get_real_path(tmp, lr->comp_dir, &lr->path);
651 free(tmp); /* Free old path */ 653 free(tmp); /* Free old path */
652 if (ret < 0) { 654 if (ret < 0) {
653 pr_warning("Failed to find source file path.\n"); 655 pr_warning("Failed to find source file path.\n");
654 return ret; 656 return ret;
655 } 657 }
656 658
657 setup_pager(); 659 setup_pager();
658 660
659 if (lr->function) 661 if (lr->function)
660 fprintf(stdout, "<%s@%s:%d>\n", lr->function, lr->path, 662 fprintf(stdout, "<%s@%s:%d>\n", lr->function, lr->path,
661 lr->start - lr->offset); 663 lr->start - lr->offset);
662 else 664 else
663 fprintf(stdout, "<%s:%d>\n", lr->path, lr->start); 665 fprintf(stdout, "<%s:%d>\n", lr->path, lr->start);
664 666
665 fp = fopen(lr->path, "r"); 667 fp = fopen(lr->path, "r");
666 if (fp == NULL) { 668 if (fp == NULL) {
667 pr_warning("Failed to open %s: %s\n", lr->path, 669 pr_warning("Failed to open %s: %s\n", lr->path,
668 strerror_r(errno, sbuf, sizeof(sbuf))); 670 strerror_r(errno, sbuf, sizeof(sbuf)));
669 return -errno; 671 return -errno;
670 } 672 }
671 /* Skip to starting line number */ 673 /* Skip to starting line number */
672 while (l < lr->start) { 674 while (l < lr->start) {
673 ret = skip_one_line(fp, l++); 675 ret = skip_one_line(fp, l++);
674 if (ret < 0) 676 if (ret < 0)
675 goto end; 677 goto end;
676 } 678 }
677 679
678 intlist__for_each(ln, lr->line_list) { 680 intlist__for_each(ln, lr->line_list) {
679 for (; ln->i > l; l++) { 681 for (; ln->i > l; l++) {
680 ret = show_one_line(fp, l - lr->offset); 682 ret = show_one_line(fp, l - lr->offset);
681 if (ret < 0) 683 if (ret < 0)
682 goto end; 684 goto end;
683 } 685 }
684 ret = show_one_line_with_num(fp, l++ - lr->offset); 686 ret = show_one_line_with_num(fp, l++ - lr->offset);
685 if (ret < 0) 687 if (ret < 0)
686 goto end; 688 goto end;
687 } 689 }
688 690
689 if (lr->end == INT_MAX) 691 if (lr->end == INT_MAX)
690 lr->end = l + NR_ADDITIONAL_LINES; 692 lr->end = l + NR_ADDITIONAL_LINES;
691 while (l <= lr->end) { 693 while (l <= lr->end) {
692 ret = show_one_line_or_eof(fp, l++ - lr->offset); 694 ret = show_one_line_or_eof(fp, l++ - lr->offset);
693 if (ret <= 0) 695 if (ret <= 0)
694 break; 696 break;
695 } 697 }
696 end: 698 end:
697 fclose(fp); 699 fclose(fp);
698 return ret; 700 return ret;
699 } 701 }
700 702
701 int show_line_range(struct line_range *lr, const char *module, bool user) 703 int show_line_range(struct line_range *lr, const char *module, bool user)
702 { 704 {
703 int ret; 705 int ret;
704 706
705 ret = init_symbol_maps(user); 707 ret = init_symbol_maps(user);
706 if (ret < 0) 708 if (ret < 0)
707 return ret; 709 return ret;
708 ret = __show_line_range(lr, module); 710 ret = __show_line_range(lr, module);
709 exit_symbol_maps(); 711 exit_symbol_maps();
710 712
711 return ret; 713 return ret;
712 } 714 }
713 715
714 static int show_available_vars_at(struct debuginfo *dinfo, 716 static int show_available_vars_at(struct debuginfo *dinfo,
715 struct perf_probe_event *pev, 717 struct perf_probe_event *pev,
716 int max_vls, struct strfilter *_filter, 718 int max_vls, struct strfilter *_filter,
717 bool externs) 719 bool externs)
718 { 720 {
719 char *buf; 721 char *buf;
720 int ret, i, nvars; 722 int ret, i, nvars;
721 struct str_node *node; 723 struct str_node *node;
722 struct variable_list *vls = NULL, *vl; 724 struct variable_list *vls = NULL, *vl;
723 const char *var; 725 const char *var;
724 726
725 buf = synthesize_perf_probe_point(&pev->point); 727 buf = synthesize_perf_probe_point(&pev->point);
726 if (!buf) 728 if (!buf)
727 return -EINVAL; 729 return -EINVAL;
728 pr_debug("Searching variables at %s\n", buf); 730 pr_debug("Searching variables at %s\n", buf);
729 731
730 ret = debuginfo__find_available_vars_at(dinfo, pev, &vls, 732 ret = debuginfo__find_available_vars_at(dinfo, pev, &vls,
731 max_vls, externs); 733 max_vls, externs);
732 if (ret <= 0) { 734 if (ret <= 0) {
733 if (ret == 0 || ret == -ENOENT) { 735 if (ret == 0 || ret == -ENOENT) {
734 pr_err("Failed to find the address of %s\n", buf); 736 pr_err("Failed to find the address of %s\n", buf);
735 ret = -ENOENT; 737 ret = -ENOENT;
736 } else 738 } else
737 pr_warning("Debuginfo analysis failed.\n"); 739 pr_warning("Debuginfo analysis failed.\n");
738 goto end; 740 goto end;
739 } 741 }
740 742
741 /* Some variables are found */ 743 /* Some variables are found */
742 fprintf(stdout, "Available variables at %s\n", buf); 744 fprintf(stdout, "Available variables at %s\n", buf);
743 for (i = 0; i < ret; i++) { 745 for (i = 0; i < ret; i++) {
744 vl = &vls[i]; 746 vl = &vls[i];
745 /* 747 /*
746 * A probe point might be converted to 748 * A probe point might be converted to
747 * several trace points. 749 * several trace points.
748 */ 750 */
749 fprintf(stdout, "\t@<%s+%lu>\n", vl->point.symbol, 751 fprintf(stdout, "\t@<%s+%lu>\n", vl->point.symbol,
750 vl->point.offset); 752 vl->point.offset);
751 zfree(&vl->point.symbol); 753 zfree(&vl->point.symbol);
752 nvars = 0; 754 nvars = 0;
753 if (vl->vars) { 755 if (vl->vars) {
754 strlist__for_each(node, vl->vars) { 756 strlist__for_each(node, vl->vars) {
755 var = strchr(node->s, '\t') + 1; 757 var = strchr(node->s, '\t') + 1;
756 if (strfilter__compare(_filter, var)) { 758 if (strfilter__compare(_filter, var)) {
757 fprintf(stdout, "\t\t%s\n", node->s); 759 fprintf(stdout, "\t\t%s\n", node->s);
758 nvars++; 760 nvars++;
759 } 761 }
760 } 762 }
761 strlist__delete(vl->vars); 763 strlist__delete(vl->vars);
762 } 764 }
763 if (nvars == 0) 765 if (nvars == 0)
764 fprintf(stdout, "\t\t(No matched variables)\n"); 766 fprintf(stdout, "\t\t(No matched variables)\n");
765 } 767 }
766 free(vls); 768 free(vls);
767 end: 769 end:
768 free(buf); 770 free(buf);
769 return ret; 771 return ret;
770 } 772 }
771 773
772 /* Show available variables on given probe point */ 774 /* Show available variables on given probe point */
773 int show_available_vars(struct perf_probe_event *pevs, int npevs, 775 int show_available_vars(struct perf_probe_event *pevs, int npevs,
774 int max_vls, const char *module, 776 int max_vls, const char *module,
775 struct strfilter *_filter, bool externs) 777 struct strfilter *_filter, bool externs)
776 { 778 {
777 int i, ret = 0; 779 int i, ret = 0;
778 struct debuginfo *dinfo; 780 struct debuginfo *dinfo;
779 781
780 ret = init_symbol_maps(pevs->uprobes); 782 ret = init_symbol_maps(pevs->uprobes);
781 if (ret < 0) 783 if (ret < 0)
782 return ret; 784 return ret;
783 785
784 dinfo = open_debuginfo(module, false); 786 dinfo = open_debuginfo(module, false);
785 if (!dinfo) { 787 if (!dinfo) {
786 ret = -ENOENT; 788 ret = -ENOENT;
787 goto out; 789 goto out;
788 } 790 }
789 791
790 setup_pager(); 792 setup_pager();
791 793
792 for (i = 0; i < npevs && ret >= 0; i++) 794 for (i = 0; i < npevs && ret >= 0; i++)
793 ret = show_available_vars_at(dinfo, &pevs[i], max_vls, _filter, 795 ret = show_available_vars_at(dinfo, &pevs[i], max_vls, _filter,
794 externs); 796 externs);
795 797
796 debuginfo__delete(dinfo); 798 debuginfo__delete(dinfo);
797 out: 799 out:
798 exit_symbol_maps(); 800 exit_symbol_maps();
799 return ret; 801 return ret;
800 } 802 }
801 803
802 #else /* !HAVE_DWARF_SUPPORT */ 804 #else /* !HAVE_DWARF_SUPPORT */
803 805
804 static int 806 static int
805 find_perf_probe_point_from_dwarf(struct probe_trace_point *tp __maybe_unused, 807 find_perf_probe_point_from_dwarf(struct probe_trace_point *tp __maybe_unused,
806 struct perf_probe_point *pp __maybe_unused, 808 struct perf_probe_point *pp __maybe_unused,
807 bool is_kprobe __maybe_unused) 809 bool is_kprobe __maybe_unused)
808 { 810 {
809 return -ENOSYS; 811 return -ENOSYS;
810 } 812 }
811 813
812 static int try_to_find_probe_trace_events(struct perf_probe_event *pev, 814 static int try_to_find_probe_trace_events(struct perf_probe_event *pev,
813 struct probe_trace_event **tevs __maybe_unused, 815 struct probe_trace_event **tevs __maybe_unused,
814 int max_tevs __maybe_unused, 816 int max_tevs __maybe_unused,
815 const char *target __maybe_unused) 817 const char *target __maybe_unused)
816 { 818 {
817 if (perf_probe_event_need_dwarf(pev)) { 819 if (perf_probe_event_need_dwarf(pev)) {
818 pr_warning("Debuginfo-analysis is not supported.\n"); 820 pr_warning("Debuginfo-analysis is not supported.\n");
819 return -ENOSYS; 821 return -ENOSYS;
820 } 822 }
821 823
822 return 0; 824 return 0;
823 } 825 }
824 826
825 int show_line_range(struct line_range *lr __maybe_unused, 827 int show_line_range(struct line_range *lr __maybe_unused,
826 const char *module __maybe_unused, 828 const char *module __maybe_unused,
827 bool user __maybe_unused) 829 bool user __maybe_unused)
828 { 830 {
829 pr_warning("Debuginfo-analysis is not supported.\n"); 831 pr_warning("Debuginfo-analysis is not supported.\n");
830 return -ENOSYS; 832 return -ENOSYS;
831 } 833 }
832 834
833 int show_available_vars(struct perf_probe_event *pevs __maybe_unused, 835 int show_available_vars(struct perf_probe_event *pevs __maybe_unused,
834 int npevs __maybe_unused, int max_vls __maybe_unused, 836 int npevs __maybe_unused, int max_vls __maybe_unused,
835 const char *module __maybe_unused, 837 const char *module __maybe_unused,
836 struct strfilter *filter __maybe_unused, 838 struct strfilter *filter __maybe_unused,
837 bool externs __maybe_unused) 839 bool externs __maybe_unused)
838 { 840 {
839 pr_warning("Debuginfo-analysis is not supported.\n"); 841 pr_warning("Debuginfo-analysis is not supported.\n");
840 return -ENOSYS; 842 return -ENOSYS;
841 } 843 }
842 #endif 844 #endif
843 845
844 void line_range__clear(struct line_range *lr) 846 void line_range__clear(struct line_range *lr)
845 { 847 {
846 free(lr->function); 848 free(lr->function);
847 free(lr->file); 849 free(lr->file);
848 free(lr->path); 850 free(lr->path);
849 free(lr->comp_dir); 851 free(lr->comp_dir);
850 intlist__delete(lr->line_list); 852 intlist__delete(lr->line_list);
851 memset(lr, 0, sizeof(*lr)); 853 memset(lr, 0, sizeof(*lr));
852 } 854 }
853 855
854 int line_range__init(struct line_range *lr) 856 int line_range__init(struct line_range *lr)
855 { 857 {
856 memset(lr, 0, sizeof(*lr)); 858 memset(lr, 0, sizeof(*lr));
857 lr->line_list = intlist__new(NULL); 859 lr->line_list = intlist__new(NULL);
858 if (!lr->line_list) 860 if (!lr->line_list)
859 return -ENOMEM; 861 return -ENOMEM;
860 else 862 else
861 return 0; 863 return 0;
862 } 864 }
863 865
864 static int parse_line_num(char **ptr, int *val, const char *what) 866 static int parse_line_num(char **ptr, int *val, const char *what)
865 { 867 {
866 const char *start = *ptr; 868 const char *start = *ptr;
867 869
868 errno = 0; 870 errno = 0;
869 *val = strtol(*ptr, ptr, 0); 871 *val = strtol(*ptr, ptr, 0);
870 if (errno || *ptr == start) { 872 if (errno || *ptr == start) {
871 semantic_error("'%s' is not a valid number.\n", what); 873 semantic_error("'%s' is not a valid number.\n", what);
872 return -EINVAL; 874 return -EINVAL;
873 } 875 }
874 return 0; 876 return 0;
875 } 877 }
876 878
877 /* 879 /*
878 * Stuff 'lr' according to the line range described by 'arg'. 880 * Stuff 'lr' according to the line range described by 'arg'.
879 * The line range syntax is described by: 881 * The line range syntax is described by:
880 * 882 *
881 * SRC[:SLN[+NUM|-ELN]] 883 * SRC[:SLN[+NUM|-ELN]]
882 * FNC[@SRC][:SLN[+NUM|-ELN]] 884 * FNC[@SRC][:SLN[+NUM|-ELN]]
883 */ 885 */
884 int parse_line_range_desc(const char *arg, struct line_range *lr) 886 int parse_line_range_desc(const char *arg, struct line_range *lr)
885 { 887 {
886 char *range, *file, *name = strdup(arg); 888 char *range, *file, *name = strdup(arg);
887 int err; 889 int err;
888 890
889 if (!name) 891 if (!name)
890 return -ENOMEM; 892 return -ENOMEM;
891 893
892 lr->start = 0; 894 lr->start = 0;
893 lr->end = INT_MAX; 895 lr->end = INT_MAX;
894 896
895 range = strchr(name, ':'); 897 range = strchr(name, ':');
896 if (range) { 898 if (range) {
897 *range++ = '\0'; 899 *range++ = '\0';
898 900
899 err = parse_line_num(&range, &lr->start, "start line"); 901 err = parse_line_num(&range, &lr->start, "start line");
900 if (err) 902 if (err)
901 goto err; 903 goto err;
902 904
903 if (*range == '+' || *range == '-') { 905 if (*range == '+' || *range == '-') {
904 const char c = *range++; 906 const char c = *range++;
905 907
906 err = parse_line_num(&range, &lr->end, "end line"); 908 err = parse_line_num(&range, &lr->end, "end line");
907 if (err) 909 if (err)
908 goto err; 910 goto err;
909 911
910 if (c == '+') { 912 if (c == '+') {
911 lr->end += lr->start; 913 lr->end += lr->start;
912 /* 914 /*
913 * Adjust the number of lines here. 915 * Adjust the number of lines here.
914 * If the number of lines == 1, the 916 * If the number of lines == 1, the
915 * the end of line should be equal to 917 * the end of line should be equal to
916 * the start of line. 918 * the start of line.
917 */ 919 */
918 lr->end--; 920 lr->end--;
919 } 921 }
920 } 922 }
921 923
922 pr_debug("Line range is %d to %d\n", lr->start, lr->end); 924 pr_debug("Line range is %d to %d\n", lr->start, lr->end);
923 925
924 err = -EINVAL; 926 err = -EINVAL;
925 if (lr->start > lr->end) { 927 if (lr->start > lr->end) {
926 semantic_error("Start line must be smaller" 928 semantic_error("Start line must be smaller"
927 " than end line.\n"); 929 " than end line.\n");
928 goto err; 930 goto err;
929 } 931 }
930 if (*range != '\0') { 932 if (*range != '\0') {
931 semantic_error("Tailing with invalid str '%s'.\n", range); 933 semantic_error("Tailing with invalid str '%s'.\n", range);
932 goto err; 934 goto err;
933 } 935 }
934 } 936 }
935 937
936 file = strchr(name, '@'); 938 file = strchr(name, '@');
937 if (file) { 939 if (file) {
938 *file = '\0'; 940 *file = '\0';
939 lr->file = strdup(++file); 941 lr->file = strdup(++file);
940 if (lr->file == NULL) { 942 if (lr->file == NULL) {
941 err = -ENOMEM; 943 err = -ENOMEM;
942 goto err; 944 goto err;
943 } 945 }
944 lr->function = name; 946 lr->function = name;
945 } else if (strchr(name, '.')) 947 } else if (strchr(name, '.'))
946 lr->file = name; 948 lr->file = name;
947 else 949 else
948 lr->function = name; 950 lr->function = name;
949 951
950 return 0; 952 return 0;
951 err: 953 err:
952 free(name); 954 free(name);
953 return err; 955 return err;
954 } 956 }
955 957
956 /* Check the name is good for event/group */ 958 /* Check the name is good for event/group */
957 static bool check_event_name(const char *name) 959 static bool check_event_name(const char *name)
958 { 960 {
959 if (!isalpha(*name) && *name != '_') 961 if (!isalpha(*name) && *name != '_')
960 return false; 962 return false;
961 while (*++name != '\0') { 963 while (*++name != '\0') {
962 if (!isalpha(*name) && !isdigit(*name) && *name != '_') 964 if (!isalpha(*name) && !isdigit(*name) && *name != '_')
963 return false; 965 return false;
964 } 966 }
965 return true; 967 return true;
966 } 968 }
967 969
968 /* Parse probepoint definition. */ 970 /* Parse probepoint definition. */
969 static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) 971 static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev)
970 { 972 {
971 struct perf_probe_point *pp = &pev->point; 973 struct perf_probe_point *pp = &pev->point;
972 char *ptr, *tmp; 974 char *ptr, *tmp;
973 char c, nc = 0; 975 char c, nc = 0;
974 /* 976 /*
975 * <Syntax> 977 * <Syntax>
976 * perf probe [EVENT=]SRC[:LN|;PTN] 978 * perf probe [EVENT=]SRC[:LN|;PTN]
977 * perf probe [EVENT=]FUNC[@SRC][+OFFS|%return|:LN|;PAT] 979 * perf probe [EVENT=]FUNC[@SRC][+OFFS|%return|:LN|;PAT]
978 * 980 *
979 * TODO:Group name support 981 * TODO:Group name support
980 */ 982 */
981 983
982 ptr = strpbrk(arg, ";=@+%"); 984 ptr = strpbrk(arg, ";=@+%");
983 if (ptr && *ptr == '=') { /* Event name */ 985 if (ptr && *ptr == '=') { /* Event name */
984 *ptr = '\0'; 986 *ptr = '\0';
985 tmp = ptr + 1; 987 tmp = ptr + 1;
986 if (strchr(arg, ':')) { 988 if (strchr(arg, ':')) {
987 semantic_error("Group name is not supported yet.\n"); 989 semantic_error("Group name is not supported yet.\n");
988 return -ENOTSUP; 990 return -ENOTSUP;
989 } 991 }
990 if (!check_event_name(arg)) { 992 if (!check_event_name(arg)) {
991 semantic_error("%s is bad for event name -it must " 993 semantic_error("%s is bad for event name -it must "
992 "follow C symbol-naming rule.\n", arg); 994 "follow C symbol-naming rule.\n", arg);
993 return -EINVAL; 995 return -EINVAL;
994 } 996 }
995 pev->event = strdup(arg); 997 pev->event = strdup(arg);
996 if (pev->event == NULL) 998 if (pev->event == NULL)
997 return -ENOMEM; 999 return -ENOMEM;
998 pev->group = NULL; 1000 pev->group = NULL;
999 arg = tmp; 1001 arg = tmp;
1000 } 1002 }
1001 1003
1002 ptr = strpbrk(arg, ";:+@%"); 1004 ptr = strpbrk(arg, ";:+@%");
1003 if (ptr) { 1005 if (ptr) {
1004 nc = *ptr; 1006 nc = *ptr;
1005 *ptr++ = '\0'; 1007 *ptr++ = '\0';
1006 } 1008 }
1007 1009
1008 tmp = strdup(arg); 1010 tmp = strdup(arg);
1009 if (tmp == NULL) 1011 if (tmp == NULL)
1010 return -ENOMEM; 1012 return -ENOMEM;
1011 1013
1012 /* Check arg is function or file and copy it */ 1014 /* Check arg is function or file and copy it */
1013 if (strchr(tmp, '.')) /* File */ 1015 if (strchr(tmp, '.')) /* File */
1014 pp->file = tmp; 1016 pp->file = tmp;
1015 else /* Function */ 1017 else /* Function */
1016 pp->function = tmp; 1018 pp->function = tmp;
1017 1019
1018 /* Parse other options */ 1020 /* Parse other options */
1019 while (ptr) { 1021 while (ptr) {
1020 arg = ptr; 1022 arg = ptr;
1021 c = nc; 1023 c = nc;
1022 if (c == ';') { /* Lazy pattern must be the last part */ 1024 if (c == ';') { /* Lazy pattern must be the last part */
1023 pp->lazy_line = strdup(arg); 1025 pp->lazy_line = strdup(arg);
1024 if (pp->lazy_line == NULL) 1026 if (pp->lazy_line == NULL)
1025 return -ENOMEM; 1027 return -ENOMEM;
1026 break; 1028 break;
1027 } 1029 }
1028 ptr = strpbrk(arg, ";:+@%"); 1030 ptr = strpbrk(arg, ";:+@%");
1029 if (ptr) { 1031 if (ptr) {
1030 nc = *ptr; 1032 nc = *ptr;
1031 *ptr++ = '\0'; 1033 *ptr++ = '\0';
1032 } 1034 }
1033 switch (c) { 1035 switch (c) {
1034 case ':': /* Line number */ 1036 case ':': /* Line number */
1035 pp->line = strtoul(arg, &tmp, 0); 1037 pp->line = strtoul(arg, &tmp, 0);
1036 if (*tmp != '\0') { 1038 if (*tmp != '\0') {
1037 semantic_error("There is non-digit char" 1039 semantic_error("There is non-digit char"
1038 " in line number.\n"); 1040 " in line number.\n");
1039 return -EINVAL; 1041 return -EINVAL;
1040 } 1042 }
1041 break; 1043 break;
1042 case '+': /* Byte offset from a symbol */ 1044 case '+': /* Byte offset from a symbol */
1043 pp->offset = strtoul(arg, &tmp, 0); 1045 pp->offset = strtoul(arg, &tmp, 0);
1044 if (*tmp != '\0') { 1046 if (*tmp != '\0') {
1045 semantic_error("There is non-digit character" 1047 semantic_error("There is non-digit character"
1046 " in offset.\n"); 1048 " in offset.\n");
1047 return -EINVAL; 1049 return -EINVAL;
1048 } 1050 }
1049 break; 1051 break;
1050 case '@': /* File name */ 1052 case '@': /* File name */
1051 if (pp->file) { 1053 if (pp->file) {
1052 semantic_error("SRC@SRC is not allowed.\n"); 1054 semantic_error("SRC@SRC is not allowed.\n");
1053 return -EINVAL; 1055 return -EINVAL;
1054 } 1056 }
1055 pp->file = strdup(arg); 1057 pp->file = strdup(arg);
1056 if (pp->file == NULL) 1058 if (pp->file == NULL)
1057 return -ENOMEM; 1059 return -ENOMEM;
1058 break; 1060 break;
1059 case '%': /* Probe places */ 1061 case '%': /* Probe places */
1060 if (strcmp(arg, "return") == 0) { 1062 if (strcmp(arg, "return") == 0) {
1061 pp->retprobe = 1; 1063 pp->retprobe = 1;
1062 } else { /* Others not supported yet */ 1064 } else { /* Others not supported yet */
1063 semantic_error("%%%s is not supported.\n", arg); 1065 semantic_error("%%%s is not supported.\n", arg);
1064 return -ENOTSUP; 1066 return -ENOTSUP;
1065 } 1067 }
1066 break; 1068 break;
1067 default: /* Buggy case */ 1069 default: /* Buggy case */
1068 pr_err("This program has a bug at %s:%d.\n", 1070 pr_err("This program has a bug at %s:%d.\n",
1069 __FILE__, __LINE__); 1071 __FILE__, __LINE__);
1070 return -ENOTSUP; 1072 return -ENOTSUP;
1071 break; 1073 break;
1072 } 1074 }
1073 } 1075 }
1074 1076
1075 /* Exclusion check */ 1077 /* Exclusion check */
1076 if (pp->lazy_line && pp->line) { 1078 if (pp->lazy_line && pp->line) {
1077 semantic_error("Lazy pattern can't be used with" 1079 semantic_error("Lazy pattern can't be used with"
1078 " line number.\n"); 1080 " line number.\n");
1079 return -EINVAL; 1081 return -EINVAL;
1080 } 1082 }
1081 1083
1082 if (pp->lazy_line && pp->offset) { 1084 if (pp->lazy_line && pp->offset) {
1083 semantic_error("Lazy pattern can't be used with offset.\n"); 1085 semantic_error("Lazy pattern can't be used with offset.\n");
1084 return -EINVAL; 1086 return -EINVAL;
1085 } 1087 }
1086 1088
1087 if (pp->line && pp->offset) { 1089 if (pp->line && pp->offset) {
1088 semantic_error("Offset can't be used with line number.\n"); 1090 semantic_error("Offset can't be used with line number.\n");
1089 return -EINVAL; 1091 return -EINVAL;
1090 } 1092 }
1091 1093
1092 if (!pp->line && !pp->lazy_line && pp->file && !pp->function) { 1094 if (!pp->line && !pp->lazy_line && pp->file && !pp->function) {
1093 semantic_error("File always requires line number or " 1095 semantic_error("File always requires line number or "
1094 "lazy pattern.\n"); 1096 "lazy pattern.\n");
1095 return -EINVAL; 1097 return -EINVAL;
1096 } 1098 }
1097 1099
1098 if (pp->offset && !pp->function) { 1100 if (pp->offset && !pp->function) {
1099 semantic_error("Offset requires an entry function.\n"); 1101 semantic_error("Offset requires an entry function.\n");
1100 return -EINVAL; 1102 return -EINVAL;
1101 } 1103 }
1102 1104
1103 if (pp->retprobe && !pp->function) { 1105 if (pp->retprobe && !pp->function) {
1104 semantic_error("Return probe requires an entry function.\n"); 1106 semantic_error("Return probe requires an entry function.\n");
1105 return -EINVAL; 1107 return -EINVAL;
1106 } 1108 }
1107 1109
1108 if ((pp->offset || pp->line || pp->lazy_line) && pp->retprobe) { 1110 if ((pp->offset || pp->line || pp->lazy_line) && pp->retprobe) {
1109 semantic_error("Offset/Line/Lazy pattern can't be used with " 1111 semantic_error("Offset/Line/Lazy pattern can't be used with "
1110 "return probe.\n"); 1112 "return probe.\n");
1111 return -EINVAL; 1113 return -EINVAL;
1112 } 1114 }
1113 1115
1114 pr_debug("symbol:%s file:%s line:%d offset:%lu return:%d lazy:%s\n", 1116 pr_debug("symbol:%s file:%s line:%d offset:%lu return:%d lazy:%s\n",
1115 pp->function, pp->file, pp->line, pp->offset, pp->retprobe, 1117 pp->function, pp->file, pp->line, pp->offset, pp->retprobe,
1116 pp->lazy_line); 1118 pp->lazy_line);
1117 return 0; 1119 return 0;
1118 } 1120 }
1119 1121
1120 /* Parse perf-probe event argument */ 1122 /* Parse perf-probe event argument */
1121 static int parse_perf_probe_arg(char *str, struct perf_probe_arg *arg) 1123 static int parse_perf_probe_arg(char *str, struct perf_probe_arg *arg)
1122 { 1124 {
1123 char *tmp, *goodname; 1125 char *tmp, *goodname;
1124 struct perf_probe_arg_field **fieldp; 1126 struct perf_probe_arg_field **fieldp;
1125 1127
1126 pr_debug("parsing arg: %s into ", str); 1128 pr_debug("parsing arg: %s into ", str);
1127 1129
1128 tmp = strchr(str, '='); 1130 tmp = strchr(str, '=');
1129 if (tmp) { 1131 if (tmp) {
1130 arg->name = strndup(str, tmp - str); 1132 arg->name = strndup(str, tmp - str);
1131 if (arg->name == NULL) 1133 if (arg->name == NULL)
1132 return -ENOMEM; 1134 return -ENOMEM;
1133 pr_debug("name:%s ", arg->name); 1135 pr_debug("name:%s ", arg->name);
1134 str = tmp + 1; 1136 str = tmp + 1;
1135 } 1137 }
1136 1138
1137 tmp = strchr(str, ':'); 1139 tmp = strchr(str, ':');
1138 if (tmp) { /* Type setting */ 1140 if (tmp) { /* Type setting */
1139 *tmp = '\0'; 1141 *tmp = '\0';
1140 arg->type = strdup(tmp + 1); 1142 arg->type = strdup(tmp + 1);
1141 if (arg->type == NULL) 1143 if (arg->type == NULL)
1142 return -ENOMEM; 1144 return -ENOMEM;
1143 pr_debug("type:%s ", arg->type); 1145 pr_debug("type:%s ", arg->type);
1144 } 1146 }
1145 1147
1146 tmp = strpbrk(str, "-.["); 1148 tmp = strpbrk(str, "-.[");
1147 if (!is_c_varname(str) || !tmp) { 1149 if (!is_c_varname(str) || !tmp) {
1148 /* A variable, register, symbol or special value */ 1150 /* A variable, register, symbol or special value */
1149 arg->var = strdup(str); 1151 arg->var = strdup(str);
1150 if (arg->var == NULL) 1152 if (arg->var == NULL)
1151 return -ENOMEM; 1153 return -ENOMEM;
1152 pr_debug("%s\n", arg->var); 1154 pr_debug("%s\n", arg->var);
1153 return 0; 1155 return 0;
1154 } 1156 }
1155 1157
1156 /* Structure fields or array element */ 1158 /* Structure fields or array element */
1157 arg->var = strndup(str, tmp - str); 1159 arg->var = strndup(str, tmp - str);
1158 if (arg->var == NULL) 1160 if (arg->var == NULL)
1159 return -ENOMEM; 1161 return -ENOMEM;
1160 goodname = arg->var; 1162 goodname = arg->var;
1161 pr_debug("%s, ", arg->var); 1163 pr_debug("%s, ", arg->var);
1162 fieldp = &arg->field; 1164 fieldp = &arg->field;
1163 1165
1164 do { 1166 do {
1165 *fieldp = zalloc(sizeof(struct perf_probe_arg_field)); 1167 *fieldp = zalloc(sizeof(struct perf_probe_arg_field));
1166 if (*fieldp == NULL) 1168 if (*fieldp == NULL)
1167 return -ENOMEM; 1169 return -ENOMEM;
1168 if (*tmp == '[') { /* Array */ 1170 if (*tmp == '[') { /* Array */
1169 str = tmp; 1171 str = tmp;
1170 (*fieldp)->index = strtol(str + 1, &tmp, 0); 1172 (*fieldp)->index = strtol(str + 1, &tmp, 0);
1171 (*fieldp)->ref = true; 1173 (*fieldp)->ref = true;
1172 if (*tmp != ']' || tmp == str + 1) { 1174 if (*tmp != ']' || tmp == str + 1) {
1173 semantic_error("Array index must be a" 1175 semantic_error("Array index must be a"
1174 " number.\n"); 1176 " number.\n");
1175 return -EINVAL; 1177 return -EINVAL;
1176 } 1178 }
1177 tmp++; 1179 tmp++;
1178 if (*tmp == '\0') 1180 if (*tmp == '\0')
1179 tmp = NULL; 1181 tmp = NULL;
1180 } else { /* Structure */ 1182 } else { /* Structure */
1181 if (*tmp == '.') { 1183 if (*tmp == '.') {
1182 str = tmp + 1; 1184 str = tmp + 1;
1183 (*fieldp)->ref = false; 1185 (*fieldp)->ref = false;
1184 } else if (tmp[1] == '>') { 1186 } else if (tmp[1] == '>') {
1185 str = tmp + 2; 1187 str = tmp + 2;
1186 (*fieldp)->ref = true; 1188 (*fieldp)->ref = true;
1187 } else { 1189 } else {
1188 semantic_error("Argument parse error: %s\n", 1190 semantic_error("Argument parse error: %s\n",
1189 str); 1191 str);
1190 return -EINVAL; 1192 return -EINVAL;
1191 } 1193 }
1192 tmp = strpbrk(str, "-.["); 1194 tmp = strpbrk(str, "-.[");
1193 } 1195 }
1194 if (tmp) { 1196 if (tmp) {
1195 (*fieldp)->name = strndup(str, tmp - str); 1197 (*fieldp)->name = strndup(str, tmp - str);
1196 if ((*fieldp)->name == NULL) 1198 if ((*fieldp)->name == NULL)
1197 return -ENOMEM; 1199 return -ENOMEM;
1198 if (*str != '[') 1200 if (*str != '[')
1199 goodname = (*fieldp)->name; 1201 goodname = (*fieldp)->name;
1200 pr_debug("%s(%d), ", (*fieldp)->name, (*fieldp)->ref); 1202 pr_debug("%s(%d), ", (*fieldp)->name, (*fieldp)->ref);
1201 fieldp = &(*fieldp)->next; 1203 fieldp = &(*fieldp)->next;
1202 } 1204 }
1203 } while (tmp); 1205 } while (tmp);
1204 (*fieldp)->name = strdup(str); 1206 (*fieldp)->name = strdup(str);
1205 if ((*fieldp)->name == NULL) 1207 if ((*fieldp)->name == NULL)
1206 return -ENOMEM; 1208 return -ENOMEM;
1207 if (*str != '[') 1209 if (*str != '[')
1208 goodname = (*fieldp)->name; 1210 goodname = (*fieldp)->name;
1209 pr_debug("%s(%d)\n", (*fieldp)->name, (*fieldp)->ref); 1211 pr_debug("%s(%d)\n", (*fieldp)->name, (*fieldp)->ref);
1210 1212
1211 /* If no name is specified, set the last field name (not array index)*/ 1213 /* If no name is specified, set the last field name (not array index)*/
1212 if (!arg->name) { 1214 if (!arg->name) {
1213 arg->name = strdup(goodname); 1215 arg->name = strdup(goodname);
1214 if (arg->name == NULL) 1216 if (arg->name == NULL)
1215 return -ENOMEM; 1217 return -ENOMEM;
1216 } 1218 }
1217 return 0; 1219 return 0;
1218 } 1220 }
1219 1221
1220 /* Parse perf-probe event command */ 1222 /* Parse perf-probe event command */
1221 int parse_perf_probe_command(const char *cmd, struct perf_probe_event *pev) 1223 int parse_perf_probe_command(const char *cmd, struct perf_probe_event *pev)
1222 { 1224 {
1223 char **argv; 1225 char **argv;
1224 int argc, i, ret = 0; 1226 int argc, i, ret = 0;
1225 1227
1226 argv = argv_split(cmd, &argc); 1228 argv = argv_split(cmd, &argc);
1227 if (!argv) { 1229 if (!argv) {
1228 pr_debug("Failed to split arguments.\n"); 1230 pr_debug("Failed to split arguments.\n");
1229 return -ENOMEM; 1231 return -ENOMEM;
1230 } 1232 }
1231 if (argc - 1 > MAX_PROBE_ARGS) { 1233 if (argc - 1 > MAX_PROBE_ARGS) {
1232 semantic_error("Too many probe arguments (%d).\n", argc - 1); 1234 semantic_error("Too many probe arguments (%d).\n", argc - 1);
1233 ret = -ERANGE; 1235 ret = -ERANGE;
1234 goto out; 1236 goto out;
1235 } 1237 }
1236 /* Parse probe point */ 1238 /* Parse probe point */
1237 ret = parse_perf_probe_point(argv[0], pev); 1239 ret = parse_perf_probe_point(argv[0], pev);
1238 if (ret < 0) 1240 if (ret < 0)
1239 goto out; 1241 goto out;
1240 1242
1241 /* Copy arguments and ensure return probe has no C argument */ 1243 /* Copy arguments and ensure return probe has no C argument */
1242 pev->nargs = argc - 1; 1244 pev->nargs = argc - 1;
1243 pev->args = zalloc(sizeof(struct perf_probe_arg) * pev->nargs); 1245 pev->args = zalloc(sizeof(struct perf_probe_arg) * pev->nargs);
1244 if (pev->args == NULL) { 1246 if (pev->args == NULL) {
1245 ret = -ENOMEM; 1247 ret = -ENOMEM;
1246 goto out; 1248 goto out;
1247 } 1249 }
1248 for (i = 0; i < pev->nargs && ret >= 0; i++) { 1250 for (i = 0; i < pev->nargs && ret >= 0; i++) {
1249 ret = parse_perf_probe_arg(argv[i + 1], &pev->args[i]); 1251 ret = parse_perf_probe_arg(argv[i + 1], &pev->args[i]);
1250 if (ret >= 0 && 1252 if (ret >= 0 &&
1251 is_c_varname(pev->args[i].var) && pev->point.retprobe) { 1253 is_c_varname(pev->args[i].var) && pev->point.retprobe) {
1252 semantic_error("You can't specify local variable for" 1254 semantic_error("You can't specify local variable for"
1253 " kretprobe.\n"); 1255 " kretprobe.\n");
1254 ret = -EINVAL; 1256 ret = -EINVAL;
1255 } 1257 }
1256 } 1258 }
1257 out: 1259 out:
1258 argv_free(argv); 1260 argv_free(argv);
1259 1261
1260 return ret; 1262 return ret;
1261 } 1263 }
1262 1264
1263 /* Return true if this perf_probe_event requires debuginfo */ 1265 /* Return true if this perf_probe_event requires debuginfo */
1264 bool perf_probe_event_need_dwarf(struct perf_probe_event *pev) 1266 bool perf_probe_event_need_dwarf(struct perf_probe_event *pev)
1265 { 1267 {
1266 int i; 1268 int i;
1267 1269
1268 if (pev->point.file || pev->point.line || pev->point.lazy_line) 1270 if (pev->point.file || pev->point.line || pev->point.lazy_line)
1269 return true; 1271 return true;
1270 1272
1271 for (i = 0; i < pev->nargs; i++) 1273 for (i = 0; i < pev->nargs; i++)
1272 if (is_c_varname(pev->args[i].var)) 1274 if (is_c_varname(pev->args[i].var))
1273 return true; 1275 return true;
1274 1276
1275 return false; 1277 return false;
1276 } 1278 }
1277 1279
1278 /* Parse probe_events event into struct probe_point */ 1280 /* Parse probe_events event into struct probe_point */
1279 static int parse_probe_trace_command(const char *cmd, 1281 static int parse_probe_trace_command(const char *cmd,
1280 struct probe_trace_event *tev) 1282 struct probe_trace_event *tev)
1281 { 1283 {
1282 struct probe_trace_point *tp = &tev->point; 1284 struct probe_trace_point *tp = &tev->point;
1283 char pr; 1285 char pr;
1284 char *p; 1286 char *p;
1285 char *argv0_str = NULL, *fmt, *fmt1_str, *fmt2_str, *fmt3_str; 1287 char *argv0_str = NULL, *fmt, *fmt1_str, *fmt2_str, *fmt3_str;
1286 int ret, i, argc; 1288 int ret, i, argc;
1287 char **argv; 1289 char **argv;
1288 1290
1289 pr_debug("Parsing probe_events: %s\n", cmd); 1291 pr_debug("Parsing probe_events: %s\n", cmd);
1290 argv = argv_split(cmd, &argc); 1292 argv = argv_split(cmd, &argc);
1291 if (!argv) { 1293 if (!argv) {
1292 pr_debug("Failed to split arguments.\n"); 1294 pr_debug("Failed to split arguments.\n");
1293 return -ENOMEM; 1295 return -ENOMEM;
1294 } 1296 }
1295 if (argc < 2) { 1297 if (argc < 2) {
1296 semantic_error("Too few probe arguments.\n"); 1298 semantic_error("Too few probe arguments.\n");
1297 ret = -ERANGE; 1299 ret = -ERANGE;
1298 goto out; 1300 goto out;
1299 } 1301 }
1300 1302
1301 /* Scan event and group name. */ 1303 /* Scan event and group name. */
1302 argv0_str = strdup(argv[0]); 1304 argv0_str = strdup(argv[0]);
1303 if (argv0_str == NULL) { 1305 if (argv0_str == NULL) {
1304 ret = -ENOMEM; 1306 ret = -ENOMEM;
1305 goto out; 1307 goto out;
1306 } 1308 }
1307 fmt1_str = strtok_r(argv0_str, ":", &fmt); 1309 fmt1_str = strtok_r(argv0_str, ":", &fmt);
1308 fmt2_str = strtok_r(NULL, "/", &fmt); 1310 fmt2_str = strtok_r(NULL, "/", &fmt);
1309 fmt3_str = strtok_r(NULL, " \t", &fmt); 1311 fmt3_str = strtok_r(NULL, " \t", &fmt);
1310 if (fmt1_str == NULL || strlen(fmt1_str) != 1 || fmt2_str == NULL 1312 if (fmt1_str == NULL || strlen(fmt1_str) != 1 || fmt2_str == NULL
1311 || fmt3_str == NULL) { 1313 || fmt3_str == NULL) {
1312 semantic_error("Failed to parse event name: %s\n", argv[0]); 1314 semantic_error("Failed to parse event name: %s\n", argv[0]);
1313 ret = -EINVAL; 1315 ret = -EINVAL;
1314 goto out; 1316 goto out;
1315 } 1317 }
1316 pr = fmt1_str[0]; 1318 pr = fmt1_str[0];
1317 tev->group = strdup(fmt2_str); 1319 tev->group = strdup(fmt2_str);
1318 tev->event = strdup(fmt3_str); 1320 tev->event = strdup(fmt3_str);
1319 if (tev->group == NULL || tev->event == NULL) { 1321 if (tev->group == NULL || tev->event == NULL) {
1320 ret = -ENOMEM; 1322 ret = -ENOMEM;
1321 goto out; 1323 goto out;
1322 } 1324 }
1323 pr_debug("Group:%s Event:%s probe:%c\n", tev->group, tev->event, pr); 1325 pr_debug("Group:%s Event:%s probe:%c\n", tev->group, tev->event, pr);
1324 1326
1325 tp->retprobe = (pr == 'r'); 1327 tp->retprobe = (pr == 'r');
1326 1328
1327 /* Scan module name(if there), function name and offset */ 1329 /* Scan module name(if there), function name and offset */
1328 p = strchr(argv[1], ':'); 1330 p = strchr(argv[1], ':');
1329 if (p) { 1331 if (p) {
1330 tp->module = strndup(argv[1], p - argv[1]); 1332 tp->module = strndup(argv[1], p - argv[1]);
1331 p++; 1333 p++;
1332 } else 1334 } else
1333 p = argv[1]; 1335 p = argv[1];
1334 fmt1_str = strtok_r(p, "+", &fmt); 1336 fmt1_str = strtok_r(p, "+", &fmt);
1335 if (fmt1_str[0] == '0') /* only the address started with 0x */ 1337 if (fmt1_str[0] == '0') /* only the address started with 0x */
1336 tp->address = strtoul(fmt1_str, NULL, 0); 1338 tp->address = strtoul(fmt1_str, NULL, 0);
1337 else { 1339 else {
1338 /* Only the symbol-based probe has offset */ 1340 /* Only the symbol-based probe has offset */
1339 tp->symbol = strdup(fmt1_str); 1341 tp->symbol = strdup(fmt1_str);
1340 if (tp->symbol == NULL) { 1342 if (tp->symbol == NULL) {
1341 ret = -ENOMEM; 1343 ret = -ENOMEM;
1342 goto out; 1344 goto out;
1343 } 1345 }
1344 fmt2_str = strtok_r(NULL, "", &fmt); 1346 fmt2_str = strtok_r(NULL, "", &fmt);
1345 if (fmt2_str == NULL) 1347 if (fmt2_str == NULL)
1346 tp->offset = 0; 1348 tp->offset = 0;
1347 else 1349 else
1348 tp->offset = strtoul(fmt2_str, NULL, 10); 1350 tp->offset = strtoul(fmt2_str, NULL, 10);
1349 } 1351 }
1350 1352
1351 tev->nargs = argc - 2; 1353 tev->nargs = argc - 2;
1352 tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs); 1354 tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs);
1353 if (tev->args == NULL) { 1355 if (tev->args == NULL) {
1354 ret = -ENOMEM; 1356 ret = -ENOMEM;
1355 goto out; 1357 goto out;
1356 } 1358 }
1357 for (i = 0; i < tev->nargs; i++) { 1359 for (i = 0; i < tev->nargs; i++) {
1358 p = strchr(argv[i + 2], '='); 1360 p = strchr(argv[i + 2], '=');
1359 if (p) /* We don't need which register is assigned. */ 1361 if (p) /* We don't need which register is assigned. */
1360 *p++ = '\0'; 1362 *p++ = '\0';
1361 else 1363 else
1362 p = argv[i + 2]; 1364 p = argv[i + 2];
1363 tev->args[i].name = strdup(argv[i + 2]); 1365 tev->args[i].name = strdup(argv[i + 2]);
1364 /* TODO: parse regs and offset */ 1366 /* TODO: parse regs and offset */
1365 tev->args[i].value = strdup(p); 1367 tev->args[i].value = strdup(p);
1366 if (tev->args[i].name == NULL || tev->args[i].value == NULL) { 1368 if (tev->args[i].name == NULL || tev->args[i].value == NULL) {
1367 ret = -ENOMEM; 1369 ret = -ENOMEM;
1368 goto out; 1370 goto out;
1369 } 1371 }
1370 } 1372 }
1371 ret = 0; 1373 ret = 0;
1372 out: 1374 out:
1373 free(argv0_str); 1375 free(argv0_str);
1374 argv_free(argv); 1376 argv_free(argv);
1375 return ret; 1377 return ret;
1376 } 1378 }
1377 1379
1378 /* Compose only probe arg */ 1380 /* Compose only probe arg */
1379 int synthesize_perf_probe_arg(struct perf_probe_arg *pa, char *buf, size_t len) 1381 int synthesize_perf_probe_arg(struct perf_probe_arg *pa, char *buf, size_t len)
1380 { 1382 {
1381 struct perf_probe_arg_field *field = pa->field; 1383 struct perf_probe_arg_field *field = pa->field;
1382 int ret; 1384 int ret;
1383 char *tmp = buf; 1385 char *tmp = buf;
1384 1386
1385 if (pa->name && pa->var) 1387 if (pa->name && pa->var)
1386 ret = e_snprintf(tmp, len, "%s=%s", pa->name, pa->var); 1388 ret = e_snprintf(tmp, len, "%s=%s", pa->name, pa->var);
1387 else 1389 else
1388 ret = e_snprintf(tmp, len, "%s", pa->name ? pa->name : pa->var); 1390 ret = e_snprintf(tmp, len, "%s", pa->name ? pa->name : pa->var);
1389 if (ret <= 0) 1391 if (ret <= 0)
1390 goto error; 1392 goto error;
1391 tmp += ret; 1393 tmp += ret;
1392 len -= ret; 1394 len -= ret;
1393 1395
1394 while (field) { 1396 while (field) {
1395 if (field->name[0] == '[') 1397 if (field->name[0] == '[')
1396 ret = e_snprintf(tmp, len, "%s", field->name); 1398 ret = e_snprintf(tmp, len, "%s", field->name);
1397 else 1399 else
1398 ret = e_snprintf(tmp, len, "%s%s", 1400 ret = e_snprintf(tmp, len, "%s%s",
1399 field->ref ? "->" : ".", field->name); 1401 field->ref ? "->" : ".", field->name);
1400 if (ret <= 0) 1402 if (ret <= 0)
1401 goto error; 1403 goto error;
1402 tmp += ret; 1404 tmp += ret;
1403 len -= ret; 1405 len -= ret;
1404 field = field->next; 1406 field = field->next;
1405 } 1407 }
1406 1408
1407 if (pa->type) { 1409 if (pa->type) {
1408 ret = e_snprintf(tmp, len, ":%s", pa->type); 1410 ret = e_snprintf(tmp, len, ":%s", pa->type);
1409 if (ret <= 0) 1411 if (ret <= 0)
1410 goto error; 1412 goto error;
1411 tmp += ret; 1413 tmp += ret;
1412 len -= ret; 1414 len -= ret;
1413 } 1415 }
1414 1416
1415 return tmp - buf; 1417 return tmp - buf;
1416 error: 1418 error:
1417 pr_debug("Failed to synthesize perf probe argument: %d\n", ret); 1419 pr_debug("Failed to synthesize perf probe argument: %d\n", ret);
1418 return ret; 1420 return ret;
1419 } 1421 }
1420 1422
1421 /* Compose only probe point (not argument) */ 1423 /* Compose only probe point (not argument) */
1422 static char *synthesize_perf_probe_point(struct perf_probe_point *pp) 1424 static char *synthesize_perf_probe_point(struct perf_probe_point *pp)
1423 { 1425 {
1424 char *buf, *tmp; 1426 char *buf, *tmp;
1425 char offs[32] = "", line[32] = "", file[32] = ""; 1427 char offs[32] = "", line[32] = "", file[32] = "";
1426 int ret, len; 1428 int ret, len;
1427 1429
1428 buf = zalloc(MAX_CMDLEN); 1430 buf = zalloc(MAX_CMDLEN);
1429 if (buf == NULL) { 1431 if (buf == NULL) {
1430 ret = -ENOMEM; 1432 ret = -ENOMEM;
1431 goto error; 1433 goto error;
1432 } 1434 }
1433 if (pp->offset) { 1435 if (pp->offset) {
1434 ret = e_snprintf(offs, 32, "+%lu", pp->offset); 1436 ret = e_snprintf(offs, 32, "+%lu", pp->offset);
1435 if (ret <= 0) 1437 if (ret <= 0)
1436 goto error; 1438 goto error;
1437 } 1439 }
1438 if (pp->line) { 1440 if (pp->line) {
1439 ret = e_snprintf(line, 32, ":%d", pp->line); 1441 ret = e_snprintf(line, 32, ":%d", pp->line);
1440 if (ret <= 0) 1442 if (ret <= 0)
1441 goto error; 1443 goto error;
1442 } 1444 }
1443 if (pp->file) { 1445 if (pp->file) {
1444 tmp = pp->file; 1446 tmp = pp->file;
1445 len = strlen(tmp); 1447 len = strlen(tmp);
1446 if (len > 30) { 1448 if (len > 30) {
1447 tmp = strchr(pp->file + len - 30, '/'); 1449 tmp = strchr(pp->file + len - 30, '/');
1448 tmp = tmp ? tmp + 1 : pp->file + len - 30; 1450 tmp = tmp ? tmp + 1 : pp->file + len - 30;
1449 } 1451 }
1450 ret = e_snprintf(file, 32, "@%s", tmp); 1452 ret = e_snprintf(file, 32, "@%s", tmp);
1451 if (ret <= 0) 1453 if (ret <= 0)
1452 goto error; 1454 goto error;
1453 } 1455 }
1454 1456
1455 if (pp->function) 1457 if (pp->function)
1456 ret = e_snprintf(buf, MAX_CMDLEN, "%s%s%s%s%s", pp->function, 1458 ret = e_snprintf(buf, MAX_CMDLEN, "%s%s%s%s%s", pp->function,
1457 offs, pp->retprobe ? "%return" : "", line, 1459 offs, pp->retprobe ? "%return" : "", line,
1458 file); 1460 file);
1459 else 1461 else
1460 ret = e_snprintf(buf, MAX_CMDLEN, "%s%s", file, line); 1462 ret = e_snprintf(buf, MAX_CMDLEN, "%s%s", file, line);
1461 if (ret <= 0) 1463 if (ret <= 0)
1462 goto error; 1464 goto error;
1463 1465
1464 return buf; 1466 return buf;
1465 error: 1467 error:
1466 pr_debug("Failed to synthesize perf probe point: %d\n", ret); 1468 pr_debug("Failed to synthesize perf probe point: %d\n", ret);
1467 free(buf); 1469 free(buf);
1468 return NULL; 1470 return NULL;
1469 } 1471 }
1470 1472
1471 #if 0 1473 #if 0
1472 char *synthesize_perf_probe_command(struct perf_probe_event *pev) 1474 char *synthesize_perf_probe_command(struct perf_probe_event *pev)
1473 { 1475 {
1474 char *buf; 1476 char *buf;
1475 int i, len, ret; 1477 int i, len, ret;
1476 1478
1477 buf = synthesize_perf_probe_point(&pev->point); 1479 buf = synthesize_perf_probe_point(&pev->point);
1478 if (!buf) 1480 if (!buf)
1479 return NULL; 1481 return NULL;
1480 1482
1481 len = strlen(buf); 1483 len = strlen(buf);
1482 for (i = 0; i < pev->nargs; i++) { 1484 for (i = 0; i < pev->nargs; i++) {
1483 ret = e_snprintf(&buf[len], MAX_CMDLEN - len, " %s", 1485 ret = e_snprintf(&buf[len], MAX_CMDLEN - len, " %s",
1484 pev->args[i].name); 1486 pev->args[i].name);
1485 if (ret <= 0) { 1487 if (ret <= 0) {
1486 free(buf); 1488 free(buf);
1487 return NULL; 1489 return NULL;
1488 } 1490 }
1489 len += ret; 1491 len += ret;
1490 } 1492 }
1491 1493
1492 return buf; 1494 return buf;
1493 } 1495 }
1494 #endif 1496 #endif
1495 1497
1496 static int __synthesize_probe_trace_arg_ref(struct probe_trace_arg_ref *ref, 1498 static int __synthesize_probe_trace_arg_ref(struct probe_trace_arg_ref *ref,
1497 char **buf, size_t *buflen, 1499 char **buf, size_t *buflen,
1498 int depth) 1500 int depth)
1499 { 1501 {
1500 int ret; 1502 int ret;
1501 if (ref->next) { 1503 if (ref->next) {
1502 depth = __synthesize_probe_trace_arg_ref(ref->next, buf, 1504 depth = __synthesize_probe_trace_arg_ref(ref->next, buf,
1503 buflen, depth + 1); 1505 buflen, depth + 1);
1504 if (depth < 0) 1506 if (depth < 0)
1505 goto out; 1507 goto out;
1506 } 1508 }
1507 1509
1508 ret = e_snprintf(*buf, *buflen, "%+ld(", ref->offset); 1510 ret = e_snprintf(*buf, *buflen, "%+ld(", ref->offset);
1509 if (ret < 0) 1511 if (ret < 0)
1510 depth = ret; 1512 depth = ret;
1511 else { 1513 else {
1512 *buf += ret; 1514 *buf += ret;
1513 *buflen -= ret; 1515 *buflen -= ret;
1514 } 1516 }
1515 out: 1517 out:
1516 return depth; 1518 return depth;
1517 1519
1518 } 1520 }
1519 1521
1520 static int synthesize_probe_trace_arg(struct probe_trace_arg *arg, 1522 static int synthesize_probe_trace_arg(struct probe_trace_arg *arg,
1521 char *buf, size_t buflen) 1523 char *buf, size_t buflen)
1522 { 1524 {
1523 struct probe_trace_arg_ref *ref = arg->ref; 1525 struct probe_trace_arg_ref *ref = arg->ref;
1524 int ret, depth = 0; 1526 int ret, depth = 0;
1525 char *tmp = buf; 1527 char *tmp = buf;
1526 1528
1527 /* Argument name or separator */ 1529 /* Argument name or separator */
1528 if (arg->name) 1530 if (arg->name)
1529 ret = e_snprintf(buf, buflen, " %s=", arg->name); 1531 ret = e_snprintf(buf, buflen, " %s=", arg->name);
1530 else 1532 else
1531 ret = e_snprintf(buf, buflen, " "); 1533 ret = e_snprintf(buf, buflen, " ");
1532 if (ret < 0) 1534 if (ret < 0)
1533 return ret; 1535 return ret;
1534 buf += ret; 1536 buf += ret;
1535 buflen -= ret; 1537 buflen -= ret;
1536 1538
1537 /* Special case: @XXX */ 1539 /* Special case: @XXX */
1538 if (arg->value[0] == '@' && arg->ref) 1540 if (arg->value[0] == '@' && arg->ref)
1539 ref = ref->next; 1541 ref = ref->next;
1540 1542
1541 /* Dereferencing arguments */ 1543 /* Dereferencing arguments */
1542 if (ref) { 1544 if (ref) {
1543 depth = __synthesize_probe_trace_arg_ref(ref, &buf, 1545 depth = __synthesize_probe_trace_arg_ref(ref, &buf,
1544 &buflen, 1); 1546 &buflen, 1);
1545 if (depth < 0) 1547 if (depth < 0)
1546 return depth; 1548 return depth;
1547 } 1549 }
1548 1550
1549 /* Print argument value */ 1551 /* Print argument value */
1550 if (arg->value[0] == '@' && arg->ref) 1552 if (arg->value[0] == '@' && arg->ref)
1551 ret = e_snprintf(buf, buflen, "%s%+ld", arg->value, 1553 ret = e_snprintf(buf, buflen, "%s%+ld", arg->value,
1552 arg->ref->offset); 1554 arg->ref->offset);
1553 else 1555 else
1554 ret = e_snprintf(buf, buflen, "%s", arg->value); 1556 ret = e_snprintf(buf, buflen, "%s", arg->value);
1555 if (ret < 0) 1557 if (ret < 0)
1556 return ret; 1558 return ret;
1557 buf += ret; 1559 buf += ret;
1558 buflen -= ret; 1560 buflen -= ret;
1559 1561
1560 /* Closing */ 1562 /* Closing */
1561 while (depth--) { 1563 while (depth--) {
1562 ret = e_snprintf(buf, buflen, ")"); 1564 ret = e_snprintf(buf, buflen, ")");
1563 if (ret < 0) 1565 if (ret < 0)
1564 return ret; 1566 return ret;
1565 buf += ret; 1567 buf += ret;
1566 buflen -= ret; 1568 buflen -= ret;
1567 } 1569 }
1568 /* Print argument type */ 1570 /* Print argument type */
1569 if (arg->type) { 1571 if (arg->type) {
1570 ret = e_snprintf(buf, buflen, ":%s", arg->type); 1572 ret = e_snprintf(buf, buflen, ":%s", arg->type);
1571 if (ret <= 0) 1573 if (ret <= 0)
1572 return ret; 1574 return ret;
1573 buf += ret; 1575 buf += ret;
1574 } 1576 }
1575 1577
1576 return buf - tmp; 1578 return buf - tmp;
1577 } 1579 }
1578 1580
1579 char *synthesize_probe_trace_command(struct probe_trace_event *tev) 1581 char *synthesize_probe_trace_command(struct probe_trace_event *tev)
1580 { 1582 {
1581 struct probe_trace_point *tp = &tev->point; 1583 struct probe_trace_point *tp = &tev->point;
1582 char *buf; 1584 char *buf;
1583 int i, len, ret; 1585 int i, len, ret;
1584 1586
1585 buf = zalloc(MAX_CMDLEN); 1587 buf = zalloc(MAX_CMDLEN);
1586 if (buf == NULL) 1588 if (buf == NULL)
1587 return NULL; 1589 return NULL;
1588 1590
1589 len = e_snprintf(buf, MAX_CMDLEN, "%c:%s/%s ", tp->retprobe ? 'r' : 'p', 1591 len = e_snprintf(buf, MAX_CMDLEN, "%c:%s/%s ", tp->retprobe ? 'r' : 'p',
1590 tev->group, tev->event); 1592 tev->group, tev->event);
1591 if (len <= 0) 1593 if (len <= 0)
1592 goto error; 1594 goto error;
1593 1595
1594 /* Uprobes must have tp->address and tp->module */ 1596 /* Uprobes must have tp->address and tp->module */
1595 if (tev->uprobes && (!tp->address || !tp->module)) 1597 if (tev->uprobes && (!tp->address || !tp->module))
1596 goto error; 1598 goto error;
1597 1599
1598 /* Use the tp->address for uprobes */ 1600 /* Use the tp->address for uprobes */
1599 if (tev->uprobes) 1601 if (tev->uprobes)
1600 ret = e_snprintf(buf + len, MAX_CMDLEN - len, "%s:0x%lx", 1602 ret = e_snprintf(buf + len, MAX_CMDLEN - len, "%s:0x%lx",
1601 tp->module, tp->address); 1603 tp->module, tp->address);
1602 else 1604 else
1603 ret = e_snprintf(buf + len, MAX_CMDLEN - len, "%s%s%s+%lu", 1605 ret = e_snprintf(buf + len, MAX_CMDLEN - len, "%s%s%s+%lu",
1604 tp->module ?: "", tp->module ? ":" : "", 1606 tp->module ?: "", tp->module ? ":" : "",
1605 tp->symbol, tp->offset); 1607 tp->symbol, tp->offset);
1606 1608
1607 if (ret <= 0) 1609 if (ret <= 0)
1608 goto error; 1610 goto error;
1609 len += ret; 1611 len += ret;
1610 1612
1611 for (i = 0; i < tev->nargs; i++) { 1613 for (i = 0; i < tev->nargs; i++) {
1612 ret = synthesize_probe_trace_arg(&tev->args[i], buf + len, 1614 ret = synthesize_probe_trace_arg(&tev->args[i], buf + len,
1613 MAX_CMDLEN - len); 1615 MAX_CMDLEN - len);
1614 if (ret <= 0) 1616 if (ret <= 0)
1615 goto error; 1617 goto error;
1616 len += ret; 1618 len += ret;
1617 } 1619 }
1618 1620
1619 return buf; 1621 return buf;
1620 error: 1622 error:
1621 free(buf); 1623 free(buf);
1622 return NULL; 1624 return NULL;
1623 } 1625 }
1624 1626
1625 static int find_perf_probe_point_from_map(struct probe_trace_point *tp, 1627 static int find_perf_probe_point_from_map(struct probe_trace_point *tp,
1626 struct perf_probe_point *pp, 1628 struct perf_probe_point *pp,
1627 bool is_kprobe) 1629 bool is_kprobe)
1628 { 1630 {
1629 struct symbol *sym = NULL; 1631 struct symbol *sym = NULL;
1630 struct map *map; 1632 struct map *map;
1631 u64 addr; 1633 u64 addr;
1632 int ret = -ENOENT; 1634 int ret = -ENOENT;
1633 1635
1634 if (!is_kprobe) { 1636 if (!is_kprobe) {
1635 map = dso__new_map(tp->module); 1637 map = dso__new_map(tp->module);
1636 if (!map) 1638 if (!map)
1637 goto out; 1639 goto out;
1638 addr = tp->address; 1640 addr = tp->address;
1639 sym = map__find_symbol(map, addr, NULL); 1641 sym = map__find_symbol(map, addr, NULL);
1640 } else { 1642 } else {
1641 addr = kernel_get_symbol_address_by_name(tp->symbol, true); 1643 addr = kernel_get_symbol_address_by_name(tp->symbol, true);
1642 if (addr) { 1644 if (addr) {
1643 addr += tp->offset; 1645 addr += tp->offset;
1644 sym = __find_kernel_function(addr, &map); 1646 sym = __find_kernel_function(addr, &map);
1645 } 1647 }
1646 } 1648 }
1647 if (!sym) 1649 if (!sym)
1648 goto out; 1650 goto out;
1649 1651
1650 pp->retprobe = tp->retprobe; 1652 pp->retprobe = tp->retprobe;
1651 pp->offset = addr - map->unmap_ip(map, sym->start); 1653 pp->offset = addr - map->unmap_ip(map, sym->start);
1652 pp->function = strdup(sym->name); 1654 pp->function = strdup(sym->name);
1653 ret = pp->function ? 0 : -ENOMEM; 1655 ret = pp->function ? 0 : -ENOMEM;
1654 1656
1655 out: 1657 out:
1656 if (map && !is_kprobe) { 1658 if (map && !is_kprobe) {
1657 dso__delete(map->dso); 1659 dso__delete(map->dso);
1658 map__delete(map); 1660 map__delete(map);
1659 } 1661 }
1660 1662
1661 return ret; 1663 return ret;
1662 } 1664 }
1663 1665
1664 static int convert_to_perf_probe_point(struct probe_trace_point *tp, 1666 static int convert_to_perf_probe_point(struct probe_trace_point *tp,
1665 struct perf_probe_point *pp, 1667 struct perf_probe_point *pp,
1666 bool is_kprobe) 1668 bool is_kprobe)
1667 { 1669 {
1668 char buf[128]; 1670 char buf[128];
1669 int ret; 1671 int ret;
1670 1672
1671 ret = find_perf_probe_point_from_dwarf(tp, pp, is_kprobe); 1673 ret = find_perf_probe_point_from_dwarf(tp, pp, is_kprobe);
1672 if (!ret) 1674 if (!ret)
1673 return 0; 1675 return 0;
1674 ret = find_perf_probe_point_from_map(tp, pp, is_kprobe); 1676 ret = find_perf_probe_point_from_map(tp, pp, is_kprobe);
1675 if (!ret) 1677 if (!ret)
1676 return 0; 1678 return 0;
1677 1679
1678 pr_debug("Failed to find probe point from both of dwarf and map.\n"); 1680 pr_debug("Failed to find probe point from both of dwarf and map.\n");
1679 1681
1680 if (tp->symbol) { 1682 if (tp->symbol) {
1681 pp->function = strdup(tp->symbol); 1683 pp->function = strdup(tp->symbol);
1682 pp->offset = tp->offset; 1684 pp->offset = tp->offset;
1683 } else if (!tp->module && !is_kprobe) { 1685 } else if (!tp->module && !is_kprobe) {
1684 ret = e_snprintf(buf, 128, "0x%" PRIx64, (u64)tp->address); 1686 ret = e_snprintf(buf, 128, "0x%" PRIx64, (u64)tp->address);
1685 if (ret < 0) 1687 if (ret < 0)
1686 return ret; 1688 return ret;
1687 pp->function = strdup(buf); 1689 pp->function = strdup(buf);
1688 pp->offset = 0; 1690 pp->offset = 0;
1689 } 1691 }
1690 if (pp->function == NULL) 1692 if (pp->function == NULL)
1691 return -ENOMEM; 1693 return -ENOMEM;
1692 1694
1693 pp->retprobe = tp->retprobe; 1695 pp->retprobe = tp->retprobe;
1694 1696
1695 return 0; 1697 return 0;
1696 } 1698 }
1697 1699
1698 static int convert_to_perf_probe_event(struct probe_trace_event *tev, 1700 static int convert_to_perf_probe_event(struct probe_trace_event *tev,
1699 struct perf_probe_event *pev, bool is_kprobe) 1701 struct perf_probe_event *pev, bool is_kprobe)
1700 { 1702 {
1701 char buf[64] = ""; 1703 char buf[64] = "";
1702 int i, ret; 1704 int i, ret;
1703 1705
1704 /* Convert event/group name */ 1706 /* Convert event/group name */
1705 pev->event = strdup(tev->event); 1707 pev->event = strdup(tev->event);
1706 pev->group = strdup(tev->group); 1708 pev->group = strdup(tev->group);
1707 if (pev->event == NULL || pev->group == NULL) 1709 if (pev->event == NULL || pev->group == NULL)
1708 return -ENOMEM; 1710 return -ENOMEM;
1709 1711
1710 /* Convert trace_point to probe_point */ 1712 /* Convert trace_point to probe_point */
1711 ret = convert_to_perf_probe_point(&tev->point, &pev->point, is_kprobe); 1713 ret = convert_to_perf_probe_point(&tev->point, &pev->point, is_kprobe);
1712 if (ret < 0) 1714 if (ret < 0)
1713 return ret; 1715 return ret;
1714 1716
1715 /* Convert trace_arg to probe_arg */ 1717 /* Convert trace_arg to probe_arg */
1716 pev->nargs = tev->nargs; 1718 pev->nargs = tev->nargs;
1717 pev->args = zalloc(sizeof(struct perf_probe_arg) * pev->nargs); 1719 pev->args = zalloc(sizeof(struct perf_probe_arg) * pev->nargs);
1718 if (pev->args == NULL) 1720 if (pev->args == NULL)
1719 return -ENOMEM; 1721 return -ENOMEM;
1720 for (i = 0; i < tev->nargs && ret >= 0; i++) { 1722 for (i = 0; i < tev->nargs && ret >= 0; i++) {
1721 if (tev->args[i].name) 1723 if (tev->args[i].name)
1722 pev->args[i].name = strdup(tev->args[i].name); 1724 pev->args[i].name = strdup(tev->args[i].name);
1723 else { 1725 else {
1724 ret = synthesize_probe_trace_arg(&tev->args[i], 1726 ret = synthesize_probe_trace_arg(&tev->args[i],
1725 buf, 64); 1727 buf, 64);
1726 pev->args[i].name = strdup(buf); 1728 pev->args[i].name = strdup(buf);
1727 } 1729 }
1728 if (pev->args[i].name == NULL && ret >= 0) 1730 if (pev->args[i].name == NULL && ret >= 0)
1729 ret = -ENOMEM; 1731 ret = -ENOMEM;
1730 } 1732 }
1731 1733
1732 if (ret < 0) 1734 if (ret < 0)
1733 clear_perf_probe_event(pev); 1735 clear_perf_probe_event(pev);
1734 1736
1735 return ret; 1737 return ret;
1736 } 1738 }
1737 1739
1738 void clear_perf_probe_event(struct perf_probe_event *pev) 1740 void clear_perf_probe_event(struct perf_probe_event *pev)
1739 { 1741 {
1740 struct perf_probe_point *pp = &pev->point; 1742 struct perf_probe_point *pp = &pev->point;
1741 struct perf_probe_arg_field *field, *next; 1743 struct perf_probe_arg_field *field, *next;
1742 int i; 1744 int i;
1743 1745
1744 free(pev->event); 1746 free(pev->event);
1745 free(pev->group); 1747 free(pev->group);
1746 free(pp->file); 1748 free(pp->file);
1747 free(pp->function); 1749 free(pp->function);
1748 free(pp->lazy_line); 1750 free(pp->lazy_line);
1749 1751
1750 for (i = 0; i < pev->nargs; i++) { 1752 for (i = 0; i < pev->nargs; i++) {
1751 free(pev->args[i].name); 1753 free(pev->args[i].name);
1752 free(pev->args[i].var); 1754 free(pev->args[i].var);
1753 free(pev->args[i].type); 1755 free(pev->args[i].type);
1754 field = pev->args[i].field; 1756 field = pev->args[i].field;
1755 while (field) { 1757 while (field) {
1756 next = field->next; 1758 next = field->next;
1757 zfree(&field->name); 1759 zfree(&field->name);
1758 free(field); 1760 free(field);
1759 field = next; 1761 field = next;
1760 } 1762 }
1761 } 1763 }
1762 free(pev->args); 1764 free(pev->args);
1763 memset(pev, 0, sizeof(*pev)); 1765 memset(pev, 0, sizeof(*pev));
1764 } 1766 }
1765 1767
1766 static void clear_probe_trace_event(struct probe_trace_event *tev) 1768 static void clear_probe_trace_event(struct probe_trace_event *tev)
1767 { 1769 {
1768 struct probe_trace_arg_ref *ref, *next; 1770 struct probe_trace_arg_ref *ref, *next;
1769 int i; 1771 int i;
1770 1772
1771 free(tev->event); 1773 free(tev->event);
1772 free(tev->group); 1774 free(tev->group);
1773 free(tev->point.symbol); 1775 free(tev->point.symbol);
1774 free(tev->point.module); 1776 free(tev->point.module);
1775 for (i = 0; i < tev->nargs; i++) { 1777 for (i = 0; i < tev->nargs; i++) {
1776 free(tev->args[i].name); 1778 free(tev->args[i].name);
1777 free(tev->args[i].value); 1779 free(tev->args[i].value);
1778 free(tev->args[i].type); 1780 free(tev->args[i].type);
1779 ref = tev->args[i].ref; 1781 ref = tev->args[i].ref;
1780 while (ref) { 1782 while (ref) {
1781 next = ref->next; 1783 next = ref->next;
1782 free(ref); 1784 free(ref);
1783 ref = next; 1785 ref = next;
1784 } 1786 }
1785 } 1787 }
1786 free(tev->args); 1788 free(tev->args);
1787 memset(tev, 0, sizeof(*tev)); 1789 memset(tev, 0, sizeof(*tev));
1788 } 1790 }
1789 1791
1790 static void print_open_warning(int err, bool is_kprobe) 1792 static void print_open_warning(int err, bool is_kprobe)
1791 { 1793 {
1792 char sbuf[STRERR_BUFSIZE]; 1794 char sbuf[STRERR_BUFSIZE];
1793 1795
1794 if (err == -ENOENT) { 1796 if (err == -ENOENT) {
1795 const char *config; 1797 const char *config;
1796 1798
1797 if (!is_kprobe) 1799 if (!is_kprobe)
1798 config = "CONFIG_UPROBE_EVENTS"; 1800 config = "CONFIG_UPROBE_EVENTS";
1799 else 1801 else
1800 config = "CONFIG_KPROBE_EVENTS"; 1802 config = "CONFIG_KPROBE_EVENTS";
1801 1803
1802 pr_warning("%cprobe_events file does not exist" 1804 pr_warning("%cprobe_events file does not exist"
1803 " - please rebuild kernel with %s.\n", 1805 " - please rebuild kernel with %s.\n",
1804 is_kprobe ? 'k' : 'u', config); 1806 is_kprobe ? 'k' : 'u', config);
1805 } else if (err == -ENOTSUP) 1807 } else if (err == -ENOTSUP)
1806 pr_warning("Debugfs is not mounted.\n"); 1808 pr_warning("Debugfs is not mounted.\n");
1807 else 1809 else
1808 pr_warning("Failed to open %cprobe_events: %s\n", 1810 pr_warning("Failed to open %cprobe_events: %s\n",
1809 is_kprobe ? 'k' : 'u', 1811 is_kprobe ? 'k' : 'u',
1810 strerror_r(-err, sbuf, sizeof(sbuf))); 1812 strerror_r(-err, sbuf, sizeof(sbuf)));
1811 } 1813 }
1812 1814
1813 static void print_both_open_warning(int kerr, int uerr) 1815 static void print_both_open_warning(int kerr, int uerr)
1814 { 1816 {
1815 /* Both kprobes and uprobes are disabled, warn it. */ 1817 /* Both kprobes and uprobes are disabled, warn it. */
1816 if (kerr == -ENOTSUP && uerr == -ENOTSUP) 1818 if (kerr == -ENOTSUP && uerr == -ENOTSUP)
1817 pr_warning("Debugfs is not mounted.\n"); 1819 pr_warning("Debugfs is not mounted.\n");
1818 else if (kerr == -ENOENT && uerr == -ENOENT) 1820 else if (kerr == -ENOENT && uerr == -ENOENT)
1819 pr_warning("Please rebuild kernel with CONFIG_KPROBE_EVENTS " 1821 pr_warning("Please rebuild kernel with CONFIG_KPROBE_EVENTS "
1820 "or/and CONFIG_UPROBE_EVENTS.\n"); 1822 "or/and CONFIG_UPROBE_EVENTS.\n");
1821 else { 1823 else {
1822 char sbuf[STRERR_BUFSIZE]; 1824 char sbuf[STRERR_BUFSIZE];
1823 pr_warning("Failed to open kprobe events: %s.\n", 1825 pr_warning("Failed to open kprobe events: %s.\n",
1824 strerror_r(-kerr, sbuf, sizeof(sbuf))); 1826 strerror_r(-kerr, sbuf, sizeof(sbuf)));
1825 pr_warning("Failed to open uprobe events: %s.\n", 1827 pr_warning("Failed to open uprobe events: %s.\n",
1826 strerror_r(-uerr, sbuf, sizeof(sbuf))); 1828 strerror_r(-uerr, sbuf, sizeof(sbuf)));
1827 } 1829 }
1828 } 1830 }
1829 1831
1830 static int open_probe_events(const char *trace_file, bool readwrite) 1832 static int open_probe_events(const char *trace_file, bool readwrite)
1831 { 1833 {
1832 char buf[PATH_MAX]; 1834 char buf[PATH_MAX];
1833 const char *__debugfs; 1835 const char *__debugfs;
1834 int ret; 1836 int ret;
1835 1837
1836 __debugfs = debugfs_find_mountpoint(); 1838 __debugfs = debugfs_find_mountpoint();
1837 if (__debugfs == NULL) 1839 if (__debugfs == NULL)
1838 return -ENOTSUP; 1840 return -ENOTSUP;
1839 1841
1840 ret = e_snprintf(buf, PATH_MAX, "%s/%s", __debugfs, trace_file); 1842 ret = e_snprintf(buf, PATH_MAX, "%s/%s", __debugfs, trace_file);
1841 if (ret >= 0) { 1843 if (ret >= 0) {
1842 pr_debug("Opening %s write=%d\n", buf, readwrite); 1844 pr_debug("Opening %s write=%d\n", buf, readwrite);
1843 if (readwrite && !probe_event_dry_run) 1845 if (readwrite && !probe_event_dry_run)
1844 ret = open(buf, O_RDWR, O_APPEND); 1846 ret = open(buf, O_RDWR, O_APPEND);
1845 else 1847 else
1846 ret = open(buf, O_RDONLY, 0); 1848 ret = open(buf, O_RDONLY, 0);
1847 1849
1848 if (ret < 0) 1850 if (ret < 0)
1849 ret = -errno; 1851 ret = -errno;
1850 } 1852 }
1851 return ret; 1853 return ret;
1852 } 1854 }
1853 1855
1854 static int open_kprobe_events(bool readwrite) 1856 static int open_kprobe_events(bool readwrite)
1855 { 1857 {
1856 return open_probe_events("tracing/kprobe_events", readwrite); 1858 return open_probe_events("tracing/kprobe_events", readwrite);
1857 } 1859 }
1858 1860
1859 static int open_uprobe_events(bool readwrite) 1861 static int open_uprobe_events(bool readwrite)
1860 { 1862 {
1861 return open_probe_events("tracing/uprobe_events", readwrite); 1863 return open_probe_events("tracing/uprobe_events", readwrite);
1862 } 1864 }
1863 1865
1864 /* Get raw string list of current kprobe_events or uprobe_events */ 1866 /* Get raw string list of current kprobe_events or uprobe_events */
1865 static struct strlist *get_probe_trace_command_rawlist(int fd) 1867 static struct strlist *get_probe_trace_command_rawlist(int fd)
1866 { 1868 {
1867 int ret, idx; 1869 int ret, idx;
1868 FILE *fp; 1870 FILE *fp;
1869 char buf[MAX_CMDLEN]; 1871 char buf[MAX_CMDLEN];
1870 char *p; 1872 char *p;
1871 struct strlist *sl; 1873 struct strlist *sl;
1872 1874
1873 sl = strlist__new(true, NULL); 1875 sl = strlist__new(true, NULL);
1874 1876
1875 fp = fdopen(dup(fd), "r"); 1877 fp = fdopen(dup(fd), "r");
1876 while (!feof(fp)) { 1878 while (!feof(fp)) {
1877 p = fgets(buf, MAX_CMDLEN, fp); 1879 p = fgets(buf, MAX_CMDLEN, fp);
1878 if (!p) 1880 if (!p)
1879 break; 1881 break;
1880 1882
1881 idx = strlen(p) - 1; 1883 idx = strlen(p) - 1;
1882 if (p[idx] == '\n') 1884 if (p[idx] == '\n')
1883 p[idx] = '\0'; 1885 p[idx] = '\0';
1884 ret = strlist__add(sl, buf); 1886 ret = strlist__add(sl, buf);
1885 if (ret < 0) { 1887 if (ret < 0) {
1886 pr_debug("strlist__add failed (%d)\n", ret); 1888 pr_debug("strlist__add failed (%d)\n", ret);
1887 strlist__delete(sl); 1889 strlist__delete(sl);
1888 return NULL; 1890 return NULL;
1889 } 1891 }
1890 } 1892 }
1891 fclose(fp); 1893 fclose(fp);
1892 1894
1893 return sl; 1895 return sl;
1894 } 1896 }
1895 1897
1896 /* Show an event */ 1898 /* Show an event */
1897 static int show_perf_probe_event(struct perf_probe_event *pev, 1899 static int show_perf_probe_event(struct perf_probe_event *pev,
1898 const char *module) 1900 const char *module)
1899 { 1901 {
1900 int i, ret; 1902 int i, ret;
1901 char buf[128]; 1903 char buf[128];
1902 char *place; 1904 char *place;
1903 1905
1904 /* Synthesize only event probe point */ 1906 /* Synthesize only event probe point */
1905 place = synthesize_perf_probe_point(&pev->point); 1907 place = synthesize_perf_probe_point(&pev->point);
1906 if (!place) 1908 if (!place)
1907 return -EINVAL; 1909 return -EINVAL;
1908 1910
1909 ret = e_snprintf(buf, 128, "%s:%s", pev->group, pev->event); 1911 ret = e_snprintf(buf, 128, "%s:%s", pev->group, pev->event);
1910 if (ret < 0) 1912 if (ret < 0)
1911 return ret; 1913 return ret;
1912 1914
1913 pr_info(" %-20s (on %s", buf, place); 1915 pr_info(" %-20s (on %s", buf, place);
1914 if (module) 1916 if (module)
1915 pr_info(" in %s", module); 1917 pr_info(" in %s", module);
1916 1918
1917 if (pev->nargs > 0) { 1919 if (pev->nargs > 0) {
1918 pr_info(" with"); 1920 pr_info(" with");
1919 for (i = 0; i < pev->nargs; i++) { 1921 for (i = 0; i < pev->nargs; i++) {
1920 ret = synthesize_perf_probe_arg(&pev->args[i], 1922 ret = synthesize_perf_probe_arg(&pev->args[i],
1921 buf, 128); 1923 buf, 128);
1922 if (ret < 0) 1924 if (ret < 0)
1923 break; 1925 break;
1924 pr_info(" %s", buf); 1926 pr_info(" %s", buf);
1925 } 1927 }
1926 } 1928 }
1927 pr_info(")\n"); 1929 pr_info(")\n");
1928 free(place); 1930 free(place);
1929 return ret; 1931 return ret;
1930 } 1932 }
1931 1933
1932 static int __show_perf_probe_events(int fd, bool is_kprobe) 1934 static int __show_perf_probe_events(int fd, bool is_kprobe)
1933 { 1935 {
1934 int ret = 0; 1936 int ret = 0;
1935 struct probe_trace_event tev; 1937 struct probe_trace_event tev;
1936 struct perf_probe_event pev; 1938 struct perf_probe_event pev;
1937 struct strlist *rawlist; 1939 struct strlist *rawlist;
1938 struct str_node *ent; 1940 struct str_node *ent;
1939 1941
1940 memset(&tev, 0, sizeof(tev)); 1942 memset(&tev, 0, sizeof(tev));
1941 memset(&pev, 0, sizeof(pev)); 1943 memset(&pev, 0, sizeof(pev));
1942 1944
1943 rawlist = get_probe_trace_command_rawlist(fd); 1945 rawlist = get_probe_trace_command_rawlist(fd);
1944 if (!rawlist) 1946 if (!rawlist)
1945 return -ENOMEM; 1947 return -ENOMEM;
1946 1948
1947 strlist__for_each(ent, rawlist) { 1949 strlist__for_each(ent, rawlist) {
1948 ret = parse_probe_trace_command(ent->s, &tev); 1950 ret = parse_probe_trace_command(ent->s, &tev);
1949 if (ret >= 0) { 1951 if (ret >= 0) {
1950 ret = convert_to_perf_probe_event(&tev, &pev, 1952 ret = convert_to_perf_probe_event(&tev, &pev,
1951 is_kprobe); 1953 is_kprobe);
1952 if (ret >= 0) 1954 if (ret >= 0)
1953 ret = show_perf_probe_event(&pev, 1955 ret = show_perf_probe_event(&pev,
1954 tev.point.module); 1956 tev.point.module);
1955 } 1957 }
1956 clear_perf_probe_event(&pev); 1958 clear_perf_probe_event(&pev);
1957 clear_probe_trace_event(&tev); 1959 clear_probe_trace_event(&tev);
1958 if (ret < 0) 1960 if (ret < 0)
1959 break; 1961 break;
1960 } 1962 }
1961 strlist__delete(rawlist); 1963 strlist__delete(rawlist);
1962 1964
1963 return ret; 1965 return ret;
1964 } 1966 }
1965 1967
1966 /* List up current perf-probe events */ 1968 /* List up current perf-probe events */
1967 int show_perf_probe_events(void) 1969 int show_perf_probe_events(void)
1968 { 1970 {
1969 int kp_fd, up_fd, ret; 1971 int kp_fd, up_fd, ret;
1970 1972
1971 setup_pager(); 1973 setup_pager();
1972 1974
1973 ret = init_symbol_maps(false); 1975 ret = init_symbol_maps(false);
1974 if (ret < 0) 1976 if (ret < 0)
1975 return ret; 1977 return ret;
1976 1978
1977 kp_fd = open_kprobe_events(false); 1979 kp_fd = open_kprobe_events(false);
1978 if (kp_fd >= 0) { 1980 if (kp_fd >= 0) {
1979 ret = __show_perf_probe_events(kp_fd, true); 1981 ret = __show_perf_probe_events(kp_fd, true);
1980 close(kp_fd); 1982 close(kp_fd);
1981 if (ret < 0) 1983 if (ret < 0)
1982 goto out; 1984 goto out;
1983 } 1985 }
1984 1986
1985 up_fd = open_uprobe_events(false); 1987 up_fd = open_uprobe_events(false);
1986 if (kp_fd < 0 && up_fd < 0) { 1988 if (kp_fd < 0 && up_fd < 0) {
1987 print_both_open_warning(kp_fd, up_fd); 1989 print_both_open_warning(kp_fd, up_fd);
1988 ret = kp_fd; 1990 ret = kp_fd;
1989 goto out; 1991 goto out;
1990 } 1992 }
1991 1993
1992 if (up_fd >= 0) { 1994 if (up_fd >= 0) {
1993 ret = __show_perf_probe_events(up_fd, false); 1995 ret = __show_perf_probe_events(up_fd, false);
1994 close(up_fd); 1996 close(up_fd);
1995 } 1997 }
1996 out: 1998 out:
1997 exit_symbol_maps(); 1999 exit_symbol_maps();
1998 return ret; 2000 return ret;
1999 } 2001 }
2000 2002
2001 /* Get current perf-probe event names */ 2003 /* Get current perf-probe event names */
2002 static struct strlist *get_probe_trace_event_names(int fd, bool include_group) 2004 static struct strlist *get_probe_trace_event_names(int fd, bool include_group)
2003 { 2005 {
2004 char buf[128]; 2006 char buf[128];
2005 struct strlist *sl, *rawlist; 2007 struct strlist *sl, *rawlist;
2006 struct str_node *ent; 2008 struct str_node *ent;
2007 struct probe_trace_event tev; 2009 struct probe_trace_event tev;
2008 int ret = 0; 2010 int ret = 0;
2009 2011
2010 memset(&tev, 0, sizeof(tev)); 2012 memset(&tev, 0, sizeof(tev));
2011 rawlist = get_probe_trace_command_rawlist(fd); 2013 rawlist = get_probe_trace_command_rawlist(fd);
2012 if (!rawlist) 2014 if (!rawlist)
2013 return NULL; 2015 return NULL;
2014 sl = strlist__new(true, NULL); 2016 sl = strlist__new(true, NULL);
2015 strlist__for_each(ent, rawlist) { 2017 strlist__for_each(ent, rawlist) {
2016 ret = parse_probe_trace_command(ent->s, &tev); 2018 ret = parse_probe_trace_command(ent->s, &tev);
2017 if (ret < 0) 2019 if (ret < 0)
2018 break; 2020 break;
2019 if (include_group) { 2021 if (include_group) {
2020 ret = e_snprintf(buf, 128, "%s:%s", tev.group, 2022 ret = e_snprintf(buf, 128, "%s:%s", tev.group,
2021 tev.event); 2023 tev.event);
2022 if (ret >= 0) 2024 if (ret >= 0)
2023 ret = strlist__add(sl, buf); 2025 ret = strlist__add(sl, buf);
2024 } else 2026 } else
2025 ret = strlist__add(sl, tev.event); 2027 ret = strlist__add(sl, tev.event);
2026 clear_probe_trace_event(&tev); 2028 clear_probe_trace_event(&tev);
2027 if (ret < 0) 2029 if (ret < 0)
2028 break; 2030 break;
2029 } 2031 }
2030 strlist__delete(rawlist); 2032 strlist__delete(rawlist);
2031 2033
2032 if (ret < 0) { 2034 if (ret < 0) {
2033 strlist__delete(sl); 2035 strlist__delete(sl);
2034 return NULL; 2036 return NULL;
2035 } 2037 }
2036 return sl; 2038 return sl;
2037 } 2039 }
2038 2040
2039 static int write_probe_trace_event(int fd, struct probe_trace_event *tev) 2041 static int write_probe_trace_event(int fd, struct probe_trace_event *tev)
2040 { 2042 {
2041 int ret = 0; 2043 int ret = 0;
2042 char *buf = synthesize_probe_trace_command(tev); 2044 char *buf = synthesize_probe_trace_command(tev);
2043 char sbuf[STRERR_BUFSIZE]; 2045 char sbuf[STRERR_BUFSIZE];
2044 2046
2045 if (!buf) { 2047 if (!buf) {
2046 pr_debug("Failed to synthesize probe trace event.\n"); 2048 pr_debug("Failed to synthesize probe trace event.\n");
2047 return -EINVAL; 2049 return -EINVAL;
2048 } 2050 }
2049 2051
2050 pr_debug("Writing event: %s\n", buf); 2052 pr_debug("Writing event: %s\n", buf);
2051 if (!probe_event_dry_run) { 2053 if (!probe_event_dry_run) {
2052 ret = write(fd, buf, strlen(buf)); 2054 ret = write(fd, buf, strlen(buf));
2053 if (ret <= 0) 2055 if (ret <= 0)
2054 pr_warning("Failed to write event: %s\n", 2056 pr_warning("Failed to write event: %s\n",
2055 strerror_r(errno, sbuf, sizeof(sbuf))); 2057 strerror_r(errno, sbuf, sizeof(sbuf)));
2056 } 2058 }
2057 free(buf); 2059 free(buf);
2058 return ret; 2060 return ret;
2059 } 2061 }
2060 2062
2061 static int get_new_event_name(char *buf, size_t len, const char *base, 2063 static int get_new_event_name(char *buf, size_t len, const char *base,
2062 struct strlist *namelist, bool allow_suffix) 2064 struct strlist *namelist, bool allow_suffix)
2063 { 2065 {
2064 int i, ret; 2066 int i, ret;
2065 2067
2066 /* Try no suffix */ 2068 /* Try no suffix */
2067 ret = e_snprintf(buf, len, "%s", base); 2069 ret = e_snprintf(buf, len, "%s", base);
2068 if (ret < 0) { 2070 if (ret < 0) {
2069 pr_debug("snprintf() failed: %d\n", ret); 2071 pr_debug("snprintf() failed: %d\n", ret);
2070 return ret; 2072 return ret;
2071 } 2073 }
2072 if (!strlist__has_entry(namelist, buf)) 2074 if (!strlist__has_entry(namelist, buf))
2073 return 0; 2075 return 0;
2074 2076
2075 if (!allow_suffix) { 2077 if (!allow_suffix) {
2076 pr_warning("Error: event \"%s\" already exists. " 2078 pr_warning("Error: event \"%s\" already exists. "
2077 "(Use -f to force duplicates.)\n", base); 2079 "(Use -f to force duplicates.)\n", base);
2078 return -EEXIST; 2080 return -EEXIST;
2079 } 2081 }
2080 2082
2081 /* Try to add suffix */ 2083 /* Try to add suffix */
2082 for (i = 1; i < MAX_EVENT_INDEX; i++) { 2084 for (i = 1; i < MAX_EVENT_INDEX; i++) {
2083 ret = e_snprintf(buf, len, "%s_%d", base, i); 2085 ret = e_snprintf(buf, len, "%s_%d", base, i);
2084 if (ret < 0) { 2086 if (ret < 0) {
2085 pr_debug("snprintf() failed: %d\n", ret); 2087 pr_debug("snprintf() failed: %d\n", ret);
2086 return ret; 2088 return ret;
2087 } 2089 }
2088 if (!strlist__has_entry(namelist, buf)) 2090 if (!strlist__has_entry(namelist, buf))
2089 break; 2091 break;
2090 } 2092 }
2091 if (i == MAX_EVENT_INDEX) { 2093 if (i == MAX_EVENT_INDEX) {
2092 pr_warning("Too many events are on the same function.\n"); 2094 pr_warning("Too many events are on the same function.\n");
2093 ret = -ERANGE; 2095 ret = -ERANGE;
2094 } 2096 }
2095 2097
2096 return ret; 2098 return ret;
2097 } 2099 }
2098 2100
2099 static int __add_probe_trace_events(struct perf_probe_event *pev, 2101 static int __add_probe_trace_events(struct perf_probe_event *pev,
2100 struct probe_trace_event *tevs, 2102 struct probe_trace_event *tevs,
2101 int ntevs, bool allow_suffix) 2103 int ntevs, bool allow_suffix)
2102 { 2104 {
2103 int i, fd, ret; 2105 int i, fd, ret;
2104 struct probe_trace_event *tev = NULL; 2106 struct probe_trace_event *tev = NULL;
2105 char buf[64]; 2107 char buf[64];
2106 const char *event, *group; 2108 const char *event, *group;
2107 struct strlist *namelist; 2109 struct strlist *namelist;
2108 2110
2109 if (pev->uprobes) 2111 if (pev->uprobes)
2110 fd = open_uprobe_events(true); 2112 fd = open_uprobe_events(true);
2111 else 2113 else
2112 fd = open_kprobe_events(true); 2114 fd = open_kprobe_events(true);
2113 2115
2114 if (fd < 0) { 2116 if (fd < 0) {
2115 print_open_warning(fd, !pev->uprobes); 2117 print_open_warning(fd, !pev->uprobes);
2116 return fd; 2118 return fd;
2117 } 2119 }
2118 2120
2119 /* Get current event names */ 2121 /* Get current event names */
2120 namelist = get_probe_trace_event_names(fd, false); 2122 namelist = get_probe_trace_event_names(fd, false);
2121 if (!namelist) { 2123 if (!namelist) {
2122 pr_debug("Failed to get current event list.\n"); 2124 pr_debug("Failed to get current event list.\n");
2123 return -EIO; 2125 return -EIO;
2124 } 2126 }
2125 2127
2126 ret = 0; 2128 ret = 0;
2127 pr_info("Added new event%s\n", (ntevs > 1) ? "s:" : ":"); 2129 pr_info("Added new event%s\n", (ntevs > 1) ? "s:" : ":");
2128 for (i = 0; i < ntevs; i++) { 2130 for (i = 0; i < ntevs; i++) {
2129 tev = &tevs[i]; 2131 tev = &tevs[i];
2130 if (pev->event) 2132 if (pev->event)
2131 event = pev->event; 2133 event = pev->event;
2132 else 2134 else
2133 if (pev->point.function) 2135 if (pev->point.function)
2134 event = pev->point.function; 2136 event = pev->point.function;
2135 else 2137 else
2136 event = tev->point.symbol; 2138 event = tev->point.symbol;
2137 if (pev->group) 2139 if (pev->group)
2138 group = pev->group; 2140 group = pev->group;
2139 else 2141 else
2140 group = PERFPROBE_GROUP; 2142 group = PERFPROBE_GROUP;
2141 2143
2142 /* Get an unused new event name */ 2144 /* Get an unused new event name */
2143 ret = get_new_event_name(buf, 64, event, 2145 ret = get_new_event_name(buf, 64, event,
2144 namelist, allow_suffix); 2146 namelist, allow_suffix);
2145 if (ret < 0) 2147 if (ret < 0)
2146 break; 2148 break;
2147 event = buf; 2149 event = buf;
2148 2150
2149 tev->event = strdup(event); 2151 tev->event = strdup(event);
2150 tev->group = strdup(group); 2152 tev->group = strdup(group);
2151 if (tev->event == NULL || tev->group == NULL) { 2153 if (tev->event == NULL || tev->group == NULL) {
2152 ret = -ENOMEM; 2154 ret = -ENOMEM;
2153 break; 2155 break;
2154 } 2156 }
2155 ret = write_probe_trace_event(fd, tev); 2157 ret = write_probe_trace_event(fd, tev);
2156 if (ret < 0) 2158 if (ret < 0)
2157 break; 2159 break;
2158 /* Add added event name to namelist */ 2160 /* Add added event name to namelist */
2159 strlist__add(namelist, event); 2161 strlist__add(namelist, event);
2160 2162
2161 /* Trick here - save current event/group */ 2163 /* Trick here - save current event/group */
2162 event = pev->event; 2164 event = pev->event;
2163 group = pev->group; 2165 group = pev->group;
2164 pev->event = tev->event; 2166 pev->event = tev->event;
2165 pev->group = tev->group; 2167 pev->group = tev->group;
2166 show_perf_probe_event(pev, tev->point.module); 2168 show_perf_probe_event(pev, tev->point.module);
2167 /* Trick here - restore current event/group */ 2169 /* Trick here - restore current event/group */
2168 pev->event = (char *)event; 2170 pev->event = (char *)event;
2169 pev->group = (char *)group; 2171 pev->group = (char *)group;
2170 2172
2171 /* 2173 /*
2172 * Probes after the first probe which comes from same 2174 * Probes after the first probe which comes from same
2173 * user input are always allowed to add suffix, because 2175 * user input are always allowed to add suffix, because
2174 * there might be several addresses corresponding to 2176 * there might be several addresses corresponding to
2175 * one code line. 2177 * one code line.
2176 */ 2178 */
2177 allow_suffix = true; 2179 allow_suffix = true;
2178 } 2180 }
2179 2181
2180 if (ret >= 0) { 2182 if (ret >= 0) {
2181 /* Show how to use the event. */ 2183 /* Show how to use the event. */
2182 pr_info("\nYou can now use it in all perf tools, such as:\n\n"); 2184 pr_info("\nYou can now use it in all perf tools, such as:\n\n");
2183 pr_info("\tperf record -e %s:%s -aR sleep 1\n\n", tev->group, 2185 pr_info("\tperf record -e %s:%s -aR sleep 1\n\n", tev->group,
2184 tev->event); 2186 tev->event);
2185 } 2187 }
2186 2188
2187 strlist__delete(namelist); 2189 strlist__delete(namelist);
2188 close(fd); 2190 close(fd);
2189 return ret; 2191 return ret;
2190 } 2192 }
2191 2193
2192 static char *looking_function_name; 2194 static char *looking_function_name;
2193 static int num_matched_functions; 2195 static int num_matched_functions;
2194 2196
2195 static int probe_function_filter(struct map *map __maybe_unused, 2197 static int probe_function_filter(struct map *map __maybe_unused,
2196 struct symbol *sym) 2198 struct symbol *sym)
2197 { 2199 {
2198 if ((sym->binding == STB_GLOBAL || sym->binding == STB_LOCAL) && 2200 if ((sym->binding == STB_GLOBAL || sym->binding == STB_LOCAL) &&
2199 strcmp(looking_function_name, sym->name) == 0) { 2201 strcmp(looking_function_name, sym->name) == 0) {
2200 num_matched_functions++; 2202 num_matched_functions++;
2201 return 0; 2203 return 0;
2202 } 2204 }
2203 return 1; 2205 return 1;
2204 } 2206 }
2205 2207
2206 #define strdup_or_goto(str, label) \ 2208 #define strdup_or_goto(str, label) \
2207 ({ char *__p = strdup(str); if (!__p) goto label; __p; }) 2209 ({ char *__p = strdup(str); if (!__p) goto label; __p; })
2208 2210
2209 /* 2211 /*
2210 * Find probe function addresses from map. 2212 * Find probe function addresses from map.
2211 * Return an error or the number of found probe_trace_event 2213 * Return an error or the number of found probe_trace_event
2212 */ 2214 */
2213 static int find_probe_trace_events_from_map(struct perf_probe_event *pev, 2215 static int find_probe_trace_events_from_map(struct perf_probe_event *pev,
2214 struct probe_trace_event **tevs, 2216 struct probe_trace_event **tevs,
2215 int max_tevs, const char *target) 2217 int max_tevs, const char *target)
2216 { 2218 {
2217 struct map *map = NULL; 2219 struct map *map = NULL;
2218 struct kmap *kmap = NULL; 2220 struct kmap *kmap = NULL;
2219 struct ref_reloc_sym *reloc_sym = NULL; 2221 struct ref_reloc_sym *reloc_sym = NULL;
2220 struct symbol *sym; 2222 struct symbol *sym;
2221 struct rb_node *nd; 2223 struct rb_node *nd;
2222 struct probe_trace_event *tev; 2224 struct probe_trace_event *tev;
2223 struct perf_probe_point *pp = &pev->point; 2225 struct perf_probe_point *pp = &pev->point;
2224 struct probe_trace_point *tp; 2226 struct probe_trace_point *tp;
2225 int ret, i; 2227 int ret, i;
2226 2228
2227 /* Init maps of given executable or kernel */ 2229 /* Init maps of given executable or kernel */
2228 if (pev->uprobes) 2230 if (pev->uprobes)
2229 map = dso__new_map(target); 2231 map = dso__new_map(target);
2230 else 2232 else
2231 map = kernel_get_module_map(target); 2233 map = kernel_get_module_map(target);
2232 if (!map) { 2234 if (!map) {
2233 ret = -EINVAL; 2235 ret = -EINVAL;
2234 goto out; 2236 goto out;
2235 } 2237 }
2236 2238
2237 /* 2239 /*
2238 * Load matched symbols: Since the different local symbols may have 2240 * Load matched symbols: Since the different local symbols may have
2239 * same name but different addresses, this lists all the symbols. 2241 * same name but different addresses, this lists all the symbols.
2240 */ 2242 */
2241 num_matched_functions = 0; 2243 num_matched_functions = 0;
2242 looking_function_name = pp->function; 2244 looking_function_name = pp->function;
2243 ret = map__load(map, probe_function_filter); 2245 ret = map__load(map, probe_function_filter);
2244 if (ret || num_matched_functions == 0) { 2246 if (ret || num_matched_functions == 0) {
2245 pr_err("Failed to find symbol %s in %s\n", pp->function, 2247 pr_err("Failed to find symbol %s in %s\n", pp->function,
2246 target ? : "kernel"); 2248 target ? : "kernel");
2247 ret = -ENOENT; 2249 ret = -ENOENT;
2248 goto out; 2250 goto out;
2249 } else if (num_matched_functions > max_tevs) { 2251 } else if (num_matched_functions > max_tevs) {
2250 pr_err("Too many functions matched in %s\n", 2252 pr_err("Too many functions matched in %s\n",
2251 target ? : "kernel"); 2253 target ? : "kernel");
2252 ret = -E2BIG; 2254 ret = -E2BIG;
2253 goto out; 2255 goto out;
2254 } 2256 }
2255 2257
2256 if (!pev->uprobes) { 2258 if (!pev->uprobes) {
2257 kmap = map__kmap(map); 2259 kmap = map__kmap(map);
2258 reloc_sym = kmap->ref_reloc_sym; 2260 reloc_sym = kmap->ref_reloc_sym;
2259 if (!reloc_sym) { 2261 if (!reloc_sym) {
2260 pr_warning("Relocated base symbol is not found!\n"); 2262 pr_warning("Relocated base symbol is not found!\n");
2261 ret = -EINVAL; 2263 ret = -EINVAL;
2262 goto out; 2264 goto out;
2263 } 2265 }
2264 } 2266 }
2265 2267
2266 /* Setup result trace-probe-events */ 2268 /* Setup result trace-probe-events */
2267 *tevs = zalloc(sizeof(*tev) * num_matched_functions); 2269 *tevs = zalloc(sizeof(*tev) * num_matched_functions);
2268 if (!*tevs) { 2270 if (!*tevs) {
2269 ret = -ENOMEM; 2271 ret = -ENOMEM;
2270 goto out; 2272 goto out;
2271 } 2273 }
2272 2274
2273 ret = 0; 2275 ret = 0;
2274 map__for_each_symbol(map, sym, nd) { 2276 map__for_each_symbol(map, sym, nd) {
2275 tev = (*tevs) + ret; 2277 tev = (*tevs) + ret;
2276 tp = &tev->point; 2278 tp = &tev->point;
2277 if (ret == num_matched_functions) { 2279 if (ret == num_matched_functions) {
2278 pr_warning("Too many symbols are listed. Skip it.\n"); 2280 pr_warning("Too many symbols are listed. Skip it.\n");
2279 break; 2281 break;
2280 } 2282 }
2281 ret++; 2283 ret++;
2282 2284
2283 if (pp->offset > sym->end - sym->start) { 2285 if (pp->offset > sym->end - sym->start) {
2284 pr_warning("Offset %ld is bigger than the size of %s\n", 2286 pr_warning("Offset %ld is bigger than the size of %s\n",
2285 pp->offset, sym->name); 2287 pp->offset, sym->name);
2286 ret = -ENOENT; 2288 ret = -ENOENT;
2287 goto err_out; 2289 goto err_out;
2288 } 2290 }
2289 /* Add one probe point */ 2291 /* Add one probe point */
2290 tp->address = map->unmap_ip(map, sym->start) + pp->offset; 2292 tp->address = map->unmap_ip(map, sym->start) + pp->offset;
2291 if (reloc_sym) { 2293 if (reloc_sym) {
2292 tp->symbol = strdup_or_goto(reloc_sym->name, nomem_out); 2294 tp->symbol = strdup_or_goto(reloc_sym->name, nomem_out);
2293 tp->offset = tp->address - reloc_sym->addr; 2295 tp->offset = tp->address - reloc_sym->addr;
2294 } else { 2296 } else {
2295 tp->symbol = strdup_or_goto(sym->name, nomem_out); 2297 tp->symbol = strdup_or_goto(sym->name, nomem_out);
2296 tp->offset = pp->offset; 2298 tp->offset = pp->offset;
2297 } 2299 }
2298 tp->retprobe = pp->retprobe; 2300 tp->retprobe = pp->retprobe;
2299 if (target) 2301 if (target)
2300 tev->point.module = strdup_or_goto(target, nomem_out); 2302 tev->point.module = strdup_or_goto(target, nomem_out);
2301 tev->uprobes = pev->uprobes; 2303 tev->uprobes = pev->uprobes;
2302 tev->nargs = pev->nargs; 2304 tev->nargs = pev->nargs;
2303 if (tev->nargs) { 2305 if (tev->nargs) {
2304 tev->args = zalloc(sizeof(struct probe_trace_arg) * 2306 tev->args = zalloc(sizeof(struct probe_trace_arg) *
2305 tev->nargs); 2307 tev->nargs);
2306 if (tev->args == NULL) 2308 if (tev->args == NULL)
2307 goto nomem_out; 2309 goto nomem_out;
2308 } 2310 }
2309 for (i = 0; i < tev->nargs; i++) { 2311 for (i = 0; i < tev->nargs; i++) {
2310 if (pev->args[i].name) 2312 if (pev->args[i].name)
2311 tev->args[i].name = 2313 tev->args[i].name =
2312 strdup_or_goto(pev->args[i].name, 2314 strdup_or_goto(pev->args[i].name,
2313 nomem_out); 2315 nomem_out);
2314 2316
2315 tev->args[i].value = strdup_or_goto(pev->args[i].var, 2317 tev->args[i].value = strdup_or_goto(pev->args[i].var,
2316 nomem_out); 2318 nomem_out);
2317 if (pev->args[i].type) 2319 if (pev->args[i].type)
2318 tev->args[i].type = 2320 tev->args[i].type =
2319 strdup_or_goto(pev->args[i].type, 2321 strdup_or_goto(pev->args[i].type,
2320 nomem_out); 2322 nomem_out);
2321 } 2323 }
2322 } 2324 }
2323 2325
2324 out: 2326 out:
2325 if (map && pev->uprobes) { 2327 if (map && pev->uprobes) {
2326 /* Only when using uprobe(exec) map needs to be released */ 2328 /* Only when using uprobe(exec) map needs to be released */
2327 dso__delete(map->dso); 2329 dso__delete(map->dso);
2328 map__delete(map); 2330 map__delete(map);
2329 } 2331 }
2330 return ret; 2332 return ret;
2331 2333
2332 nomem_out: 2334 nomem_out:
2333 ret = -ENOMEM; 2335 ret = -ENOMEM;
2334 err_out: 2336 err_out:
2335 clear_probe_trace_events(*tevs, num_matched_functions); 2337 clear_probe_trace_events(*tevs, num_matched_functions);
2336 zfree(tevs); 2338 zfree(tevs);
2337 goto out; 2339 goto out;
2338 } 2340 }
2339 2341
2340 static int convert_to_probe_trace_events(struct perf_probe_event *pev, 2342 static int convert_to_probe_trace_events(struct perf_probe_event *pev,
2341 struct probe_trace_event **tevs, 2343 struct probe_trace_event **tevs,
2342 int max_tevs, const char *target) 2344 int max_tevs, const char *target)
2343 { 2345 {
2344 int ret; 2346 int ret;
2345 2347
2346 if (pev->uprobes && !pev->group) { 2348 if (pev->uprobes && !pev->group) {
2347 /* Replace group name if not given */ 2349 /* Replace group name if not given */
2348 ret = convert_exec_to_group(target, &pev->group); 2350 ret = convert_exec_to_group(target, &pev->group);
2349 if (ret != 0) { 2351 if (ret != 0) {
2350 pr_warning("Failed to make a group name.\n"); 2352 pr_warning("Failed to make a group name.\n");
2351 return ret; 2353 return ret;
2352 } 2354 }
2353 } 2355 }
2354 2356
2355 /* Convert perf_probe_event with debuginfo */ 2357 /* Convert perf_probe_event with debuginfo */
2356 ret = try_to_find_probe_trace_events(pev, tevs, max_tevs, target); 2358 ret = try_to_find_probe_trace_events(pev, tevs, max_tevs, target);
2357 if (ret != 0) 2359 if (ret != 0)
2358 return ret; /* Found in debuginfo or got an error */ 2360 return ret; /* Found in debuginfo or got an error */
2359 2361
2360 return find_probe_trace_events_from_map(pev, tevs, max_tevs, target); 2362 return find_probe_trace_events_from_map(pev, tevs, max_tevs, target);
2361 } 2363 }
2362 2364
2363 struct __event_package { 2365 struct __event_package {
2364 struct perf_probe_event *pev; 2366 struct perf_probe_event *pev;
2365 struct probe_trace_event *tevs; 2367 struct probe_trace_event *tevs;
2366 int ntevs; 2368 int ntevs;
2367 }; 2369 };
2368 2370
2369 int add_perf_probe_events(struct perf_probe_event *pevs, int npevs, 2371 int add_perf_probe_events(struct perf_probe_event *pevs, int npevs,
2370 int max_tevs, const char *target, bool force_add) 2372 int max_tevs, const char *target, bool force_add)
2371 { 2373 {
2372 int i, j, ret; 2374 int i, j, ret;
2373 struct __event_package *pkgs; 2375 struct __event_package *pkgs;
2374 2376
2375 ret = 0; 2377 ret = 0;
2376 pkgs = zalloc(sizeof(struct __event_package) * npevs); 2378 pkgs = zalloc(sizeof(struct __event_package) * npevs);
2377 2379
2378 if (pkgs == NULL) 2380 if (pkgs == NULL)
2379 return -ENOMEM; 2381 return -ENOMEM;
2380 2382
2381 ret = init_symbol_maps(pevs->uprobes); 2383 ret = init_symbol_maps(pevs->uprobes);
2382 if (ret < 0) { 2384 if (ret < 0) {
2383 free(pkgs); 2385 free(pkgs);
2384 return ret; 2386 return ret;
2385 } 2387 }
2386 2388
2387 /* Loop 1: convert all events */ 2389 /* Loop 1: convert all events */
2388 for (i = 0; i < npevs; i++) { 2390 for (i = 0; i < npevs; i++) {
2389 pkgs[i].pev = &pevs[i]; 2391 pkgs[i].pev = &pevs[i];
2390 /* Convert with or without debuginfo */ 2392 /* Convert with or without debuginfo */
2391 ret = convert_to_probe_trace_events(pkgs[i].pev, 2393 ret = convert_to_probe_trace_events(pkgs[i].pev,
2392 &pkgs[i].tevs, 2394 &pkgs[i].tevs,
2393 max_tevs, 2395 max_tevs,
2394 target); 2396 target);
2395 if (ret < 0) 2397 if (ret < 0)
2396 goto end; 2398 goto end;
2397 pkgs[i].ntevs = ret; 2399 pkgs[i].ntevs = ret;
2398 } 2400 }
2399 2401
2400 /* Loop 2: add all events */ 2402 /* Loop 2: add all events */
2401 for (i = 0; i < npevs; i++) { 2403 for (i = 0; i < npevs; i++) {
2402 ret = __add_probe_trace_events(pkgs[i].pev, pkgs[i].tevs, 2404 ret = __add_probe_trace_events(pkgs[i].pev, pkgs[i].tevs,
2403 pkgs[i].ntevs, force_add); 2405 pkgs[i].ntevs, force_add);
2404 if (ret < 0) 2406 if (ret < 0)
2405 break; 2407 break;
2406 } 2408 }
2407 end: 2409 end:
2408 /* Loop 3: cleanup and free trace events */ 2410 /* Loop 3: cleanup and free trace events */
2409 for (i = 0; i < npevs; i++) { 2411 for (i = 0; i < npevs; i++) {
2410 for (j = 0; j < pkgs[i].ntevs; j++) 2412 for (j = 0; j < pkgs[i].ntevs; j++)
2411 clear_probe_trace_event(&pkgs[i].tevs[j]); 2413 clear_probe_trace_event(&pkgs[i].tevs[j]);
2412 zfree(&pkgs[i].tevs); 2414 zfree(&pkgs[i].tevs);
2413 } 2415 }
2414 free(pkgs); 2416 free(pkgs);
2415 exit_symbol_maps(); 2417 exit_symbol_maps();
2416 2418
2417 return ret; 2419 return ret;
2418 } 2420 }
2419 2421
2420 static int __del_trace_probe_event(int fd, struct str_node *ent) 2422 static int __del_trace_probe_event(int fd, struct str_node *ent)
2421 { 2423 {
2422 char *p; 2424 char *p;
2423 char buf[128]; 2425 char buf[128];
2424 int ret; 2426 int ret;
2425 2427
2426 /* Convert from perf-probe event to trace-probe event */ 2428 /* Convert from perf-probe event to trace-probe event */
2427 ret = e_snprintf(buf, 128, "-:%s", ent->s); 2429 ret = e_snprintf(buf, 128, "-:%s", ent->s);
2428 if (ret < 0) 2430 if (ret < 0)
2429 goto error; 2431 goto error;
2430 2432
2431 p = strchr(buf + 2, ':'); 2433 p = strchr(buf + 2, ':');
2432 if (!p) { 2434 if (!p) {
2433 pr_debug("Internal error: %s should have ':' but not.\n", 2435 pr_debug("Internal error: %s should have ':' but not.\n",
2434 ent->s); 2436 ent->s);
2435 ret = -ENOTSUP; 2437 ret = -ENOTSUP;
2436 goto error; 2438 goto error;
2437 } 2439 }
2438 *p = '/'; 2440 *p = '/';
2439 2441
2440 pr_debug("Writing event: %s\n", buf); 2442 pr_debug("Writing event: %s\n", buf);
2441 ret = write(fd, buf, strlen(buf)); 2443 ret = write(fd, buf, strlen(buf));
2442 if (ret < 0) { 2444 if (ret < 0) {
2443 ret = -errno; 2445 ret = -errno;
2444 goto error; 2446 goto error;
2445 } 2447 }
2446 2448
2447 pr_info("Removed event: %s\n", ent->s); 2449 pr_info("Removed event: %s\n", ent->s);
2448 return 0; 2450 return 0;
2449 error: 2451 error:
2450 pr_warning("Failed to delete event: %s\n", 2452 pr_warning("Failed to delete event: %s\n",
2451 strerror_r(-ret, buf, sizeof(buf))); 2453 strerror_r(-ret, buf, sizeof(buf)));
2452 return ret; 2454 return ret;
2453 } 2455 }
2454 2456
2455 static int del_trace_probe_event(int fd, const char *buf, 2457 static int del_trace_probe_event(int fd, const char *buf,
2456 struct strlist *namelist) 2458 struct strlist *namelist)
2457 { 2459 {
2458 struct str_node *ent, *n; 2460 struct str_node *ent, *n;
2459 int ret = -1; 2461 int ret = -1;
2460 2462
2461 if (strpbrk(buf, "*?")) { /* Glob-exp */ 2463 if (strpbrk(buf, "*?")) { /* Glob-exp */
2462 strlist__for_each_safe(ent, n, namelist) 2464 strlist__for_each_safe(ent, n, namelist)
2463 if (strglobmatch(ent->s, buf)) { 2465 if (strglobmatch(ent->s, buf)) {
2464 ret = __del_trace_probe_event(fd, ent); 2466 ret = __del_trace_probe_event(fd, ent);
2465 if (ret < 0) 2467 if (ret < 0)
2466 break; 2468 break;
2467 strlist__remove(namelist, ent); 2469 strlist__remove(namelist, ent);
2468 } 2470 }
2469 } else { 2471 } else {
2470 ent = strlist__find(namelist, buf); 2472 ent = strlist__find(namelist, buf);
2471 if (ent) { 2473 if (ent) {
2472 ret = __del_trace_probe_event(fd, ent); 2474 ret = __del_trace_probe_event(fd, ent);
2473 if (ret >= 0) 2475 if (ret >= 0)
2474 strlist__remove(namelist, ent); 2476 strlist__remove(namelist, ent);
2475 } 2477 }
2476 } 2478 }
2477 2479
2478 return ret; 2480 return ret;
2479 } 2481 }
2480 2482
2481 int del_perf_probe_events(struct strlist *dellist) 2483 int del_perf_probe_events(struct strlist *dellist)
2482 { 2484 {
2483 int ret = -1, ufd = -1, kfd = -1; 2485 int ret = -1, ufd = -1, kfd = -1;
2484 char buf[128]; 2486 char buf[128];
2485 const char *group, *event; 2487 const char *group, *event;
2486 char *p, *str; 2488 char *p, *str;
2487 struct str_node *ent; 2489 struct str_node *ent;
2488 struct strlist *namelist = NULL, *unamelist = NULL; 2490 struct strlist *namelist = NULL, *unamelist = NULL;
2489 2491
2490 /* Get current event names */ 2492 /* Get current event names */
2491 kfd = open_kprobe_events(true); 2493 kfd = open_kprobe_events(true);
2492 if (kfd >= 0) 2494 if (kfd >= 0)
2493 namelist = get_probe_trace_event_names(kfd, true); 2495 namelist = get_probe_trace_event_names(kfd, true);
2494 2496
2495 ufd = open_uprobe_events(true); 2497 ufd = open_uprobe_events(true);
2496 if (ufd >= 0) 2498 if (ufd >= 0)
2497 unamelist = get_probe_trace_event_names(ufd, true); 2499 unamelist = get_probe_trace_event_names(ufd, true);
2498 2500
2499 if (kfd < 0 && ufd < 0) { 2501 if (kfd < 0 && ufd < 0) {
2500 print_both_open_warning(kfd, ufd); 2502 print_both_open_warning(kfd, ufd);
2501 goto error; 2503 goto error;
2502 } 2504 }
2503 2505
2504 if (namelist == NULL && unamelist == NULL) 2506 if (namelist == NULL && unamelist == NULL)
2505 goto error; 2507 goto error;
2506 2508
2507 strlist__for_each(ent, dellist) { 2509 strlist__for_each(ent, dellist) {
2508 str = strdup(ent->s); 2510 str = strdup(ent->s);
2509 if (str == NULL) { 2511 if (str == NULL) {
2510 ret = -ENOMEM; 2512 ret = -ENOMEM;
2511 goto error; 2513 goto error;
2512 } 2514 }
2513 pr_debug("Parsing: %s\n", str); 2515 pr_debug("Parsing: %s\n", str);
2514 p = strchr(str, ':'); 2516 p = strchr(str, ':');
2515 if (p) { 2517 if (p) {
2516 group = str; 2518 group = str;
2517 *p = '\0'; 2519 *p = '\0';
2518 event = p + 1; 2520 event = p + 1;
2519 } else { 2521 } else {
2520 group = "*"; 2522 group = "*";
2521 event = str; 2523 event = str;
2522 } 2524 }
2523 2525
2524 ret = e_snprintf(buf, 128, "%s:%s", group, event); 2526 ret = e_snprintf(buf, 128, "%s:%s", group, event);
2525 if (ret < 0) { 2527 if (ret < 0) {
2526 pr_err("Failed to copy event."); 2528 pr_err("Failed to copy event.");
2527 free(str); 2529 free(str);
2528 goto error; 2530 goto error;
2529 } 2531 }
2530 2532
2531 pr_debug("Group: %s, Event: %s\n", group, event); 2533 pr_debug("Group: %s, Event: %s\n", group, event);
2532 2534
2533 if (namelist) 2535 if (namelist)
2534 ret = del_trace_probe_event(kfd, buf, namelist); 2536 ret = del_trace_probe_event(kfd, buf, namelist);
2535 2537
2536 if (unamelist && ret != 0) 2538 if (unamelist && ret != 0)
2537 ret = del_trace_probe_event(ufd, buf, unamelist); 2539 ret = del_trace_probe_event(ufd, buf, unamelist);
2538 2540
2539 if (ret != 0) 2541 if (ret != 0)
2540 pr_info("Info: Event \"%s\" does not exist.\n", buf); 2542 pr_info("Info: Event \"%s\" does not exist.\n", buf);
2541 2543
2542 free(str); 2544 free(str);
2543 } 2545 }
2544 2546
2545 error: 2547 error:
2546 if (kfd >= 0) { 2548 if (kfd >= 0) {
2547 strlist__delete(namelist); 2549 strlist__delete(namelist);
2548 close(kfd); 2550 close(kfd);
2549 } 2551 }
2550 2552
2551 if (ufd >= 0) { 2553 if (ufd >= 0) {
2552 strlist__delete(unamelist); 2554 strlist__delete(unamelist);
2553 close(ufd); 2555 close(ufd);
2554 } 2556 }
2555 2557
2556 return ret; 2558 return ret;
2557 } 2559 }
2558 2560
2559 /* TODO: don't use a global variable for filter ... */ 2561 /* TODO: don't use a global variable for filter ... */
2560 static struct strfilter *available_func_filter; 2562 static struct strfilter *available_func_filter;
2561 2563
2562 /* 2564 /*
2563 * If a symbol corresponds to a function with global binding and 2565 * If a symbol corresponds to a function with global binding and
2564 * matches filter return 0. For all others return 1. 2566 * matches filter return 0. For all others return 1.
2565 */ 2567 */
2566 static int filter_available_functions(struct map *map __maybe_unused, 2568 static int filter_available_functions(struct map *map __maybe_unused,
2567 struct symbol *sym) 2569 struct symbol *sym)
2568 { 2570 {
2569 if ((sym->binding == STB_GLOBAL || sym->binding == STB_LOCAL) && 2571 if ((sym->binding == STB_GLOBAL || sym->binding == STB_LOCAL) &&
2570 strfilter__compare(available_func_filter, sym->name)) 2572 strfilter__compare(available_func_filter, sym->name))
2571 return 0; 2573 return 0;
2572 return 1; 2574 return 1;
2573 } 2575 }
2574 2576
2575 int show_available_funcs(const char *target, struct strfilter *_filter, 2577 int show_available_funcs(const char *target, struct strfilter *_filter,
2576 bool user) 2578 bool user)
2577 { 2579 {
2578 struct map *map; 2580 struct map *map;
2579 int ret; 2581 int ret;
2580 2582
2581 ret = init_symbol_maps(user); 2583 ret = init_symbol_maps(user);
2582 if (ret < 0) 2584 if (ret < 0)
2583 return ret; 2585 return ret;
2584 2586
2585 /* Get a symbol map */ 2587 /* Get a symbol map */
2586 if (user) 2588 if (user)
2587 map = dso__new_map(target); 2589 map = dso__new_map(target);
2588 else 2590 else
2589 map = kernel_get_module_map(target); 2591 map = kernel_get_module_map(target);
2590 if (!map) { 2592 if (!map) {
2591 pr_err("Failed to get a map for %s\n", (target) ? : "kernel"); 2593 pr_err("Failed to get a map for %s\n", (target) ? : "kernel");
2592 return -EINVAL; 2594 return -EINVAL;
2593 } 2595 }
2594 2596
2595 /* Load symbols with given filter */ 2597 /* Load symbols with given filter */
2596 available_func_filter = _filter; 2598 available_func_filter = _filter;
2597 if (map__load(map, filter_available_functions)) { 2599 if (map__load(map, filter_available_functions)) {
2598 pr_err("Failed to load symbols in %s\n", (target) ? : "kernel"); 2600 pr_err("Failed to load symbols in %s\n", (target) ? : "kernel");
2599 goto end; 2601 goto end;
2600 } 2602 }
2601 if (!dso__sorted_by_name(map->dso, map->type)) 2603 if (!dso__sorted_by_name(map->dso, map->type))
2602 dso__sort_by_name(map->dso, map->type); 2604 dso__sort_by_name(map->dso, map->type);
2603 2605
2604 /* Show all (filtered) symbols */ 2606 /* Show all (filtered) symbols */
2605 setup_pager(); 2607 setup_pager();
2606 dso__fprintf_symbols_by_name(map->dso, map->type, stdout); 2608 dso__fprintf_symbols_by_name(map->dso, map->type, stdout);
2607 end: 2609 end:
2608 if (user) { 2610 if (user) {
2609 dso__delete(map->dso); 2611 dso__delete(map->dso);
2610 map__delete(map); 2612 map__delete(map);
2611 } 2613 }
2612 exit_symbol_maps(); 2614 exit_symbol_maps();
2613 2615
2614 return ret; 2616 return ret;
2615 } 2617 }
2616 2618
2617 2619
tools/perf/util/probe-finder.c
Diff comments   View file @ ddb321a
1 /* 1 /*
2 * probe-finder.c : C expression to kprobe event converter 2 * probe-finder.c : C expression to kprobe event converter
3 * 3 *
4 * Written by Masami Hiramatsu <mhiramat@redhat.com> 4 * Written by Masami Hiramatsu <mhiramat@redhat.com>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or 8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version. 9 * (at your option) any later version.
10 * 10 *
11 * This program is distributed in the hope that it will be useful, 11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details. 14 * GNU General Public License for more details.
15 * 15 *
16 * You should have received a copy of the GNU General Public License 16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software 17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 * 19 *
20 */ 20 */
21 21
22 #include <sys/utsname.h> 22 #include <sys/utsname.h>
23 #include <sys/types.h> 23 #include <sys/types.h>
24 #include <sys/stat.h> 24 #include <sys/stat.h>
25 #include <fcntl.h> 25 #include <fcntl.h>
26 #include <errno.h> 26 #include <errno.h>
27 #include <stdio.h> 27 #include <stdio.h>
28 #include <unistd.h> 28 #include <unistd.h>
29 #include <stdlib.h> 29 #include <stdlib.h>
30 #include <string.h> 30 #include <string.h>
31 #include <stdarg.h> 31 #include <stdarg.h>
32 #include <dwarf-regs.h> 32 #include <dwarf-regs.h>
33 33
34 #include <linux/bitops.h> 34 #include <linux/bitops.h>
35 #include "event.h" 35 #include "event.h"
36 #include "dso.h" 36 #include "dso.h"
37 #include "debug.h" 37 #include "debug.h"
38 #include "intlist.h" 38 #include "intlist.h"
39 #include "util.h" 39 #include "util.h"
40 #include "symbol.h" 40 #include "symbol.h"
41 #include "probe-finder.h" 41 #include "probe-finder.h"
42 42
43 /* Kprobe tracer basic type is up to u64 */ 43 /* Kprobe tracer basic type is up to u64 */
44 #define MAX_BASIC_TYPE_BITS 64 44 #define MAX_BASIC_TYPE_BITS 64
45 45
46 /* Dwarf FL wrappers */ 46 /* Dwarf FL wrappers */
47 static char *debuginfo_path; /* Currently dummy */ 47 static char *debuginfo_path; /* Currently dummy */
48 48
49 static const Dwfl_Callbacks offline_callbacks = { 49 static const Dwfl_Callbacks offline_callbacks = {
50 .find_debuginfo = dwfl_standard_find_debuginfo, 50 .find_debuginfo = dwfl_standard_find_debuginfo,
51 .debuginfo_path = &debuginfo_path, 51 .debuginfo_path = &debuginfo_path,
52 52
53 .section_address = dwfl_offline_section_address, 53 .section_address = dwfl_offline_section_address,
54 54
55 /* We use this table for core files too. */ 55 /* We use this table for core files too. */
56 .find_elf = dwfl_build_id_find_elf, 56 .find_elf = dwfl_build_id_find_elf,
57 }; 57 };
58 58
59 /* Get a Dwarf from offline image */ 59 /* Get a Dwarf from offline image */
60 static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, 60 static int debuginfo__init_offline_dwarf(struct debuginfo *dbg,
61 const char *path) 61 const char *path)
62 { 62 {
63 int fd; 63 int fd;
64 64
65 fd = open(path, O_RDONLY); 65 fd = open(path, O_RDONLY);
66 if (fd < 0) 66 if (fd < 0)
67 return fd; 67 return fd;
68 68
69 dbg->dwfl = dwfl_begin(&offline_callbacks); 69 dbg->dwfl = dwfl_begin(&offline_callbacks);
70 if (!dbg->dwfl) 70 if (!dbg->dwfl)
71 goto error; 71 goto error;
72 72
73 dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd); 73 dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd);
74 if (!dbg->mod) 74 if (!dbg->mod)
75 goto error; 75 goto error;
76 76
77 dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias); 77 dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias);
78 if (!dbg->dbg) 78 if (!dbg->dbg)
79 goto error; 79 goto error;
80 80
81 return 0; 81 return 0;
82 error: 82 error:
83 if (dbg->dwfl) 83 if (dbg->dwfl)
84 dwfl_end(dbg->dwfl); 84 dwfl_end(dbg->dwfl);
85 else 85 else
86 close(fd); 86 close(fd);
87 memset(dbg, 0, sizeof(*dbg)); 87 memset(dbg, 0, sizeof(*dbg));
88 88
89 return -ENOENT; 89 return -ENOENT;
90 } 90 }
91 91
92 static struct debuginfo *__debuginfo__new(const char *path) 92 static struct debuginfo *__debuginfo__new(const char *path)
93 { 93 {
94 struct debuginfo *dbg = zalloc(sizeof(*dbg)); 94 struct debuginfo *dbg = zalloc(sizeof(*dbg));
95 if (!dbg) 95 if (!dbg)
96 return NULL; 96 return NULL;
97 97
98 if (debuginfo__init_offline_dwarf(dbg, path) < 0) 98 if (debuginfo__init_offline_dwarf(dbg, path) < 0)
99 zfree(&dbg); 99 zfree(&dbg);
100 if (dbg) 100 if (dbg)
101 pr_debug("Open Debuginfo file: %s\n", path); 101 pr_debug("Open Debuginfo file: %s\n", path);
102 return dbg; 102 return dbg;
103 } 103 }
104 104
105 enum dso_binary_type distro_dwarf_types[] = { 105 enum dso_binary_type distro_dwarf_types[] = {
106 DSO_BINARY_TYPE__FEDORA_DEBUGINFO, 106 DSO_BINARY_TYPE__FEDORA_DEBUGINFO,
107 DSO_BINARY_TYPE__UBUNTU_DEBUGINFO, 107 DSO_BINARY_TYPE__UBUNTU_DEBUGINFO,
108 DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO, 108 DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO,
109 DSO_BINARY_TYPE__BUILDID_DEBUGINFO, 109 DSO_BINARY_TYPE__BUILDID_DEBUGINFO,
110 DSO_BINARY_TYPE__NOT_FOUND, 110 DSO_BINARY_TYPE__NOT_FOUND,
111 }; 111 };
112 112
113 struct debuginfo *debuginfo__new(const char *path) 113 struct debuginfo *debuginfo__new(const char *path)
114 { 114 {
115 enum dso_binary_type *type; 115 enum dso_binary_type *type;
116 char buf[PATH_MAX], nil = '\0'; 116 char buf[PATH_MAX], nil = '\0';
117 struct dso *dso; 117 struct dso *dso;
118 struct debuginfo *dinfo = NULL; 118 struct debuginfo *dinfo = NULL;
119 119
120 /* Try to open distro debuginfo files */ 120 /* Try to open distro debuginfo files */
121 dso = dso__new(path); 121 dso = dso__new(path);
122 if (!dso) 122 if (!dso)
123 goto out; 123 goto out;
124 124
125 for (type = distro_dwarf_types; 125 for (type = distro_dwarf_types;
126 !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND; 126 !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND;
127 type++) { 127 type++) {
128 if (dso__read_binary_type_filename(dso, *type, &nil, 128 if (dso__read_binary_type_filename(dso, *type, &nil,
129 buf, PATH_MAX) < 0) 129 buf, PATH_MAX) < 0)
130 continue; 130 continue;
131 dinfo = __debuginfo__new(buf); 131 dinfo = __debuginfo__new(buf);
132 } 132 }
133 dso__delete(dso); 133 dso__delete(dso);
134 134
135 out: 135 out:
136 /* if failed to open all distro debuginfo, open given binary */ 136 /* if failed to open all distro debuginfo, open given binary */
137 return dinfo ? : __debuginfo__new(path); 137 return dinfo ? : __debuginfo__new(path);
138 } 138 }
139 139
140 void debuginfo__delete(struct debuginfo *dbg) 140 void debuginfo__delete(struct debuginfo *dbg)
141 { 141 {
142 if (dbg) { 142 if (dbg) {
143 if (dbg->dwfl) 143 if (dbg->dwfl)
144 dwfl_end(dbg->dwfl); 144 dwfl_end(dbg->dwfl);
145 free(dbg); 145 free(dbg);
146 } 146 }
147 } 147 }
148 148
149 /* 149 /*
150 * Probe finder related functions 150 * Probe finder related functions
151 */ 151 */
152 152
153 static struct probe_trace_arg_ref *alloc_trace_arg_ref(long offs) 153 static struct probe_trace_arg_ref *alloc_trace_arg_ref(long offs)
154 { 154 {
155 struct probe_trace_arg_ref *ref; 155 struct probe_trace_arg_ref *ref;
156 ref = zalloc(sizeof(struct probe_trace_arg_ref)); 156 ref = zalloc(sizeof(struct probe_trace_arg_ref));
157 if (ref != NULL) 157 if (ref != NULL)
158 ref->offset = offs; 158 ref->offset = offs;
159 return ref; 159 return ref;
160 } 160 }
161 161
162 /* 162 /*
163 * Convert a location into trace_arg. 163 * Convert a location into trace_arg.
164 * If tvar == NULL, this just checks variable can be converted. 164 * If tvar == NULL, this just checks variable can be converted.
165 * If fentry == true and vr_die is a parameter, do huristic search 165 * If fentry == true and vr_die is a parameter, do huristic search
166 * for the location fuzzed by function entry mcount. 166 * for the location fuzzed by function entry mcount.
167 */ 167 */
168 static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, 168 static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr,
169 Dwarf_Op *fb_ops, Dwarf_Die *sp_die, 169 Dwarf_Op *fb_ops, Dwarf_Die *sp_die,
170 struct probe_trace_arg *tvar) 170 struct probe_trace_arg *tvar)
171 { 171 {
172 Dwarf_Attribute attr; 172 Dwarf_Attribute attr;
173 Dwarf_Addr tmp = 0; 173 Dwarf_Addr tmp = 0;
174 Dwarf_Op *op; 174 Dwarf_Op *op;
175 size_t nops; 175 size_t nops;
176 unsigned int regn; 176 unsigned int regn;
177 Dwarf_Word offs = 0; 177 Dwarf_Word offs = 0;
178 bool ref = false; 178 bool ref = false;
179 const char *regs; 179 const char *regs;
180 int ret; 180 int ret;
181 181
182 if (dwarf_attr(vr_die, DW_AT_external, &attr) != NULL) 182 if (dwarf_attr(vr_die, DW_AT_external, &attr) != NULL)
183 goto static_var; 183 goto static_var;
184 184
185 /* TODO: handle more than 1 exprs */ 185 /* TODO: handle more than 1 exprs */
186 if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL) 186 if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL)
187 return -EINVAL; /* Broken DIE ? */ 187 return -EINVAL; /* Broken DIE ? */
188 if (dwarf_getlocation_addr(&attr, addr, &op, &nops, 1) <= 0) { 188 if (dwarf_getlocation_addr(&attr, addr, &op, &nops, 1) <= 0) {
189 ret = dwarf_entrypc(sp_die, &tmp); 189 ret = dwarf_entrypc(sp_die, &tmp);
190 if (ret || addr != tmp || 190 if (ret || addr != tmp ||
191 dwarf_tag(vr_die) != DW_TAG_formal_parameter || 191 dwarf_tag(vr_die) != DW_TAG_formal_parameter ||
192 dwarf_highpc(sp_die, &tmp)) 192 dwarf_highpc(sp_die, &tmp))
193 return -ENOENT; 193 return -ENOENT;
194 /* 194 /*
195 * This is fuzzed by fentry mcount. We try to find the 195 * This is fuzzed by fentry mcount. We try to find the
196 * parameter location at the earliest address. 196 * parameter location at the earliest address.
197 */ 197 */
198 for (addr += 1; addr <= tmp; addr++) { 198 for (addr += 1; addr <= tmp; addr++) {
199 if (dwarf_getlocation_addr(&attr, addr, &op, 199 if (dwarf_getlocation_addr(&attr, addr, &op,
200 &nops, 1) > 0) 200 &nops, 1) > 0)
201 goto found; 201 goto found;
202 } 202 }
203 return -ENOENT; 203 return -ENOENT;
204 } 204 }
205 found: 205 found:
206 if (nops == 0) 206 if (nops == 0)
207 /* TODO: Support const_value */ 207 /* TODO: Support const_value */
208 return -ENOENT; 208 return -ENOENT;
209 209
210 if (op->atom == DW_OP_addr) { 210 if (op->atom == DW_OP_addr) {
211 static_var: 211 static_var:
212 if (!tvar) 212 if (!tvar)
213 return 0; 213 return 0;
214 /* Static variables on memory (not stack), make @varname */ 214 /* Static variables on memory (not stack), make @varname */
215 ret = strlen(dwarf_diename(vr_die)); 215 ret = strlen(dwarf_diename(vr_die));
216 tvar->value = zalloc(ret + 2); 216 tvar->value = zalloc(ret + 2);
217 if (tvar->value == NULL) 217 if (tvar->value == NULL)
218 return -ENOMEM; 218 return -ENOMEM;
219 snprintf(tvar->value, ret + 2, "@%s", dwarf_diename(vr_die)); 219 snprintf(tvar->value, ret + 2, "@%s", dwarf_diename(vr_die));
220 tvar->ref = alloc_trace_arg_ref((long)offs); 220 tvar->ref = alloc_trace_arg_ref((long)offs);
221 if (tvar->ref == NULL) 221 if (tvar->ref == NULL)
222 return -ENOMEM; 222 return -ENOMEM;
223 return 0; 223 return 0;
224 } 224 }
225 225
226 /* If this is based on frame buffer, set the offset */ 226 /* If this is based on frame buffer, set the offset */
227 if (op->atom == DW_OP_fbreg) { 227 if (op->atom == DW_OP_fbreg) {
228 if (fb_ops == NULL) 228 if (fb_ops == NULL)
229 return -ENOTSUP; 229 return -ENOTSUP;
230 ref = true; 230 ref = true;
231 offs = op->number; 231 offs = op->number;
232 op = &fb_ops[0]; 232 op = &fb_ops[0];
233 } 233 }
234 234
235 if (op->atom >= DW_OP_breg0 && op->atom <= DW_OP_breg31) { 235 if (op->atom >= DW_OP_breg0 && op->atom <= DW_OP_breg31) {
236 regn = op->atom - DW_OP_breg0; 236 regn = op->atom - DW_OP_breg0;
237 offs += op->number; 237 offs += op->number;
238 ref = true; 238 ref = true;
239 } else if (op->atom >= DW_OP_reg0 && op->atom <= DW_OP_reg31) { 239 } else if (op->atom >= DW_OP_reg0 && op->atom <= DW_OP_reg31) {
240 regn = op->atom - DW_OP_reg0; 240 regn = op->atom - DW_OP_reg0;
241 } else if (op->atom == DW_OP_bregx) { 241 } else if (op->atom == DW_OP_bregx) {
242 regn = op->number; 242 regn = op->number;
243 offs += op->number2; 243 offs += op->number2;
244 ref = true; 244 ref = true;
245 } else if (op->atom == DW_OP_regx) { 245 } else if (op->atom == DW_OP_regx) {
246 regn = op->number; 246 regn = op->number;
247 } else { 247 } else {
248 pr_debug("DW_OP %x is not supported.\n", op->atom); 248 pr_debug("DW_OP %x is not supported.\n", op->atom);
249 return -ENOTSUP; 249 return -ENOTSUP;
250 } 250 }
251 251
252 if (!tvar) 252 if (!tvar)
253 return 0; 253 return 0;
254 254
255 regs = get_arch_regstr(regn); 255 regs = get_arch_regstr(regn);
256 if (!regs) { 256 if (!regs) {
257 /* This should be a bug in DWARF or this tool */ 257 /* This should be a bug in DWARF or this tool */
258 pr_warning("Mapping for the register number %u " 258 pr_warning("Mapping for the register number %u "
259 "missing on this architecture.\n", regn); 259 "missing on this architecture.\n", regn);
260 return -ERANGE; 260 return -ERANGE;
261 } 261 }
262 262
263 tvar->value = strdup(regs); 263 tvar->value = strdup(regs);
264 if (tvar->value == NULL) 264 if (tvar->value == NULL)
265 return -ENOMEM; 265 return -ENOMEM;
266 266
267 if (ref) { 267 if (ref) {
268 tvar->ref = alloc_trace_arg_ref((long)offs); 268 tvar->ref = alloc_trace_arg_ref((long)offs);
269 if (tvar->ref == NULL) 269 if (tvar->ref == NULL)
270 return -ENOMEM; 270 return -ENOMEM;
271 } 271 }
272 return 0; 272 return 0;
273 } 273 }
274 274
275 #define BYTES_TO_BITS(nb) ((nb) * BITS_PER_LONG / sizeof(long)) 275 #define BYTES_TO_BITS(nb) ((nb) * BITS_PER_LONG / sizeof(long))
276 276
277 static int convert_variable_type(Dwarf_Die *vr_die, 277 static int convert_variable_type(Dwarf_Die *vr_die,
278 struct probe_trace_arg *tvar, 278 struct probe_trace_arg *tvar,
279 const char *cast) 279 const char *cast)
280 { 280 {
281 struct probe_trace_arg_ref **ref_ptr = &tvar->ref; 281 struct probe_trace_arg_ref **ref_ptr = &tvar->ref;
282 Dwarf_Die type; 282 Dwarf_Die type;
283 char buf[16]; 283 char buf[16];
284 char sbuf[STRERR_BUFSIZE]; 284 char sbuf[STRERR_BUFSIZE];
285 int bsize, boffs, total; 285 int bsize, boffs, total;
286 int ret; 286 int ret;
287 287
288 /* TODO: check all types */ 288 /* TODO: check all types */
289 if (cast && strcmp(cast, "string") != 0) { 289 if (cast && strcmp(cast, "string") != 0) {
290 /* Non string type is OK */ 290 /* Non string type is OK */
291 tvar->type = strdup(cast); 291 tvar->type = strdup(cast);
292 return (tvar->type == NULL) ? -ENOMEM : 0; 292 return (tvar->type == NULL) ? -ENOMEM : 0;
293 } 293 }
294 294
295 bsize = dwarf_bitsize(vr_die); 295 bsize = dwarf_bitsize(vr_die);
296 if (bsize > 0) { 296 if (bsize > 0) {
297 /* This is a bitfield */ 297 /* This is a bitfield */
298 boffs = dwarf_bitoffset(vr_die); 298 boffs = dwarf_bitoffset(vr_die);
299 total = dwarf_bytesize(vr_die); 299 total = dwarf_bytesize(vr_die);
300 if (boffs < 0 || total < 0) 300 if (boffs < 0 || total < 0)
301 return -ENOENT; 301 return -ENOENT;
302 ret = snprintf(buf, 16, "b%d@%d/%zd", bsize, boffs, 302 ret = snprintf(buf, 16, "b%d@%d/%zd", bsize, boffs,
303 BYTES_TO_BITS(total)); 303 BYTES_TO_BITS(total));
304 goto formatted; 304 goto formatted;
305 } 305 }
306 306
307 if (die_get_real_type(vr_die, &type) == NULL) { 307 if (die_get_real_type(vr_die, &type) == NULL) {
308 pr_warning("Failed to get a type information of %s.\n", 308 pr_warning("Failed to get a type information of %s.\n",
309 dwarf_diename(vr_die)); 309 dwarf_diename(vr_die));
310 return -ENOENT; 310 return -ENOENT;
311 } 311 }
312 312
313 pr_debug("%s type is %s.\n", 313 pr_debug("%s type is %s.\n",
314 dwarf_diename(vr_die), dwarf_diename(&type)); 314 dwarf_diename(vr_die), dwarf_diename(&type));
315 315
316 if (cast && strcmp(cast, "string") == 0) { /* String type */ 316 if (cast && strcmp(cast, "string") == 0) { /* String type */
317 ret = dwarf_tag(&type); 317 ret = dwarf_tag(&type);
318 if (ret != DW_TAG_pointer_type && 318 if (ret != DW_TAG_pointer_type &&
319 ret != DW_TAG_array_type) { 319 ret != DW_TAG_array_type) {
320 pr_warning("Failed to cast into string: " 320 pr_warning("Failed to cast into string: "
321 "%s(%s) is not a pointer nor array.\n", 321 "%s(%s) is not a pointer nor array.\n",
322 dwarf_diename(vr_die), dwarf_diename(&type)); 322 dwarf_diename(vr_die), dwarf_diename(&type));
323 return -EINVAL; 323 return -EINVAL;
324 } 324 }
325 if (die_get_real_type(&type, &type) == NULL) { 325 if (die_get_real_type(&type, &type) == NULL) {
326 pr_warning("Failed to get a type" 326 pr_warning("Failed to get a type"
327 " information.\n"); 327 " information.\n");
328 return -ENOENT; 328 return -ENOENT;
329 } 329 }
330 if (ret == DW_TAG_pointer_type) { 330 if (ret == DW_TAG_pointer_type) {
331 while (*ref_ptr) 331 while (*ref_ptr)
332 ref_ptr = &(*ref_ptr)->next; 332 ref_ptr = &(*ref_ptr)->next;
333 /* Add new reference with offset +0 */ 333 /* Add new reference with offset +0 */
334 *ref_ptr = zalloc(sizeof(struct probe_trace_arg_ref)); 334 *ref_ptr = zalloc(sizeof(struct probe_trace_arg_ref));
335 if (*ref_ptr == NULL) { 335 if (*ref_ptr == NULL) {
336 pr_warning("Out of memory error\n"); 336 pr_warning("Out of memory error\n");
337 return -ENOMEM; 337 return -ENOMEM;
338 } 338 }
339 } 339 }
340 if (!die_compare_name(&type, "char") && 340 if (!die_compare_name(&type, "char") &&
341 !die_compare_name(&type, "unsigned char")) { 341 !die_compare_name(&type, "unsigned char")) {
342 pr_warning("Failed to cast into string: " 342 pr_warning("Failed to cast into string: "
343 "%s is not (unsigned) char *.\n", 343 "%s is not (unsigned) char *.\n",
344 dwarf_diename(vr_die)); 344 dwarf_diename(vr_die));
345 return -EINVAL; 345 return -EINVAL;
346 } 346 }
347 tvar->type = strdup(cast); 347 tvar->type = strdup(cast);
348 return (tvar->type == NULL) ? -ENOMEM : 0; 348 return (tvar->type == NULL) ? -ENOMEM : 0;
349 } 349 }
350 350
351 ret = dwarf_bytesize(&type); 351 ret = dwarf_bytesize(&type);
352 if (ret <= 0) 352 if (ret <= 0)
353 /* No size ... try to use default type */ 353 /* No size ... try to use default type */
354 return 0; 354 return 0;
355 ret = BYTES_TO_BITS(ret); 355 ret = BYTES_TO_BITS(ret);
356 356
357 /* Check the bitwidth */ 357 /* Check the bitwidth */
358 if (ret > MAX_BASIC_TYPE_BITS) { 358 if (ret > MAX_BASIC_TYPE_BITS) {
359 pr_info("%s exceeds max-bitwidth. Cut down to %d bits.\n", 359 pr_info("%s exceeds max-bitwidth. Cut down to %d bits.\n",
360 dwarf_diename(&type), MAX_BASIC_TYPE_BITS); 360 dwarf_diename(&type), MAX_BASIC_TYPE_BITS);
361 ret = MAX_BASIC_TYPE_BITS; 361 ret = MAX_BASIC_TYPE_BITS;
362 } 362 }
363 ret = snprintf(buf, 16, "%c%d", 363 ret = snprintf(buf, 16, "%c%d",
364 die_is_signed_type(&type) ? 's' : 'u', ret); 364 die_is_signed_type(&type) ? 's' : 'u', ret);
365 365
366 formatted: 366 formatted:
367 if (ret < 0 || ret >= 16) { 367 if (ret < 0 || ret >= 16) {
368 if (ret >= 16) 368 if (ret >= 16)
369 ret = -E2BIG; 369 ret = -E2BIG;
370 pr_warning("Failed to convert variable type: %s\n", 370 pr_warning("Failed to convert variable type: %s\n",
371 strerror_r(-ret, sbuf, sizeof(sbuf))); 371 strerror_r(-ret, sbuf, sizeof(sbuf)));
372 return ret; 372 return ret;
373 } 373 }
374 tvar->type = strdup(buf); 374 tvar->type = strdup(buf);
375 if (tvar->type == NULL) 375 if (tvar->type == NULL)
376 return -ENOMEM; 376 return -ENOMEM;
377 return 0; 377 return 0;
378 } 378 }
379 379
380 static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname, 380 static int convert_variable_fields(Dwarf_Die *vr_die, const char *varname,
381 struct perf_probe_arg_field *field, 381 struct perf_probe_arg_field *field,
382 struct probe_trace_arg_ref **ref_ptr, 382 struct probe_trace_arg_ref **ref_ptr,
383 Dwarf_Die *die_mem) 383 Dwarf_Die *die_mem)
384 { 384 {
385 struct probe_trace_arg_ref *ref = *ref_ptr; 385 struct probe_trace_arg_ref *ref = *ref_ptr;
386 Dwarf_Die type; 386 Dwarf_Die type;
387 Dwarf_Word offs; 387 Dwarf_Word offs;
388 int ret, tag; 388 int ret, tag;
389 389
390 pr_debug("converting %s in %s\n", field->name, varname); 390 pr_debug("converting %s in %s\n", field->name, varname);
391 if (die_get_real_type(vr_die, &type) == NULL) { 391 if (die_get_real_type(vr_die, &type) == NULL) {
392 pr_warning("Failed to get the type of %s.\n", varname); 392 pr_warning("Failed to get the type of %s.\n", varname);
393 return -ENOENT; 393 return -ENOENT;
394 } 394 }
395 pr_debug2("Var real type: (%x)\n", (unsigned)dwarf_dieoffset(&type)); 395 pr_debug2("Var real type: (%x)\n", (unsigned)dwarf_dieoffset(&type));
396 tag = dwarf_tag(&type); 396 tag = dwarf_tag(&type);
397 397
398 if (field->name[0] == '[' && 398 if (field->name[0] == '[' &&
399 (tag == DW_TAG_array_type || tag == DW_TAG_pointer_type)) { 399 (tag == DW_TAG_array_type || tag == DW_TAG_pointer_type)) {
400 if (field->next) 400 if (field->next)
401 /* Save original type for next field */ 401 /* Save original type for next field */
402 memcpy(die_mem, &type, sizeof(*die_mem)); 402 memcpy(die_mem, &type, sizeof(*die_mem));
403 /* Get the type of this array */ 403 /* Get the type of this array */
404 if (die_get_real_type(&type, &type) == NULL) { 404 if (die_get_real_type(&type, &type) == NULL) {
405 pr_warning("Failed to get the type of %s.\n", varname); 405 pr_warning("Failed to get the type of %s.\n", varname);
406 return -ENOENT; 406 return -ENOENT;
407 } 407 }
408 pr_debug2("Array real type: (%x)\n", 408 pr_debug2("Array real type: (%x)\n",
409 (unsigned)dwarf_dieoffset(&type)); 409 (unsigned)dwarf_dieoffset(&type));
410 if (tag == DW_TAG_pointer_type) { 410 if (tag == DW_TAG_pointer_type) {
411 ref = zalloc(sizeof(struct probe_trace_arg_ref)); 411 ref = zalloc(sizeof(struct probe_trace_arg_ref));
412 if (ref == NULL) 412 if (ref == NULL)
413 return -ENOMEM; 413 return -ENOMEM;
414 if (*ref_ptr) 414 if (*ref_ptr)
415 (*ref_ptr)->next = ref; 415 (*ref_ptr)->next = ref;
416 else 416 else
417 *ref_ptr = ref; 417 *ref_ptr = ref;
418 } 418 }
419 ref->offset += dwarf_bytesize(&type) * field->index; 419 ref->offset += dwarf_bytesize(&type) * field->index;
420 if (!field->next) 420 if (!field->next)
421 /* Save vr_die for converting types */ 421 /* Save vr_die for converting types */
422 memcpy(die_mem, vr_die, sizeof(*die_mem)); 422 memcpy(die_mem, vr_die, sizeof(*die_mem));
423 goto next; 423 goto next;
424 } else if (tag == DW_TAG_pointer_type) { 424 } else if (tag == DW_TAG_pointer_type) {
425 /* Check the pointer and dereference */ 425 /* Check the pointer and dereference */
426 if (!field->ref) { 426 if (!field->ref) {
427 pr_err("Semantic error: %s must be referred by '->'\n", 427 pr_err("Semantic error: %s must be referred by '->'\n",
428 field->name); 428 field->name);
429 return -EINVAL; 429 return -EINVAL;
430 } 430 }
431 /* Get the type pointed by this pointer */ 431 /* Get the type pointed by this pointer */
432 if (die_get_real_type(&type, &type) == NULL) { 432 if (die_get_real_type(&type, &type) == NULL) {
433 pr_warning("Failed to get the type of %s.\n", varname); 433 pr_warning("Failed to get the type of %s.\n", varname);
434 return -ENOENT; 434 return -ENOENT;
435 } 435 }
436 /* Verify it is a data structure */ 436 /* Verify it is a data structure */
437 tag = dwarf_tag(&type); 437 tag = dwarf_tag(&type);
438 if (tag != DW_TAG_structure_type && tag != DW_TAG_union_type) { 438 if (tag != DW_TAG_structure_type && tag != DW_TAG_union_type) {
439 pr_warning("%s is not a data structure nor an union.\n", 439 pr_warning("%s is not a data structure nor an union.\n",
440 varname); 440 varname);
441 return -EINVAL; 441 return -EINVAL;
442 } 442 }
443 443
444 ref = zalloc(sizeof(struct probe_trace_arg_ref)); 444 ref = zalloc(sizeof(struct probe_trace_arg_ref));
445 if (ref == NULL) 445 if (ref == NULL)
446 return -ENOMEM; 446 return -ENOMEM;
447 if (*ref_ptr) 447 if (*ref_ptr)
448 (*ref_ptr)->next = ref; 448 (*ref_ptr)->next = ref;
449 else 449 else
450 *ref_ptr = ref; 450 *ref_ptr = ref;
451 } else { 451 } else {
452 /* Verify it is a data structure */ 452 /* Verify it is a data structure */
453 if (tag != DW_TAG_structure_type && tag != DW_TAG_union_type) { 453 if (tag != DW_TAG_structure_type && tag != DW_TAG_union_type) {
454 pr_warning("%s is not a data structure nor an union.\n", 454 pr_warning("%s is not a data structure nor an union.\n",
455 varname); 455 varname);
456 return -EINVAL; 456 return -EINVAL;
457 } 457 }
458 if (field->name[0] == '[') { 458 if (field->name[0] == '[') {
459 pr_err("Semantic error: %s is not a pointor" 459 pr_err("Semantic error: %s is not a pointor"
460 " nor array.\n", varname); 460 " nor array.\n", varname);
461 return -EINVAL; 461 return -EINVAL;
462 } 462 }
463 if (field->ref) { 463 if (field->ref) {
464 pr_err("Semantic error: %s must be referred by '.'\n", 464 pr_err("Semantic error: %s must be referred by '.'\n",
465 field->name); 465 field->name);
466 return -EINVAL; 466 return -EINVAL;
467 } 467 }
468 if (!ref) { 468 if (!ref) {
469 pr_warning("Structure on a register is not " 469 pr_warning("Structure on a register is not "
470 "supported yet.\n"); 470 "supported yet.\n");
471 return -ENOTSUP; 471 return -ENOTSUP;
472 } 472 }
473 } 473 }
474 474
475 if (die_find_member(&type, field->name, die_mem) == NULL) { 475 if (die_find_member(&type, field->name, die_mem) == NULL) {
476 pr_warning("%s(type:%s) has no member %s.\n", varname, 476 pr_warning("%s(type:%s) has no member %s.\n", varname,
477 dwarf_diename(&type), field->name); 477 dwarf_diename(&type), field->name);
478 return -EINVAL; 478 return -EINVAL;
479 } 479 }
480 480
481 /* Get the offset of the field */ 481 /* Get the offset of the field */
482 if (tag == DW_TAG_union_type) { 482 if (tag == DW_TAG_union_type) {
483 offs = 0; 483 offs = 0;
484 } else { 484 } else {
485 ret = die_get_data_member_location(die_mem, &offs); 485 ret = die_get_data_member_location(die_mem, &offs);
486 if (ret < 0) { 486 if (ret < 0) {
487 pr_warning("Failed to get the offset of %s.\n", 487 pr_warning("Failed to get the offset of %s.\n",
488 field->name); 488 field->name);
489 return ret; 489 return ret;
490 } 490 }
491 } 491 }
492 ref->offset += (long)offs; 492 ref->offset += (long)offs;
493 493
494 next: 494 next:
495 /* Converting next field */ 495 /* Converting next field */
496 if (field->next) 496 if (field->next)
497 return convert_variable_fields(die_mem, field->name, 497 return convert_variable_fields(die_mem, field->name,
498 field->next, &ref, die_mem); 498 field->next, &ref, die_mem);
499 else 499 else
500 return 0; 500 return 0;
501 } 501 }
502 502
503 /* Show a variables in kprobe event format */ 503 /* Show a variables in kprobe event format */
504 static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf) 504 static int convert_variable(Dwarf_Die *vr_die, struct probe_finder *pf)
505 { 505 {
506 Dwarf_Die die_mem; 506 Dwarf_Die die_mem;
507 int ret; 507 int ret;
508 508
509 pr_debug("Converting variable %s into trace event.\n", 509 pr_debug("Converting variable %s into trace event.\n",
510 dwarf_diename(vr_die)); 510 dwarf_diename(vr_die));
511 511
512 ret = convert_variable_location(vr_die, pf->addr, pf->fb_ops, 512 ret = convert_variable_location(vr_die, pf->addr, pf->fb_ops,
513 &pf->sp_die, pf->tvar); 513 &pf->sp_die, pf->tvar);
514 if (ret == -ENOENT || ret == -EINVAL) 514 if (ret == -ENOENT || ret == -EINVAL)
515 pr_err("Failed to find the location of %s at this address.\n" 515 pr_err("Failed to find the location of %s at this address.\n"
516 " Perhaps, it has been optimized out.\n", pf->pvar->var); 516 " Perhaps, it has been optimized out.\n", pf->pvar->var);
517 else if (ret == -ENOTSUP) 517 else if (ret == -ENOTSUP)
518 pr_err("Sorry, we don't support this variable location yet.\n"); 518 pr_err("Sorry, we don't support this variable location yet.\n");
519 else if (ret == 0 && pf->pvar->field) { 519 else if (ret == 0 && pf->pvar->field) {
520 ret = convert_variable_fields(vr_die, pf->pvar->var, 520 ret = convert_variable_fields(vr_die, pf->pvar->var,
521 pf->pvar->field, &pf->tvar->ref, 521 pf->pvar->field, &pf->tvar->ref,
522 &die_mem); 522 &die_mem);
523 vr_die = &die_mem; 523 vr_die = &die_mem;
524 } 524 }
525 if (ret == 0) 525 if (ret == 0)
526 ret = convert_variable_type(vr_die, pf->tvar, pf->pvar->type); 526 ret = convert_variable_type(vr_die, pf->tvar, pf->pvar->type);
527 /* *expr will be cached in libdw. Don't free it. */ 527 /* *expr will be cached in libdw. Don't free it. */
528 return ret; 528 return ret;
529 } 529 }
530 530
531 /* Find a variable in a scope DIE */ 531 /* Find a variable in a scope DIE */
532 static int find_variable(Dwarf_Die *sc_die, struct probe_finder *pf) 532 static int find_variable(Dwarf_Die *sc_die, struct probe_finder *pf)
533 { 533 {
534 Dwarf_Die vr_die; 534 Dwarf_Die vr_die;
535 char buf[32], *ptr; 535 char buf[32], *ptr;
536 int ret = 0; 536 int ret = 0;
537 537
538 if (!is_c_varname(pf->pvar->var)) { 538 if (!is_c_varname(pf->pvar->var)) {
539 /* Copy raw parameters */ 539 /* Copy raw parameters */
540 pf->tvar->value = strdup(pf->pvar->var); 540 pf->tvar->value = strdup(pf->pvar->var);
541 if (pf->tvar->value == NULL) 541 if (pf->tvar->value == NULL)
542 return -ENOMEM; 542 return -ENOMEM;
543 if (pf->pvar->type) { 543 if (pf->pvar->type) {
544 pf->tvar->type = strdup(pf->pvar->type); 544 pf->tvar->type = strdup(pf->pvar->type);
545 if (pf->tvar->type == NULL) 545 if (pf->tvar->type == NULL)
546 return -ENOMEM; 546 return -ENOMEM;
547 } 547 }
548 if (pf->pvar->name) { 548 if (pf->pvar->name) {
549 pf->tvar->name = strdup(pf->pvar->name); 549 pf->tvar->name = strdup(pf->pvar->name);
550 if (pf->tvar->name == NULL) 550 if (pf->tvar->name == NULL)
551 return -ENOMEM; 551 return -ENOMEM;
552 } else 552 } else
553 pf->tvar->name = NULL; 553 pf->tvar->name = NULL;
554 return 0; 554 return 0;
555 } 555 }
556 556
557 if (pf->pvar->name) 557 if (pf->pvar->name)
558 pf->tvar->name = strdup(pf->pvar->name); 558 pf->tvar->name = strdup(pf->pvar->name);
559 else { 559 else {
560 ret = synthesize_perf_probe_arg(pf->pvar, buf, 32); 560 ret = synthesize_perf_probe_arg(pf->pvar, buf, 32);
561 if (ret < 0) 561 if (ret < 0)
562 return ret; 562 return ret;
563 ptr = strchr(buf, ':'); /* Change type separator to _ */ 563 ptr = strchr(buf, ':'); /* Change type separator to _ */
564 if (ptr) 564 if (ptr)
565 *ptr = '_'; 565 *ptr = '_';
566 pf->tvar->name = strdup(buf); 566 pf->tvar->name = strdup(buf);
567 } 567 }
568 if (pf->tvar->name == NULL) 568 if (pf->tvar->name == NULL)
569 return -ENOMEM; 569 return -ENOMEM;
570 570
571 pr_debug("Searching '%s' variable in context.\n", pf->pvar->var); 571 pr_debug("Searching '%s' variable in context.\n", pf->pvar->var);
572 /* Search child die for local variables and parameters. */ 572 /* Search child die for local variables and parameters. */
573 if (!die_find_variable_at(sc_die, pf->pvar->var, pf->addr, &vr_die)) { 573 if (!die_find_variable_at(sc_die, pf->pvar->var, pf->addr, &vr_die)) {
574 /* Search again in global variables */ 574 /* Search again in global variables */
575 if (!die_find_variable_at(&pf->cu_die, pf->pvar->var, 0, &vr_die)) 575 if (!die_find_variable_at(&pf->cu_die, pf->pvar->var, 0, &vr_die))
576 pr_warning("Failed to find '%s' in this function.\n", 576 pr_warning("Failed to find '%s' in this function.\n",
577 pf->pvar->var); 577 pf->pvar->var);
578 ret = -ENOENT; 578 ret = -ENOENT;
579 } 579 }
580 if (ret >= 0) 580 if (ret >= 0)
581 ret = convert_variable(&vr_die, pf); 581 ret = convert_variable(&vr_die, pf);
582 582
583 return ret; 583 return ret;
584 } 584 }
585 585
586 /* Convert subprogram DIE to trace point */ 586 /* Convert subprogram DIE to trace point */
587 static int convert_to_trace_point(Dwarf_Die *sp_die, Dwfl_Module *mod, 587 static int convert_to_trace_point(Dwarf_Die *sp_die, Dwfl_Module *mod,
588 Dwarf_Addr paddr, bool retprobe, 588 Dwarf_Addr paddr, bool retprobe,
589 struct probe_trace_point *tp) 589 struct probe_trace_point *tp)
590 { 590 {
591 Dwarf_Addr eaddr, highaddr; 591 Dwarf_Addr eaddr, highaddr;
592 GElf_Sym sym; 592 GElf_Sym sym;
593 const char *symbol; 593 const char *symbol;
594 594
595 /* Verify the address is correct */ 595 /* Verify the address is correct */
596 if (dwarf_entrypc(sp_die, &eaddr) != 0) { 596 if (dwarf_entrypc(sp_die, &eaddr) != 0) {
597 pr_warning("Failed to get entry address of %s\n", 597 pr_warning("Failed to get entry address of %s\n",
598 dwarf_diename(sp_die)); 598 dwarf_diename(sp_die));
599 return -ENOENT; 599 return -ENOENT;
600 } 600 }
601 if (dwarf_highpc(sp_die, &highaddr) != 0) { 601 if (dwarf_highpc(sp_die, &highaddr) != 0) {
602 pr_warning("Failed to get end address of %s\n", 602 pr_warning("Failed to get end address of %s\n",
603 dwarf_diename(sp_die)); 603 dwarf_diename(sp_die));
604 return -ENOENT; 604 return -ENOENT;
605 } 605 }
606 if (paddr > highaddr) { 606 if (paddr > highaddr) {
607 pr_warning("Offset specified is greater than size of %s\n", 607 pr_warning("Offset specified is greater than size of %s\n",
608 dwarf_diename(sp_die)); 608 dwarf_diename(sp_die));
609 return -EINVAL; 609 return -EINVAL;
610 } 610 }
611 611
612 symbol = dwarf_diename(sp_die); 612 symbol = dwarf_diename(sp_die);
613 if (!symbol) { 613 if (!symbol) {
614 /* Try to get the symbol name from symtab */ 614 /* Try to get the symbol name from symtab */
615 symbol = dwfl_module_addrsym(mod, paddr, &sym, NULL); 615 symbol = dwfl_module_addrsym(mod, paddr, &sym, NULL);
616 if (!symbol) { 616 if (!symbol) {
617 pr_warning("Failed to find symbol at 0x%lx\n", 617 pr_warning("Failed to find symbol at 0x%lx\n",
618 (unsigned long)paddr); 618 (unsigned long)paddr);
619 return -ENOENT; 619 return -ENOENT;
620 } 620 }
621 eaddr = sym.st_value; 621 eaddr = sym.st_value;
622 } 622 }
623 tp->offset = (unsigned long)(paddr - eaddr); 623 tp->offset = (unsigned long)(paddr - eaddr);
624 tp->address = (unsigned long)paddr; 624 tp->address = (unsigned long)paddr;
625 tp->symbol = strdup(symbol); 625 tp->symbol = strdup(symbol);
626 if (!tp->symbol) 626 if (!tp->symbol)
627 return -ENOMEM; 627 return -ENOMEM;
628 628
629 /* Return probe must be on the head of a subprogram */ 629 /* Return probe must be on the head of a subprogram */
630 if (retprobe) { 630 if (retprobe) {
631 if (eaddr != paddr) { 631 if (eaddr != paddr) {
632 pr_warning("Return probe must be on the head of" 632 pr_warning("Return probe must be on the head of"
633 " a real function.\n"); 633 " a real function.\n");
634 return -EINVAL; 634 return -EINVAL;
635 } 635 }
636 tp->retprobe = true; 636 tp->retprobe = true;
637 } 637 }
638 638
639 return 0; 639 return 0;
640 } 640 }
641 641
642 /* Call probe_finder callback with scope DIE */ 642 /* Call probe_finder callback with scope DIE */
643 static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf) 643 static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf)
644 { 644 {
645 Dwarf_Attribute fb_attr; 645 Dwarf_Attribute fb_attr;
646 size_t nops; 646 size_t nops;
647 int ret; 647 int ret;
648 648
649 if (!sc_die) { 649 if (!sc_die) {
650 pr_err("Caller must pass a scope DIE. Program error.\n"); 650 pr_err("Caller must pass a scope DIE. Program error.\n");
651 return -EINVAL; 651 return -EINVAL;
652 } 652 }
653 653
654 /* If not a real subprogram, find a real one */ 654 /* If not a real subprogram, find a real one */
655 if (!die_is_func_def(sc_die)) { 655 if (!die_is_func_def(sc_die)) {
656 if (!die_find_realfunc(&pf->cu_die, pf->addr, &pf->sp_die)) { 656 if (!die_find_realfunc(&pf->cu_die, pf->addr, &pf->sp_die)) {
657 pr_warning("Failed to find probe point in any " 657 pr_warning("Failed to find probe point in any "
658 "functions.\n"); 658 "functions.\n");
659 return -ENOENT; 659 return -ENOENT;
660 } 660 }
661 } else 661 } else
662 memcpy(&pf->sp_die, sc_die, sizeof(Dwarf_Die)); 662 memcpy(&pf->sp_die, sc_die, sizeof(Dwarf_Die));
663 663
664 /* Get the frame base attribute/ops from subprogram */ 664 /* Get the frame base attribute/ops from subprogram */
665 dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr); 665 dwarf_attr(&pf->sp_die, DW_AT_frame_base, &fb_attr);
666 ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1); 666 ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1);
667 if (ret <= 0 || nops == 0) { 667 if (ret <= 0 || nops == 0) {
668 pf->fb_ops = NULL; 668 pf->fb_ops = NULL;
669 #if _ELFUTILS_PREREQ(0, 142) 669 #if _ELFUTILS_PREREQ(0, 142)
670 } else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa && 670 } else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa &&
671 pf->cfi != NULL) { 671 pf->cfi != NULL) {
672 Dwarf_Frame *frame; 672 Dwarf_Frame *frame;
673 if (dwarf_cfi_addrframe(pf->cfi, pf->addr, &frame) != 0 || 673 if (dwarf_cfi_addrframe(pf->cfi, pf->addr, &frame) != 0 ||
674 dwarf_frame_cfa(frame, &pf->fb_ops, &nops) != 0) { 674 dwarf_frame_cfa(frame, &pf->fb_ops, &nops) != 0) {
675 pr_warning("Failed to get call frame on 0x%jx\n", 675 pr_warning("Failed to get call frame on 0x%jx\n",
676 (uintmax_t)pf->addr); 676 (uintmax_t)pf->addr);
677 return -ENOENT; 677 return -ENOENT;
678 } 678 }
679 #endif 679 #endif
680 } 680 }
681 681
682 /* Call finder's callback handler */ 682 /* Call finder's callback handler */
683 ret = pf->callback(sc_die, pf); 683 ret = pf->callback(sc_die, pf);
684 684
685 /* *pf->fb_ops will be cached in libdw. Don't free it. */ 685 /* *pf->fb_ops will be cached in libdw. Don't free it. */
686 pf->fb_ops = NULL; 686 pf->fb_ops = NULL;
687 687
688 return ret; 688 return ret;
689 } 689 }
690 690
691 struct find_scope_param { 691 struct find_scope_param {
692 const char *function; 692 const char *function;
693 const char *file; 693 const char *file;
694 int line; 694 int line;
695 int diff; 695 int diff;
696 Dwarf_Die *die_mem; 696 Dwarf_Die *die_mem;
697 bool found; 697 bool found;
698 }; 698 };
699 699
700 static int find_best_scope_cb(Dwarf_Die *fn_die, void *data) 700 static int find_best_scope_cb(Dwarf_Die *fn_die, void *data)
701 { 701 {
702 struct find_scope_param *fsp = data; 702 struct find_scope_param *fsp = data;
703 const char *file; 703 const char *file;
704 int lno; 704 int lno;
705 705
706 /* Skip if declared file name does not match */ 706 /* Skip if declared file name does not match */
707 if (fsp->file) { 707 if (fsp->file) {
708 file = dwarf_decl_file(fn_die); 708 file = dwarf_decl_file(fn_die);
709 if (!file || strcmp(fsp->file, file) != 0) 709 if (!file || strcmp(fsp->file, file) != 0)
710 return 0; 710 return 0;
711 } 711 }
712 /* If the function name is given, that's what user expects */ 712 /* If the function name is given, that's what user expects */
713 if (fsp->function) { 713 if (fsp->function) {
714 if (die_compare_name(fn_die, fsp->function)) { 714 if (die_compare_name(fn_die, fsp->function)) {
715 memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die)); 715 memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die));
716 fsp->found = true; 716 fsp->found = true;
717 return 1; 717 return 1;
718 } 718 }
719 } else { 719 } else {
720 /* With the line number, find the nearest declared DIE */ 720 /* With the line number, find the nearest declared DIE */
721 dwarf_decl_line(fn_die, &lno); 721 dwarf_decl_line(fn_die, &lno);
722 if (lno < fsp->line && fsp->diff > fsp->line - lno) { 722 if (lno < fsp->line && fsp->diff > fsp->line - lno) {
723 /* Keep a candidate and continue */ 723 /* Keep a candidate and continue */
724 fsp->diff = fsp->line - lno; 724 fsp->diff = fsp->line - lno;
725 memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die)); 725 memcpy(fsp->die_mem, fn_die, sizeof(Dwarf_Die));
726 fsp->found = true; 726 fsp->found = true;
727 } 727 }
728 } 728 }
729 return 0; 729 return 0;
730 } 730 }
731 731
732 /* Find an appropriate scope fits to given conditions */ 732 /* Find an appropriate scope fits to given conditions */
733 static Dwarf_Die *find_best_scope(struct probe_finder *pf, Dwarf_Die *die_mem) 733 static Dwarf_Die *find_best_scope(struct probe_finder *pf, Dwarf_Die *die_mem)
734 { 734 {
735 struct find_scope_param fsp = { 735 struct find_scope_param fsp = {
736 .function = pf->pev->point.function, 736 .function = pf->pev->point.function,
737 .file = pf->fname, 737 .file = pf->fname,
738 .line = pf->lno, 738 .line = pf->lno,
739 .diff = INT_MAX, 739 .diff = INT_MAX,
740 .die_mem = die_mem, 740 .die_mem = die_mem,
741 .found = false, 741 .found = false,
742 }; 742 };
743 743
744 cu_walk_functions_at(&pf->cu_die, pf->addr, find_best_scope_cb, &fsp); 744 cu_walk_functions_at(&pf->cu_die, pf->addr, find_best_scope_cb, &fsp);
745 745
746 return fsp.found ? die_mem : NULL; 746 return fsp.found ? die_mem : NULL;
747 } 747 }
748 748
749 static int probe_point_line_walker(const char *fname, int lineno, 749 static int probe_point_line_walker(const char *fname, int lineno,
750 Dwarf_Addr addr, void *data) 750 Dwarf_Addr addr, void *data)
751 { 751 {
752 struct probe_finder *pf = data; 752 struct probe_finder *pf = data;
753 Dwarf_Die *sc_die, die_mem; 753 Dwarf_Die *sc_die, die_mem;
754 int ret; 754 int ret;
755 755
756 if (lineno != pf->lno || strtailcmp(fname, pf->fname) != 0) 756 if (lineno != pf->lno || strtailcmp(fname, pf->fname) != 0)
757 return 0; 757 return 0;
758 758
759 pf->addr = addr; 759 pf->addr = addr;
760 sc_die = find_best_scope(pf, &die_mem); 760 sc_die = find_best_scope(pf, &die_mem);
761 if (!sc_die) { 761 if (!sc_die) {
762 pr_warning("Failed to find scope of probe point.\n"); 762 pr_warning("Failed to find scope of probe point.\n");
763 return -ENOENT; 763 return -ENOENT;
764 } 764 }
765 765
766 ret = call_probe_finder(sc_die, pf); 766 ret = call_probe_finder(sc_die, pf);
767 767
768 /* Continue if no error, because the line will be in inline function */ 768 /* Continue if no error, because the line will be in inline function */
769 return ret < 0 ? ret : 0; 769 return ret < 0 ? ret : 0;
770 } 770 }
771 771
772 /* Find probe point from its line number */ 772 /* Find probe point from its line number */
773 static int find_probe_point_by_line(struct probe_finder *pf) 773 static int find_probe_point_by_line(struct probe_finder *pf)
774 { 774 {
775 return die_walk_lines(&pf->cu_die, probe_point_line_walker, pf); 775 return die_walk_lines(&pf->cu_die, probe_point_line_walker, pf);
776 } 776 }
777 777
778 /* Find lines which match lazy pattern */ 778 /* Find lines which match lazy pattern */
779 static int find_lazy_match_lines(struct intlist *list, 779 static int find_lazy_match_lines(struct intlist *list,
780 const char *fname, const char *pat) 780 const char *fname, const char *pat)
781 { 781 {
782 FILE *fp; 782 FILE *fp;
783 char *line = NULL; 783 char *line = NULL;
784 size_t line_len; 784 size_t line_len;
785 ssize_t len; 785 ssize_t len;
786 int count = 0, linenum = 1; 786 int count = 0, linenum = 1;
787 char sbuf[STRERR_BUFSIZE]; 787 char sbuf[STRERR_BUFSIZE];
788 788
789 fp = fopen(fname, "r"); 789 fp = fopen(fname, "r");
790 if (!fp) { 790 if (!fp) {
791 pr_warning("Failed to open %s: %s\n", fname, 791 pr_warning("Failed to open %s: %s\n", fname,
792 strerror_r(errno, sbuf, sizeof(sbuf))); 792 strerror_r(errno, sbuf, sizeof(sbuf)));
793 return -errno; 793 return -errno;
794 } 794 }
795 795
796 while ((len = getline(&line, &line_len, fp)) > 0) { 796 while ((len = getline(&line, &line_len, fp)) > 0) {
797 797
798 if (line[len - 1] == '\n') 798 if (line[len - 1] == '\n')
799 line[len - 1] = '\0'; 799 line[len - 1] = '\0';
800 800
801 if (strlazymatch(line, pat)) { 801 if (strlazymatch(line, pat)) {
802 intlist__add(list, linenum); 802 intlist__add(list, linenum);
803 count++; 803 count++;
804 } 804 }
805 linenum++; 805 linenum++;
806 } 806 }
807 807
808 if (ferror(fp)) 808 if (ferror(fp))
809 count = -errno; 809 count = -errno;
810 free(line); 810 free(line);
811 fclose(fp); 811 fclose(fp);
812 812
813 if (count == 0) 813 if (count == 0)
814 pr_debug("No matched lines found in %s.\n", fname); 814 pr_debug("No matched lines found in %s.\n", fname);
815 return count; 815 return count;
816 } 816 }
817 817
818 static int probe_point_lazy_walker(const char *fname, int lineno, 818 static int probe_point_lazy_walker(const char *fname, int lineno,
819 Dwarf_Addr addr, void *data) 819 Dwarf_Addr addr, void *data)
820 { 820 {
821 struct probe_finder *pf = data; 821 struct probe_finder *pf = data;
822 Dwarf_Die *sc_die, die_mem; 822 Dwarf_Die *sc_die, die_mem;
823 int ret; 823 int ret;
824 824
825 if (!intlist__has_entry(pf->lcache, lineno) || 825 if (!intlist__has_entry(pf->lcache, lineno) ||
826 strtailcmp(fname, pf->fname) != 0) 826 strtailcmp(fname, pf->fname) != 0)
827 return 0; 827 return 0;
828 828
829 pr_debug("Probe line found: line:%d addr:0x%llx\n", 829 pr_debug("Probe line found: line:%d addr:0x%llx\n",
830 lineno, (unsigned long long)addr); 830 lineno, (unsigned long long)addr);
831 pf->addr = addr; 831 pf->addr = addr;
832 pf->lno = lineno; 832 pf->lno = lineno;
833 sc_die = find_best_scope(pf, &die_mem); 833 sc_die = find_best_scope(pf, &die_mem);
834 if (!sc_die) { 834 if (!sc_die) {
835 pr_warning("Failed to find scope of probe point.\n"); 835 pr_warning("Failed to find scope of probe point.\n");
836 return -ENOENT; 836 return -ENOENT;
837 } 837 }
838 838
839 ret = call_probe_finder(sc_die, pf); 839 ret = call_probe_finder(sc_die, pf);
840 840
841 /* 841 /*
842 * Continue if no error, because the lazy pattern will match 842 * Continue if no error, because the lazy pattern will match
843 * to other lines 843 * to other lines
844 */ 844 */
845 return ret < 0 ? ret : 0; 845 return ret < 0 ? ret : 0;
846 } 846 }
847 847
848 /* Find probe points from lazy pattern */ 848 /* Find probe points from lazy pattern */
849 static int find_probe_point_lazy(Dwarf_Die *sp_die, struct probe_finder *pf) 849 static int find_probe_point_lazy(Dwarf_Die *sp_die, struct probe_finder *pf)
850 { 850 {
851 int ret = 0; 851 int ret = 0;
852 852
853 if (intlist__empty(pf->lcache)) { 853 if (intlist__empty(pf->lcache)) {
854 /* Matching lazy line pattern */ 854 /* Matching lazy line pattern */
855 ret = find_lazy_match_lines(pf->lcache, pf->fname, 855 ret = find_lazy_match_lines(pf->lcache, pf->fname,
856 pf->pev->point.lazy_line); 856 pf->pev->point.lazy_line);
857 if (ret <= 0) 857 if (ret <= 0)
858 return ret; 858 return ret;
859 } 859 }
860 860
861 return die_walk_lines(sp_die, probe_point_lazy_walker, pf); 861 return die_walk_lines(sp_die, probe_point_lazy_walker, pf);
862 } 862 }
863 863
864 static int probe_point_inline_cb(Dwarf_Die *in_die, void *data) 864 static int probe_point_inline_cb(Dwarf_Die *in_die, void *data)
865 { 865 {
866 struct probe_finder *pf = data; 866 struct probe_finder *pf = data;
867 struct perf_probe_point *pp = &pf->pev->point; 867 struct perf_probe_point *pp = &pf->pev->point;
868 Dwarf_Addr addr; 868 Dwarf_Addr addr;
869 int ret; 869 int ret;
870 870
871 if (pp->lazy_line) 871 if (pp->lazy_line)
872 ret = find_probe_point_lazy(in_die, pf); 872 ret = find_probe_point_lazy(in_die, pf);
873 else { 873 else {
874 /* Get probe address */ 874 /* Get probe address */
875 if (dwarf_entrypc(in_die, &addr) != 0) { 875 if (dwarf_entrypc(in_die, &addr) != 0) {
876 pr_warning("Failed to get entry address of %s.\n", 876 pr_warning("Failed to get entry address of %s.\n",
877 dwarf_diename(in_die)); 877 dwarf_diename(in_die));
878 return -ENOENT; 878 return -ENOENT;
879 } 879 }
880 pf->addr = addr; 880 pf->addr = addr;
881 pf->addr += pp->offset; 881 pf->addr += pp->offset;
882 pr_debug("found inline addr: 0x%jx\n", 882 pr_debug("found inline addr: 0x%jx\n",
883 (uintmax_t)pf->addr); 883 (uintmax_t)pf->addr);
884 884
885 ret = call_probe_finder(in_die, pf); 885 ret = call_probe_finder(in_die, pf);
886 } 886 }
887 887
888 return ret; 888 return ret;
889 } 889 }
890 890
891 /* Callback parameter with return value for libdw */ 891 /* Callback parameter with return value for libdw */
892 struct dwarf_callback_param { 892 struct dwarf_callback_param {
893 void *data; 893 void *data;
894 int retval; 894 int retval;
895 }; 895 };
896 896
897 /* Search function from function name */ 897 /* Search function from function name */
898 static int probe_point_search_cb(Dwarf_Die *sp_die, void *data) 898 static int probe_point_search_cb(Dwarf_Die *sp_die, void *data)
899 { 899 {
900 struct dwarf_callback_param *param = data; 900 struct dwarf_callback_param *param = data;
901 struct probe_finder *pf = param->data; 901 struct probe_finder *pf = param->data;
902 struct perf_probe_point *pp = &pf->pev->point; 902 struct perf_probe_point *pp = &pf->pev->point;
903 903
904 /* Check tag and diename */ 904 /* Check tag and diename */
905 if (!die_is_func_def(sp_die) || 905 if (!die_is_func_def(sp_die) ||
906 !die_compare_name(sp_die, pp->function)) 906 !die_compare_name(sp_die, pp->function))
907 return DWARF_CB_OK; 907 return DWARF_CB_OK;
908 908
909 /* Check declared file */ 909 /* Check declared file */
910 if (pp->file && strtailcmp(pp->file, dwarf_decl_file(sp_die))) 910 if (pp->file && strtailcmp(pp->file, dwarf_decl_file(sp_die)))
911 return DWARF_CB_OK; 911 return DWARF_CB_OK;
912 912
913 pf->fname = dwarf_decl_file(sp_die); 913 pf->fname = dwarf_decl_file(sp_die);
914 if (pp->line) { /* Function relative line */ 914 if (pp->line) { /* Function relative line */
915 dwarf_decl_line(sp_die, &pf->lno); 915 dwarf_decl_line(sp_die, &pf->lno);
916 pf->lno += pp->line; 916 pf->lno += pp->line;
917 param->retval = find_probe_point_by_line(pf); 917 param->retval = find_probe_point_by_line(pf);
918 } else if (!dwarf_func_inline(sp_die)) { 918 } else if (!dwarf_func_inline(sp_die)) {
919 /* Real function */ 919 /* Real function */
920 if (pp->lazy_line) 920 if (pp->lazy_line)
921 param->retval = find_probe_point_lazy(sp_die, pf); 921 param->retval = find_probe_point_lazy(sp_die, pf);
922 else { 922 else {
923 if (dwarf_entrypc(sp_die, &pf->addr) != 0) { 923 if (dwarf_entrypc(sp_die, &pf->addr) != 0) {
924 pr_warning("Failed to get entry address of " 924 pr_warning("Failed to get entry address of "
925 "%s.\n", dwarf_diename(sp_die)); 925 "%s.\n", dwarf_diename(sp_die));
926 param->retval = -ENOENT; 926 param->retval = -ENOENT;
927 return DWARF_CB_ABORT; 927 return DWARF_CB_ABORT;
928 } 928 }
929 pf->addr += pp->offset; 929 pf->addr += pp->offset;
930 /* TODO: Check the address in this function */ 930 /* TODO: Check the address in this function */
931 param->retval = call_probe_finder(sp_die, pf); 931 param->retval = call_probe_finder(sp_die, pf);
932 } 932 }
933 } else 933 } else
934 /* Inlined function: search instances */ 934 /* Inlined function: search instances */
935 param->retval = die_walk_instances(sp_die, 935 param->retval = die_walk_instances(sp_die,
936 probe_point_inline_cb, (void *)pf); 936 probe_point_inline_cb, (void *)pf);
937 937
938 return DWARF_CB_ABORT; /* Exit; no same symbol in this CU. */ 938 return DWARF_CB_ABORT; /* Exit; no same symbol in this CU. */
939 } 939 }
940 940
941 static int find_probe_point_by_func(struct probe_finder *pf) 941 static int find_probe_point_by_func(struct probe_finder *pf)
942 { 942 {
943 struct dwarf_callback_param _param = {.data = (void *)pf, 943 struct dwarf_callback_param _param = {.data = (void *)pf,
944 .retval = 0}; 944 .retval = 0};
945 dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0); 945 dwarf_getfuncs(&pf->cu_die, probe_point_search_cb, &_param, 0);
946 return _param.retval; 946 return _param.retval;
947 } 947 }
948 948
949 struct pubname_callback_param { 949 struct pubname_callback_param {
950 char *function; 950 char *function;
951 char *file; 951 char *file;
952 Dwarf_Die *cu_die; 952 Dwarf_Die *cu_die;
953 Dwarf_Die *sp_die; 953 Dwarf_Die *sp_die;
954 int found; 954 int found;
955 }; 955 };
956 956
957 static int pubname_search_cb(Dwarf *dbg, Dwarf_Global *gl, void *data) 957 static int pubname_search_cb(Dwarf *dbg, Dwarf_Global *gl, void *data)
958 { 958 {
959 struct pubname_callback_param *param = data; 959 struct pubname_callback_param *param = data;
960 960
961 if (dwarf_offdie(dbg, gl->die_offset, param->sp_die)) { 961 if (dwarf_offdie(dbg, gl->die_offset, param->sp_die)) {
962 if (dwarf_tag(param->sp_die) != DW_TAG_subprogram) 962 if (dwarf_tag(param->sp_die) != DW_TAG_subprogram)
963 return DWARF_CB_OK; 963 return DWARF_CB_OK;
964 964
965 if (die_compare_name(param->sp_die, param->function)) { 965 if (die_compare_name(param->sp_die, param->function)) {
966 if (!dwarf_offdie(dbg, gl->cu_offset, param->cu_die)) 966 if (!dwarf_offdie(dbg, gl->cu_offset, param->cu_die))
967 return DWARF_CB_OK; 967 return DWARF_CB_OK;
968 968
969 if (param->file && 969 if (param->file &&
970 strtailcmp(param->file, dwarf_decl_file(param->sp_die))) 970 strtailcmp(param->file, dwarf_decl_file(param->sp_die)))
971 return DWARF_CB_OK; 971 return DWARF_CB_OK;
972 972
973 param->found = 1; 973 param->found = 1;
974 return DWARF_CB_ABORT; 974 return DWARF_CB_ABORT;
975 } 975 }
976 } 976 }
977 977
978 return DWARF_CB_OK; 978 return DWARF_CB_OK;
979 } 979 }
980 980
981 /* Find probe points from debuginfo */ 981 /* Find probe points from debuginfo */
982 static int debuginfo__find_probes(struct debuginfo *dbg, 982 static int debuginfo__find_probes(struct debuginfo *dbg,
983 struct probe_finder *pf) 983 struct probe_finder *pf)
984 { 984 {
985 struct perf_probe_point *pp = &pf->pev->point; 985 struct perf_probe_point *pp = &pf->pev->point;
986 Dwarf_Off off, noff; 986 Dwarf_Off off, noff;
987 size_t cuhl; 987 size_t cuhl;
988 Dwarf_Die *diep; 988 Dwarf_Die *diep;
989 int ret = 0; 989 int ret = 0;
990 990
991 #if _ELFUTILS_PREREQ(0, 142) 991 #if _ELFUTILS_PREREQ(0, 142)
992 Elf *elf;
993 GElf_Ehdr ehdr;
994 GElf_Shdr shdr;
995
992 /* Get the call frame information from this dwarf */ 996 /* Get the call frame information from this dwarf */
993 pf->cfi = dwarf_getcfi_elf(dwarf_getelf(dbg->dbg)); 997 elf = dwarf_getelf(dbg->dbg);
998 if (elf == NULL)
999 return -EINVAL;
1000
1001 if (gelf_getehdr(elf, &ehdr) == NULL)
1002 return -EINVAL;
1003
1004 if (elf_section_by_name(elf, &ehdr, &shdr, ".eh_frame", NULL) &&
1005 shdr.sh_type == SHT_PROGBITS) {
1006 pf->cfi = dwarf_getcfi_elf(elf);
1007 } else {
1008 pf->cfi = dwarf_getcfi(dbg->dbg);
1009 }
994 #endif 1010 #endif
995 1011
996 off = 0; 1012 off = 0;
997 pf->lcache = intlist__new(NULL); 1013 pf->lcache = intlist__new(NULL);
998 if (!pf->lcache) 1014 if (!pf->lcache)
999 return -ENOMEM; 1015 return -ENOMEM;
1000 1016
1001 /* Fastpath: lookup by function name from .debug_pubnames section */ 1017 /* Fastpath: lookup by function name from .debug_pubnames section */
1002 if (pp->function) { 1018 if (pp->function) {
1003 struct pubname_callback_param pubname_param = { 1019 struct pubname_callback_param pubname_param = {
1004 .function = pp->function, 1020 .function = pp->function,
1005 .file = pp->file, 1021 .file = pp->file,
1006 .cu_die = &pf->cu_die, 1022 .cu_die = &pf->cu_die,
1007 .sp_die = &pf->sp_die, 1023 .sp_die = &pf->sp_die,
1008 .found = 0, 1024 .found = 0,
1009 }; 1025 };
1010 struct dwarf_callback_param probe_param = { 1026 struct dwarf_callback_param probe_param = {
1011 .data = pf, 1027 .data = pf,
1012 }; 1028 };
1013 1029
1014 dwarf_getpubnames(dbg->dbg, pubname_search_cb, 1030 dwarf_getpubnames(dbg->dbg, pubname_search_cb,
1015 &pubname_param, 0); 1031 &pubname_param, 0);
1016 if (pubname_param.found) { 1032 if (pubname_param.found) {
1017 ret = probe_point_search_cb(&pf->sp_die, &probe_param); 1033 ret = probe_point_search_cb(&pf->sp_die, &probe_param);
1018 if (ret) 1034 if (ret)
1019 goto found; 1035 goto found;
1020 } 1036 }
1021 } 1037 }
1022 1038
1023 /* Loop on CUs (Compilation Unit) */ 1039 /* Loop on CUs (Compilation Unit) */
1024 while (!dwarf_nextcu(dbg->dbg, off, &noff, &cuhl, NULL, NULL, NULL)) { 1040 while (!dwarf_nextcu(dbg->dbg, off, &noff, &cuhl, NULL, NULL, NULL)) {
1025 /* Get the DIE(Debugging Information Entry) of this CU */ 1041 /* Get the DIE(Debugging Information Entry) of this CU */
1026 diep = dwarf_offdie(dbg->dbg, off + cuhl, &pf->cu_die); 1042 diep = dwarf_offdie(dbg->dbg, off + cuhl, &pf->cu_die);
1027 if (!diep) 1043 if (!diep)
1028 continue; 1044 continue;
1029 1045
1030 /* Check if target file is included. */ 1046 /* Check if target file is included. */
1031 if (pp->file) 1047 if (pp->file)
1032 pf->fname = cu_find_realpath(&pf->cu_die, pp->file); 1048 pf->fname = cu_find_realpath(&pf->cu_die, pp->file);
1033 else 1049 else
1034 pf->fname = NULL; 1050 pf->fname = NULL;
1035 1051
1036 if (!pp->file || pf->fname) { 1052 if (!pp->file || pf->fname) {
1037 if (pp->function) 1053 if (pp->function)
1038 ret = find_probe_point_by_func(pf); 1054 ret = find_probe_point_by_func(pf);
1039 else if (pp->lazy_line) 1055 else if (pp->lazy_line)
1040 ret = find_probe_point_lazy(NULL, pf); 1056 ret = find_probe_point_lazy(NULL, pf);
1041 else { 1057 else {
1042 pf->lno = pp->line; 1058 pf->lno = pp->line;
1043 ret = find_probe_point_by_line(pf); 1059 ret = find_probe_point_by_line(pf);
1044 } 1060 }
1045 if (ret < 0) 1061 if (ret < 0)
1046 break; 1062 break;
1047 } 1063 }
1048 off = noff; 1064 off = noff;
1049 } 1065 }
1050 1066
1051 found: 1067 found:
1052 intlist__delete(pf->lcache); 1068 intlist__delete(pf->lcache);
1053 pf->lcache = NULL; 1069 pf->lcache = NULL;
1054 1070
1055 return ret; 1071 return ret;
1056 } 1072 }
1057 1073
1058 struct local_vars_finder { 1074 struct local_vars_finder {
1059 struct probe_finder *pf; 1075 struct probe_finder *pf;
1060 struct perf_probe_arg *args; 1076 struct perf_probe_arg *args;
1061 int max_args; 1077 int max_args;
1062 int nargs; 1078 int nargs;
1063 int ret; 1079 int ret;
1064 }; 1080 };
1065 1081
1066 /* Collect available variables in this scope */ 1082 /* Collect available variables in this scope */
1067 static int copy_variables_cb(Dwarf_Die *die_mem, void *data) 1083 static int copy_variables_cb(Dwarf_Die *die_mem, void *data)
1068 { 1084 {
1069 struct local_vars_finder *vf = data; 1085 struct local_vars_finder *vf = data;
1070 struct probe_finder *pf = vf->pf; 1086 struct probe_finder *pf = vf->pf;
1071 int tag; 1087 int tag;
1072 1088
1073 tag = dwarf_tag(die_mem); 1089 tag = dwarf_tag(die_mem);
1074 if (tag == DW_TAG_formal_parameter || 1090 if (tag == DW_TAG_formal_parameter ||
1075 tag == DW_TAG_variable) { 1091 tag == DW_TAG_variable) {
1076 if (convert_variable_location(die_mem, vf->pf->addr, 1092 if (convert_variable_location(die_mem, vf->pf->addr,
1077 vf->pf->fb_ops, &pf->sp_die, 1093 vf->pf->fb_ops, &pf->sp_die,
1078 NULL) == 0) { 1094 NULL) == 0) {
1079 vf->args[vf->nargs].var = (char *)dwarf_diename(die_mem); 1095 vf->args[vf->nargs].var = (char *)dwarf_diename(die_mem);
1080 if (vf->args[vf->nargs].var == NULL) { 1096 if (vf->args[vf->nargs].var == NULL) {
1081 vf->ret = -ENOMEM; 1097 vf->ret = -ENOMEM;
1082 return DIE_FIND_CB_END; 1098 return DIE_FIND_CB_END;
1083 } 1099 }
1084 pr_debug(" %s", vf->args[vf->nargs].var); 1100 pr_debug(" %s", vf->args[vf->nargs].var);
1085 vf->nargs++; 1101 vf->nargs++;
1086 } 1102 }
1087 } 1103 }
1088 1104
1089 if (dwarf_haspc(die_mem, vf->pf->addr)) 1105 if (dwarf_haspc(die_mem, vf->pf->addr))
1090 return DIE_FIND_CB_CONTINUE; 1106 return DIE_FIND_CB_CONTINUE;
1091 else 1107 else
1092 return DIE_FIND_CB_SIBLING; 1108 return DIE_FIND_CB_SIBLING;
1093 } 1109 }
1094 1110
1095 static int expand_probe_args(Dwarf_Die *sc_die, struct probe_finder *pf, 1111 static int expand_probe_args(Dwarf_Die *sc_die, struct probe_finder *pf,
1096 struct perf_probe_arg *args) 1112 struct perf_probe_arg *args)
1097 { 1113 {
1098 Dwarf_Die die_mem; 1114 Dwarf_Die die_mem;
1099 int i; 1115 int i;
1100 int n = 0; 1116 int n = 0;
1101 struct local_vars_finder vf = {.pf = pf, .args = args, 1117 struct local_vars_finder vf = {.pf = pf, .args = args,
1102 .max_args = MAX_PROBE_ARGS, .ret = 0}; 1118 .max_args = MAX_PROBE_ARGS, .ret = 0};
1103 1119
1104 for (i = 0; i < pf->pev->nargs; i++) { 1120 for (i = 0; i < pf->pev->nargs; i++) {
1105 /* var never be NULL */ 1121 /* var never be NULL */
1106 if (strcmp(pf->pev->args[i].var, "$vars") == 0) { 1122 if (strcmp(pf->pev->args[i].var, "$vars") == 0) {
1107 pr_debug("Expanding $vars into:"); 1123 pr_debug("Expanding $vars into:");
1108 vf.nargs = n; 1124 vf.nargs = n;
1109 /* Special local variables */ 1125 /* Special local variables */
1110 die_find_child(sc_die, copy_variables_cb, (void *)&vf, 1126 die_find_child(sc_die, copy_variables_cb, (void *)&vf,
1111 &die_mem); 1127 &die_mem);
1112 pr_debug(" (%d)\n", vf.nargs - n); 1128 pr_debug(" (%d)\n", vf.nargs - n);
1113 if (vf.ret < 0) 1129 if (vf.ret < 0)
1114 return vf.ret; 1130 return vf.ret;
1115 n = vf.nargs; 1131 n = vf.nargs;
1116 } else { 1132 } else {
1117 /* Copy normal argument */ 1133 /* Copy normal argument */
1118 args[n] = pf->pev->args[i]; 1134 args[n] = pf->pev->args[i];
1119 n++; 1135 n++;
1120 } 1136 }
1121 } 1137 }
1122 return n; 1138 return n;
1123 } 1139 }
1124 1140
1125 /* Add a found probe point into trace event list */ 1141 /* Add a found probe point into trace event list */
1126 static int add_probe_trace_event(Dwarf_Die *sc_die, struct probe_finder *pf) 1142 static int add_probe_trace_event(Dwarf_Die *sc_die, struct probe_finder *pf)
1127 { 1143 {
1128 struct trace_event_finder *tf = 1144 struct trace_event_finder *tf =
1129 container_of(pf, struct trace_event_finder, pf); 1145 container_of(pf, struct trace_event_finder, pf);
1130 struct probe_trace_event *tev; 1146 struct probe_trace_event *tev;
1131 struct perf_probe_arg *args; 1147 struct perf_probe_arg *args;
1132 int ret, i; 1148 int ret, i;
1133 1149
1134 /* Check number of tevs */ 1150 /* Check number of tevs */
1135 if (tf->ntevs == tf->max_tevs) { 1151 if (tf->ntevs == tf->max_tevs) {
1136 pr_warning("Too many( > %d) probe point found.\n", 1152 pr_warning("Too many( > %d) probe point found.\n",
1137 tf->max_tevs); 1153 tf->max_tevs);
1138 return -ERANGE; 1154 return -ERANGE;
1139 } 1155 }
1140 tev = &tf->tevs[tf->ntevs++]; 1156 tev = &tf->tevs[tf->ntevs++];
1141 1157
1142 /* Trace point should be converted from subprogram DIE */ 1158 /* Trace point should be converted from subprogram DIE */
1143 ret = convert_to_trace_point(&pf->sp_die, tf->mod, pf->addr, 1159 ret = convert_to_trace_point(&pf->sp_die, tf->mod, pf->addr,
1144 pf->pev->point.retprobe, &tev->point); 1160 pf->pev->point.retprobe, &tev->point);
1145 if (ret < 0) 1161 if (ret < 0)
1146 return ret; 1162 return ret;
1147 1163
1148 pr_debug("Probe point found: %s+%lu\n", tev->point.symbol, 1164 pr_debug("Probe point found: %s+%lu\n", tev->point.symbol,
1149 tev->point.offset); 1165 tev->point.offset);
1150 1166
1151 /* Expand special probe argument if exist */ 1167 /* Expand special probe argument if exist */
1152 args = zalloc(sizeof(struct perf_probe_arg) * MAX_PROBE_ARGS); 1168 args = zalloc(sizeof(struct perf_probe_arg) * MAX_PROBE_ARGS);
1153 if (args == NULL) 1169 if (args == NULL)
1154 return -ENOMEM; 1170 return -ENOMEM;
1155 1171
1156 ret = expand_probe_args(sc_die, pf, args); 1172 ret = expand_probe_args(sc_die, pf, args);
1157 if (ret < 0) 1173 if (ret < 0)
1158 goto end; 1174 goto end;
1159 1175
1160 tev->nargs = ret; 1176 tev->nargs = ret;
1161 tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs); 1177 tev->args = zalloc(sizeof(struct probe_trace_arg) * tev->nargs);
1162 if (tev->args == NULL) { 1178 if (tev->args == NULL) {
1163 ret = -ENOMEM; 1179 ret = -ENOMEM;
1164 goto end; 1180 goto end;
1165 } 1181 }
1166 1182
1167 /* Find each argument */ 1183 /* Find each argument */
1168 for (i = 0; i < tev->nargs; i++) { 1184 for (i = 0; i < tev->nargs; i++) {
1169 pf->pvar = &args[i]; 1185 pf->pvar = &args[i];
1170 pf->tvar = &tev->args[i]; 1186 pf->tvar = &tev->args[i];
1171 /* Variable should be found from scope DIE */ 1187 /* Variable should be found from scope DIE */
1172 ret = find_variable(sc_die, pf); 1188 ret = find_variable(sc_die, pf);
1173 if (ret != 0) 1189 if (ret != 0)
1174 break; 1190 break;
1175 } 1191 }
1176 1192
1177 end: 1193 end:
1178 free(args); 1194 free(args);
1179 return ret; 1195 return ret;
1180 } 1196 }
1181 1197
1182 /* Find probe_trace_events specified by perf_probe_event from debuginfo */ 1198 /* Find probe_trace_events specified by perf_probe_event from debuginfo */
1183 int debuginfo__find_trace_events(struct debuginfo *dbg, 1199 int debuginfo__find_trace_events(struct debuginfo *dbg,
1184 struct perf_probe_event *pev, 1200 struct perf_probe_event *pev,
1185 struct probe_trace_event **tevs, int max_tevs) 1201 struct probe_trace_event **tevs, int max_tevs)
1186 { 1202 {
1187 struct trace_event_finder tf = { 1203 struct trace_event_finder tf = {
1188 .pf = {.pev = pev, .callback = add_probe_trace_event}, 1204 .pf = {.pev = pev, .callback = add_probe_trace_event},
1189 .mod = dbg->mod, .max_tevs = max_tevs}; 1205 .mod = dbg->mod, .max_tevs = max_tevs};
1190 int ret; 1206 int ret;
1191 1207
1192 /* Allocate result tevs array */ 1208 /* Allocate result tevs array */
1193 *tevs = zalloc(sizeof(struct probe_trace_event) * max_tevs); 1209 *tevs = zalloc(sizeof(struct probe_trace_event) * max_tevs);
1194 if (*tevs == NULL) 1210 if (*tevs == NULL)
1195 return -ENOMEM; 1211 return -ENOMEM;
1196 1212
1197 tf.tevs = *tevs; 1213 tf.tevs = *tevs;
1198 tf.ntevs = 0; 1214 tf.ntevs = 0;
1199 1215
1200 ret = debuginfo__find_probes(dbg, &tf.pf); 1216 ret = debuginfo__find_probes(dbg, &tf.pf);
1201 if (ret < 0) { 1217 if (ret < 0) {
1202 zfree(tevs); 1218 zfree(tevs);
1203 return ret; 1219 return ret;
1204 } 1220 }
1205 1221
1206 return (ret < 0) ? ret : tf.ntevs; 1222 return (ret < 0) ? ret : tf.ntevs;
1207 } 1223 }
1208 1224
1209 #define MAX_VAR_LEN 64 1225 #define MAX_VAR_LEN 64
1210 1226
1211 /* Collect available variables in this scope */ 1227 /* Collect available variables in this scope */
1212 static int collect_variables_cb(Dwarf_Die *die_mem, void *data) 1228 static int collect_variables_cb(Dwarf_Die *die_mem, void *data)
1213 { 1229 {
1214 struct available_var_finder *af = data; 1230 struct available_var_finder *af = data;
1215 struct variable_list *vl; 1231 struct variable_list *vl;
1216 char buf[MAX_VAR_LEN]; 1232 char buf[MAX_VAR_LEN];
1217 int tag, ret; 1233 int tag, ret;
1218 1234
1219 vl = &af->vls[af->nvls - 1]; 1235 vl = &af->vls[af->nvls - 1];
1220 1236
1221 tag = dwarf_tag(die_mem); 1237 tag = dwarf_tag(die_mem);
1222 if (tag == DW_TAG_formal_parameter || 1238 if (tag == DW_TAG_formal_parameter ||
1223 tag == DW_TAG_variable) { 1239 tag == DW_TAG_variable) {
1224 ret = convert_variable_location(die_mem, af->pf.addr, 1240 ret = convert_variable_location(die_mem, af->pf.addr,
1225 af->pf.fb_ops, &af->pf.sp_die, 1241 af->pf.fb_ops, &af->pf.sp_die,
1226 NULL); 1242 NULL);
1227 if (ret == 0) { 1243 if (ret == 0) {
1228 ret = die_get_varname(die_mem, buf, MAX_VAR_LEN); 1244 ret = die_get_varname(die_mem, buf, MAX_VAR_LEN);
1229 pr_debug2("Add new var: %s\n", buf); 1245 pr_debug2("Add new var: %s\n", buf);
1230 if (ret > 0) 1246 if (ret > 0)
1231 strlist__add(vl->vars, buf); 1247 strlist__add(vl->vars, buf);
1232 } 1248 }
1233 } 1249 }
1234 1250
1235 if (af->child && dwarf_haspc(die_mem, af->pf.addr)) 1251 if (af->child && dwarf_haspc(die_mem, af->pf.addr))
1236 return DIE_FIND_CB_CONTINUE; 1252 return DIE_FIND_CB_CONTINUE;
1237 else 1253 else
1238 return DIE_FIND_CB_SIBLING; 1254 return DIE_FIND_CB_SIBLING;
1239 } 1255 }
1240 1256
1241 /* Add a found vars into available variables list */ 1257 /* Add a found vars into available variables list */
1242 static int add_available_vars(Dwarf_Die *sc_die, struct probe_finder *pf) 1258 static int add_available_vars(Dwarf_Die *sc_die, struct probe_finder *pf)
1243 { 1259 {
1244 struct available_var_finder *af = 1260 struct available_var_finder *af =
1245 container_of(pf, struct available_var_finder, pf); 1261 container_of(pf, struct available_var_finder, pf);
1246 struct variable_list *vl; 1262 struct variable_list *vl;
1247 Dwarf_Die die_mem; 1263 Dwarf_Die die_mem;
1248 int ret; 1264 int ret;
1249 1265
1250 /* Check number of tevs */ 1266 /* Check number of tevs */
1251 if (af->nvls == af->max_vls) { 1267 if (af->nvls == af->max_vls) {
1252 pr_warning("Too many( > %d) probe point found.\n", af->max_vls); 1268 pr_warning("Too many( > %d) probe point found.\n", af->max_vls);
1253 return -ERANGE; 1269 return -ERANGE;
1254 } 1270 }
1255 vl = &af->vls[af->nvls++]; 1271 vl = &af->vls[af->nvls++];
1256 1272
1257 /* Trace point should be converted from subprogram DIE */ 1273 /* Trace point should be converted from subprogram DIE */
1258 ret = convert_to_trace_point(&pf->sp_die, af->mod, pf->addr, 1274 ret = convert_to_trace_point(&pf->sp_die, af->mod, pf->addr,
1259 pf->pev->point.retprobe, &vl->point); 1275 pf->pev->point.retprobe, &vl->point);
1260 if (ret < 0) 1276 if (ret < 0)
1261 return ret; 1277 return ret;
1262 1278
1263 pr_debug("Probe point found: %s+%lu\n", vl->point.symbol, 1279 pr_debug("Probe point found: %s+%lu\n", vl->point.symbol,
1264 vl->point.offset); 1280 vl->point.offset);
1265 1281
1266 /* Find local variables */ 1282 /* Find local variables */
1267 vl->vars = strlist__new(true, NULL); 1283 vl->vars = strlist__new(true, NULL);
1268 if (vl->vars == NULL) 1284 if (vl->vars == NULL)
1269 return -ENOMEM; 1285 return -ENOMEM;
1270 af->child = true; 1286 af->child = true;
1271 die_find_child(sc_die, collect_variables_cb, (void *)af, &die_mem); 1287 die_find_child(sc_die, collect_variables_cb, (void *)af, &die_mem);
1272 1288
1273 /* Find external variables */ 1289 /* Find external variables */
1274 if (!af->externs) 1290 if (!af->externs)
1275 goto out; 1291 goto out;
1276 /* Don't need to search child DIE for externs. */ 1292 /* Don't need to search child DIE for externs. */
1277 af->child = false; 1293 af->child = false;
1278 die_find_child(&pf->cu_die, collect_variables_cb, (void *)af, &die_mem); 1294 die_find_child(&pf->cu_die, collect_variables_cb, (void *)af, &die_mem);
1279 1295
1280 out: 1296 out:
1281 if (strlist__empty(vl->vars)) { 1297 if (strlist__empty(vl->vars)) {
1282 strlist__delete(vl->vars); 1298 strlist__delete(vl->vars);
1283 vl->vars = NULL; 1299 vl->vars = NULL;
1284 } 1300 }
1285 1301
1286 return ret; 1302 return ret;
1287 } 1303 }
1288 1304
1289 /* 1305 /*
1290 * Find available variables at given probe point 1306 * Find available variables at given probe point
1291 * Return the number of found probe points. Return 0 if there is no 1307 * Return the number of found probe points. Return 0 if there is no
1292 * matched probe point. Return <0 if an error occurs. 1308 * matched probe point. Return <0 if an error occurs.
1293 */ 1309 */
1294 int debuginfo__find_available_vars_at(struct debuginfo *dbg, 1310 int debuginfo__find_available_vars_at(struct debuginfo *dbg,
1295 struct perf_probe_event *pev, 1311 struct perf_probe_event *pev,
1296 struct variable_list **vls, 1312 struct variable_list **vls,
1297 int max_vls, bool externs) 1313 int max_vls, bool externs)
1298 { 1314 {
1299 struct available_var_finder af = { 1315 struct available_var_finder af = {
1300 .pf = {.pev = pev, .callback = add_available_vars}, 1316 .pf = {.pev = pev, .callback = add_available_vars},
1301 .mod = dbg->mod, 1317 .mod = dbg->mod,
1302 .max_vls = max_vls, .externs = externs}; 1318 .max_vls = max_vls, .externs = externs};
1303 int ret; 1319 int ret;
1304 1320
1305 /* Allocate result vls array */ 1321 /* Allocate result vls array */
1306 *vls = zalloc(sizeof(struct variable_list) * max_vls); 1322 *vls = zalloc(sizeof(struct variable_list) * max_vls);
1307 if (*vls == NULL) 1323 if (*vls == NULL)
1308 return -ENOMEM; 1324 return -ENOMEM;
1309 1325
1310 af.vls = *vls; 1326 af.vls = *vls;
1311 af.nvls = 0; 1327 af.nvls = 0;
1312 1328
1313 ret = debuginfo__find_probes(dbg, &af.pf); 1329 ret = debuginfo__find_probes(dbg, &af.pf);
1314 if (ret < 0) { 1330 if (ret < 0) {
1315 /* Free vlist for error */ 1331 /* Free vlist for error */
1316 while (af.nvls--) { 1332 while (af.nvls--) {
1317 zfree(&af.vls[af.nvls].point.symbol); 1333 zfree(&af.vls[af.nvls].point.symbol);
1318 strlist__delete(af.vls[af.nvls].vars); 1334 strlist__delete(af.vls[af.nvls].vars);
1319 } 1335 }
1320 zfree(vls); 1336 zfree(vls);
1321 return ret; 1337 return ret;
1322 } 1338 }
1323 1339
1324 return (ret < 0) ? ret : af.nvls; 1340 return (ret < 0) ? ret : af.nvls;
1325 } 1341 }
1326 1342
1327 /* Reverse search */ 1343 /* Reverse search */
1328 int debuginfo__find_probe_point(struct debuginfo *dbg, unsigned long addr, 1344 int debuginfo__find_probe_point(struct debuginfo *dbg, unsigned long addr,
1329 struct perf_probe_point *ppt) 1345 struct perf_probe_point *ppt)
1330 { 1346 {
1331 Dwarf_Die cudie, spdie, indie; 1347 Dwarf_Die cudie, spdie, indie;
1332 Dwarf_Addr _addr = 0, baseaddr = 0; 1348 Dwarf_Addr _addr = 0, baseaddr = 0;
1333 const char *fname = NULL, *func = NULL, *basefunc = NULL, *tmp; 1349 const char *fname = NULL, *func = NULL, *basefunc = NULL, *tmp;
1334 int baseline = 0, lineno = 0, ret = 0; 1350 int baseline = 0, lineno = 0, ret = 0;
1335 1351
1336 /* Adjust address with bias */ 1352 /* Adjust address with bias */
1337 addr += dbg->bias; 1353 addr += dbg->bias;
1338 1354
1339 /* Find cu die */ 1355 /* Find cu die */
1340 if (!dwarf_addrdie(dbg->dbg, (Dwarf_Addr)addr - dbg->bias, &cudie)) { 1356 if (!dwarf_addrdie(dbg->dbg, (Dwarf_Addr)addr - dbg->bias, &cudie)) {
1341 pr_warning("Failed to find debug information for address %lx\n", 1357 pr_warning("Failed to find debug information for address %lx\n",
1342 addr); 1358 addr);
1343 ret = -EINVAL; 1359 ret = -EINVAL;
1344 goto end; 1360 goto end;
1345 } 1361 }
1346 1362
1347 /* Find a corresponding line (filename and lineno) */ 1363 /* Find a corresponding line (filename and lineno) */
1348 cu_find_lineinfo(&cudie, addr, &fname, &lineno); 1364 cu_find_lineinfo(&cudie, addr, &fname, &lineno);
1349 /* Don't care whether it failed or not */ 1365 /* Don't care whether it failed or not */
1350 1366
1351 /* Find a corresponding function (name, baseline and baseaddr) */ 1367 /* Find a corresponding function (name, baseline and baseaddr) */
1352 if (die_find_realfunc(&cudie, (Dwarf_Addr)addr, &spdie)) { 1368 if (die_find_realfunc(&cudie, (Dwarf_Addr)addr, &spdie)) {
1353 /* Get function entry information */ 1369 /* Get function entry information */
1354 func = basefunc = dwarf_diename(&spdie); 1370 func = basefunc = dwarf_diename(&spdie);
1355 if (!func || 1371 if (!func ||
1356 dwarf_entrypc(&spdie, &baseaddr) != 0 || 1372 dwarf_entrypc(&spdie, &baseaddr) != 0 ||
1357 dwarf_decl_line(&spdie, &baseline) != 0) { 1373 dwarf_decl_line(&spdie, &baseline) != 0) {
1358 lineno = 0; 1374 lineno = 0;
1359 goto post; 1375 goto post;
1360 } 1376 }
1361 1377
1362 fname = dwarf_decl_file(&spdie); 1378 fname = dwarf_decl_file(&spdie);
1363 if (addr == (unsigned long)baseaddr) { 1379 if (addr == (unsigned long)baseaddr) {
1364 /* Function entry - Relative line number is 0 */ 1380 /* Function entry - Relative line number is 0 */
1365 lineno = baseline; 1381 lineno = baseline;
1366 goto post; 1382 goto post;
1367 } 1383 }
1368 1384
1369 /* Track down the inline functions step by step */ 1385 /* Track down the inline functions step by step */
1370 while (die_find_top_inlinefunc(&spdie, (Dwarf_Addr)addr, 1386 while (die_find_top_inlinefunc(&spdie, (Dwarf_Addr)addr,
1371 &indie)) { 1387 &indie)) {
1372 /* There is an inline function */ 1388 /* There is an inline function */
1373 if (dwarf_entrypc(&indie, &_addr) == 0 && 1389 if (dwarf_entrypc(&indie, &_addr) == 0 &&
1374 _addr == addr) { 1390 _addr == addr) {
1375 /* 1391 /*
1376 * addr is at an inline function entry. 1392 * addr is at an inline function entry.
1377 * In this case, lineno should be the call-site 1393 * In this case, lineno should be the call-site
1378 * line number. (overwrite lineinfo) 1394 * line number. (overwrite lineinfo)
1379 */ 1395 */
1380 lineno = die_get_call_lineno(&indie); 1396 lineno = die_get_call_lineno(&indie);
1381 fname = die_get_call_file(&indie); 1397 fname = die_get_call_file(&indie);
1382 break; 1398 break;
1383 } else { 1399 } else {
1384 /* 1400 /*
1385 * addr is in an inline function body. 1401 * addr is in an inline function body.
1386 * Since lineno points one of the lines 1402 * Since lineno points one of the lines
1387 * of the inline function, baseline should 1403 * of the inline function, baseline should
1388 * be the entry line of the inline function. 1404 * be the entry line of the inline function.
1389 */ 1405 */
1390 tmp = dwarf_diename(&indie); 1406 tmp = dwarf_diename(&indie);
1391 if (!tmp || 1407 if (!tmp ||
1392 dwarf_decl_line(&indie, &baseline) != 0) 1408 dwarf_decl_line(&indie, &baseline) != 0)
1393 break; 1409 break;
1394 func = tmp; 1410 func = tmp;
1395 spdie = indie; 1411 spdie = indie;
1396 } 1412 }
1397 } 1413 }
1398 /* Verify the lineno and baseline are in a same file */ 1414 /* Verify the lineno and baseline are in a same file */
1399 tmp = dwarf_decl_file(&spdie); 1415 tmp = dwarf_decl_file(&spdie);
1400 if (!tmp || strcmp(tmp, fname) != 0) 1416 if (!tmp || strcmp(tmp, fname) != 0)
1401 lineno = 0; 1417 lineno = 0;
1402 } 1418 }
1403 1419
1404 post: 1420 post:
1405 /* Make a relative line number or an offset */ 1421 /* Make a relative line number or an offset */
1406 if (lineno) 1422 if (lineno)
1407 ppt->line = lineno - baseline; 1423 ppt->line = lineno - baseline;
1408 else if (basefunc) { 1424 else if (basefunc) {
1409 ppt->offset = addr - (unsigned long)baseaddr; 1425 ppt->offset = addr - (unsigned long)baseaddr;
1410 func = basefunc; 1426 func = basefunc;
1411 } 1427 }
1412 1428
1413 /* Duplicate strings */ 1429 /* Duplicate strings */
1414 if (func) { 1430 if (func) {
1415 ppt->function = strdup(func); 1431 ppt->function = strdup(func);
1416 if (ppt->function == NULL) { 1432 if (ppt->function == NULL) {
1417 ret = -ENOMEM; 1433 ret = -ENOMEM;
1418 goto end; 1434 goto end;
1419 } 1435 }
1420 } 1436 }
1421 if (fname) { 1437 if (fname) {
1422 ppt->file = strdup(fname); 1438 ppt->file = strdup(fname);
1423 if (ppt->file == NULL) { 1439 if (ppt->file == NULL) {
1424 zfree(&ppt->function); 1440 zfree(&ppt->function);
1425 ret = -ENOMEM; 1441 ret = -ENOMEM;
1426 goto end; 1442 goto end;
1427 } 1443 }
1428 } 1444 }
1429 end: 1445 end:
1430 if (ret == 0 && (fname || func)) 1446 if (ret == 0 && (fname || func))
1431 ret = 1; /* Found a point */ 1447 ret = 1; /* Found a point */
1432 return ret; 1448 return ret;
1433 } 1449 }
1434 1450
1435 /* Add a line and store the src path */ 1451 /* Add a line and store the src path */
1436 static int line_range_add_line(const char *src, unsigned int lineno, 1452 static int line_range_add_line(const char *src, unsigned int lineno,
1437 struct line_range *lr) 1453 struct line_range *lr)
1438 { 1454 {
1439 /* Copy source path */ 1455 /* Copy source path */
1440 if (!lr->path) { 1456 if (!lr->path) {
1441 lr->path = strdup(src); 1457 lr->path = strdup(src);
1442 if (lr->path == NULL) 1458 if (lr->path == NULL)
1443 return -ENOMEM; 1459 return -ENOMEM;
1444 } 1460 }
1445 return intlist__add(lr->line_list, lineno); 1461 return intlist__add(lr->line_list, lineno);
1446 } 1462 }
1447 1463
1448 static int line_range_walk_cb(const char *fname, int lineno, 1464 static int line_range_walk_cb(const char *fname, int lineno,
1449 Dwarf_Addr addr __maybe_unused, 1465 Dwarf_Addr addr __maybe_unused,
1450 void *data) 1466 void *data)
1451 { 1467 {
1452 struct line_finder *lf = data; 1468 struct line_finder *lf = data;
1453 int err; 1469 int err;
1454 1470
1455 if ((strtailcmp(fname, lf->fname) != 0) || 1471 if ((strtailcmp(fname, lf->fname) != 0) ||
1456 (lf->lno_s > lineno || lf->lno_e < lineno)) 1472 (lf->lno_s > lineno || lf->lno_e < lineno))
1457 return 0; 1473 return 0;
1458 1474
1459 err = line_range_add_line(fname, lineno, lf->lr); 1475 err = line_range_add_line(fname, lineno, lf->lr);
1460 if (err < 0 && err != -EEXIST) 1476 if (err < 0 && err != -EEXIST)
1461 return err; 1477 return err;
1462 1478
1463 return 0; 1479 return 0;
1464 } 1480 }
1465 1481
1466 /* Find line range from its line number */ 1482 /* Find line range from its line number */
1467 static int find_line_range_by_line(Dwarf_Die *sp_die, struct line_finder *lf) 1483 static int find_line_range_by_line(Dwarf_Die *sp_die, struct line_finder *lf)
1468 { 1484 {
1469 int ret; 1485 int ret;
1470 1486
1471 ret = die_walk_lines(sp_die ?: &lf->cu_die, line_range_walk_cb, lf); 1487 ret = die_walk_lines(sp_die ?: &lf->cu_die, line_range_walk_cb, lf);
1472 1488
1473 /* Update status */ 1489 /* Update status */
1474 if (ret >= 0) 1490 if (ret >= 0)
1475 if (!intlist__empty(lf->lr->line_list)) 1491 if (!intlist__empty(lf->lr->line_list))
1476 ret = lf->found = 1; 1492 ret = lf->found = 1;
1477 else 1493 else
1478 ret = 0; /* Lines are not found */ 1494 ret = 0; /* Lines are not found */
1479 else { 1495 else {
1480 zfree(&lf->lr->path); 1496 zfree(&lf->lr->path);
1481 } 1497 }
1482 return ret; 1498 return ret;
1483 } 1499 }
1484 1500
1485 static int line_range_inline_cb(Dwarf_Die *in_die, void *data) 1501 static int line_range_inline_cb(Dwarf_Die *in_die, void *data)
1486 { 1502 {
1487 int ret = find_line_range_by_line(in_die, data); 1503 int ret = find_line_range_by_line(in_die, data);
1488 1504
1489 /* 1505 /*
1490 * We have to check all instances of inlined function, because 1506 * We have to check all instances of inlined function, because
1491 * some execution paths can be optimized out depends on the 1507 * some execution paths can be optimized out depends on the
1492 * function argument of instances. However, if an error occurs, 1508 * function argument of instances. However, if an error occurs,
1493 * it should be handled by the caller. 1509 * it should be handled by the caller.
1494 */ 1510 */
1495 return ret < 0 ? ret : 0; 1511 return ret < 0 ? ret : 0;
1496 } 1512 }
1497 1513
1498 /* Search function definition from function name */ 1514 /* Search function definition from function name */
1499 static int line_range_search_cb(Dwarf_Die *sp_die, void *data) 1515 static int line_range_search_cb(Dwarf_Die *sp_die, void *data)
1500 { 1516 {
1501 struct dwarf_callback_param *param = data; 1517 struct dwarf_callback_param *param = data;
1502 struct line_finder *lf = param->data; 1518 struct line_finder *lf = param->data;
1503 struct line_range *lr = lf->lr; 1519 struct line_range *lr = lf->lr;
1504 1520
1505 /* Check declared file */ 1521 /* Check declared file */
1506 if (lr->file && strtailcmp(lr->file, dwarf_decl_file(sp_die))) 1522 if (lr->file && strtailcmp(lr->file, dwarf_decl_file(sp_die)))
1507 return DWARF_CB_OK; 1523 return DWARF_CB_OK;
1508 1524
1509 if (die_is_func_def(sp_die) && 1525 if (die_is_func_def(sp_die) &&
1510 die_compare_name(sp_die, lr->function)) { 1526 die_compare_name(sp_die, lr->function)) {
1511 lf->fname = dwarf_decl_file(sp_die); 1527 lf->fname = dwarf_decl_file(sp_die);
1512 dwarf_decl_line(sp_die, &lr->offset); 1528 dwarf_decl_line(sp_die, &lr->offset);
1513 pr_debug("fname: %s, lineno:%d\n", lf->fname, lr->offset); 1529 pr_debug("fname: %s, lineno:%d\n", lf->fname, lr->offset);
1514 lf->lno_s = lr->offset + lr->start; 1530 lf->lno_s = lr->offset + lr->start;
1515 if (lf->lno_s < 0) /* Overflow */ 1531 if (lf->lno_s < 0) /* Overflow */
1516 lf->lno_s = INT_MAX; 1532 lf->lno_s = INT_MAX;
1517 lf->lno_e = lr->offset + lr->end; 1533 lf->lno_e = lr->offset + lr->end;
1518 if (lf->lno_e < 0) /* Overflow */ 1534 if (lf->lno_e < 0) /* Overflow */
1519 lf->lno_e = INT_MAX; 1535 lf->lno_e = INT_MAX;
1520 pr_debug("New line range: %d to %d\n", lf->lno_s, lf->lno_e); 1536 pr_debug("New line range: %d to %d\n", lf->lno_s, lf->lno_e);
1521 lr->start = lf->lno_s; 1537 lr->start = lf->lno_s;
1522 lr->end = lf->lno_e; 1538 lr->end = lf->lno_e;
1523 if (dwarf_func_inline(sp_die)) 1539 if (dwarf_func_inline(sp_die))
1524 param->retval = die_walk_instances(sp_die, 1540 param->retval = die_walk_instances(sp_die,
1525 line_range_inline_cb, lf); 1541 line_range_inline_cb, lf);
1526 else 1542 else
1527 param->retval = find_line_range_by_line(sp_die, lf); 1543 param->retval = find_line_range_by_line(sp_die, lf);
1528 return DWARF_CB_ABORT; 1544 return DWARF_CB_ABORT;
1529 } 1545 }
1530 return DWARF_CB_OK; 1546 return DWARF_CB_OK;
1531 } 1547 }
1532 1548
1533 static int find_line_range_by_func(struct line_finder *lf) 1549 static int find_line_range_by_func(struct line_finder *lf)
1534 { 1550 {
1535 struct dwarf_callback_param param = {.data = (void *)lf, .retval = 0}; 1551 struct dwarf_callback_param param = {.data = (void *)lf, .retval = 0};
1536 dwarf_getfuncs(&lf->cu_die, line_range_search_cb, &param, 0); 1552 dwarf_getfuncs(&lf->cu_die, line_range_search_cb, &param, 0);
1537 return param.retval; 1553 return param.retval;
1538 } 1554 }
1539 1555
1540 int debuginfo__find_line_range(struct debuginfo *dbg, struct line_range *lr) 1556 int debuginfo__find_line_range(struct debuginfo *dbg, struct line_range *lr)
1541 { 1557 {
1542 struct line_finder lf = {.lr = lr, .found = 0}; 1558 struct line_finder lf = {.lr = lr, .found = 0};
1543 int ret = 0; 1559 int ret = 0;
1544 Dwarf_Off off = 0, noff; 1560 Dwarf_Off off = 0, noff;
1545 size_t cuhl; 1561 size_t cuhl;
1546 Dwarf_Die *diep; 1562 Dwarf_Die *diep;
1547 const char *comp_dir; 1563 const char *comp_dir;
1548 1564
1549 /* Fastpath: lookup by function name from .debug_pubnames section */ 1565 /* Fastpath: lookup by function name from .debug_pubnames section */
1550 if (lr->function) { 1566 if (lr->function) {
1551 struct pubname_callback_param pubname_param = { 1567 struct pubname_callback_param pubname_param = {
1552 .function = lr->function, .file = lr->file, 1568 .function = lr->function, .file = lr->file,
1553 .cu_die = &lf.cu_die, .sp_die = &lf.sp_die, .found = 0}; 1569 .cu_die = &lf.cu_die, .sp_die = &lf.sp_die, .found = 0};
1554 struct dwarf_callback_param line_range_param = { 1570 struct dwarf_callback_param line_range_param = {
1555 .data = (void *)&lf, .retval = 0}; 1571 .data = (void *)&lf, .retval = 0};
1556 1572
1557 dwarf_getpubnames(dbg->dbg, pubname_search_cb, 1573 dwarf_getpubnames(dbg->dbg, pubname_search_cb,
1558 &pubname_param, 0); 1574 &pubname_param, 0);
1559 if (pubname_param.found) { 1575 if (pubname_param.found) {
1560 line_range_search_cb(&lf.sp_die, &line_range_param); 1576 line_range_search_cb(&lf.sp_die, &line_range_param);
1561 if (lf.found) 1577 if (lf.found)
1562 goto found; 1578 goto found;
1563 } 1579 }
1564 } 1580 }
1565 1581
1566 /* Loop on CUs (Compilation Unit) */ 1582 /* Loop on CUs (Compilation Unit) */
1567 while (!lf.found && ret >= 0) { 1583 while (!lf.found && ret >= 0) {
1568 if (dwarf_nextcu(dbg->dbg, off, &noff, &cuhl, 1584 if (dwarf_nextcu(dbg->dbg, off, &noff, &cuhl,
1569 NULL, NULL, NULL) != 0) 1585 NULL, NULL, NULL) != 0)
1570 break; 1586 break;
1571 1587
1572 /* Get the DIE(Debugging Information Entry) of this CU */ 1588 /* Get the DIE(Debugging Information Entry) of this CU */
1573 diep = dwarf_offdie(dbg->dbg, off + cuhl, &lf.cu_die); 1589 diep = dwarf_offdie(dbg->dbg, off + cuhl, &lf.cu_die);
1574 if (!diep) 1590 if (!diep)
1575 continue; 1591 continue;
1576 1592
1577 /* Check if target file is included. */ 1593 /* Check if target file is included. */
1578 if (lr->file) 1594 if (lr->file)
1579 lf.fname = cu_find_realpath(&lf.cu_die, lr->file); 1595 lf.fname = cu_find_realpath(&lf.cu_die, lr->file);
1580 else 1596 else
1581 lf.fname = 0; 1597 lf.fname = 0;
1582 1598
1583 if (!lr->file || lf.fname) { 1599 if (!lr->file || lf.fname) {
1584 if (lr->function) 1600 if (lr->function)
1585 ret = find_line_range_by_func(&lf); 1601 ret = find_line_range_by_func(&lf);
1586 else { 1602 else {
1587 lf.lno_s = lr->start; 1603 lf.lno_s = lr->start;
1588 lf.lno_e = lr->end; 1604 lf.lno_e = lr->end;
1589 ret = find_line_range_by_line(NULL, &lf); 1605 ret = find_line_range_by_line(NULL, &lf);
1590 } 1606 }
1591 } 1607 }
1592 off = noff; 1608 off = noff;
1593 } 1609 }
1594 1610
1595 found: 1611 found:
1596 /* Store comp_dir */ 1612 /* Store comp_dir */
1597 if (lf.found) { 1613 if (lf.found) {
1598 comp_dir = cu_get_comp_dir(&lf.cu_die); 1614 comp_dir = cu_get_comp_dir(&lf.cu_die);
1599 if (comp_dir) { 1615 if (comp_dir) {
1600 lr->comp_dir = strdup(comp_dir); 1616 lr->comp_dir = strdup(comp_dir);
1601 if (!lr->comp_dir) 1617 if (!lr->comp_dir)
1602 ret = -ENOMEM; 1618 ret = -ENOMEM;
1603 } 1619 }
1604 } 1620 }
1605 1621
1606 pr_debug("path: %s\n", lr->path); 1622 pr_debug("path: %s\n", lr->path);
1607 return (ret < 0) ? ret : lf.found; 1623 return (ret < 0) ? ret : lf.found;
1608 } 1624 }
1609 1625
1610 1626