1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
3 * lttng-context-perf-counters.c
5 * LTTng performance monitoring counters (perf-counters) integration module.
7 * Copyright (C) 2009-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
10 #include <linux/module.h>
11 #include <linux/slab.h>
12 #include <linux/perf_event.h>
13 #include <linux/list.h>
14 #include <linux/string.h>
15 #include <linux/cpu.h>
17 #include <lttng-events.h>
18 #include <wrapper/ringbuffer/frontend_types.h>
19 #include <lttng-tracer.h>
22 size_t perf_counter_get_size(size_t offset
)
26 size
+= lib_ring_buffer_align(offset
, lttng_alignof(uint64_t));
27 size
+= sizeof(uint64_t);
32 void perf_counter_record(struct lttng_ctx_field
*field
,
33 struct lib_ring_buffer_ctx
*ctx
,
34 struct lttng_channel
*chan
)
36 struct perf_event
*event
;
39 event
= field
->u
.perf_counter
->e
[ctx
->cpu
];
41 if (unlikely(event
->state
== PERF_EVENT_STATE_ERROR
)) {
44 event
->pmu
->read(event
);
45 value
= local64_read(&event
->count
);
49 * Perf chooses not to be clever and not to support enabling a
50 * perf counter before the cpu is brought up. Therefore, we need
51 * to support having events coming (e.g. scheduler events)
52 * before the counter is setup. Write an arbitrary 0 in this
57 lib_ring_buffer_align_ctx(ctx
, lttng_alignof(value
));
58 chan
->ops
->event_write(ctx
, &value
, sizeof(value
));
61 #if defined(CONFIG_PERF_EVENTS) && (LINUX_VERSION_CODE >= KERNEL_VERSION(3,0,99))
63 void overflow_callback(struct perf_event
*event
,
64 struct perf_sample_data
*data
,
70 void overflow_callback(struct perf_event
*event
, int nmi
,
71 struct perf_sample_data
*data
,
78 void lttng_destroy_perf_counter_field(struct lttng_ctx_field
*field
)
80 struct perf_event
**events
= field
->u
.perf_counter
->e
;
82 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
86 ret
= cpuhp_state_remove_instance(lttng_hp_online
,
87 &field
->u
.perf_counter
->cpuhp_online
.node
);
89 ret
= cpuhp_state_remove_instance(lttng_hp_prepare
,
90 &field
->u
.perf_counter
->cpuhp_prepare
.node
);
93 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
98 for_each_online_cpu(cpu
)
99 perf_event_release_kernel(events
[cpu
]);
101 #ifdef CONFIG_HOTPLUG_CPU
102 unregister_cpu_notifier(&field
->u
.perf_counter
->nb
);
105 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
106 kfree(field
->event_field
.name
);
107 kfree(field
->u
.perf_counter
->attr
);
109 kfree(field
->u
.perf_counter
);
112 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
114 int lttng_cpuhp_perf_counter_online(unsigned int cpu
,
115 struct lttng_cpuhp_node
*node
)
117 struct lttng_perf_counter_field
*perf_field
=
118 container_of(node
, struct lttng_perf_counter_field
,
120 struct perf_event
**events
= perf_field
->e
;
121 struct perf_event_attr
*attr
= perf_field
->attr
;
122 struct perf_event
*pevent
;
124 pevent
= perf_event_create_kernel_counter(attr
,
125 cpu
, NULL
, overflow_callback
, NULL
);
126 if (!pevent
|| IS_ERR(pevent
))
128 if (pevent
->state
== PERF_EVENT_STATE_ERROR
) {
129 perf_event_release_kernel(pevent
);
132 barrier(); /* Create perf counter before setting event */
133 events
[cpu
] = pevent
;
137 int lttng_cpuhp_perf_counter_dead(unsigned int cpu
,
138 struct lttng_cpuhp_node
*node
)
140 struct lttng_perf_counter_field
*perf_field
=
141 container_of(node
, struct lttng_perf_counter_field
,
143 struct perf_event
**events
= perf_field
->e
;
144 struct perf_event
*pevent
;
146 pevent
= events
[cpu
];
148 barrier(); /* NULLify event before perf counter teardown */
149 perf_event_release_kernel(pevent
);
153 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
155 #ifdef CONFIG_HOTPLUG_CPU
158 * lttng_perf_counter_hp_callback - CPU hotplug callback
159 * @nb: notifier block
160 * @action: hotplug action to take
163 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
165 * We can setup perf counters when the cpu is online (up prepare seems to be too
169 int lttng_perf_counter_cpu_hp_callback(struct notifier_block
*nb
,
170 unsigned long action
,
173 unsigned int cpu
= (unsigned long) hcpu
;
174 struct lttng_perf_counter_field
*perf_field
=
175 container_of(nb
, struct lttng_perf_counter_field
, nb
);
176 struct perf_event
**events
= perf_field
->e
;
177 struct perf_event_attr
*attr
= perf_field
->attr
;
178 struct perf_event
*pevent
;
180 if (!perf_field
->hp_enable
)
185 case CPU_ONLINE_FROZEN
:
186 pevent
= perf_event_create_kernel_counter(attr
,
187 cpu
, NULL
, overflow_callback
, NULL
);
188 if (!pevent
|| IS_ERR(pevent
))
190 if (pevent
->state
== PERF_EVENT_STATE_ERROR
) {
191 perf_event_release_kernel(pevent
);
194 barrier(); /* Create perf counter before setting event */
195 events
[cpu
] = pevent
;
197 case CPU_UP_CANCELED
:
198 case CPU_UP_CANCELED_FROZEN
:
200 case CPU_DEAD_FROZEN
:
201 pevent
= events
[cpu
];
203 barrier(); /* NULLify event before perf counter teardown */
204 perf_event_release_kernel(pevent
);
212 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
214 int lttng_add_perf_counter_to_ctx(uint32_t type
,
217 struct lttng_ctx
**ctx
)
219 struct lttng_ctx_field
*field
;
220 struct lttng_perf_counter_field
*perf_field
;
221 struct perf_event
**events
;
222 struct perf_event_attr
*attr
;
226 events
= kvzalloc(num_possible_cpus() * sizeof(*events
), GFP_KERNEL
);
230 attr
= kzalloc(sizeof(struct perf_event_attr
), GFP_KERNEL
);
237 attr
->config
= config
;
238 attr
->size
= sizeof(struct perf_event_attr
);
242 perf_field
= kzalloc(sizeof(struct lttng_perf_counter_field
), GFP_KERNEL
);
245 goto error_alloc_perf_field
;
247 perf_field
->e
= events
;
248 perf_field
->attr
= attr
;
250 name_alloc
= kstrdup(name
, GFP_KERNEL
);
253 goto name_alloc_error
;
256 field
= lttng_append_context(ctx
);
259 goto append_context_error
;
261 if (lttng_find_context(*ctx
, name_alloc
)) {
266 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
268 perf_field
->cpuhp_prepare
.component
= LTTNG_CONTEXT_PERF_COUNTERS
;
269 ret
= cpuhp_state_add_instance(lttng_hp_prepare
,
270 &perf_field
->cpuhp_prepare
.node
);
272 goto cpuhp_prepare_error
;
274 perf_field
->cpuhp_online
.component
= LTTNG_CONTEXT_PERF_COUNTERS
;
275 ret
= cpuhp_state_add_instance(lttng_hp_online
,
276 &perf_field
->cpuhp_online
.node
);
278 goto cpuhp_online_error
;
280 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
284 #ifdef CONFIG_HOTPLUG_CPU
285 perf_field
->nb
.notifier_call
=
286 lttng_perf_counter_cpu_hp_callback
;
287 perf_field
->nb
.priority
= 0;
288 register_cpu_notifier(&perf_field
->nb
);
291 for_each_online_cpu(cpu
) {
292 events
[cpu
] = perf_event_create_kernel_counter(attr
,
293 cpu
, NULL
, overflow_callback
, NULL
);
294 if (!events
[cpu
] || IS_ERR(events
[cpu
])) {
298 if (events
[cpu
]->state
== PERF_EVENT_STATE_ERROR
) {
304 perf_field
->hp_enable
= 1;
306 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
308 field
->destroy
= lttng_destroy_perf_counter_field
;
310 field
->event_field
.name
= name_alloc
;
311 field
->event_field
.type
.atype
= atype_integer
;
312 field
->event_field
.type
.u
.integer
.size
= sizeof(uint64_t) * CHAR_BIT
;
313 field
->event_field
.type
.u
.integer
.alignment
= lttng_alignof(uint64_t) * CHAR_BIT
;
314 field
->event_field
.type
.u
.integer
.signedness
= lttng_is_signed_type(uint64_t);
315 field
->event_field
.type
.u
.integer
.reverse_byte_order
= 0;
316 field
->event_field
.type
.u
.integer
.base
= 10;
317 field
->event_field
.type
.u
.integer
.encoding
= lttng_encode_none
;
318 field
->get_size
= perf_counter_get_size
;
319 field
->record
= perf_counter_record
;
320 field
->u
.perf_counter
= perf_field
;
321 lttng_context_update(*ctx
);
325 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
330 remove_ret
= cpuhp_state_remove_instance(lttng_hp_prepare
,
331 &perf_field
->cpuhp_prepare
.node
);
335 #else /* #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
341 for_each_online_cpu(cpu
) {
342 if (events
[cpu
] && !IS_ERR(events
[cpu
]))
343 perf_event_release_kernel(events
[cpu
]);
346 #ifdef CONFIG_HOTPLUG_CPU
347 unregister_cpu_notifier(&perf_field
->nb
);
350 #endif /* #else #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0)) */
352 lttng_remove_context_field(ctx
, field
);
353 append_context_error
:
357 error_alloc_perf_field
: