Fix: pass private data to context callbacks
[lttng-ust.git] / liblttng-ust / lttng-context-perf-counters.c
CommitLineData
d58d1454 1/*
c0c0989a 2 * SPDX-License-Identifier: LGPL-2.1-only
d58d1454
MD
3 *
4 * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
5 *
c0c0989a 6 * LTTng UST performance monitoring counters (perf-counters) integration.
d58d1454
MD
7 */
8
3fbec7dc 9#define _LGPL_SOURCE
9af5d97a 10#include <limits.h>
d58d1454
MD
11#include <sys/types.h>
12#include <unistd.h>
13#include <string.h>
14#include <stdlib.h>
15#include <stdio.h>
649fb6b3 16#include <stdbool.h>
b4051ad8 17#include <stddef.h>
fb31eb73 18#include <stdint.h>
d58d1454
MD
19#include <sys/mman.h>
20#include <sys/syscall.h>
2eba8e39 21#include <lttng/ust-arch.h>
d58d1454
MD
22#include <lttng/ust-events.h>
23#include <lttng/ust-tracer.h>
0466ac28 24#include <lttng/ringbuffer-context.h>
d58d1454
MD
25#include <urcu/system.h>
26#include <urcu/arch.h>
27#include <urcu/rculist.h>
864a1eda 28#include <ust-helper.h>
d58d1454
MD
29#include <urcu/ref.h>
30#include <usterr-signal-safe.h>
31#include <signal.h>
20142124 32#include <urcu/tls-compat.h>
77d7fa98 33#include "perf_event.h"
fc80554e
MJ
34
35#include "context-internal.h"
d58d1454 36#include "lttng-tracer-core.h"
bd8c1787 37#include "ust-events-internal.h"
d58d1454
MD
38
39/*
40 * We use a global perf counter key and iterate on per-thread RCU lists
41 * of fields in the fast path, even though this is not strictly speaking
42 * what would provide the best fast-path complexity, to ensure teardown
43 * of sessions vs thread exit is handled racelessly.
44 *
45 * Updates and traversals of thread_list are protected by UST lock.
46 * Updates to rcu_field_list are protected by UST lock.
47 */
48
49struct lttng_perf_counter_thread_field {
50 struct lttng_perf_counter_field *field; /* Back reference */
51 struct perf_event_mmap_page *pc;
52 struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
53 struct cds_list_head rcu_field_node; /* RCU per-thread list of fields (node) */
b9389e6e 54 int fd; /* Perf FD */
d58d1454
MD
55};
56
57struct lttng_perf_counter_thread {
58 struct cds_list_head rcu_field_list; /* RCU per-thread list of fields */
59};
60
61struct lttng_perf_counter_field {
62 struct perf_event_attr attr;
63 struct cds_list_head thread_field_list; /* Per-field list of thread fields */
4e48b5d2 64 char *name;
d58d1454
MD
65};
66
67static pthread_key_t perf_counter_key;
68
20142124
MD
69/*
70 * lttng_perf_lock - Protect lttng-ust perf counter data structures
71 *
72 * Nests within the ust_lock, and therefore within the libc dl lock.
73 * Therefore, we need to fixup the TLS before nesting into this lock.
74 * Nests inside RCU bp read-side lock. Protects against concurrent
75 * fork.
76 */
77static pthread_mutex_t ust_perf_mutex = PTHREAD_MUTEX_INITIALIZER;
78
79/*
80 * Cancel state when grabbing the ust_perf_mutex. Saved when locking,
81 * restored on unlock. Protected by ust_perf_mutex.
82 */
83static int ust_perf_saved_cancelstate;
84
85/*
86 * Track whether we are tracing from a signal handler nested on an
87 * application thread.
88 */
89static DEFINE_URCU_TLS(int, ust_perf_mutex_nest);
90
91/*
92 * Force a read (imply TLS fixup for dlopen) of TLS variables.
93 */
94void lttng_ust_fixup_perf_counter_tls(void)
95{
96 asm volatile ("" : : "m" (URCU_TLS(ust_perf_mutex_nest)));
97}
98
99void lttng_perf_lock(void)
100{
101 sigset_t sig_all_blocked, orig_mask;
102 int ret, oldstate;
103
104 ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);
105 if (ret) {
106 ERR("pthread_setcancelstate: %s", strerror(ret));
107 }
108 sigfillset(&sig_all_blocked);
109 ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask);
110 if (ret) {
111 ERR("pthread_sigmask: %s", strerror(ret));
112 }
113 if (!URCU_TLS(ust_perf_mutex_nest)++) {
114 /*
115 * Ensure the compiler don't move the store after the close()
116 * call in case close() would be marked as leaf.
117 */
118 cmm_barrier();
119 pthread_mutex_lock(&ust_perf_mutex);
120 ust_perf_saved_cancelstate = oldstate;
121 }
122 ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL);
123 if (ret) {
124 ERR("pthread_sigmask: %s", strerror(ret));
125 }
126}
127
128void lttng_perf_unlock(void)
129{
130 sigset_t sig_all_blocked, orig_mask;
131 int ret, newstate, oldstate;
132 bool restore_cancel = false;
133
134 sigfillset(&sig_all_blocked);
135 ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask);
136 if (ret) {
137 ERR("pthread_sigmask: %s", strerror(ret));
138 }
139 /*
140 * Ensure the compiler don't move the store before the close()
141 * call, in case close() would be marked as leaf.
142 */
143 cmm_barrier();
144 if (!--URCU_TLS(ust_perf_mutex_nest)) {
145 newstate = ust_perf_saved_cancelstate;
146 restore_cancel = true;
147 pthread_mutex_unlock(&ust_perf_mutex);
148 }
149 ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL);
150 if (ret) {
151 ERR("pthread_sigmask: %s", strerror(ret));
152 }
153 if (restore_cancel) {
154 ret = pthread_setcancelstate(newstate, &oldstate);
155 if (ret) {
156 ERR("pthread_setcancelstate: %s", strerror(ret));
157 }
158 }
159}
160
d58d1454 161static
4e48b5d2 162size_t perf_counter_get_size(void *priv __attribute__((unused)),
2208d8b5 163 size_t offset)
d58d1454
MD
164{
165 size_t size = 0;
166
dc325c1d 167 size += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
d58d1454
MD
168 size += sizeof(uint64_t);
169 return size;
170}
171
a3a8d943
MD
172static
173uint64_t read_perf_counter_syscall(
174 struct lttng_perf_counter_thread_field *thread_field)
175{
176 uint64_t count;
177
178 if (caa_unlikely(thread_field->fd < 0))
179 return 0;
180
181 if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
182 < sizeof(count)))
183 return 0;
184
185 return count;
186}
187
2eba8e39 188#if defined(LTTNG_UST_ARCH_X86)
d58d1454
MD
189
190static
191uint64_t rdpmc(unsigned int counter)
192{
193 unsigned int low, high;
194
195 asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
196
197 return low | ((uint64_t) high) << 32;
198}
199
77d7fa98
MD
200static
201bool has_rdpmc(struct perf_event_mmap_page *pc)
202{
203 if (caa_unlikely(!pc->cap_bit0_is_deprecated))
204 return false;
205 /* Since Linux kernel 3.12. */
206 return pc->cap_user_rdpmc;
207}
208
d58d1454 209static
a3a8d943 210uint64_t arch_read_perf_counter(
d286ad50 211 struct lttng_perf_counter_thread_field *thread_field)
d58d1454
MD
212{
213 uint32_t seq, idx;
214 uint64_t count;
d286ad50 215 struct perf_event_mmap_page *pc = thread_field->pc;
d58d1454
MD
216
217 if (caa_unlikely(!pc))
218 return 0;
219
220 do {
221 seq = CMM_LOAD_SHARED(pc->lock);
222 cmm_barrier();
223
224 idx = pc->index;
77d7fa98 225 if (caa_likely(has_rdpmc(pc) && idx)) {
4f58f54f
MD
226 int64_t pmcval;
227
228 pmcval = rdpmc(idx - 1);
229 /* Sign-extend the pmc register result. */
230 pmcval <<= 64 - pc->pmc_width;
231 pmcval >>= 64 - pc->pmc_width;
232 count = pc->offset + pmcval;
233 } else {
a3a8d943
MD
234 /* Fall-back on system call if rdpmc cannot be used. */
235 return read_perf_counter_syscall(thread_field);
4f58f54f 236 }
d58d1454
MD
237 cmm_barrier();
238 } while (CMM_LOAD_SHARED(pc->lock) != seq);
239
240 return count;
241}
242
d286ad50 243static
a3a8d943 244int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
d286ad50 245{
a3a8d943 246 struct perf_event_mmap_page *pc = thread_field->pc;
d286ad50 247
a3a8d943 248 if (!pc)
d286ad50 249 return 0;
77d7fa98 250 return !has_rdpmc(pc);
a3a8d943 251}
d286ad50 252
a3a8d943 253#else
d286ad50 254
a3a8d943
MD
255/* Generic (slow) implementation using a read system call. */
256static
257uint64_t arch_read_perf_counter(
258 struct lttng_perf_counter_thread_field *thread_field)
259{
260 return read_perf_counter_syscall(thread_field);
d286ad50
JD
261}
262
a3a8d943
MD
263static
264int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
265{
266 return 1;
267}
d286ad50 268
a3a8d943 269#endif
d286ad50 270
d58d1454
MD
271static
272int sys_perf_event_open(struct perf_event_attr *attr,
273 pid_t pid, int cpu, int group_fd,
274 unsigned long flags)
275{
276 return syscall(SYS_perf_event_open, attr, pid, cpu,
277 group_fd, flags);
278}
279
280static
b9389e6e 281int open_perf_fd(struct perf_event_attr *attr)
d58d1454 282{
b9389e6e 283 int fd;
d58d1454
MD
284
285 fd = sys_perf_event_open(attr, 0, -1, -1, 0);
286 if (fd < 0)
b9389e6e
JD
287 return -1;
288
289 return fd;
290}
291
d286ad50
JD
292static
293void close_perf_fd(int fd)
294{
295 int ret;
296
297 if (fd < 0)
298 return;
299
300 ret = close(fd);
301 if (ret) {
302 perror("Error closing LTTng-UST perf memory mapping FD");
303 }
304}
305
77d7fa98 306static void setup_perf(struct lttng_perf_counter_thread_field *thread_field)
b9389e6e
JD
307{
308 void *perf_addr;
d58d1454
MD
309
310 perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
b9389e6e 311 PROT_READ, MAP_SHARED, thread_field->fd, 0);
d58d1454 312 if (perf_addr == MAP_FAILED)
b9389e6e 313 perf_addr = NULL;
77d7fa98 314 thread_field->pc = perf_addr;
b9389e6e 315
a3a8d943 316 if (!arch_perf_keep_fd(thread_field)) {
b9389e6e
JD
317 close_perf_fd(thread_field->fd);
318 thread_field->fd = -1;
6c2125af 319 }
d58d1454
MD
320}
321
322static
323void unmap_perf_page(struct perf_event_mmap_page *pc)
324{
325 int ret;
326
327 if (!pc)
328 return;
329 ret = munmap(pc, sizeof(struct perf_event_mmap_page));
330 if (ret < 0) {
331 PERROR("Error in munmap");
332 abort();
333 }
334}
335
336static
337struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
338{
339 struct lttng_perf_counter_thread *perf_thread;
340 sigset_t newmask, oldmask;
341 int ret;
342
343 ret = sigfillset(&newmask);
344 if (ret)
345 abort();
346 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
347 if (ret)
348 abort();
349 /* Check again with signals disabled */
350 perf_thread = pthread_getspecific(perf_counter_key);
351 if (perf_thread)
352 goto skip;
353 perf_thread = zmalloc(sizeof(*perf_thread));
354 if (!perf_thread)
355 abort();
356 CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
357 ret = pthread_setspecific(perf_counter_key, perf_thread);
358 if (ret)
359 abort();
360skip:
361 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
362 if (ret)
363 abort();
364 return perf_thread;
365}
366
367static
368struct lttng_perf_counter_thread_field *
369 add_thread_field(struct lttng_perf_counter_field *perf_field,
370 struct lttng_perf_counter_thread *perf_thread)
371{
372 struct lttng_perf_counter_thread_field *thread_field;
373 sigset_t newmask, oldmask;
374 int ret;
375
376 ret = sigfillset(&newmask);
377 if (ret)
378 abort();
379 ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
380 if (ret)
381 abort();
382 /* Check again with signals disabled */
383 cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
384 rcu_field_node) {
385 if (thread_field->field == perf_field)
386 goto skip;
387 }
388 thread_field = zmalloc(sizeof(*thread_field));
389 if (!thread_field)
390 abort();
391 thread_field->field = perf_field;
b9389e6e
JD
392 thread_field->fd = open_perf_fd(&perf_field->attr);
393 if (thread_field->fd >= 0)
77d7fa98 394 setup_perf(thread_field);
b9389e6e
JD
395 /*
396 * Note: thread_field->pc can be NULL if setup_perf() fails.
397 * Also, thread_field->fd can be -1 if open_perf_fd() fails.
398 */
20142124 399 lttng_perf_lock();
d58d1454
MD
400 cds_list_add_rcu(&thread_field->rcu_field_node,
401 &perf_thread->rcu_field_list);
402 cds_list_add(&thread_field->thread_field_node,
403 &perf_field->thread_field_list);
20142124 404 lttng_perf_unlock();
d58d1454
MD
405skip:
406 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
407 if (ret)
408 abort();
409 return thread_field;
410}
411
412static
413struct lttng_perf_counter_thread_field *
414 get_thread_field(struct lttng_perf_counter_field *field)
415{
416 struct lttng_perf_counter_thread *perf_thread;
417 struct lttng_perf_counter_thread_field *thread_field;
418
419 perf_thread = pthread_getspecific(perf_counter_key);
420 if (!perf_thread)
421 perf_thread = alloc_perf_counter_thread();
422 cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
423 rcu_field_node) {
424 if (thread_field->field == field)
425 return thread_field;
426 }
427 /* perf_counter_thread_field not found, need to add one */
428 return add_thread_field(field, perf_thread);
429}
430
431static
4e48b5d2 432uint64_t wrapper_perf_counter_read(void *priv)
d58d1454
MD
433{
434 struct lttng_perf_counter_field *perf_field;
435 struct lttng_perf_counter_thread_field *perf_thread_field;
436
4e48b5d2 437 perf_field = (struct lttng_perf_counter_field *) priv;
d58d1454 438 perf_thread_field = get_thread_field(perf_field);
a3a8d943 439 return arch_read_perf_counter(perf_thread_field);
d58d1454
MD
440}
441
442static
4e48b5d2 443void perf_counter_record(void *priv,
d58d1454 444 struct lttng_ust_lib_ring_buffer_ctx *ctx,
e7bc0ef6 445 struct lttng_ust_channel_buffer *chan)
d58d1454
MD
446{
447 uint64_t value;
448
4e48b5d2 449 value = wrapper_perf_counter_read(priv);
8936b6c0 450 chan->ops->event_write(ctx, &value, sizeof(value), lttng_ust_rb_alignof(value));
d58d1454
MD
451}
452
453static
4e48b5d2 454void perf_counter_get_value(void *priv,
daacdbfc 455 struct lttng_ust_ctx_value *value)
d58d1454 456{
4e48b5d2 457 value->u.s64 = wrapper_perf_counter_read(priv);
d58d1454
MD
458}
459
20142124 460/* Called with perf lock held */
d58d1454
MD
461static
462void lttng_destroy_perf_thread_field(
463 struct lttng_perf_counter_thread_field *thread_field)
464{
b9389e6e 465 close_perf_fd(thread_field->fd);
d58d1454
MD
466 unmap_perf_page(thread_field->pc);
467 cds_list_del_rcu(&thread_field->rcu_field_node);
468 cds_list_del(&thread_field->thread_field_node);
469 free(thread_field);
470}
471
472static
473void lttng_destroy_perf_thread_key(void *_key)
474{
475 struct lttng_perf_counter_thread *perf_thread = _key;
476 struct lttng_perf_counter_thread_field *pos, *p;
477
20142124 478 lttng_perf_lock();
d58d1454
MD
479 cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
480 rcu_field_node)
481 lttng_destroy_perf_thread_field(pos);
20142124 482 lttng_perf_unlock();
d58d1454
MD
483 free(perf_thread);
484}
485
486/* Called with UST lock held */
487static
4e48b5d2 488void lttng_destroy_perf_counter_ctx_field(void *priv)
d58d1454
MD
489{
490 struct lttng_perf_counter_field *perf_field;
491 struct lttng_perf_counter_thread_field *pos, *p;
492
4e48b5d2
MD
493 perf_field = (struct lttng_perf_counter_field *) priv;
494 free(perf_field->name);
d58d1454
MD
495 /*
496 * This put is performed when no threads can concurrently
497 * perform a "get" concurrently, thanks to urcu-bp grace
20142124
MD
498 * period. Holding the lttng perf lock protects against
499 * concurrent modification of the per-thread thread field
500 * list.
d58d1454 501 */
20142124 502 lttng_perf_lock();
d58d1454
MD
503 cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
504 thread_field_node)
505 lttng_destroy_perf_thread_field(pos);
20142124 506 lttng_perf_unlock();
d58d1454
MD
507 free(perf_field);
508}
509
2eba8e39 510#ifdef LTTNG_UST_ARCH_ARMV7
d286ad50
JD
511
512static
513int perf_get_exclude_kernel(void)
514{
515 return 0;
516}
517
2eba8e39 518#else /* LTTNG_UST_ARCH_ARMV7 */
d286ad50
JD
519
520static
521int perf_get_exclude_kernel(void)
522{
523 return 1;
524}
525
2eba8e39 526#endif /* LTTNG_UST_ARCH_ARMV7 */
d286ad50 527
4e48b5d2
MD
528static const struct lttng_ust_type_common *ust_type =
529 lttng_ust_static_type_integer(sizeof(uint64_t) * CHAR_BIT,
530 lttng_ust_rb_alignof(uint64_t) * CHAR_BIT,
531 lttng_ust_is_signed_type(uint64_t),
532 BYTE_ORDER, 10);
533
d58d1454
MD
534/* Called with UST lock held */
535int lttng_add_perf_counter_to_ctx(uint32_t type,
536 uint64_t config,
537 const char *name,
daacdbfc 538 struct lttng_ust_ctx **ctx)
d58d1454 539{
4e48b5d2
MD
540 struct lttng_ust_ctx_field ctx_field;
541 struct lttng_ust_event_field *event_field;
d58d1454 542 struct lttng_perf_counter_field *perf_field;
d58d1454
MD
543 char *name_alloc;
544 int ret;
545
4e48b5d2
MD
546 if (lttng_find_context(*ctx, name)) {
547 ret = -EEXIST;
548 goto find_error;
549 }
d58d1454
MD
550 name_alloc = strdup(name);
551 if (!name_alloc) {
552 ret = -ENOMEM;
553 goto name_alloc_error;
554 }
4e48b5d2
MD
555 event_field = zmalloc(sizeof(*event_field));
556 if (!event_field) {
557 ret = -ENOMEM;
558 goto event_field_alloc_error;
559 }
560 event_field->name = name_alloc;
561 event_field->type = ust_type;
562
d58d1454
MD
563 perf_field = zmalloc(sizeof(*perf_field));
564 if (!perf_field) {
565 ret = -ENOMEM;
566 goto perf_field_alloc_error;
567 }
d58d1454
MD
568 perf_field->attr.type = type;
569 perf_field->attr.config = config;
d286ad50 570 perf_field->attr.exclude_kernel = perf_get_exclude_kernel();
d58d1454 571 CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
4e48b5d2 572 perf_field->name = name_alloc;
d58d1454
MD
573
574 /* Ensure that this perf counter can be used in this process. */
b9389e6e
JD
575 ret = open_perf_fd(&perf_field->attr);
576 if (ret < 0) {
d58d1454
MD
577 ret = -ENODEV;
578 goto setup_error;
579 }
b9389e6e 580 close_perf_fd(ret);
d58d1454 581
4e48b5d2
MD
582 ctx_field.event_field = event_field;
583 ctx_field.get_size = perf_counter_get_size;
584 ctx_field.record = perf_counter_record;
585 ctx_field.get_value = perf_counter_get_value;
586 ctx_field.destroy = lttng_destroy_perf_counter_ctx_field;
587 ctx_field.priv = perf_field;
d58d1454 588
4e48b5d2
MD
589 ret = lttng_ust_context_append(ctx, &ctx_field);
590 if (ret) {
591 ret = -ENOMEM;
592 goto append_context_error;
593 }
d58d1454
MD
594 return 0;
595
d58d1454 596append_context_error:
4e48b5d2 597setup_error:
d58d1454
MD
598 free(perf_field);
599perf_field_alloc_error:
4e48b5d2
MD
600 free(event_field);
601event_field_alloc_error:
d58d1454
MD
602 free(name_alloc);
603name_alloc_error:
4e48b5d2 604find_error:
d58d1454
MD
605 return ret;
606}
607
608int lttng_perf_counter_init(void)
609{
610 int ret;
611
612 ret = pthread_key_create(&perf_counter_key,
613 lttng_destroy_perf_thread_key);
614 if (ret)
615 ret = -ret;
616 return ret;
617}
618
619void lttng_perf_counter_exit(void)
620{
621 int ret;
622
623 ret = pthread_key_delete(perf_counter_key);
624 if (ret) {
625 errno = ret;
626 PERROR("Error in pthread_key_delete");
627 }
628}
This page took 0.058174 seconds and 4 git commands to generate.