liblttng-ust/lttng-context-perf-counters.c

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * LTTng UST performance monitoring counters (perf-counters) integration.
   7  */
   8
   9 #define _LGPL_SOURCE
  10 #include <sys/types.h>
  11 #include <unistd.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdbool.h>
  16 #include <stddef.h>
  17 #include <stdint.h>
  18 #include <sys/mman.h>
  19 #include <sys/syscall.h>
  20 #include <lttng/ust-arch.h>
  21 #include <lttng/ust-events.h>
  22 #include <lttng/ust-tracer.h>
  23 #include <lttng/ringbuffer-context.h>
  24 #include <urcu/system.h>
  25 #include <urcu/arch.h>
  26 #include <urcu/rculist.h>
  27 #include <ust-helper.h>
  28 #include <urcu/ref.h>
  29 #include <usterr-signal-safe.h>
  30 #include <signal.h>
  31 #include <urcu/tls-compat.h>
  32 #include "perf_event.h"
  33
  34 #include "context-internal.h"
  35 #include "lttng-tracer-core.h"
  36 #include "ust-events-internal.h"
  37
  38 /*
  39  * We use a global perf counter key and iterate on per-thread RCU lists
  40  * of fields in the fast path, even though this is not strictly speaking
  41  * what would provide the best fast-path complexity, to ensure teardown
  42  * of sessions vs thread exit is handled racelessly.
  43  *
  44  * Updates and traversals of thread_list are protected by UST lock.
  45  * Updates to rcu_field_list are protected by UST lock.
  46  */
  47
  48 struct lttng_perf_counter_thread_field {
  49         struct lttng_perf_counter_field *field; /* Back reference */
  50         struct perf_event_mmap_page *pc;
  51         struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
  52         struct cds_list_head rcu_field_node;    /* RCU per-thread list of fields (node) */
  53         int fd;                                 /* Perf FD */
  54 };
  55
  56 struct lttng_perf_counter_thread {
  57         struct cds_list_head rcu_field_list;    /* RCU per-thread list of fields */
  58 };
  59
  60 struct lttng_perf_counter_field {
  61         struct perf_event_attr attr;
  62         struct cds_list_head thread_field_list; /* Per-field list of thread fields */
  63 };
  64
  65 static pthread_key_t perf_counter_key;
  66
  67 /*
  68  * lttng_perf_lock - Protect lttng-ust perf counter data structures
  69  *
  70  * Nests within the ust_lock, and therefore within the libc dl lock.
  71  * Therefore, we need to fixup the TLS before nesting into this lock.
  72  * Nests inside RCU bp read-side lock. Protects against concurrent
  73  * fork.
  74  */
  75 static pthread_mutex_t ust_perf_mutex = PTHREAD_MUTEX_INITIALIZER;
  76
  77 /*
  78  * Cancel state when grabbing the ust_perf_mutex. Saved when locking,
  79  * restored on unlock. Protected by ust_perf_mutex.
  80  */
  81 static int ust_perf_saved_cancelstate;
  82
  83 /*
  84  * Track whether we are tracing from a signal handler nested on an
  85  * application thread.
  86  */
  87 static DEFINE_URCU_TLS(int, ust_perf_mutex_nest);
  88
  89 /*
  90  * Force a read (imply TLS fixup for dlopen) of TLS variables.
  91  */
  92 void lttng_ust_fixup_perf_counter_tls(void)
  93 {
  94         asm volatile ("" : : "m" (URCU_TLS(ust_perf_mutex_nest)));
  95 }
  96
  97 void lttng_perf_lock(void)
  98 {
  99         sigset_t sig_all_blocked, orig_mask;
 100         int ret, oldstate;
 101
 102         ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate);
 103         if (ret) {
 104                 ERR("pthread_setcancelstate: %s", strerror(ret));
 105         }
 106         sigfillset(&sig_all_blocked);
 107         ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask);
 108         if (ret) {
 109                 ERR("pthread_sigmask: %s", strerror(ret));
 110         }
 111         if (!URCU_TLS(ust_perf_mutex_nest)++) {
 112                 /*
 113                  * Ensure the compiler don't move the store after the close()
 114                  * call in case close() would be marked as leaf.
 115                  */
 116                 cmm_barrier();
 117                 pthread_mutex_lock(&ust_perf_mutex);
 118                 ust_perf_saved_cancelstate = oldstate;
 119         }
 120         ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL);
 121         if (ret) {
 122                 ERR("pthread_sigmask: %s", strerror(ret));
 123         }
 124 }
 125
 126 void lttng_perf_unlock(void)
 127 {
 128         sigset_t sig_all_blocked, orig_mask;
 129         int ret, newstate, oldstate;
 130         bool restore_cancel = false;
 131
 132         sigfillset(&sig_all_blocked);
 133         ret = pthread_sigmask(SIG_SETMASK, &sig_all_blocked, &orig_mask);
 134         if (ret) {
 135                 ERR("pthread_sigmask: %s", strerror(ret));
 136         }
 137         /*
 138          * Ensure the compiler don't move the store before the close()
 139          * call, in case close() would be marked as leaf.
 140          */
 141         cmm_barrier();
 142         if (!--URCU_TLS(ust_perf_mutex_nest)) {
 143                 newstate = ust_perf_saved_cancelstate;
 144                 restore_cancel = true;
 145                 pthread_mutex_unlock(&ust_perf_mutex);
 146         }
 147         ret = pthread_sigmask(SIG_SETMASK, &orig_mask, NULL);
 148         if (ret) {
 149                 ERR("pthread_sigmask: %s", strerror(ret));
 150         }
 151         if (restore_cancel) {
 152                 ret = pthread_setcancelstate(newstate, &oldstate);
 153                 if (ret) {
 154                         ERR("pthread_setcancelstate: %s", strerror(ret));
 155                 }
 156         }
 157 }
 158
 159 static
 160 size_t perf_counter_get_size(struct lttng_ust_ctx_field *field, size_t offset)
 161 {
 162         size_t size = 0;
 163
 164         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
 165         size += sizeof(uint64_t);
 166         return size;
 167 }
 168
 169 static
 170 uint64_t read_perf_counter_syscall(
 171                 struct lttng_perf_counter_thread_field *thread_field)
 172 {
 173         uint64_t count;
 174
 175         if (caa_unlikely(thread_field->fd < 0))
 176                 return 0;
 177
 178         if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
 179                                 < sizeof(count)))
 180                 return 0;
 181
 182         return count;
 183 }
 184
 185 #if defined(LTTNG_UST_ARCH_X86)
 186
 187 static
 188 uint64_t rdpmc(unsigned int counter)
 189 {
 190         unsigned int low, high;
 191
 192         asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
 193
 194         return low | ((uint64_t) high) << 32;
 195 }
 196
 197 static
 198 bool has_rdpmc(struct perf_event_mmap_page *pc)
 199 {
 200         if (caa_unlikely(!pc->cap_bit0_is_deprecated))
 201                 return false;
 202         /* Since Linux kernel 3.12. */
 203         return pc->cap_user_rdpmc;
 204 }
 205
 206 static
 207 uint64_t arch_read_perf_counter(
 208                 struct lttng_perf_counter_thread_field *thread_field)
 209 {
 210         uint32_t seq, idx;
 211         uint64_t count;
 212         struct perf_event_mmap_page *pc = thread_field->pc;
 213
 214         if (caa_unlikely(!pc))
 215                 return 0;
 216
 217         do {
 218                 seq = CMM_LOAD_SHARED(pc->lock);
 219                 cmm_barrier();
 220
 221                 idx = pc->index;
 222                 if (caa_likely(has_rdpmc(pc) && idx)) {
 223                         int64_t pmcval;
 224
 225                         pmcval = rdpmc(idx - 1);
 226                         /* Sign-extend the pmc register result. */
 227                         pmcval <<= 64 - pc->pmc_width;
 228                         pmcval >>= 64 - pc->pmc_width;
 229                         count = pc->offset + pmcval;
 230                 } else {
 231                         /* Fall-back on system call if rdpmc cannot be used. */
 232                         return read_perf_counter_syscall(thread_field);
 233                 }
 234                 cmm_barrier();
 235         } while (CMM_LOAD_SHARED(pc->lock) != seq);
 236
 237         return count;
 238 }
 239
 240 static
 241 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
 242 {
 243         struct perf_event_mmap_page *pc = thread_field->pc;
 244
 245         if (!pc)
 246                 return 0;
 247         return !has_rdpmc(pc);
 248 }
 249
 250 #else
 251
 252 /* Generic (slow) implementation using a read system call. */
 253 static
 254 uint64_t arch_read_perf_counter(
 255                 struct lttng_perf_counter_thread_field *thread_field)
 256 {
 257         return read_perf_counter_syscall(thread_field);
 258 }
 259
 260 static
 261 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
 262 {
 263         return 1;
 264 }
 265
 266 #endif
 267
 268 static
 269 int sys_perf_event_open(struct perf_event_attr *attr,
 270                 pid_t pid, int cpu, int group_fd,
 271                 unsigned long flags)
 272 {
 273         return syscall(SYS_perf_event_open, attr, pid, cpu,
 274                         group_fd, flags);
 275 }
 276
 277 static
 278 int open_perf_fd(struct perf_event_attr *attr)
 279 {
 280         int fd;
 281
 282         fd = sys_perf_event_open(attr, 0, -1, -1, 0);
 283         if (fd < 0)
 284                 return -1;
 285
 286         return fd;
 287 }
 288
 289 static
 290 void close_perf_fd(int fd)
 291 {
 292         int ret;
 293
 294         if (fd < 0)
 295                 return;
 296
 297         ret = close(fd);
 298         if (ret) {
 299                 perror("Error closing LTTng-UST perf memory mapping FD");
 300         }
 301 }
 302
 303 static void setup_perf(struct lttng_perf_counter_thread_field *thread_field)
 304 {
 305         void *perf_addr;
 306
 307         perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
 308                         PROT_READ, MAP_SHARED, thread_field->fd, 0);
 309         if (perf_addr == MAP_FAILED)
 310                 perf_addr = NULL;
 311         thread_field->pc = perf_addr;
 312
 313         if (!arch_perf_keep_fd(thread_field)) {
 314                 close_perf_fd(thread_field->fd);
 315                 thread_field->fd = -1;
 316         }
 317 }
 318
 319 static
 320 void unmap_perf_page(struct perf_event_mmap_page *pc)
 321 {
 322         int ret;
 323
 324         if (!pc)
 325                 return;
 326         ret = munmap(pc, sizeof(struct perf_event_mmap_page));
 327         if (ret < 0) {
 328                 PERROR("Error in munmap");
 329                 abort();
 330         }
 331 }
 332
 333 static
 334 struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
 335 {
 336         struct lttng_perf_counter_thread *perf_thread;
 337         sigset_t newmask, oldmask;
 338         int ret;
 339
 340         ret = sigfillset(&newmask);
 341         if (ret)
 342                 abort();
 343         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 344         if (ret)
 345                 abort();
 346         /* Check again with signals disabled */
 347         perf_thread = pthread_getspecific(perf_counter_key);
 348         if (perf_thread)
 349                 goto skip;
 350         perf_thread = zmalloc(sizeof(*perf_thread));
 351         if (!perf_thread)
 352                 abort();
 353         CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
 354         ret = pthread_setspecific(perf_counter_key, perf_thread);
 355         if (ret)
 356                 abort();
 357 skip:
 358         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 359         if (ret)
 360                 abort();
 361         return perf_thread;
 362 }
 363
 364 static
 365 struct lttng_perf_counter_thread_field *
 366         add_thread_field(struct lttng_perf_counter_field *perf_field,
 367                 struct lttng_perf_counter_thread *perf_thread)
 368 {
 369         struct lttng_perf_counter_thread_field *thread_field;
 370         sigset_t newmask, oldmask;
 371         int ret;
 372
 373         ret = sigfillset(&newmask);
 374         if (ret)
 375                 abort();
 376         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 377         if (ret)
 378                 abort();
 379         /* Check again with signals disabled */
 380         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 381                         rcu_field_node) {
 382                 if (thread_field->field == perf_field)
 383                         goto skip;
 384         }
 385         thread_field = zmalloc(sizeof(*thread_field));
 386         if (!thread_field)
 387                 abort();
 388         thread_field->field = perf_field;
 389         thread_field->fd = open_perf_fd(&perf_field->attr);
 390         if (thread_field->fd >= 0)
 391                 setup_perf(thread_field);
 392         /*
 393          * Note: thread_field->pc can be NULL if setup_perf() fails.
 394          * Also, thread_field->fd can be -1 if open_perf_fd() fails.
 395          */
 396         lttng_perf_lock();
 397         cds_list_add_rcu(&thread_field->rcu_field_node,
 398                         &perf_thread->rcu_field_list);
 399         cds_list_add(&thread_field->thread_field_node,
 400                         &perf_field->thread_field_list);
 401         lttng_perf_unlock();
 402 skip:
 403         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 404         if (ret)
 405                 abort();
 406         return thread_field;
 407 }
 408
 409 static
 410 struct lttng_perf_counter_thread_field *
 411                 get_thread_field(struct lttng_perf_counter_field *field)
 412 {
 413         struct lttng_perf_counter_thread *perf_thread;
 414         struct lttng_perf_counter_thread_field *thread_field;
 415
 416         perf_thread = pthread_getspecific(perf_counter_key);
 417         if (!perf_thread)
 418                 perf_thread = alloc_perf_counter_thread();
 419         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 420                         rcu_field_node) {
 421                 if (thread_field->field == field)
 422                         return thread_field;
 423         }
 424         /* perf_counter_thread_field not found, need to add one */
 425         return add_thread_field(field, perf_thread);
 426 }
 427
 428 static
 429 uint64_t wrapper_perf_counter_read(struct lttng_ust_ctx_field *field)
 430 {
 431         struct lttng_perf_counter_field *perf_field;
 432         struct lttng_perf_counter_thread_field *perf_thread_field;
 433
 434         perf_field = (struct lttng_perf_counter_field *) field->priv;
 435         perf_thread_field = get_thread_field(perf_field);
 436         return arch_read_perf_counter(perf_thread_field);
 437 }
 438
 439 static
 440 void perf_counter_record(struct lttng_ust_ctx_field *field,
 441                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 442                  struct lttng_ust_channel_buffer *chan)
 443 {
 444         uint64_t value;
 445
 446         value = wrapper_perf_counter_read(field);
 447         lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
 448         chan->ops->event_write(ctx, &value, sizeof(value));
 449 }
 450
 451 static
 452 void perf_counter_get_value(struct lttng_ust_ctx_field *field,
 453                 struct lttng_ust_ctx_value *value)
 454 {
 455         value->u.s64 = wrapper_perf_counter_read(field);
 456 }
 457
 458 /* Called with perf lock held */
 459 static
 460 void lttng_destroy_perf_thread_field(
 461                 struct lttng_perf_counter_thread_field *thread_field)
 462 {
 463         close_perf_fd(thread_field->fd);
 464         unmap_perf_page(thread_field->pc);
 465         cds_list_del_rcu(&thread_field->rcu_field_node);
 466         cds_list_del(&thread_field->thread_field_node);
 467         free(thread_field);
 468 }
 469
 470 static
 471 void lttng_destroy_perf_thread_key(void *_key)
 472 {
 473         struct lttng_perf_counter_thread *perf_thread = _key;
 474         struct lttng_perf_counter_thread_field *pos, *p;
 475
 476         lttng_perf_lock();
 477         cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
 478                         rcu_field_node)
 479                 lttng_destroy_perf_thread_field(pos);
 480         lttng_perf_unlock();
 481         free(perf_thread);
 482 }
 483
 484 /* Called with UST lock held */
 485 static
 486 void lttng_destroy_perf_counter_field(struct lttng_ust_ctx_field *field)
 487 {
 488         struct lttng_perf_counter_field *perf_field;
 489         struct lttng_perf_counter_thread_field *pos, *p;
 490
 491         free((char *) field->event_field->name);
 492         perf_field = (struct lttng_perf_counter_field *) field->priv;
 493         /*
 494          * This put is performed when no threads can concurrently
 495          * perform a "get" concurrently, thanks to urcu-bp grace
 496          * period. Holding the lttng perf lock protects against
 497          * concurrent modification of the per-thread thread field
 498          * list.
 499          */
 500         lttng_perf_lock();
 501         cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
 502                         thread_field_node)
 503                 lttng_destroy_perf_thread_field(pos);
 504         lttng_perf_unlock();
 505         free(perf_field);
 506 }
 507
 508 #ifdef LTTNG_UST_ARCH_ARMV7
 509
 510 static
 511 int perf_get_exclude_kernel(void)
 512 {
 513         return 0;
 514 }
 515
 516 #else /* LTTNG_UST_ARCH_ARMV7 */
 517
 518 static
 519 int perf_get_exclude_kernel(void)
 520 {
 521         return 1;
 522 }
 523
 524 #endif /* LTTNG_UST_ARCH_ARMV7 */
 525
 526 /* Called with UST lock held */
 527 int lttng_add_perf_counter_to_ctx(uint32_t type,
 528                                 uint64_t config,
 529                                 const char *name,
 530                                 struct lttng_ust_ctx **ctx)
 531 {
 532         struct lttng_ust_ctx_field *field;
 533         struct lttng_ust_type_common *ust_type;
 534         struct lttng_perf_counter_field *perf_field;
 535         char *name_alloc;
 536         int ret;
 537
 538         name_alloc = strdup(name);
 539         if (!name_alloc) {
 540                 ret = -ENOMEM;
 541                 goto name_alloc_error;
 542         }
 543         perf_field = zmalloc(sizeof(*perf_field));
 544         if (!perf_field) {
 545                 ret = -ENOMEM;
 546                 goto perf_field_alloc_error;
 547         }
 548         ust_type = lttng_ust_create_type_integer(sizeof(uint64_t) * CHAR_BIT,
 549                         lttng_alignof(uint64_t) * CHAR_BIT,
 550                         lttng_ust_is_signed_type(uint64_t),
 551                         BYTE_ORDER, 10);
 552         if (!ust_type) {
 553                 ret = -ENOMEM;
 554                 goto type_alloc_error;
 555         }
 556         field = lttng_append_context(ctx);
 557         if (!field) {
 558                 ret = -ENOMEM;
 559                 goto append_context_error;
 560         }
 561         if (lttng_find_context(*ctx, name_alloc)) {
 562                 ret = -EEXIST;
 563                 goto find_error;
 564         }
 565
 566         field->destroy = lttng_destroy_perf_counter_field;
 567
 568         field->event_field->name = name_alloc;
 569         field->event_field->type = ust_type;
 570         field->get_size = perf_counter_get_size;
 571         field->record = perf_counter_record;
 572         field->get_value = perf_counter_get_value;
 573
 574         perf_field->attr.type = type;
 575         perf_field->attr.config = config;
 576         perf_field->attr.exclude_kernel = perf_get_exclude_kernel();
 577         CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
 578         field->priv = perf_field;
 579
 580         /* Ensure that this perf counter can be used in this process. */
 581         ret = open_perf_fd(&perf_field->attr);
 582         if (ret < 0) {
 583                 ret = -ENODEV;
 584                 goto setup_error;
 585         }
 586         close_perf_fd(ret);
 587
 588         /*
 589          * Contexts can only be added before tracing is started, so we
 590          * don't have to synchronize against concurrent threads using
 591          * the field here.
 592          */
 593
 594         lttng_context_update(*ctx);
 595         return 0;
 596
 597 setup_error:
 598 find_error:
 599         lttng_remove_context_field(ctx, field);
 600 append_context_error:
 601         lttng_ust_destroy_type(ust_type);
 602 type_alloc_error:
 603         free(perf_field);
 604 perf_field_alloc_error:
 605         free(name_alloc);
 606 name_alloc_error:
 607         return ret;
 608 }
 609
 610 int lttng_perf_counter_init(void)
 611 {
 612         int ret;
 613
 614         ret = pthread_key_create(&perf_counter_key,
 615                         lttng_destroy_perf_thread_key);
 616         if (ret)
 617                 ret = -ret;
 618         return ret;
 619 }
 620
 621 void lttng_perf_counter_exit(void)
 622 {
 623         int ret;
 624
 625         ret = pthread_key_delete(perf_counter_key);
 626         if (ret) {
 627                 errno = ret;
 628                 PERROR("Error in pthread_key_delete");
 629         }
 630 }