liblttng-ust/lttng-context-perf-counters.c

   1 /*
   2  * lttng-context-perf-counters.c
   3  *
   4  * LTTng UST performance monitoring counters (perf-counters) integration.
   5  *
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; only
  11  * version 2.1 of the License.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include <sys/types.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <stdbool.h>
  29 #include <sys/mman.h>
  30 #include <sys/syscall.h>
  31 #include <lttng/ust-events.h>
  32 #include <lttng/ust-tracer.h>
  33 #include <lttng/ringbuffer-config.h>
  34 #include <urcu/system.h>
  35 #include <urcu/arch.h>
  36 #include <urcu/rculist.h>
  37 #include <helper.h>
  38 #include <urcu/ref.h>
  39 #include <usterr-signal-safe.h>
  40 #include <signal.h>
  41 #include "perf_event.h"
  42 #include "lttng-tracer-core.h"
  43
  44 /*
  45  * We use a global perf counter key and iterate on per-thread RCU lists
  46  * of fields in the fast path, even though this is not strictly speaking
  47  * what would provide the best fast-path complexity, to ensure teardown
  48  * of sessions vs thread exit is handled racelessly.
  49  *
  50  * Updates and traversals of thread_list are protected by UST lock.
  51  * Updates to rcu_field_list are protected by UST lock.
  52  */
  53
  54 struct lttng_perf_counter_thread_field {
  55         struct lttng_perf_counter_field *field; /* Back reference */
  56         struct perf_event_mmap_page *pc;
  57         struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
  58         struct cds_list_head rcu_field_node;    /* RCU per-thread list of fields (node) */
  59         int fd;                                 /* Perf FD */
  60 };
  61
  62 struct lttng_perf_counter_thread {
  63         struct cds_list_head rcu_field_list;    /* RCU per-thread list of fields */
  64 };
  65
  66 struct lttng_perf_counter_field {
  67         struct perf_event_attr attr;
  68         struct cds_list_head thread_field_list; /* Per-field list of thread fields */
  69 };
  70
  71 static pthread_key_t perf_counter_key;
  72
  73 static
  74 size_t perf_counter_get_size(struct lttng_ctx_field *field, size_t offset)
  75 {
  76         size_t size = 0;
  77
  78         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
  79         size += sizeof(uint64_t);
  80         return size;
  81 }
  82
  83 static
  84 uint64_t read_perf_counter_syscall(
  85                 struct lttng_perf_counter_thread_field *thread_field)
  86 {
  87         uint64_t count;
  88
  89         if (caa_unlikely(thread_field->fd < 0))
  90                 return 0;
  91
  92         if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
  93                                 < sizeof(count)))
  94                 return 0;
  95
  96         return count;
  97 }
  98
  99 #if defined(__x86_64__) || defined(__i386__)
 100
 101 static
 102 uint64_t rdpmc(unsigned int counter)
 103 {
 104         unsigned int low, high;
 105
 106         asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
 107
 108         return low | ((uint64_t) high) << 32;
 109 }
 110
 111 static
 112 bool has_rdpmc(struct perf_event_mmap_page *pc)
 113 {
 114         if (caa_unlikely(!pc->cap_bit0_is_deprecated))
 115                 return false;
 116         /* Since Linux kernel 3.12. */
 117         return pc->cap_user_rdpmc;
 118 }
 119
 120 static
 121 uint64_t arch_read_perf_counter(
 122                 struct lttng_perf_counter_thread_field *thread_field)
 123 {
 124         uint32_t seq, idx;
 125         uint64_t count;
 126         struct perf_event_mmap_page *pc = thread_field->pc;
 127
 128         if (caa_unlikely(!pc))
 129                 return 0;
 130
 131         do {
 132                 seq = CMM_LOAD_SHARED(pc->lock);
 133                 cmm_barrier();
 134
 135                 idx = pc->index;
 136                 if (caa_likely(has_rdpmc(pc) && idx)) {
 137                         int64_t pmcval;
 138
 139                         pmcval = rdpmc(idx - 1);
 140                         /* Sign-extend the pmc register result. */
 141                         pmcval <<= 64 - pc->pmc_width;
 142                         pmcval >>= 64 - pc->pmc_width;
 143                         count = pc->offset + pmcval;
 144                 } else {
 145                         /* Fall-back on system call if rdpmc cannot be used. */
 146                         return read_perf_counter_syscall(thread_field);
 147                 }
 148                 cmm_barrier();
 149         } while (CMM_LOAD_SHARED(pc->lock) != seq);
 150
 151         return count;
 152 }
 153
 154 static
 155 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
 156 {
 157         struct perf_event_mmap_page *pc = thread_field->pc;
 158
 159         if (!pc)
 160                 return 0;
 161         return !has_rdpmc(pc);
 162 }
 163
 164 #else
 165
 166 /* Generic (slow) implementation using a read system call. */
 167 static
 168 uint64_t arch_read_perf_counter(
 169                 struct lttng_perf_counter_thread_field *thread_field)
 170 {
 171         return read_perf_counter_syscall(thread_field);
 172 }
 173
 174 static
 175 int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
 176 {
 177         return 1;
 178 }
 179
 180 #endif
 181
 182 static
 183 int sys_perf_event_open(struct perf_event_attr *attr,
 184                 pid_t pid, int cpu, int group_fd,
 185                 unsigned long flags)
 186 {
 187         return syscall(SYS_perf_event_open, attr, pid, cpu,
 188                         group_fd, flags);
 189 }
 190
 191 static
 192 int open_perf_fd(struct perf_event_attr *attr)
 193 {
 194         int fd;
 195
 196         fd = sys_perf_event_open(attr, 0, -1, -1, 0);
 197         if (fd < 0)
 198                 return -1;
 199
 200         return fd;
 201 }
 202
 203 static
 204 void close_perf_fd(int fd)
 205 {
 206         int ret;
 207
 208         if (fd < 0)
 209                 return;
 210
 211         ret = close(fd);
 212         if (ret) {
 213                 perror("Error closing LTTng-UST perf memory mapping FD");
 214         }
 215 }
 216
 217 static void setup_perf(struct lttng_perf_counter_thread_field *thread_field)
 218 {
 219         void *perf_addr;
 220
 221         perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
 222                         PROT_READ, MAP_SHARED, thread_field->fd, 0);
 223         if (perf_addr == MAP_FAILED)
 224                 perf_addr = NULL;
 225         thread_field->pc = perf_addr;
 226
 227         if (!arch_perf_keep_fd(thread_field)) {
 228                 close_perf_fd(thread_field->fd);
 229                 thread_field->fd = -1;
 230         }
 231 }
 232
 233 static
 234 void unmap_perf_page(struct perf_event_mmap_page *pc)
 235 {
 236         int ret;
 237
 238         if (!pc)
 239                 return;
 240         ret = munmap(pc, sizeof(struct perf_event_mmap_page));
 241         if (ret < 0) {
 242                 PERROR("Error in munmap");
 243                 abort();
 244         }
 245 }
 246
 247 static
 248 struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
 249 {
 250         struct lttng_perf_counter_thread *perf_thread;
 251         sigset_t newmask, oldmask;
 252         int ret;
 253
 254         ret = sigfillset(&newmask);
 255         if (ret)
 256                 abort();
 257         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 258         if (ret)
 259                 abort();
 260         /* Check again with signals disabled */
 261         perf_thread = pthread_getspecific(perf_counter_key);
 262         if (perf_thread)
 263                 goto skip;
 264         perf_thread = zmalloc(sizeof(*perf_thread));
 265         if (!perf_thread)
 266                 abort();
 267         CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
 268         ret = pthread_setspecific(perf_counter_key, perf_thread);
 269         if (ret)
 270                 abort();
 271 skip:
 272         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 273         if (ret)
 274                 abort();
 275         return perf_thread;
 276 }
 277
 278 static
 279 struct lttng_perf_counter_thread_field *
 280         add_thread_field(struct lttng_perf_counter_field *perf_field,
 281                 struct lttng_perf_counter_thread *perf_thread)
 282 {
 283         struct lttng_perf_counter_thread_field *thread_field;
 284         sigset_t newmask, oldmask;
 285         int ret;
 286
 287         ret = sigfillset(&newmask);
 288         if (ret)
 289                 abort();
 290         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 291         if (ret)
 292                 abort();
 293         /* Check again with signals disabled */
 294         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 295                         rcu_field_node) {
 296                 if (thread_field->field == perf_field)
 297                         goto skip;
 298         }
 299         thread_field = zmalloc(sizeof(*thread_field));
 300         if (!thread_field)
 301                 abort();
 302         thread_field->field = perf_field;
 303         thread_field->fd = open_perf_fd(&perf_field->attr);
 304         if (thread_field->fd >= 0)
 305                 setup_perf(thread_field);
 306         /*
 307          * Note: thread_field->pc can be NULL if setup_perf() fails.
 308          * Also, thread_field->fd can be -1 if open_perf_fd() fails.
 309          */
 310         ust_lock_nocheck();
 311         cds_list_add_rcu(&thread_field->rcu_field_node,
 312                         &perf_thread->rcu_field_list);
 313         cds_list_add(&thread_field->thread_field_node,
 314                         &perf_field->thread_field_list);
 315         ust_unlock();
 316 skip:
 317         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 318         if (ret)
 319                 abort();
 320         return thread_field;
 321 }
 322
 323 static
 324 struct lttng_perf_counter_thread_field *
 325                 get_thread_field(struct lttng_perf_counter_field *field)
 326 {
 327         struct lttng_perf_counter_thread *perf_thread;
 328         struct lttng_perf_counter_thread_field *thread_field;
 329
 330         perf_thread = pthread_getspecific(perf_counter_key);
 331         if (!perf_thread)
 332                 perf_thread = alloc_perf_counter_thread();
 333         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 334                         rcu_field_node) {
 335                 if (thread_field->field == field)
 336                         return thread_field;
 337         }
 338         /* perf_counter_thread_field not found, need to add one */
 339         return add_thread_field(field, perf_thread);
 340 }
 341
 342 static
 343 uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field)
 344 {
 345         struct lttng_perf_counter_field *perf_field;
 346         struct lttng_perf_counter_thread_field *perf_thread_field;
 347
 348         perf_field = field->u.perf_counter;
 349         perf_thread_field = get_thread_field(perf_field);
 350         return arch_read_perf_counter(perf_thread_field);
 351 }
 352
 353 static
 354 void perf_counter_record(struct lttng_ctx_field *field,
 355                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 356                  struct lttng_channel *chan)
 357 {
 358         uint64_t value;
 359
 360         value = wrapper_perf_counter_read(field);
 361         lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
 362         chan->ops->event_write(ctx, &value, sizeof(value));
 363 }
 364
 365 static
 366 void perf_counter_get_value(struct lttng_ctx_field *field,
 367                 struct lttng_ctx_value *value)
 368 {
 369         uint64_t v;
 370
 371         v = wrapper_perf_counter_read(field);
 372         value->u.s64 = v;
 373 }
 374
 375 /* Called with UST lock held */
 376 static
 377 void lttng_destroy_perf_thread_field(
 378                 struct lttng_perf_counter_thread_field *thread_field)
 379 {
 380         close_perf_fd(thread_field->fd);
 381         unmap_perf_page(thread_field->pc);
 382         cds_list_del_rcu(&thread_field->rcu_field_node);
 383         cds_list_del(&thread_field->thread_field_node);
 384         free(thread_field);
 385 }
 386
 387 static
 388 void lttng_destroy_perf_thread_key(void *_key)
 389 {
 390         struct lttng_perf_counter_thread *perf_thread = _key;
 391         struct lttng_perf_counter_thread_field *pos, *p;
 392
 393         ust_lock_nocheck();
 394         cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
 395                         rcu_field_node)
 396                 lttng_destroy_perf_thread_field(pos);
 397         ust_unlock();
 398         free(perf_thread);
 399 }
 400
 401 /* Called with UST lock held */
 402 static
 403 void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
 404 {
 405         struct lttng_perf_counter_field *perf_field;
 406         struct lttng_perf_counter_thread_field *pos, *p;
 407
 408         free((char *) field->event_field.name);
 409         perf_field = field->u.perf_counter;
 410         /*
 411          * This put is performed when no threads can concurrently
 412          * perform a "get" concurrently, thanks to urcu-bp grace
 413          * period.
 414          */
 415         cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
 416                         thread_field_node)
 417                 lttng_destroy_perf_thread_field(pos);
 418         free(perf_field);
 419 }
 420
 421 #ifdef __ARM_ARCH_7A__
 422
 423 static
 424 int perf_get_exclude_kernel(void)
 425 {
 426         return 0;
 427 }
 428
 429 #else /* __ARM_ARCH_7A__ */
 430
 431 static
 432 int perf_get_exclude_kernel(void)
 433 {
 434         return 1;
 435 }
 436
 437 #endif /* __ARM_ARCH_7A__ */
 438
 439 /* Called with UST lock held */
 440 int lttng_add_perf_counter_to_ctx(uint32_t type,
 441                                 uint64_t config,
 442                                 const char *name,
 443                                 struct lttng_ctx **ctx)
 444 {
 445         struct lttng_ctx_field *field;
 446         struct lttng_perf_counter_field *perf_field;
 447         char *name_alloc;
 448         int ret;
 449
 450         name_alloc = strdup(name);
 451         if (!name_alloc) {
 452                 ret = -ENOMEM;
 453                 goto name_alloc_error;
 454         }
 455         perf_field = zmalloc(sizeof(*perf_field));
 456         if (!perf_field) {
 457                 ret = -ENOMEM;
 458                 goto perf_field_alloc_error;
 459         }
 460         field = lttng_append_context(ctx);
 461         if (!field) {
 462                 ret = -ENOMEM;
 463                 goto append_context_error;
 464         }
 465         if (lttng_find_context(*ctx, name_alloc)) {
 466                 ret = -EEXIST;
 467                 goto find_error;
 468         }
 469
 470         field->destroy = lttng_destroy_perf_counter_field;
 471
 472         field->event_field.name = name_alloc;
 473         field->event_field.type.atype = atype_integer;
 474         field->event_field.type.u.basic.integer.size =
 475                         sizeof(uint64_t) * CHAR_BIT;
 476         field->event_field.type.u.basic.integer.alignment =
 477                         lttng_alignof(uint64_t) * CHAR_BIT;
 478         field->event_field.type.u.basic.integer.signedness =
 479                         lttng_is_signed_type(uint64_t);
 480         field->event_field.type.u.basic.integer.reverse_byte_order = 0;
 481         field->event_field.type.u.basic.integer.base = 10;
 482         field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
 483         field->get_size = perf_counter_get_size;
 484         field->record = perf_counter_record;
 485         field->get_value = perf_counter_get_value;
 486
 487         perf_field->attr.type = type;
 488         perf_field->attr.config = config;
 489         perf_field->attr.exclude_kernel = perf_get_exclude_kernel();
 490         CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
 491         field->u.perf_counter = perf_field;
 492
 493         /* Ensure that this perf counter can be used in this process. */
 494         ret = open_perf_fd(&perf_field->attr);
 495         if (ret < 0) {
 496                 ret = -ENODEV;
 497                 goto setup_error;
 498         }
 499         close_perf_fd(ret);
 500
 501         /*
 502          * Contexts can only be added before tracing is started, so we
 503          * don't have to synchronize against concurrent threads using
 504          * the field here.
 505          */
 506
 507         lttng_context_update(*ctx);
 508         return 0;
 509
 510 setup_error:
 511 find_error:
 512         lttng_remove_context_field(ctx, field);
 513 append_context_error:
 514         free(perf_field);
 515 perf_field_alloc_error:
 516         free(name_alloc);
 517 name_alloc_error:
 518         return ret;
 519 }
 520
 521 int lttng_perf_counter_init(void)
 522 {
 523         int ret;
 524
 525         ret = pthread_key_create(&perf_counter_key,
 526                         lttng_destroy_perf_thread_key);
 527         if (ret)
 528                 ret = -ret;
 529         return ret;
 530 }
 531
 532 void lttng_perf_counter_exit(void)
 533 {
 534         int ret;
 535
 536         ret = pthread_key_delete(perf_counter_key);
 537         if (ret) {
 538                 errno = ret;
 539                 PERROR("Error in pthread_key_delete");
 540         }
 541 }