liblttng-ust/lttng-context-perf-counters.c

   1 /*
   2  * lttng-context-perf-counters.c
   3  *
   4  * LTTng UST performance monitoring counters (perf-counters) integration.
   5  *
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; only
  11  * version 2.1 of the License.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include <sys/types.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <sys/mman.h>
  29 #include <sys/syscall.h>
  30 #include <linux/perf_event.h>
  31 #include <lttng/ust-events.h>
  32 #include <lttng/ust-tracer.h>
  33 #include <lttng/ringbuffer-config.h>
  34 #include <urcu/system.h>
  35 #include <urcu/arch.h>
  36 #include <urcu/rculist.h>
  37 #include <helper.h>
  38 #include <urcu/ref.h>
  39 #include <usterr-signal-safe.h>
  40 #include <signal.h>
  41 #include "lttng-tracer-core.h"
  42
  43 /*
  44  * We use a global perf counter key and iterate on per-thread RCU lists
  45  * of fields in the fast path, even though this is not strictly speaking
  46  * what would provide the best fast-path complexity, to ensure teardown
  47  * of sessions vs thread exit is handled racelessly.
  48  *
  49  * Updates and traversals of thread_list are protected by UST lock.
  50  * Updates to rcu_field_list are protected by UST lock.
  51  */
  52
  53 struct lttng_perf_counter_thread_field {
  54         struct lttng_perf_counter_field *field; /* Back reference */
  55         struct perf_event_mmap_page *pc;
  56         struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
  57         struct cds_list_head rcu_field_node;    /* RCU per-thread list of fields (node) */
  58         int fd;                                 /* Perf FD */
  59 };
  60
  61 struct lttng_perf_counter_thread {
  62         struct cds_list_head rcu_field_list;    /* RCU per-thread list of fields */
  63 };
  64
  65 struct lttng_perf_counter_field {
  66         struct perf_event_attr attr;
  67         struct cds_list_head thread_field_list; /* Per-field list of thread fields */
  68 };
  69
  70 static pthread_key_t perf_counter_key;
  71
  72 static
  73 size_t perf_counter_get_size(struct lttng_ctx_field *field, size_t offset)
  74 {
  75         size_t size = 0;
  76
  77         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
  78         size += sizeof(uint64_t);
  79         return size;
  80 }
  81
  82 #if defined(__x86_64__) || defined(__i386__)
  83
  84 static
  85 uint64_t rdpmc(unsigned int counter)
  86 {
  87         unsigned int low, high;
  88
  89         asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
  90
  91         return low | ((uint64_t) high) << 32;
  92 }
  93
  94 static bool arch_perf_use_read(void)
  95 {
  96         return false;
  97 }
  98
  99 #else /* defined(__x86_64__) || defined(__i386__) */
 100
 101 #error "Perf event counters are only supported on x86 so far."
 102
 103 #endif /* #else defined(__x86_64__) || defined(__i386__) */
 104
 105 static
 106 uint64_t read_perf_counter(struct perf_event_mmap_page *pc)
 107 {
 108         uint32_t seq, idx;
 109         uint64_t count;
 110
 111         if (caa_unlikely(!pc))
 112                 return 0;
 113
 114         do {
 115                 seq = CMM_LOAD_SHARED(pc->lock);
 116                 cmm_barrier();
 117
 118                 idx = pc->index;
 119                 if (idx)
 120                         count = pc->offset + rdpmc(idx - 1);
 121                 else
 122                         count = 0;
 123
 124                 cmm_barrier();
 125         } while (CMM_LOAD_SHARED(pc->lock) != seq);
 126
 127         return count;
 128 }
 129
 130 static
 131 int sys_perf_event_open(struct perf_event_attr *attr,
 132                 pid_t pid, int cpu, int group_fd,
 133                 unsigned long flags)
 134 {
 135         return syscall(SYS_perf_event_open, attr, pid, cpu,
 136                         group_fd, flags);
 137 }
 138
 139 static
 140 int open_perf_fd(struct perf_event_attr *attr)
 141 {
 142         int fd;
 143
 144         fd = sys_perf_event_open(attr, 0, -1, -1, 0);
 145         if (fd < 0)
 146                 return -1;
 147
 148         return fd;
 149 }
 150
 151 static
 152 struct perf_event_mmap_page *setup_perf(
 153                 struct lttng_perf_counter_thread_field *thread_field)
 154 {
 155         void *perf_addr;
 156
 157         perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
 158                         PROT_READ, MAP_SHARED, thread_field->fd, 0);
 159         if (perf_addr == MAP_FAILED)
 160                 perf_addr = NULL;
 161
 162         if (!arch_perf_use_read()) {
 163                 close_perf_fd(thread_field->fd);
 164                 thread_field->fd = -1;
 165         }
 166
 167 end:
 168         return perf_addr;
 169 }
 170
 171 static
 172 void close_perf_fd(int fd)
 173 {
 174         int ret;
 175
 176         if (fd < 0)
 177                 return;
 178
 179         ret = close(fd);
 180         if (ret)
 181                 perror("Error closing LTTng-UST perf memory mapping FD");
 182 }
 183
 184 static
 185 void unmap_perf_page(struct perf_event_mmap_page *pc)
 186 {
 187         int ret;
 188
 189         if (!pc)
 190                 return;
 191         ret = munmap(pc, sizeof(struct perf_event_mmap_page));
 192         if (ret < 0) {
 193                 PERROR("Error in munmap");
 194                 abort();
 195         }
 196 }
 197
 198 static
 199 struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
 200 {
 201         struct lttng_perf_counter_thread *perf_thread;
 202         sigset_t newmask, oldmask;
 203         int ret;
 204
 205         ret = sigfillset(&newmask);
 206         if (ret)
 207                 abort();
 208         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 209         if (ret)
 210                 abort();
 211         /* Check again with signals disabled */
 212         perf_thread = pthread_getspecific(perf_counter_key);
 213         if (perf_thread)
 214                 goto skip;
 215         perf_thread = zmalloc(sizeof(*perf_thread));
 216         if (!perf_thread)
 217                 abort();
 218         CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
 219         ret = pthread_setspecific(perf_counter_key, perf_thread);
 220         if (ret)
 221                 abort();
 222 skip:
 223         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 224         if (ret)
 225                 abort();
 226         return perf_thread;
 227 }
 228
 229 static
 230 struct lttng_perf_counter_thread_field *
 231         add_thread_field(struct lttng_perf_counter_field *perf_field,
 232                 struct lttng_perf_counter_thread *perf_thread)
 233 {
 234         struct lttng_perf_counter_thread_field *thread_field;
 235         sigset_t newmask, oldmask;
 236         int ret;
 237
 238         ret = sigfillset(&newmask);
 239         if (ret)
 240                 abort();
 241         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 242         if (ret)
 243                 abort();
 244         /* Check again with signals disabled */
 245         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 246                         rcu_field_node) {
 247                 if (thread_field->field == perf_field)
 248                         goto skip;
 249         }
 250         thread_field = zmalloc(sizeof(*thread_field));
 251         if (!thread_field)
 252                 abort();
 253         thread_field->field = perf_field;
 254         thread_field->fd = open_perf_fd(&perf_field->attr);
 255         if (thread_field->fd >= 0)
 256                 thread_field->pc = setup_perf(thread_field);
 257         /*
 258          * Note: thread_field->pc can be NULL if setup_perf() fails.
 259          * Also, thread_field->fd can be -1 if open_perf_fd() fails.
 260          */
 261         ust_lock_nocheck();
 262         cds_list_add_rcu(&thread_field->rcu_field_node,
 263                         &perf_thread->rcu_field_list);
 264         cds_list_add(&thread_field->thread_field_node,
 265                         &perf_field->thread_field_list);
 266         ust_unlock();
 267 skip:
 268         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 269         if (ret)
 270                 abort();
 271         return thread_field;
 272 }
 273
 274 static
 275 struct lttng_perf_counter_thread_field *
 276                 get_thread_field(struct lttng_perf_counter_field *field)
 277 {
 278         struct lttng_perf_counter_thread *perf_thread;
 279         struct lttng_perf_counter_thread_field *thread_field;
 280
 281         perf_thread = pthread_getspecific(perf_counter_key);
 282         if (!perf_thread)
 283                 perf_thread = alloc_perf_counter_thread();
 284         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 285                         rcu_field_node) {
 286                 if (thread_field->field == field)
 287                         return thread_field;
 288         }
 289         /* perf_counter_thread_field not found, need to add one */
 290         return add_thread_field(field, perf_thread);
 291 }
 292
 293 static
 294 uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field)
 295 {
 296         struct lttng_perf_counter_field *perf_field;
 297         struct lttng_perf_counter_thread_field *perf_thread_field;
 298
 299         perf_field = field->u.perf_counter;
 300         perf_thread_field = get_thread_field(perf_field);
 301         return read_perf_counter(perf_thread_field->pc);
 302 }
 303
 304 static
 305 void perf_counter_record(struct lttng_ctx_field *field,
 306                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 307                  struct lttng_channel *chan)
 308 {
 309         uint64_t value;
 310
 311         value = wrapper_perf_counter_read(field);
 312         lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
 313         chan->ops->event_write(ctx, &value, sizeof(value));
 314 }
 315
 316 static
 317 void perf_counter_get_value(struct lttng_ctx_field *field,
 318                 struct lttng_ctx_value *value)
 319 {
 320         uint64_t v;
 321
 322         v = wrapper_perf_counter_read(field);
 323         value->u.s64 = v;
 324 }
 325
 326 /* Called with UST lock held */
 327 static
 328 void lttng_destroy_perf_thread_field(
 329                 struct lttng_perf_counter_thread_field *thread_field)
 330 {
 331         close_perf_fd(thread_field->fd);
 332         unmap_perf_page(thread_field->pc);
 333         cds_list_del_rcu(&thread_field->rcu_field_node);
 334         cds_list_del(&thread_field->thread_field_node);
 335         free(thread_field);
 336 }
 337
 338 static
 339 void lttng_destroy_perf_thread_key(void *_key)
 340 {
 341         struct lttng_perf_counter_thread *perf_thread = _key;
 342         struct lttng_perf_counter_thread_field *pos, *p;
 343
 344         ust_lock_nocheck();
 345         cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
 346                         rcu_field_node)
 347                 lttng_destroy_perf_thread_field(pos);
 348         ust_unlock();
 349         free(perf_thread);
 350 }
 351
 352 /* Called with UST lock held */
 353 static
 354 void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
 355 {
 356         struct lttng_perf_counter_field *perf_field;
 357         struct lttng_perf_counter_thread_field *pos, *p;
 358
 359         free((char *) field->event_field.name);
 360         perf_field = field->u.perf_counter;
 361         /*
 362          * This put is performed when no threads can concurrently
 363          * perform a "get" concurrently, thanks to urcu-bp grace
 364          * period.
 365          */
 366         cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
 367                         thread_field_node)
 368                 lttng_destroy_perf_thread_field(pos);
 369         free(perf_field);
 370 }
 371
 372 /* Called with UST lock held */
 373 int lttng_add_perf_counter_to_ctx(uint32_t type,
 374                                 uint64_t config,
 375                                 const char *name,
 376                                 struct lttng_ctx **ctx)
 377 {
 378         struct lttng_ctx_field *field;
 379         struct lttng_perf_counter_field *perf_field;
 380         char *name_alloc;
 381         int ret;
 382
 383         name_alloc = strdup(name);
 384         if (!name_alloc) {
 385                 ret = -ENOMEM;
 386                 goto name_alloc_error;
 387         }
 388         perf_field = zmalloc(sizeof(*perf_field));
 389         if (!perf_field) {
 390                 ret = -ENOMEM;
 391                 goto perf_field_alloc_error;
 392         }
 393         field = lttng_append_context(ctx);
 394         if (!field) {
 395                 ret = -ENOMEM;
 396                 goto append_context_error;
 397         }
 398         if (lttng_find_context(*ctx, name_alloc)) {
 399                 ret = -EEXIST;
 400                 goto find_error;
 401         }
 402
 403         field->destroy = lttng_destroy_perf_counter_field;
 404
 405         field->event_field.name = name_alloc;
 406         field->event_field.type.atype = atype_integer;
 407         field->event_field.type.u.basic.integer.size =
 408                         sizeof(uint64_t) * CHAR_BIT;
 409         field->event_field.type.u.basic.integer.alignment =
 410                         lttng_alignof(uint64_t) * CHAR_BIT;
 411         field->event_field.type.u.basic.integer.signedness =
 412                         lttng_is_signed_type(uint64_t);
 413         field->event_field.type.u.basic.integer.reverse_byte_order = 0;
 414         field->event_field.type.u.basic.integer.base = 10;
 415         field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
 416         field->get_size = perf_counter_get_size;
 417         field->record = perf_counter_record;
 418         field->get_value = perf_counter_get_value;
 419
 420         perf_field->attr.type = type;
 421         perf_field->attr.config = config;
 422         perf_field->attr.exclude_kernel = 1;
 423         CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
 424         field->u.perf_counter = perf_field;
 425
 426         /* Ensure that this perf counter can be used in this process. */
 427         ret = open_perf_fd(&perf_field->attr);
 428         if (ret < 0) {
 429                 ret = -ENODEV;
 430                 goto setup_error;
 431         }
 432         close_perf_fd(ret);
 433
 434         /*
 435          * Contexts can only be added before tracing is started, so we
 436          * don't have to synchronize against concurrent threads using
 437          * the field here.
 438          */
 439
 440         lttng_context_update(*ctx);
 441         return 0;
 442
 443 setup_error:
 444 find_error:
 445         lttng_remove_context_field(ctx, field);
 446 append_context_error:
 447         free(perf_field);
 448 perf_field_alloc_error:
 449         free(name_alloc);
 450 name_alloc_error:
 451         return ret;
 452 }
 453
 454 int lttng_perf_counter_init(void)
 455 {
 456         int ret;
 457
 458         ret = pthread_key_create(&perf_counter_key,
 459                         lttng_destroy_perf_thread_key);
 460         if (ret)
 461                 ret = -ret;
 462         return ret;
 463 }
 464
 465 void lttng_perf_counter_exit(void)
 466 {
 467         int ret;
 468
 469         ret = pthread_key_delete(perf_counter_key);
 470         if (ret) {
 471                 errno = ret;
 472                 PERROR("Error in pthread_key_delete");
 473         }
 474 }