liblttng-ust/lttng-context-perf-counters.c

   1 /*
   2  * lttng-context-perf-counters.c
   3  *
   4  * LTTng UST performance monitoring counters (perf-counters) integration.
   5  *
   6  * Copyright (C) 2009-2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; only
  11  * version 2.1 of the License.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include <sys/types.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <sys/mman.h>
  29 #include <sys/syscall.h>
  30 #include <linux/perf_event.h>
  31 #include <lttng/ust-events.h>
  32 #include <lttng/ust-tracer.h>
  33 #include <lttng/ringbuffer-config.h>
  34 #include <urcu/system.h>
  35 #include <urcu/arch.h>
  36 #include <urcu/rculist.h>
  37 #include <helper.h>
  38 #include <urcu/ref.h>
  39 #include <usterr-signal-safe.h>
  40 #include <signal.h>
  41 #include "lttng-tracer-core.h"
  42
  43 /*
  44  * We use a global perf counter key and iterate on per-thread RCU lists
  45  * of fields in the fast path, even though this is not strictly speaking
  46  * what would provide the best fast-path complexity, to ensure teardown
  47  * of sessions vs thread exit is handled racelessly.
  48  *
  49  * Updates and traversals of thread_list are protected by UST lock.
  50  * Updates to rcu_field_list are protected by UST lock.
  51  */
  52
  53 struct lttng_perf_counter_thread_field {
  54         struct lttng_perf_counter_field *field; /* Back reference */
  55         struct perf_event_mmap_page *pc;
  56         struct cds_list_head thread_field_node; /* Per-field list of thread fields (node) */
  57         struct cds_list_head rcu_field_node;    /* RCU per-thread list of fields (node) */
  58 };
  59
  60 struct lttng_perf_counter_thread {
  61         struct cds_list_head rcu_field_list;    /* RCU per-thread list of fields */
  62 };
  63
  64 struct lttng_perf_counter_field {
  65         struct perf_event_attr attr;
  66         struct cds_list_head thread_field_list; /* Per-field list of thread fields */
  67 };
  68
  69 static pthread_key_t perf_counter_key;
  70
  71 static
  72 size_t perf_counter_get_size(struct lttng_ctx_field *field, size_t offset)
  73 {
  74         size_t size = 0;
  75
  76         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
  77         size += sizeof(uint64_t);
  78         return size;
  79 }
  80
  81 #if defined(__x86_64__) || defined(__i386__)
  82
  83 static
  84 uint64_t rdpmc(unsigned int counter)
  85 {
  86         unsigned int low, high;
  87
  88         asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter));
  89
  90         return low | ((uint64_t) high) << 32;
  91 }
  92
  93 #else /* defined(__x86_64__) || defined(__i386__) */
  94
  95 #error "Perf event counters are only supported on x86 so far."
  96
  97 #endif /* #else defined(__x86_64__) || defined(__i386__) */
  98
  99 static
 100 uint64_t read_perf_counter(struct perf_event_mmap_page *pc)
 101 {
 102         uint32_t seq, idx;
 103         uint64_t count;
 104
 105         if (caa_unlikely(!pc))
 106                 return 0;
 107
 108         do {
 109                 seq = CMM_LOAD_SHARED(pc->lock);
 110                 cmm_barrier();
 111
 112                 idx = pc->index;
 113                 if (idx) {
 114                         int64_t pmcval;
 115
 116                         pmcval = rdpmc(idx - 1);
 117                         /* Sign-extend the pmc register result. */
 118                         pmcval <<= 64 - pc->pmc_width;
 119                         pmcval >>= 64 - pc->pmc_width;
 120                         count = pc->offset + pmcval;
 121                 } else {
 122                         count = 0;
 123                 }
 124                 cmm_barrier();
 125         } while (CMM_LOAD_SHARED(pc->lock) != seq);
 126
 127         return count;
 128 }
 129
 130 static
 131 int sys_perf_event_open(struct perf_event_attr *attr,
 132                 pid_t pid, int cpu, int group_fd,
 133                 unsigned long flags)
 134 {
 135         return syscall(SYS_perf_event_open, attr, pid, cpu,
 136                         group_fd, flags);
 137 }
 138
 139 static
 140 struct perf_event_mmap_page *setup_perf(struct perf_event_attr *attr)
 141 {
 142         void *perf_addr;
 143         int fd, ret;
 144
 145         fd = sys_perf_event_open(attr, 0, -1, -1, 0);
 146         if (fd < 0)
 147                 return NULL;
 148
 149         perf_addr = mmap(NULL, sizeof(struct perf_event_mmap_page),
 150                         PROT_READ, MAP_SHARED, fd, 0);
 151         if (perf_addr == MAP_FAILED)
 152                 return NULL;
 153         ret = close(fd);
 154         if (ret) {
 155                 perror("Error closing LTTng-UST perf memory mapping FD");
 156         }
 157         return perf_addr;
 158 }
 159
 160 static
 161 void unmap_perf_page(struct perf_event_mmap_page *pc)
 162 {
 163         int ret;
 164
 165         if (!pc)
 166                 return;
 167         ret = munmap(pc, sizeof(struct perf_event_mmap_page));
 168         if (ret < 0) {
 169                 PERROR("Error in munmap");
 170                 abort();
 171         }
 172 }
 173
 174 static
 175 struct lttng_perf_counter_thread *alloc_perf_counter_thread(void)
 176 {
 177         struct lttng_perf_counter_thread *perf_thread;
 178         sigset_t newmask, oldmask;
 179         int ret;
 180
 181         ret = sigfillset(&newmask);
 182         if (ret)
 183                 abort();
 184         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 185         if (ret)
 186                 abort();
 187         /* Check again with signals disabled */
 188         perf_thread = pthread_getspecific(perf_counter_key);
 189         if (perf_thread)
 190                 goto skip;
 191         perf_thread = zmalloc(sizeof(*perf_thread));
 192         if (!perf_thread)
 193                 abort();
 194         CDS_INIT_LIST_HEAD(&perf_thread->rcu_field_list);
 195         ret = pthread_setspecific(perf_counter_key, perf_thread);
 196         if (ret)
 197                 abort();
 198 skip:
 199         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 200         if (ret)
 201                 abort();
 202         return perf_thread;
 203 }
 204
 205 static
 206 struct lttng_perf_counter_thread_field *
 207         add_thread_field(struct lttng_perf_counter_field *perf_field,
 208                 struct lttng_perf_counter_thread *perf_thread)
 209 {
 210         struct lttng_perf_counter_thread_field *thread_field;
 211         sigset_t newmask, oldmask;
 212         int ret;
 213
 214         ret = sigfillset(&newmask);
 215         if (ret)
 216                 abort();
 217         ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask);
 218         if (ret)
 219                 abort();
 220         /* Check again with signals disabled */
 221         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 222                         rcu_field_node) {
 223                 if (thread_field->field == perf_field)
 224                         goto skip;
 225         }
 226         thread_field = zmalloc(sizeof(*thread_field));
 227         if (!thread_field)
 228                 abort();
 229         thread_field->field = perf_field;
 230         thread_field->pc = setup_perf(&perf_field->attr);
 231         /* Note: thread_field->pc can be NULL if setup_perf() fails. */
 232         ust_lock_nocheck();
 233         cds_list_add_rcu(&thread_field->rcu_field_node,
 234                         &perf_thread->rcu_field_list);
 235         cds_list_add(&thread_field->thread_field_node,
 236                         &perf_field->thread_field_list);
 237         ust_unlock();
 238 skip:
 239         ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 240         if (ret)
 241                 abort();
 242         return thread_field;
 243 }
 244
 245 static
 246 struct lttng_perf_counter_thread_field *
 247                 get_thread_field(struct lttng_perf_counter_field *field)
 248 {
 249         struct lttng_perf_counter_thread *perf_thread;
 250         struct lttng_perf_counter_thread_field *thread_field;
 251
 252         perf_thread = pthread_getspecific(perf_counter_key);
 253         if (!perf_thread)
 254                 perf_thread = alloc_perf_counter_thread();
 255         cds_list_for_each_entry_rcu(thread_field, &perf_thread->rcu_field_list,
 256                         rcu_field_node) {
 257                 if (thread_field->field == field)
 258                         return thread_field;
 259         }
 260         /* perf_counter_thread_field not found, need to add one */
 261         return add_thread_field(field, perf_thread);
 262 }
 263
 264 static
 265 uint64_t wrapper_perf_counter_read(struct lttng_ctx_field *field)
 266 {
 267         struct lttng_perf_counter_field *perf_field;
 268         struct lttng_perf_counter_thread_field *perf_thread_field;
 269
 270         perf_field = field->u.perf_counter;
 271         perf_thread_field = get_thread_field(perf_field);
 272         return read_perf_counter(perf_thread_field->pc);
 273 }
 274
 275 static
 276 void perf_counter_record(struct lttng_ctx_field *field,
 277                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 278                  struct lttng_channel *chan)
 279 {
 280         uint64_t value;
 281
 282         value = wrapper_perf_counter_read(field);
 283         lib_ring_buffer_align_ctx(ctx, lttng_alignof(value));
 284         chan->ops->event_write(ctx, &value, sizeof(value));
 285 }
 286
 287 static
 288 void perf_counter_get_value(struct lttng_ctx_field *field,
 289                 struct lttng_ctx_value *value)
 290 {
 291         uint64_t v;
 292
 293         v = wrapper_perf_counter_read(field);
 294         value->u.s64 = v;
 295 }
 296
 297 /* Called with UST lock held */
 298 static
 299 void lttng_destroy_perf_thread_field(
 300                 struct lttng_perf_counter_thread_field *thread_field)
 301 {
 302         unmap_perf_page(thread_field->pc);
 303         cds_list_del_rcu(&thread_field->rcu_field_node);
 304         cds_list_del(&thread_field->thread_field_node);
 305         free(thread_field);
 306 }
 307
 308 static
 309 void lttng_destroy_perf_thread_key(void *_key)
 310 {
 311         struct lttng_perf_counter_thread *perf_thread = _key;
 312         struct lttng_perf_counter_thread_field *pos, *p;
 313
 314         ust_lock_nocheck();
 315         cds_list_for_each_entry_safe(pos, p, &perf_thread->rcu_field_list,
 316                         rcu_field_node)
 317                 lttng_destroy_perf_thread_field(pos);
 318         ust_unlock();
 319         free(perf_thread);
 320 }
 321
 322 /* Called with UST lock held */
 323 static
 324 void lttng_destroy_perf_counter_field(struct lttng_ctx_field *field)
 325 {
 326         struct lttng_perf_counter_field *perf_field;
 327         struct lttng_perf_counter_thread_field *pos, *p;
 328
 329         free((char *) field->event_field.name);
 330         perf_field = field->u.perf_counter;
 331         /*
 332          * This put is performed when no threads can concurrently
 333          * perform a "get" concurrently, thanks to urcu-bp grace
 334          * period.
 335          */
 336         cds_list_for_each_entry_safe(pos, p, &perf_field->thread_field_list,
 337                         thread_field_node)
 338                 lttng_destroy_perf_thread_field(pos);
 339         free(perf_field);
 340 }
 341
 342 /* Called with UST lock held */
 343 int lttng_add_perf_counter_to_ctx(uint32_t type,
 344                                 uint64_t config,
 345                                 const char *name,
 346                                 struct lttng_ctx **ctx)
 347 {
 348         struct lttng_ctx_field *field;
 349         struct lttng_perf_counter_field *perf_field;
 350         struct perf_event_mmap_page *tmp_pc;
 351         char *name_alloc;
 352         int ret;
 353
 354         name_alloc = strdup(name);
 355         if (!name_alloc) {
 356                 ret = -ENOMEM;
 357                 goto name_alloc_error;
 358         }
 359         perf_field = zmalloc(sizeof(*perf_field));
 360         if (!perf_field) {
 361                 ret = -ENOMEM;
 362                 goto perf_field_alloc_error;
 363         }
 364         field = lttng_append_context(ctx);
 365         if (!field) {
 366                 ret = -ENOMEM;
 367                 goto append_context_error;
 368         }
 369         if (lttng_find_context(*ctx, name_alloc)) {
 370                 ret = -EEXIST;
 371                 goto find_error;
 372         }
 373
 374         field->destroy = lttng_destroy_perf_counter_field;
 375
 376         field->event_field.name = name_alloc;
 377         field->event_field.type.atype = atype_integer;
 378         field->event_field.type.u.basic.integer.size =
 379                         sizeof(uint64_t) * CHAR_BIT;
 380         field->event_field.type.u.basic.integer.alignment =
 381                         lttng_alignof(uint64_t) * CHAR_BIT;
 382         field->event_field.type.u.basic.integer.signedness =
 383                         lttng_is_signed_type(uint64_t);
 384         field->event_field.type.u.basic.integer.reverse_byte_order = 0;
 385         field->event_field.type.u.basic.integer.base = 10;
 386         field->event_field.type.u.basic.integer.encoding = lttng_encode_none;
 387         field->get_size = perf_counter_get_size;
 388         field->record = perf_counter_record;
 389         field->get_value = perf_counter_get_value;
 390
 391         perf_field->attr.type = type;
 392         perf_field->attr.config = config;
 393         perf_field->attr.exclude_kernel = 1;
 394         CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
 395         field->u.perf_counter = perf_field;
 396
 397         /* Ensure that this perf counter can be used in this process. */
 398         tmp_pc = setup_perf(&perf_field->attr);
 399         if (!tmp_pc) {
 400                 ret = -ENODEV;
 401                 goto setup_error;
 402         }
 403         unmap_perf_page(tmp_pc);
 404
 405         /*
 406          * Contexts can only be added before tracing is started, so we
 407          * don't have to synchronize against concurrent threads using
 408          * the field here.
 409          */
 410
 411         lttng_context_update(*ctx);
 412         return 0;
 413
 414 setup_error:
 415 find_error:
 416         lttng_remove_context_field(ctx, field);
 417 append_context_error:
 418         free(perf_field);
 419 perf_field_alloc_error:
 420         free(name_alloc);
 421 name_alloc_error:
 422         return ret;
 423 }
 424
 425 int lttng_perf_counter_init(void)
 426 {
 427         int ret;
 428
 429         ret = pthread_key_create(&perf_counter_key,
 430                         lttng_destroy_perf_thread_key);
 431         if (ret)
 432                 ret = -ret;
 433         return ret;
 434 }
 435
 436 void lttng_perf_counter_exit(void)
 437 {
 438         int ret;
 439
 440         ret = pthread_key_delete(perf_counter_key);
 441         if (ret) {
 442                 errno = ret;
 443                 PERROR("Error in pthread_key_delete");
 444         }
 445 }