liblttng-ust/lttng-ring-buffer-client-template.h

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * LTTng lib ring buffer client template.
   7  */
   8
   9 #include <limits.h>
  10 #include <stddef.h>
  11 #include <stdint.h>
  12
  13 #include <ust-events-internal.h>
  14 #include <lttng/urcu/pointer.h>
  15 #include "ust-bitfield.h"
  16 #include "ust-compat.h"
  17 #include "clock.h"
  18 #include "context-internal.h"
  19 #include "lttng-tracer.h"
  20 #include "../libringbuffer/frontend_types.h"
  21 #include <urcu/tls-compat.h>
  22
  23 #define LTTNG_COMPACT_EVENT_BITS       5
  24 #define LTTNG_COMPACT_TSC_BITS         27
  25
  26 enum app_ctx_mode {
  27         APP_CTX_DISABLED,
  28         APP_CTX_ENABLED,
  29 };
  30
  31 /*
  32  * Keep the natural field alignment for _each field_ within this structure if
  33  * you ever add/remove a field from this header. Packed attribute is not used
  34  * because gcc generates poor code on at least powerpc and mips. Don't ever
  35  * let gcc add padding between the structure elements.
  36  */
  37
  38 struct packet_header {
  39         /* Trace packet header */
  40         uint32_t magic;                 /*
  41                                          * Trace magic number.
  42                                          * contains endianness information.
  43                                          */
  44         uint8_t uuid[LTTNG_UST_UUID_LEN];
  45         uint32_t stream_id;
  46         uint64_t stream_instance_id;
  47
  48         struct {
  49                 /* Stream packet context */
  50                 uint64_t timestamp_begin;       /* Cycle count at subbuffer start */
  51                 uint64_t timestamp_end;         /* Cycle count at subbuffer end */
  52                 uint64_t content_size;          /* Size of data in subbuffer */
  53                 uint64_t packet_size;           /* Subbuffer size (include padding) */
  54                 uint64_t packet_seq_num;        /* Packet sequence number */
  55                 unsigned long events_discarded; /*
  56                                                  * Events lost in this subbuffer since
  57                                                  * the beginning of the trace.
  58                                                  * (may overflow)
  59                                                  */
  60                 uint32_t cpu_id;                /* CPU id associated with stream */
  61                 uint8_t header_end;             /* End of header */
  62         } ctx;
  63 };
  64
  65 struct lttng_client_ctx {
  66         size_t packet_context_len;
  67         size_t event_context_len;
  68         struct lttng_ust_ctx *chan_ctx;
  69         struct lttng_ust_ctx *event_ctx;
  70 };
  71
  72 /*
  73  * Indexed by lib_ring_buffer_nesting_count().
  74  */
  75 typedef struct lttng_ust_lib_ring_buffer_ctx_private private_ctx_stack_t[LIB_RING_BUFFER_MAX_NESTING];
  76 static DEFINE_URCU_TLS(private_ctx_stack_t, private_ctx_stack);
  77
  78 /*
  79  * Force a read (imply TLS fixup for dlopen) of TLS variables.
  80  */
  81 void RING_BUFFER_MODE_TEMPLATE_TLS_FIXUP(void)
  82 {
  83         asm volatile ("" : : "m" (URCU_TLS(private_ctx_stack)));
  84 }
  85
  86 static inline uint64_t lib_ring_buffer_clock_read(
  87                 struct lttng_ust_lib_ring_buffer_channel *chan __attribute__((unused)))
  88 {
  89         return trace_clock_read64();
  90 }
  91
  92 static inline
  93 size_t ctx_get_aligned_size(size_t offset, struct lttng_ust_ctx *ctx,
  94                 size_t ctx_len)
  95 {
  96         size_t orig_offset = offset;
  97
  98         if (caa_likely(!ctx))
  99                 return 0;
 100         offset += lttng_ust_lib_ring_buffer_align(offset, ctx->largest_align);
 101         offset += ctx_len;
 102         return offset - orig_offset;
 103 }
 104
 105 static inline
 106 void ctx_get_struct_size(struct lttng_ust_ctx *ctx, size_t *ctx_len,
 107                 enum app_ctx_mode mode)
 108 {
 109         int i;
 110         size_t offset = 0;
 111
 112         if (caa_likely(!ctx)) {
 113                 *ctx_len = 0;
 114                 return;
 115         }
 116         for (i = 0; i < ctx->nr_fields; i++) {
 117                 if (mode == APP_CTX_ENABLED) {
 118                         offset += ctx->fields[i].get_size(&ctx->fields[i], offset);
 119                 } else {
 120                         if (lttng_context_is_app(ctx->fields[i].event_field->name)) {
 121                                 /*
 122                                  * Before UST 2.8, we cannot use the
 123                                  * application context, because we
 124                                  * cannot trust that the handler used
 125                                  * for get_size is the same used for
 126                                  * ctx_record, which would result in
 127                                  * corrupted traces when tracing
 128                                  * concurrently with application context
 129                                  * register/unregister.
 130                                  */
 131                                 offset += lttng_ust_dummy_get_size(&ctx->fields[i], offset);
 132                         } else {
 133                                 offset += ctx->fields[i].get_size(&ctx->fields[i], offset);
 134                         }
 135                 }
 136         }
 137         *ctx_len = offset;
 138 }
 139
 140 static inline
 141 void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx,
 142                 struct lttng_ust_channel_buffer *chan,
 143                 struct lttng_ust_ctx *ctx,
 144                 enum app_ctx_mode mode)
 145 {
 146         int i;
 147
 148         if (caa_likely(!ctx))
 149                 return;
 150         lttng_ust_lib_ring_buffer_align_ctx(bufctx, ctx->largest_align);
 151         for (i = 0; i < ctx->nr_fields; i++) {
 152                 if (mode == APP_CTX_ENABLED) {
 153                         ctx->fields[i].record(&ctx->fields[i], bufctx, chan);
 154                 } else {
 155                         if (lttng_context_is_app(ctx->fields[i].event_field->name)) {
 156                                 /*
 157                                  * Before UST 2.8, we cannot use the
 158                                  * application context, because we
 159                                  * cannot trust that the handler used
 160                                  * for get_size is the same used for
 161                                  * ctx_record, which would result in
 162                                  * corrupted traces when tracing
 163                                  * concurrently with application context
 164                                  * register/unregister.
 165                                  */
 166                                 lttng_ust_dummy_record(&ctx->fields[i], bufctx, chan);
 167                         } else {
 168                                 ctx->fields[i].record(&ctx->fields[i], bufctx, chan);
 169                         }
 170                 }
 171         }
 172 }
 173
 174 /*
 175  * record_header_size - Calculate the header size and padding necessary.
 176  * @config: ring buffer instance configuration
 177  * @chan: channel
 178  * @offset: offset in the write buffer
 179  * @pre_header_padding: padding to add before the header (output)
 180  * @ctx: reservation context
 181  *
 182  * Returns the event header size (including padding).
 183  *
 184  * The payload must itself determine its own alignment from the biggest type it
 185  * contains.
 186  */
 187 static __inline__
 188 size_t record_header_size(
 189                 const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)),
 190                 struct lttng_ust_lib_ring_buffer_channel *chan,
 191                 size_t offset,
 192                 size_t *pre_header_padding,
 193                 struct lttng_ust_lib_ring_buffer_ctx *ctx,
 194                 struct lttng_client_ctx *client_ctx)
 195 {
 196         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan);
 197         size_t orig_offset = offset;
 198         size_t padding;
 199
 200         switch (lttng_chan->priv->header_type) {
 201         case 1: /* compact */
 202                 padding = lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint32_t));
 203                 offset += padding;
 204                 if (!(ctx->priv->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 205                         offset += sizeof(uint32_t);     /* id and timestamp */
 206                 } else {
 207                         /* Minimum space taken by LTTNG_COMPACT_EVENT_BITS id */
 208                         offset += (LTTNG_COMPACT_EVENT_BITS + CHAR_BIT - 1) / CHAR_BIT;
 209                         /* Align extended struct on largest member */
 210                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 211                         offset += sizeof(uint32_t);     /* id */
 212                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 213                         offset += sizeof(uint64_t);     /* timestamp */
 214                 }
 215                 break;
 216         case 2: /* large */
 217                 padding = lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint16_t));
 218                 offset += padding;
 219                 offset += sizeof(uint16_t);
 220                 if (!(ctx->priv->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 221                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint32_t));
 222                         offset += sizeof(uint32_t);     /* timestamp */
 223                 } else {
 224                         /* Align extended struct on largest member */
 225                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 226                         offset += sizeof(uint32_t);     /* id */
 227                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 228                         offset += sizeof(uint64_t);     /* timestamp */
 229                 }
 230                 break;
 231         default:
 232                 padding = 0;
 233                 WARN_ON_ONCE(1);
 234         }
 235         offset += ctx_get_aligned_size(offset, client_ctx->chan_ctx,
 236                         client_ctx->packet_context_len);
 237         offset += ctx_get_aligned_size(offset, client_ctx->event_ctx,
 238                         client_ctx->event_context_len);
 239         *pre_header_padding = padding;
 240         return offset - orig_offset;
 241 }
 242
 243 #include "../libringbuffer/api.h"
 244 #include "lttng-rb-clients.h"
 245
 246 static
 247 void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
 248                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 249                                  struct lttng_client_ctx *client_ctx,
 250                                  uint32_t event_id);
 251
 252 /*
 253  * lttng_write_event_header
 254  *
 255  * Writes the event header to the offset (already aligned on 32-bits).
 256  *
 257  * @config: ring buffer instance configuration
 258  * @ctx: reservation context
 259  * @event_id: event ID
 260  */
 261 static __inline__
 262 void lttng_write_event_header(const struct lttng_ust_lib_ring_buffer_config *config,
 263                             struct lttng_ust_lib_ring_buffer_ctx *ctx,
 264                             struct lttng_client_ctx *client_ctx,
 265                             uint32_t event_id)
 266 {
 267         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(ctx->priv->chan);
 268
 269         if (caa_unlikely(ctx->priv->rflags))
 270                 goto slow_path;
 271
 272         switch (lttng_chan->priv->header_type) {
 273         case 1: /* compact */
 274         {
 275                 uint32_t id_time = 0;
 276
 277                 bt_bitfield_write(&id_time, uint32_t,
 278                                 0,
 279                                 LTTNG_COMPACT_EVENT_BITS,
 280                                 event_id);
 281                 bt_bitfield_write(&id_time, uint32_t,
 282                                 LTTNG_COMPACT_EVENT_BITS,
 283                                 LTTNG_COMPACT_TSC_BITS,
 284                                 ctx->priv->tsc);
 285                 lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
 286                 break;
 287         }
 288         case 2: /* large */
 289         {
 290                 uint32_t timestamp = (uint32_t) ctx->priv->tsc;
 291                 uint16_t id = event_id;
 292
 293                 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 294                 lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint32_t));
 295                 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 296                 break;
 297         }
 298         default:
 299                 WARN_ON_ONCE(1);
 300         }
 301
 302         ctx_record(ctx, lttng_chan, client_ctx->chan_ctx, APP_CTX_ENABLED);
 303         ctx_record(ctx, lttng_chan, client_ctx->event_ctx, APP_CTX_ENABLED);
 304         lttng_ust_lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
 305
 306         return;
 307
 308 slow_path:
 309         lttng_write_event_header_slow(config, ctx, client_ctx, event_id);
 310 }
 311
 312 static
 313 void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
 314                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 315                                  struct lttng_client_ctx *client_ctx,
 316                                  uint32_t event_id)
 317 {
 318         struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv;
 319         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(ctx->priv->chan);
 320
 321         switch (lttng_chan->priv->header_type) {
 322         case 1: /* compact */
 323                 if (!(ctx_private->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 324                         uint32_t id_time = 0;
 325
 326                         bt_bitfield_write(&id_time, uint32_t,
 327                                         0,
 328                                         LTTNG_COMPACT_EVENT_BITS,
 329                                         event_id);
 330                         bt_bitfield_write(&id_time, uint32_t,
 331                                         LTTNG_COMPACT_EVENT_BITS,
 332                                         LTTNG_COMPACT_TSC_BITS,
 333                                         ctx_private->tsc);
 334                         lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
 335                 } else {
 336                         uint8_t id = 0;
 337                         uint64_t timestamp = ctx_private->tsc;
 338
 339                         bt_bitfield_write(&id, uint8_t,
 340                                         0,
 341                                         LTTNG_COMPACT_EVENT_BITS,
 342                                         31);
 343                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 344                         /* Align extended struct on largest member */
 345                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 346                         lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
 347                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 348                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 349                 }
 350                 break;
 351         case 2: /* large */
 352         {
 353                 if (!(ctx_private->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 354                         uint32_t timestamp = (uint32_t) ctx_private->tsc;
 355                         uint16_t id = event_id;
 356
 357                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 358                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint32_t));
 359                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 360                 } else {
 361                         uint16_t id = 65535;
 362                         uint64_t timestamp = ctx_private->tsc;
 363
 364                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 365                         /* Align extended struct on largest member */
 366                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 367                         lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
 368                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 369                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 370                 }
 371                 break;
 372         }
 373         default:
 374                 WARN_ON_ONCE(1);
 375         }
 376         ctx_record(ctx, lttng_chan, client_ctx->chan_ctx, APP_CTX_ENABLED);
 377         ctx_record(ctx, lttng_chan, client_ctx->event_ctx, APP_CTX_ENABLED);
 378         lttng_ust_lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
 379 }
 380
 381 static const struct lttng_ust_lib_ring_buffer_config client_config;
 382
 383 static uint64_t client_ring_buffer_clock_read(struct lttng_ust_lib_ring_buffer_channel *chan)
 384 {
 385         return lib_ring_buffer_clock_read(chan);
 386 }
 387
 388 static
 389 size_t client_record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
 390                                  struct lttng_ust_lib_ring_buffer_channel *chan,
 391                                  size_t offset,
 392                                  size_t *pre_header_padding,
 393                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 394                                  void *client_ctx)
 395 {
 396         return record_header_size(config, chan, offset,
 397                                   pre_header_padding, ctx, client_ctx);
 398 }
 399
 400 /**
 401  * client_packet_header_size - called on buffer-switch to a new sub-buffer
 402  *
 403  * Return header size without padding after the structure. Don't use packed
 404  * structure because gcc generates inefficient code on some architectures
 405  * (powerpc, mips..)
 406  */
 407 static size_t client_packet_header_size(void)
 408 {
 409         return offsetof(struct packet_header, ctx.header_end);
 410 }
 411
 412 static void client_buffer_begin(struct lttng_ust_lib_ring_buffer *buf, uint64_t tsc,
 413                                 unsigned int subbuf_idx,
 414                                 struct lttng_ust_shm_handle *handle)
 415 {
 416         struct lttng_ust_lib_ring_buffer_channel *chan = shmp(handle, buf->backend.chan);
 417         struct packet_header *header =
 418                 (struct packet_header *)
 419                         lib_ring_buffer_offset_address(&buf->backend,
 420                                 subbuf_idx * chan->backend.subbuf_size,
 421                                 handle);
 422         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan);
 423         uint64_t cnt = shmp_index(handle, buf->backend.buf_cnt, subbuf_idx)->seq_cnt;
 424
 425         assert(header);
 426         if (!header)
 427                 return;
 428         header->magic = CTF_MAGIC_NUMBER;
 429         memcpy(header->uuid, lttng_chan->priv->uuid, sizeof(lttng_chan->priv->uuid));
 430         header->stream_id = lttng_chan->priv->id;
 431         header->stream_instance_id = buf->backend.cpu;
 432         header->ctx.timestamp_begin = tsc;
 433         header->ctx.timestamp_end = 0;
 434         header->ctx.content_size = ~0ULL; /* for debugging */
 435         header->ctx.packet_size = ~0ULL;
 436         header->ctx.packet_seq_num = chan->backend.num_subbuf * cnt + subbuf_idx;
 437         header->ctx.events_discarded = 0;
 438         header->ctx.cpu_id = buf->backend.cpu;
 439 }
 440
 441 /*
 442  * offset is assumed to never be 0 here : never deliver a completely empty
 443  * subbuffer. data_size is between 1 and subbuf_size.
 444  */
 445 static void client_buffer_end(struct lttng_ust_lib_ring_buffer *buf, uint64_t tsc,
 446                               unsigned int subbuf_idx, unsigned long data_size,
 447                               struct lttng_ust_shm_handle *handle)
 448 {
 449         struct lttng_ust_lib_ring_buffer_channel *chan = shmp(handle, buf->backend.chan);
 450         struct packet_header *header =
 451                 (struct packet_header *)
 452                         lib_ring_buffer_offset_address(&buf->backend,
 453                                 subbuf_idx * chan->backend.subbuf_size,
 454                                 handle);
 455         unsigned long records_lost = 0;
 456
 457         assert(header);
 458         if (!header)
 459                 return;
 460         header->ctx.timestamp_end = tsc;
 461         header->ctx.content_size =
 462                 (uint64_t) data_size * CHAR_BIT;                /* in bits */
 463         header->ctx.packet_size =
 464                 (uint64_t) LTTNG_UST_PAGE_ALIGN(data_size) * CHAR_BIT;  /* in bits */
 465
 466         records_lost += lib_ring_buffer_get_records_lost_full(&client_config, buf);
 467         records_lost += lib_ring_buffer_get_records_lost_wrap(&client_config, buf);
 468         records_lost += lib_ring_buffer_get_records_lost_big(&client_config, buf);
 469         header->ctx.events_discarded = records_lost;
 470 }
 471
 472 static int client_buffer_create(
 473                 struct lttng_ust_lib_ring_buffer *buf __attribute__((unused)),
 474                 void *priv __attribute__((unused)),
 475                 int cpu __attribute__((unused)),
 476                 const char *name __attribute__((unused)),
 477                 struct lttng_ust_shm_handle *handle __attribute__((unused)))
 478 {
 479         return 0;
 480 }
 481
 482 static void client_buffer_finalize(
 483                 struct lttng_ust_lib_ring_buffer *buf __attribute__((unused)),
 484                 void *priv __attribute__((unused)),
 485                 int cpu __attribute__((unused)),
 486                 struct lttng_ust_shm_handle *handle __attribute__((unused)))
 487 {
 488 }
 489
 490 static void client_content_size_field(
 491                 const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)),
 492                 size_t *offset, size_t *length)
 493 {
 494         *offset = offsetof(struct packet_header, ctx.content_size);
 495         *length = sizeof(((struct packet_header *) NULL)->ctx.content_size);
 496 }
 497
 498 static void client_packet_size_field(
 499                 const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)),
 500                 size_t *offset, size_t *length)
 501 {
 502         *offset = offsetof(struct packet_header, ctx.packet_size);
 503         *length = sizeof(((struct packet_header *) NULL)->ctx.packet_size);
 504 }
 505
 506 static struct packet_header *client_packet_header(struct lttng_ust_lib_ring_buffer *buf,
 507                 struct lttng_ust_shm_handle *handle)
 508 {
 509         return lib_ring_buffer_read_offset_address(&buf->backend, 0, handle);
 510 }
 511
 512 static int client_timestamp_begin(struct lttng_ust_lib_ring_buffer *buf,
 513                 struct lttng_ust_lib_ring_buffer_channel *chan,
 514                 uint64_t *timestamp_begin)
 515 {
 516         struct lttng_ust_shm_handle *handle = chan->handle;
 517         struct packet_header *header;
 518
 519         header = client_packet_header(buf, handle);
 520         if (!header)
 521                 return -1;
 522         *timestamp_begin = header->ctx.timestamp_begin;
 523         return 0;
 524 }
 525
 526 static int client_timestamp_end(struct lttng_ust_lib_ring_buffer *buf,
 527                 struct lttng_ust_lib_ring_buffer_channel *chan,
 528                 uint64_t *timestamp_end)
 529 {
 530         struct lttng_ust_shm_handle *handle = chan->handle;
 531         struct packet_header *header;
 532
 533         header = client_packet_header(buf, handle);
 534         if (!header)
 535                 return -1;
 536         *timestamp_end = header->ctx.timestamp_end;
 537         return 0;
 538 }
 539
 540 static int client_events_discarded(struct lttng_ust_lib_ring_buffer *buf,
 541                 struct lttng_ust_lib_ring_buffer_channel *chan,
 542                 uint64_t *events_discarded)
 543 {
 544         struct lttng_ust_shm_handle *handle = chan->handle;
 545         struct packet_header *header;
 546
 547         header = client_packet_header(buf, handle);
 548         if (!header)
 549                 return -1;
 550         *events_discarded = header->ctx.events_discarded;
 551         return 0;
 552 }
 553
 554 static int client_content_size(struct lttng_ust_lib_ring_buffer *buf,
 555                 struct lttng_ust_lib_ring_buffer_channel *chan,
 556                 uint64_t *content_size)
 557 {
 558         struct lttng_ust_shm_handle *handle = chan->handle;
 559         struct packet_header *header;
 560
 561         header = client_packet_header(buf, handle);
 562         if (!header)
 563                 return -1;
 564         *content_size = header->ctx.content_size;
 565         return 0;
 566 }
 567
 568 static int client_packet_size(struct lttng_ust_lib_ring_buffer *buf,
 569                 struct lttng_ust_lib_ring_buffer_channel *chan,
 570                 uint64_t *packet_size)
 571 {
 572         struct lttng_ust_shm_handle *handle = chan->handle;
 573         struct packet_header *header;
 574
 575         header = client_packet_header(buf, handle);
 576         if (!header)
 577                 return -1;
 578         *packet_size = header->ctx.packet_size;
 579         return 0;
 580 }
 581
 582 static int client_stream_id(struct lttng_ust_lib_ring_buffer *buf __attribute__((unused)),
 583                 struct lttng_ust_lib_ring_buffer_channel *chan,
 584                 uint64_t *stream_id)
 585 {
 586         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan);
 587
 588         *stream_id = lttng_chan->priv->id;
 589
 590         return 0;
 591 }
 592
 593 static int client_current_timestamp(
 594                 struct lttng_ust_lib_ring_buffer *buf  __attribute__((unused)),
 595                 struct lttng_ust_lib_ring_buffer_channel *chan,
 596                 uint64_t *ts)
 597 {
 598         *ts = client_ring_buffer_clock_read(chan);
 599
 600         return 0;
 601 }
 602
 603 static int client_sequence_number(struct lttng_ust_lib_ring_buffer *buf,
 604                 struct lttng_ust_lib_ring_buffer_channel *chan,
 605                 uint64_t *seq)
 606 {
 607         struct lttng_ust_shm_handle *handle = chan->handle;
 608         struct packet_header *header;
 609
 610         header = client_packet_header(buf, handle);
 611         if (!header)
 612                 return -1;
 613         *seq = header->ctx.packet_seq_num;
 614         return 0;
 615 }
 616
 617 static int client_instance_id(struct lttng_ust_lib_ring_buffer *buf,
 618                 struct lttng_ust_lib_ring_buffer_channel *chan __attribute__((unused)),
 619                 uint64_t *id)
 620 {
 621         *id = buf->backend.cpu;
 622
 623         return 0;
 624 }
 625
 626 static const
 627 struct lttng_ust_client_lib_ring_buffer_client_cb client_cb = {
 628         .parent = {
 629                 .ring_buffer_clock_read = client_ring_buffer_clock_read,
 630                 .record_header_size = client_record_header_size,
 631                 .subbuffer_header_size = client_packet_header_size,
 632                 .buffer_begin = client_buffer_begin,
 633                 .buffer_end = client_buffer_end,
 634                 .buffer_create = client_buffer_create,
 635                 .buffer_finalize = client_buffer_finalize,
 636                 .content_size_field = client_content_size_field,
 637                 .packet_size_field = client_packet_size_field,
 638         },
 639         .timestamp_begin = client_timestamp_begin,
 640         .timestamp_end = client_timestamp_end,
 641         .events_discarded = client_events_discarded,
 642         .content_size = client_content_size,
 643         .packet_size = client_packet_size,
 644         .stream_id = client_stream_id,
 645         .current_timestamp = client_current_timestamp,
 646         .sequence_number = client_sequence_number,
 647         .instance_id = client_instance_id,
 648 };
 649
 650 static const struct lttng_ust_lib_ring_buffer_config client_config = {
 651         .cb.ring_buffer_clock_read = client_ring_buffer_clock_read,
 652         .cb.record_header_size = client_record_header_size,
 653         .cb.subbuffer_header_size = client_packet_header_size,
 654         .cb.buffer_begin = client_buffer_begin,
 655         .cb.buffer_end = client_buffer_end,
 656         .cb.buffer_create = client_buffer_create,
 657         .cb.buffer_finalize = client_buffer_finalize,
 658         .cb.content_size_field = client_content_size_field,
 659         .cb.packet_size_field = client_packet_size_field,
 660
 661         .tsc_bits = LTTNG_COMPACT_TSC_BITS,
 662         .alloc = RING_BUFFER_ALLOC_PER_CPU,
 663         .sync = RING_BUFFER_SYNC_GLOBAL,
 664         .mode = RING_BUFFER_MODE_TEMPLATE,
 665         .backend = RING_BUFFER_PAGE,
 666         .output = RING_BUFFER_MMAP,
 667         .oops = RING_BUFFER_OOPS_CONSISTENCY,
 668         .ipi = RING_BUFFER_NO_IPI_BARRIER,
 669         .wakeup = LTTNG_CLIENT_WAKEUP,
 670         .client_type = LTTNG_CLIENT_TYPE,
 671
 672         .cb_ptr = &client_cb.parent,
 673 };
 674
 675 static
 676 struct lttng_ust_channel_buffer *_channel_create(const char *name,
 677                                 void *buf_addr,
 678                                 size_t subbuf_size, size_t num_subbuf,
 679                                 unsigned int switch_timer_interval,
 680                                 unsigned int read_timer_interval,
 681                                 unsigned char *uuid,
 682                                 uint32_t chan_id,
 683                                 const int *stream_fds, int nr_stream_fds,
 684                                 int64_t blocking_timeout)
 685 {
 686         struct lttng_ust_abi_channel_config chan_priv_init;
 687         struct lttng_ust_shm_handle *handle;
 688         struct lttng_ust_channel_buffer *lttng_chan_buf;
 689
 690         lttng_chan_buf = lttng_ust_alloc_channel_buffer();
 691         if (!lttng_chan_buf)
 692                 return NULL;
 693         memcpy(lttng_chan_buf->priv->uuid, uuid, LTTNG_UST_UUID_LEN);
 694         lttng_chan_buf->priv->id = chan_id;
 695
 696         memset(&chan_priv_init, 0, sizeof(chan_priv_init));
 697         memcpy(chan_priv_init.uuid, uuid, LTTNG_UST_UUID_LEN);
 698         chan_priv_init.id = chan_id;
 699
 700         handle = channel_create(&client_config, name,
 701                         __alignof__(struct lttng_ust_abi_channel_config),
 702                         sizeof(struct lttng_ust_abi_channel_config),
 703                         &chan_priv_init,
 704                         lttng_chan_buf, buf_addr, subbuf_size, num_subbuf,
 705                         switch_timer_interval, read_timer_interval,
 706                         stream_fds, nr_stream_fds, blocking_timeout);
 707         if (!handle)
 708                 goto error;
 709         lttng_chan_buf->priv->rb_chan = shmp(handle, handle->chan);
 710         return lttng_chan_buf;
 711
 712 error:
 713         lttng_ust_free_channel_common(lttng_chan_buf->parent);
 714         return NULL;
 715 }
 716
 717 static
 718 void lttng_channel_destroy(struct lttng_ust_channel_buffer *lttng_chan_buf)
 719 {
 720         channel_destroy(lttng_chan_buf->priv->rb_chan, lttng_chan_buf->priv->rb_chan->handle, 1);
 721         lttng_ust_free_channel_common(lttng_chan_buf->parent);
 722 }
 723
 724 static
 725 int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx)
 726 {
 727         struct lttng_ust_event_recorder *event_recorder = ctx->client_priv;
 728         struct lttng_ust_channel_buffer *lttng_chan = event_recorder->chan;
 729         struct lttng_client_ctx client_ctx;
 730         int ret, nesting;
 731         struct lttng_ust_lib_ring_buffer_ctx_private *private_ctx;
 732         uint32_t event_id;
 733
 734         event_id = event_recorder->priv->id;
 735         client_ctx.chan_ctx = lttng_ust_rcu_dereference(lttng_chan->priv->ctx);
 736         client_ctx.event_ctx = lttng_ust_rcu_dereference(event_recorder->priv->ctx);
 737         /* Compute internal size of context structures. */
 738         ctx_get_struct_size(client_ctx.chan_ctx, &client_ctx.packet_context_len,
 739                         APP_CTX_ENABLED);
 740         ctx_get_struct_size(client_ctx.event_ctx, &client_ctx.event_context_len,
 741                         APP_CTX_ENABLED);
 742
 743         nesting = lib_ring_buffer_nesting_inc(&client_config);
 744         if (nesting < 0)
 745                 return -EPERM;
 746
 747         private_ctx = &URCU_TLS(private_ctx_stack)[nesting];
 748         memset(private_ctx, 0, sizeof(*private_ctx));
 749         private_ctx->pub = ctx;
 750         private_ctx->chan = lttng_chan->priv->rb_chan;
 751
 752         ctx->priv = private_ctx;
 753
 754         switch (lttng_chan->priv->header_type) {
 755         case 1: /* compact */
 756                 if (event_id > 30)
 757                         private_ctx->rflags |= LTTNG_RFLAG_EXTENDED;
 758                 break;
 759         case 2: /* large */
 760                 if (event_id > 65534)
 761                         private_ctx->rflags |= LTTNG_RFLAG_EXTENDED;
 762                 break;
 763         default:
 764                 WARN_ON_ONCE(1);
 765         }
 766
 767         ret = lib_ring_buffer_reserve(&client_config, ctx, &client_ctx);
 768         if (caa_unlikely(ret))
 769                 goto put;
 770         if (lib_ring_buffer_backend_get_pages(&client_config, ctx,
 771                         &private_ctx->backend_pages)) {
 772                 ret = -EPERM;
 773                 goto put;
 774         }
 775         lttng_write_event_header(&client_config, ctx, &client_ctx, event_id);
 776         return 0;
 777 put:
 778         lib_ring_buffer_nesting_dec(&client_config);
 779         return ret;
 780 }
 781
 782 static
 783 void lttng_event_commit(struct lttng_ust_lib_ring_buffer_ctx *ctx)
 784 {
 785         lib_ring_buffer_commit(&client_config, ctx);
 786         lib_ring_buffer_nesting_dec(&client_config);
 787 }
 788
 789 static
 790 void lttng_event_write(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 791                 const void *src, size_t len, size_t alignment)
 792 {
 793         lttng_ust_lib_ring_buffer_align_ctx(ctx, alignment);
 794         lib_ring_buffer_write(&client_config, ctx, src, len);
 795 }
 796
 797 static
 798 void lttng_event_strcpy(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 799                 const char *src, size_t len)
 800 {
 801         lib_ring_buffer_strcpy(&client_config, ctx, src, len, '#');
 802 }
 803
 804 static
 805 void lttng_event_pstrcpy_pad(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 806                 const char *src, size_t len)
 807 {
 808         lib_ring_buffer_pstrcpy(&client_config, ctx, src, len, '\0');
 809 }
 810
 811 static
 812 int lttng_is_finalized(struct lttng_ust_channel_buffer *chan)
 813 {
 814         struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan;
 815
 816         return lib_ring_buffer_channel_is_finalized(rb_chan);
 817 }
 818
 819 static
 820 int lttng_is_disabled(struct lttng_ust_channel_buffer *chan)
 821 {
 822         struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan;
 823
 824         return lib_ring_buffer_channel_is_disabled(rb_chan);
 825 }
 826
 827 static
 828 int lttng_flush_buffer(struct lttng_ust_channel_buffer *chan)
 829 {
 830         struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan;
 831         struct lttng_ust_lib_ring_buffer *buf;
 832         int cpu;
 833
 834         for_each_channel_cpu(cpu, rb_chan) {
 835                 int shm_fd, wait_fd, wakeup_fd;
 836                 uint64_t memory_map_size;
 837
 838                 buf = channel_get_ring_buffer(&client_config, rb_chan,
 839                                 cpu, rb_chan->handle, &shm_fd, &wait_fd,
 840                                 &wakeup_fd, &memory_map_size);
 841                 lib_ring_buffer_switch(&client_config, buf,
 842                                 SWITCH_ACTIVE, rb_chan->handle);
 843         }
 844         return 0;
 845 }
 846
 847 static struct lttng_transport lttng_relay_transport = {
 848         .name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap",
 849         .ops = {
 850                 .struct_size = sizeof(struct lttng_ust_channel_buffer_ops),
 851                 .priv = __LTTNG_COMPOUND_LITERAL(struct lttng_ust_channel_buffer_ops_private, {
 852                         .pub = &lttng_relay_transport.ops,
 853                         .channel_create = _channel_create,
 854                         .channel_destroy = lttng_channel_destroy,
 855                         .packet_avail_size = NULL,      /* Would be racy anyway */
 856                         .is_finalized = lttng_is_finalized,
 857                         .is_disabled = lttng_is_disabled,
 858                         .flush_buffer = lttng_flush_buffer,
 859                 }),
 860                 .event_reserve = lttng_event_reserve,
 861                 .event_commit = lttng_event_commit,
 862                 .event_write = lttng_event_write,
 863                 .event_strcpy = lttng_event_strcpy,
 864                 .event_pstrcpy_pad = lttng_event_pstrcpy_pad,
 865         },
 866         .client_config = &client_config,
 867 };
 868
 869 void RING_BUFFER_MODE_TEMPLATE_INIT(void)
 870 {
 871         DBG("LTT : ltt ring buffer client \"%s\" init\n",
 872                 "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap");
 873         lttng_transport_register(&lttng_relay_transport);
 874 }
 875
 876 void RING_BUFFER_MODE_TEMPLATE_EXIT(void)
 877 {
 878         DBG("LTT : ltt ring buffer client \"%s\" exit\n",
 879                 "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap");
 880         lttng_transport_unregister(&lttng_relay_transport);
 881 }