liblttng-ust/lttng-ring-buffer-client.h

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * LTTng lib ring buffer client template.
   7  */
   8
   9 #include <limits.h>
  10 #include <stddef.h>
  11 #include <stdint.h>
  12
  13 #include <ust-events-internal.h>
  14 #include <lttng/urcu/pointer.h>
  15 #include "ust-bitfield.h"
  16 #include "ust-compat.h"
  17 #include "clock.h"
  18 #include "context-internal.h"
  19 #include "lttng-tracer.h"
  20 #include "../libringbuffer/frontend_types.h"
  21 #include <urcu/tls-compat.h>
  22
  23 #define LTTNG_COMPACT_EVENT_BITS       5
  24 #define LTTNG_COMPACT_TSC_BITS         27
  25
  26 enum app_ctx_mode {
  27         APP_CTX_DISABLED,
  28         APP_CTX_ENABLED,
  29 };
  30
  31 /*
  32  * Keep the natural field alignment for _each field_ within this structure if
  33  * you ever add/remove a field from this header. Packed attribute is not used
  34  * because gcc generates poor code on at least powerpc and mips. Don't ever
  35  * let gcc add padding between the structure elements.
  36  */
  37
  38 struct packet_header {
  39         /* Trace packet header */
  40         uint32_t magic;                 /*
  41                                          * Trace magic number.
  42                                          * contains endianness information.
  43                                          */
  44         uint8_t uuid[LTTNG_UST_UUID_LEN];
  45         uint32_t stream_id;
  46         uint64_t stream_instance_id;
  47
  48         struct {
  49                 /* Stream packet context */
  50                 uint64_t timestamp_begin;       /* Cycle count at subbuffer start */
  51                 uint64_t timestamp_end;         /* Cycle count at subbuffer end */
  52                 uint64_t content_size;          /* Size of data in subbuffer */
  53                 uint64_t packet_size;           /* Subbuffer size (include padding) */
  54                 uint64_t packet_seq_num;        /* Packet sequence number */
  55                 unsigned long events_discarded; /*
  56                                                  * Events lost in this subbuffer since
  57                                                  * the beginning of the trace.
  58                                                  * (may overflow)
  59                                                  */
  60                 uint32_t cpu_id;                /* CPU id associated with stream */
  61                 uint8_t header_end;             /* End of header */
  62         } ctx;
  63 };
  64
  65 struct lttng_client_ctx {
  66         size_t packet_context_len;
  67         size_t event_context_len;
  68         struct lttng_ust_ctx *chan_ctx;
  69         struct lttng_ust_ctx *event_ctx;
  70 };
  71
  72 /*
  73  * Indexed by lib_ring_buffer_nesting_count().
  74  */
  75 typedef struct lttng_ust_lib_ring_buffer_ctx_private private_ctx_stack_t[LIB_RING_BUFFER_MAX_NESTING];
  76 static DEFINE_URCU_TLS(private_ctx_stack_t, private_ctx_stack);
  77
  78 /*
  79  * Force a read (imply TLS fixup for dlopen) of TLS variables.
  80  */
  81 void RING_BUFFER_MODE_TEMPLATE_TLS_FIXUP(void)
  82 {
  83         asm volatile ("" : : "m" (URCU_TLS(private_ctx_stack)));
  84 }
  85
  86 static inline uint64_t lib_ring_buffer_clock_read(struct lttng_ust_lib_ring_buffer_channel *chan)
  87 {
  88         return trace_clock_read64();
  89 }
  90
  91 static inline
  92 size_t ctx_get_aligned_size(size_t offset, struct lttng_ust_ctx *ctx,
  93                 size_t ctx_len)
  94 {
  95         size_t orig_offset = offset;
  96
  97         if (caa_likely(!ctx))
  98                 return 0;
  99         offset += lttng_ust_lib_ring_buffer_align(offset, ctx->largest_align);
 100         offset += ctx_len;
 101         return offset - orig_offset;
 102 }
 103
 104 static inline
 105 void ctx_get_struct_size(struct lttng_ust_ctx *ctx, size_t *ctx_len,
 106                 enum app_ctx_mode mode)
 107 {
 108         int i;
 109         size_t offset = 0;
 110
 111         if (caa_likely(!ctx)) {
 112                 *ctx_len = 0;
 113                 return;
 114         }
 115         for (i = 0; i < ctx->nr_fields; i++) {
 116                 if (mode == APP_CTX_ENABLED) {
 117                         offset += ctx->fields[i]->get_size(ctx->fields[i], offset);
 118                 } else {
 119                         if (lttng_context_is_app(ctx->fields[i]->event_field->name)) {
 120                                 /*
 121                                  * Before UST 2.8, we cannot use the
 122                                  * application context, because we
 123                                  * cannot trust that the handler used
 124                                  * for get_size is the same used for
 125                                  * ctx_record, which would result in
 126                                  * corrupted traces when tracing
 127                                  * concurrently with application context
 128                                  * register/unregister.
 129                                  */
 130                                 offset += lttng_ust_dummy_get_size(ctx->fields[i], offset);
 131                         } else {
 132                                 offset += ctx->fields[i]->get_size(ctx->fields[i], offset);
 133                         }
 134                 }
 135         }
 136         *ctx_len = offset;
 137 }
 138
 139 static inline
 140 void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx,
 141                 struct lttng_ust_channel_buffer *chan,
 142                 struct lttng_ust_ctx *ctx,
 143                 enum app_ctx_mode mode)
 144 {
 145         int i;
 146
 147         if (caa_likely(!ctx))
 148                 return;
 149         lttng_ust_lib_ring_buffer_align_ctx(bufctx, ctx->largest_align);
 150         for (i = 0; i < ctx->nr_fields; i++) {
 151                 if (mode == APP_CTX_ENABLED) {
 152                         ctx->fields[i]->record(ctx->fields[i], bufctx, chan);
 153                 } else {
 154                         if (lttng_context_is_app(ctx->fields[i]->event_field->name)) {
 155                                 /*
 156                                  * Before UST 2.8, we cannot use the
 157                                  * application context, because we
 158                                  * cannot trust that the handler used
 159                                  * for get_size is the same used for
 160                                  * ctx_record, which would result in
 161                                  * corrupted traces when tracing
 162                                  * concurrently with application context
 163                                  * register/unregister.
 164                                  */
 165                                 lttng_ust_dummy_record(ctx->fields[i], bufctx, chan);
 166                         } else {
 167                                 ctx->fields[i]->record(ctx->fields[i], bufctx, chan);
 168                         }
 169                 }
 170         }
 171 }
 172
 173 /*
 174  * record_header_size - Calculate the header size and padding necessary.
 175  * @config: ring buffer instance configuration
 176  * @chan: channel
 177  * @offset: offset in the write buffer
 178  * @pre_header_padding: padding to add before the header (output)
 179  * @ctx: reservation context
 180  *
 181  * Returns the event header size (including padding).
 182  *
 183  * The payload must itself determine its own alignment from the biggest type it
 184  * contains.
 185  */
 186 static __inline__
 187 size_t record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
 188                                  struct lttng_ust_lib_ring_buffer_channel *chan,
 189                                  size_t offset,
 190                                  size_t *pre_header_padding,
 191                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 192                                  struct lttng_client_ctx *client_ctx)
 193 {
 194         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan);
 195         size_t orig_offset = offset;
 196         size_t padding;
 197
 198         switch (lttng_chan->priv->header_type) {
 199         case 1: /* compact */
 200                 padding = lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint32_t));
 201                 offset += padding;
 202                 if (!(ctx->priv->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 203                         offset += sizeof(uint32_t);     /* id and timestamp */
 204                 } else {
 205                         /* Minimum space taken by LTTNG_COMPACT_EVENT_BITS id */
 206                         offset += (LTTNG_COMPACT_EVENT_BITS + CHAR_BIT - 1) / CHAR_BIT;
 207                         /* Align extended struct on largest member */
 208                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 209                         offset += sizeof(uint32_t);     /* id */
 210                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 211                         offset += sizeof(uint64_t);     /* timestamp */
 212                 }
 213                 break;
 214         case 2: /* large */
 215                 padding = lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint16_t));
 216                 offset += padding;
 217                 offset += sizeof(uint16_t);
 218                 if (!(ctx->priv->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 219                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint32_t));
 220                         offset += sizeof(uint32_t);     /* timestamp */
 221                 } else {
 222                         /* Align extended struct on largest member */
 223                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 224                         offset += sizeof(uint32_t);     /* id */
 225                         offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t));
 226                         offset += sizeof(uint64_t);     /* timestamp */
 227                 }
 228                 break;
 229         default:
 230                 padding = 0;
 231                 WARN_ON_ONCE(1);
 232         }
 233         offset += ctx_get_aligned_size(offset, client_ctx->chan_ctx,
 234                         client_ctx->packet_context_len);
 235         offset += ctx_get_aligned_size(offset, client_ctx->event_ctx,
 236                         client_ctx->event_context_len);
 237         *pre_header_padding = padding;
 238         return offset - orig_offset;
 239 }
 240
 241 #include "../libringbuffer/api.h"
 242 #include "lttng-rb-clients.h"
 243
 244 static
 245 void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
 246                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 247                                  struct lttng_client_ctx *client_ctx,
 248                                  uint32_t event_id);
 249
 250 /*
 251  * lttng_write_event_header
 252  *
 253  * Writes the event header to the offset (already aligned on 32-bits).
 254  *
 255  * @config: ring buffer instance configuration
 256  * @ctx: reservation context
 257  * @event_id: event ID
 258  */
 259 static __inline__
 260 void lttng_write_event_header(const struct lttng_ust_lib_ring_buffer_config *config,
 261                             struct lttng_ust_lib_ring_buffer_ctx *ctx,
 262                             struct lttng_client_ctx *client_ctx,
 263                             uint32_t event_id)
 264 {
 265         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(ctx->priv->chan);
 266
 267         if (caa_unlikely(ctx->priv->rflags))
 268                 goto slow_path;
 269
 270         switch (lttng_chan->priv->header_type) {
 271         case 1: /* compact */
 272         {
 273                 uint32_t id_time = 0;
 274
 275                 bt_bitfield_write(&id_time, uint32_t,
 276                                 0,
 277                                 LTTNG_COMPACT_EVENT_BITS,
 278                                 event_id);
 279                 bt_bitfield_write(&id_time, uint32_t,
 280                                 LTTNG_COMPACT_EVENT_BITS,
 281                                 LTTNG_COMPACT_TSC_BITS,
 282                                 ctx->priv->tsc);
 283                 lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
 284                 break;
 285         }
 286         case 2: /* large */
 287         {
 288                 uint32_t timestamp = (uint32_t) ctx->priv->tsc;
 289                 uint16_t id = event_id;
 290
 291                 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 292                 lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint32_t));
 293                 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 294                 break;
 295         }
 296         default:
 297                 WARN_ON_ONCE(1);
 298         }
 299
 300         ctx_record(ctx, lttng_chan, client_ctx->chan_ctx, APP_CTX_ENABLED);
 301         ctx_record(ctx, lttng_chan, client_ctx->event_ctx, APP_CTX_ENABLED);
 302         lttng_ust_lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
 303
 304         return;
 305
 306 slow_path:
 307         lttng_write_event_header_slow(config, ctx, client_ctx, event_id);
 308 }
 309
 310 static
 311 void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
 312                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 313                                  struct lttng_client_ctx *client_ctx,
 314                                  uint32_t event_id)
 315 {
 316         struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv;
 317         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(ctx->priv->chan);
 318
 319         switch (lttng_chan->priv->header_type) {
 320         case 1: /* compact */
 321                 if (!(ctx_private->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 322                         uint32_t id_time = 0;
 323
 324                         bt_bitfield_write(&id_time, uint32_t,
 325                                         0,
 326                                         LTTNG_COMPACT_EVENT_BITS,
 327                                         event_id);
 328                         bt_bitfield_write(&id_time, uint32_t,
 329                                         LTTNG_COMPACT_EVENT_BITS,
 330                                         LTTNG_COMPACT_TSC_BITS,
 331                                         ctx_private->tsc);
 332                         lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
 333                 } else {
 334                         uint8_t id = 0;
 335                         uint64_t timestamp = ctx_private->tsc;
 336
 337                         bt_bitfield_write(&id, uint8_t,
 338                                         0,
 339                                         LTTNG_COMPACT_EVENT_BITS,
 340                                         31);
 341                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 342                         /* Align extended struct on largest member */
 343                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 344                         lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
 345                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 346                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 347                 }
 348                 break;
 349         case 2: /* large */
 350         {
 351                 if (!(ctx_private->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 352                         uint32_t timestamp = (uint32_t) ctx_private->tsc;
 353                         uint16_t id = event_id;
 354
 355                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 356                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint32_t));
 357                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 358                 } else {
 359                         uint16_t id = 65535;
 360                         uint64_t timestamp = ctx_private->tsc;
 361
 362                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 363                         /* Align extended struct on largest member */
 364                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 365                         lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
 366                         lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t));
 367                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 368                 }
 369                 break;
 370         }
 371         default:
 372                 WARN_ON_ONCE(1);
 373         }
 374         ctx_record(ctx, lttng_chan, client_ctx->chan_ctx, APP_CTX_ENABLED);
 375         ctx_record(ctx, lttng_chan, client_ctx->event_ctx, APP_CTX_ENABLED);
 376         lttng_ust_lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
 377 }
 378
 379 static const struct lttng_ust_lib_ring_buffer_config client_config;
 380
 381 static uint64_t client_ring_buffer_clock_read(struct lttng_ust_lib_ring_buffer_channel *chan)
 382 {
 383         return lib_ring_buffer_clock_read(chan);
 384 }
 385
 386 static
 387 size_t client_record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
 388                                  struct lttng_ust_lib_ring_buffer_channel *chan,
 389                                  size_t offset,
 390                                  size_t *pre_header_padding,
 391                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 392                                  void *client_ctx)
 393 {
 394         return record_header_size(config, chan, offset,
 395                                   pre_header_padding, ctx, client_ctx);
 396 }
 397
 398 /**
 399  * client_packet_header_size - called on buffer-switch to a new sub-buffer
 400  *
 401  * Return header size without padding after the structure. Don't use packed
 402  * structure because gcc generates inefficient code on some architectures
 403  * (powerpc, mips..)
 404  */
 405 static size_t client_packet_header_size(void)
 406 {
 407         return offsetof(struct packet_header, ctx.header_end);
 408 }
 409
 410 static void client_buffer_begin(struct lttng_ust_lib_ring_buffer *buf, uint64_t tsc,
 411                                 unsigned int subbuf_idx,
 412                                 struct lttng_ust_shm_handle *handle)
 413 {
 414         struct lttng_ust_lib_ring_buffer_channel *chan = shmp(handle, buf->backend.chan);
 415         struct packet_header *header =
 416                 (struct packet_header *)
 417                         lib_ring_buffer_offset_address(&buf->backend,
 418                                 subbuf_idx * chan->backend.subbuf_size,
 419                                 handle);
 420         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan);
 421         uint64_t cnt = shmp_index(handle, buf->backend.buf_cnt, subbuf_idx)->seq_cnt;
 422
 423         assert(header);
 424         if (!header)
 425                 return;
 426         header->magic = CTF_MAGIC_NUMBER;
 427         memcpy(header->uuid, lttng_chan->priv->uuid, sizeof(lttng_chan->priv->uuid));
 428         header->stream_id = lttng_chan->priv->id;
 429         header->stream_instance_id = buf->backend.cpu;
 430         header->ctx.timestamp_begin = tsc;
 431         header->ctx.timestamp_end = 0;
 432         header->ctx.content_size = ~0ULL; /* for debugging */
 433         header->ctx.packet_size = ~0ULL;
 434         header->ctx.packet_seq_num = chan->backend.num_subbuf * cnt + subbuf_idx;
 435         header->ctx.events_discarded = 0;
 436         header->ctx.cpu_id = buf->backend.cpu;
 437 }
 438
 439 /*
 440  * offset is assumed to never be 0 here : never deliver a completely empty
 441  * subbuffer. data_size is between 1 and subbuf_size.
 442  */
 443 static void client_buffer_end(struct lttng_ust_lib_ring_buffer *buf, uint64_t tsc,
 444                               unsigned int subbuf_idx, unsigned long data_size,
 445                               struct lttng_ust_shm_handle *handle)
 446 {
 447         struct lttng_ust_lib_ring_buffer_channel *chan = shmp(handle, buf->backend.chan);
 448         struct packet_header *header =
 449                 (struct packet_header *)
 450                         lib_ring_buffer_offset_address(&buf->backend,
 451                                 subbuf_idx * chan->backend.subbuf_size,
 452                                 handle);
 453         unsigned long records_lost = 0;
 454
 455         assert(header);
 456         if (!header)
 457                 return;
 458         header->ctx.timestamp_end = tsc;
 459         header->ctx.content_size =
 460                 (uint64_t) data_size * CHAR_BIT;                /* in bits */
 461         header->ctx.packet_size =
 462                 (uint64_t) LTTNG_UST_PAGE_ALIGN(data_size) * CHAR_BIT;  /* in bits */
 463
 464         records_lost += lib_ring_buffer_get_records_lost_full(&client_config, buf);
 465         records_lost += lib_ring_buffer_get_records_lost_wrap(&client_config, buf);
 466         records_lost += lib_ring_buffer_get_records_lost_big(&client_config, buf);
 467         header->ctx.events_discarded = records_lost;
 468 }
 469
 470 static int client_buffer_create(struct lttng_ust_lib_ring_buffer *buf, void *priv,
 471                                 int cpu, const char *name, struct lttng_ust_shm_handle *handle)
 472 {
 473         return 0;
 474 }
 475
 476 static void client_buffer_finalize(struct lttng_ust_lib_ring_buffer *buf, void *priv, int cpu, struct lttng_ust_shm_handle *handle)
 477 {
 478 }
 479
 480 static void client_content_size_field(const struct lttng_ust_lib_ring_buffer_config *config,
 481                                       size_t *offset, size_t *length)
 482 {
 483         *offset = offsetof(struct packet_header, ctx.content_size);
 484         *length = sizeof(((struct packet_header *) NULL)->ctx.content_size);
 485 }
 486
 487 static void client_packet_size_field(const struct lttng_ust_lib_ring_buffer_config *config,
 488                                       size_t *offset, size_t *length)
 489 {
 490         *offset = offsetof(struct packet_header, ctx.packet_size);
 491         *length = sizeof(((struct packet_header *) NULL)->ctx.packet_size);
 492 }
 493
 494 static struct packet_header *client_packet_header(struct lttng_ust_lib_ring_buffer *buf,
 495                 struct lttng_ust_shm_handle *handle)
 496 {
 497         return lib_ring_buffer_read_offset_address(&buf->backend, 0, handle);
 498 }
 499
 500 static int client_timestamp_begin(struct lttng_ust_lib_ring_buffer *buf,
 501                 struct lttng_ust_lib_ring_buffer_channel *chan,
 502                 uint64_t *timestamp_begin)
 503 {
 504         struct lttng_ust_shm_handle *handle = chan->handle;
 505         struct packet_header *header;
 506
 507         header = client_packet_header(buf, handle);
 508         if (!header)
 509                 return -1;
 510         *timestamp_begin = header->ctx.timestamp_begin;
 511         return 0;
 512 }
 513
 514 static int client_timestamp_end(struct lttng_ust_lib_ring_buffer *buf,
 515                 struct lttng_ust_lib_ring_buffer_channel *chan,
 516                 uint64_t *timestamp_end)
 517 {
 518         struct lttng_ust_shm_handle *handle = chan->handle;
 519         struct packet_header *header;
 520
 521         header = client_packet_header(buf, handle);
 522         if (!header)
 523                 return -1;
 524         *timestamp_end = header->ctx.timestamp_end;
 525         return 0;
 526 }
 527
 528 static int client_events_discarded(struct lttng_ust_lib_ring_buffer *buf,
 529                 struct lttng_ust_lib_ring_buffer_channel *chan,
 530                 uint64_t *events_discarded)
 531 {
 532         struct lttng_ust_shm_handle *handle = chan->handle;
 533         struct packet_header *header;
 534
 535         header = client_packet_header(buf, handle);
 536         if (!header)
 537                 return -1;
 538         *events_discarded = header->ctx.events_discarded;
 539         return 0;
 540 }
 541
 542 static int client_content_size(struct lttng_ust_lib_ring_buffer *buf,
 543                 struct lttng_ust_lib_ring_buffer_channel *chan,
 544                 uint64_t *content_size)
 545 {
 546         struct lttng_ust_shm_handle *handle = chan->handle;
 547         struct packet_header *header;
 548
 549         header = client_packet_header(buf, handle);
 550         if (!header)
 551                 return -1;
 552         *content_size = header->ctx.content_size;
 553         return 0;
 554 }
 555
 556 static int client_packet_size(struct lttng_ust_lib_ring_buffer *buf,
 557                 struct lttng_ust_lib_ring_buffer_channel *chan,
 558                 uint64_t *packet_size)
 559 {
 560         struct lttng_ust_shm_handle *handle = chan->handle;
 561         struct packet_header *header;
 562
 563         header = client_packet_header(buf, handle);
 564         if (!header)
 565                 return -1;
 566         *packet_size = header->ctx.packet_size;
 567         return 0;
 568 }
 569
 570 static int client_stream_id(struct lttng_ust_lib_ring_buffer *buf,
 571                 struct lttng_ust_lib_ring_buffer_channel *chan,
 572                 uint64_t *stream_id)
 573 {
 574         struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan);
 575
 576         *stream_id = lttng_chan->priv->id;
 577
 578         return 0;
 579 }
 580
 581 static int client_current_timestamp(struct lttng_ust_lib_ring_buffer *buf,
 582                 struct lttng_ust_lib_ring_buffer_channel *chan,
 583                 uint64_t *ts)
 584 {
 585         *ts = client_ring_buffer_clock_read(chan);
 586
 587         return 0;
 588 }
 589
 590 static int client_sequence_number(struct lttng_ust_lib_ring_buffer *buf,
 591                 struct lttng_ust_lib_ring_buffer_channel *chan,
 592                 uint64_t *seq)
 593 {
 594         struct lttng_ust_shm_handle *handle = chan->handle;
 595         struct packet_header *header;
 596
 597         header = client_packet_header(buf, handle);
 598         if (!header)
 599                 return -1;
 600         *seq = header->ctx.packet_seq_num;
 601         return 0;
 602 }
 603
 604 static int client_instance_id(struct lttng_ust_lib_ring_buffer *buf,
 605                 struct lttng_ust_lib_ring_buffer_channel *chan,
 606                 uint64_t *id)
 607 {
 608         *id = buf->backend.cpu;
 609
 610         return 0;
 611 }
 612
 613 static const
 614 struct lttng_ust_client_lib_ring_buffer_client_cb client_cb = {
 615         .parent = {
 616                 .ring_buffer_clock_read = client_ring_buffer_clock_read,
 617                 .record_header_size = client_record_header_size,
 618                 .subbuffer_header_size = client_packet_header_size,
 619                 .buffer_begin = client_buffer_begin,
 620                 .buffer_end = client_buffer_end,
 621                 .buffer_create = client_buffer_create,
 622                 .buffer_finalize = client_buffer_finalize,
 623                 .content_size_field = client_content_size_field,
 624                 .packet_size_field = client_packet_size_field,
 625         },
 626         .timestamp_begin = client_timestamp_begin,
 627         .timestamp_end = client_timestamp_end,
 628         .events_discarded = client_events_discarded,
 629         .content_size = client_content_size,
 630         .packet_size = client_packet_size,
 631         .stream_id = client_stream_id,
 632         .current_timestamp = client_current_timestamp,
 633         .sequence_number = client_sequence_number,
 634         .instance_id = client_instance_id,
 635 };
 636
 637 static const struct lttng_ust_lib_ring_buffer_config client_config = {
 638         .cb.ring_buffer_clock_read = client_ring_buffer_clock_read,
 639         .cb.record_header_size = client_record_header_size,
 640         .cb.subbuffer_header_size = client_packet_header_size,
 641         .cb.buffer_begin = client_buffer_begin,
 642         .cb.buffer_end = client_buffer_end,
 643         .cb.buffer_create = client_buffer_create,
 644         .cb.buffer_finalize = client_buffer_finalize,
 645         .cb.content_size_field = client_content_size_field,
 646         .cb.packet_size_field = client_packet_size_field,
 647
 648         .tsc_bits = LTTNG_COMPACT_TSC_BITS,
 649         .alloc = RING_BUFFER_ALLOC_PER_CPU,
 650         .sync = RING_BUFFER_SYNC_GLOBAL,
 651         .mode = RING_BUFFER_MODE_TEMPLATE,
 652         .backend = RING_BUFFER_PAGE,
 653         .output = RING_BUFFER_MMAP,
 654         .oops = RING_BUFFER_OOPS_CONSISTENCY,
 655         .ipi = RING_BUFFER_NO_IPI_BARRIER,
 656         .wakeup = LTTNG_CLIENT_WAKEUP,
 657         .client_type = LTTNG_CLIENT_TYPE,
 658
 659         .cb_ptr = &client_cb.parent,
 660 };
 661
 662 static
 663 struct lttng_ust_channel_buffer *_channel_create(const char *name,
 664                                 void *buf_addr,
 665                                 size_t subbuf_size, size_t num_subbuf,
 666                                 unsigned int switch_timer_interval,
 667                                 unsigned int read_timer_interval,
 668                                 unsigned char *uuid,
 669                                 uint32_t chan_id,
 670                                 const int *stream_fds, int nr_stream_fds,
 671                                 int64_t blocking_timeout)
 672 {
 673         struct lttng_ust_abi_channel_config chan_priv_init;
 674         struct lttng_ust_shm_handle *handle;
 675         struct lttng_ust_channel_buffer *lttng_chan_buf;
 676
 677         lttng_chan_buf = lttng_ust_alloc_channel_buffer();
 678         if (!lttng_chan_buf)
 679                 return NULL;
 680         memcpy(lttng_chan_buf->priv->uuid, uuid, LTTNG_UST_UUID_LEN);
 681         lttng_chan_buf->priv->id = chan_id;
 682
 683         memset(&chan_priv_init, 0, sizeof(chan_priv_init));
 684         memcpy(chan_priv_init.uuid, uuid, LTTNG_UST_UUID_LEN);
 685         chan_priv_init.id = chan_id;
 686
 687         handle = channel_create(&client_config, name,
 688                         __alignof__(struct lttng_ust_abi_channel_config),
 689                         sizeof(struct lttng_ust_abi_channel_config),
 690                         &chan_priv_init,
 691                         lttng_chan_buf, buf_addr, subbuf_size, num_subbuf,
 692                         switch_timer_interval, read_timer_interval,
 693                         stream_fds, nr_stream_fds, blocking_timeout);
 694         if (!handle)
 695                 goto error;
 696         lttng_chan_buf->priv->rb_chan = shmp(handle, handle->chan);
 697         return lttng_chan_buf;
 698
 699 error:
 700         lttng_ust_free_channel_common(lttng_chan_buf->parent);
 701         return NULL;
 702 }
 703
 704 static
 705 void lttng_channel_destroy(struct lttng_ust_channel_buffer *lttng_chan_buf)
 706 {
 707         channel_destroy(lttng_chan_buf->priv->rb_chan, lttng_chan_buf->priv->rb_chan->handle, 1);
 708         lttng_ust_free_channel_common(lttng_chan_buf->parent);
 709 }
 710
 711 static
 712 int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx)
 713 {
 714         struct lttng_ust_event_recorder *event_recorder = ctx->client_priv;
 715         struct lttng_ust_channel_buffer *lttng_chan = event_recorder->chan;
 716         struct lttng_client_ctx client_ctx;
 717         int ret, nesting;
 718         struct lttng_ust_lib_ring_buffer_ctx_private *private_ctx;
 719         uint32_t event_id;
 720
 721         event_id = event_recorder->priv->id;
 722         client_ctx.chan_ctx = lttng_ust_rcu_dereference(lttng_chan->priv->ctx);
 723         client_ctx.event_ctx = lttng_ust_rcu_dereference(event_recorder->priv->ctx);
 724         /* Compute internal size of context structures. */
 725         ctx_get_struct_size(client_ctx.chan_ctx, &client_ctx.packet_context_len,
 726                         APP_CTX_ENABLED);
 727         ctx_get_struct_size(client_ctx.event_ctx, &client_ctx.event_context_len,
 728                         APP_CTX_ENABLED);
 729
 730         nesting = lib_ring_buffer_nesting_inc(&client_config);
 731         if (nesting < 0)
 732                 return -EPERM;
 733
 734         private_ctx = &URCU_TLS(private_ctx_stack)[nesting];
 735         memset(private_ctx, 0, sizeof(*private_ctx));
 736         private_ctx->pub = ctx;
 737         private_ctx->chan = lttng_chan->priv->rb_chan;
 738
 739         ctx->priv = private_ctx;
 740
 741         switch (lttng_chan->priv->header_type) {
 742         case 1: /* compact */
 743                 if (event_id > 30)
 744                         private_ctx->rflags |= LTTNG_RFLAG_EXTENDED;
 745                 break;
 746         case 2: /* large */
 747                 if (event_id > 65534)
 748                         private_ctx->rflags |= LTTNG_RFLAG_EXTENDED;
 749                 break;
 750         default:
 751                 WARN_ON_ONCE(1);
 752         }
 753
 754         ret = lib_ring_buffer_reserve(&client_config, ctx, &client_ctx);
 755         if (caa_unlikely(ret))
 756                 goto put;
 757         if (lib_ring_buffer_backend_get_pages(&client_config, ctx,
 758                         &private_ctx->backend_pages)) {
 759                 ret = -EPERM;
 760                 goto put;
 761         }
 762         lttng_write_event_header(&client_config, ctx, &client_ctx, event_id);
 763         return 0;
 764 put:
 765         lib_ring_buffer_nesting_dec(&client_config);
 766         return ret;
 767 }
 768
 769 static
 770 void lttng_event_commit(struct lttng_ust_lib_ring_buffer_ctx *ctx)
 771 {
 772         lib_ring_buffer_commit(&client_config, ctx);
 773         lib_ring_buffer_nesting_dec(&client_config);
 774 }
 775
 776 static
 777 void lttng_event_write(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 778                 const void *src, size_t len, size_t alignment)
 779 {
 780         lttng_ust_lib_ring_buffer_align_ctx(ctx, alignment);
 781         lib_ring_buffer_write(&client_config, ctx, src, len);
 782 }
 783
 784 static
 785 void lttng_event_strcpy(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 786                 const char *src, size_t len)
 787 {
 788         lib_ring_buffer_strcpy(&client_config, ctx, src, len, '#');
 789 }
 790
 791 static
 792 void lttng_event_pstrcpy_pad(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 793                 const char *src, size_t len)
 794 {
 795         lib_ring_buffer_pstrcpy(&client_config, ctx, src, len, '\0');
 796 }
 797
 798 static
 799 int lttng_is_finalized(struct lttng_ust_channel_buffer *chan)
 800 {
 801         struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan;
 802
 803         return lib_ring_buffer_channel_is_finalized(rb_chan);
 804 }
 805
 806 static
 807 int lttng_is_disabled(struct lttng_ust_channel_buffer *chan)
 808 {
 809         struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan;
 810
 811         return lib_ring_buffer_channel_is_disabled(rb_chan);
 812 }
 813
 814 static
 815 int lttng_flush_buffer(struct lttng_ust_channel_buffer *chan)
 816 {
 817         struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan;
 818         struct lttng_ust_lib_ring_buffer *buf;
 819         int cpu;
 820
 821         for_each_channel_cpu(cpu, rb_chan) {
 822                 int shm_fd, wait_fd, wakeup_fd;
 823                 uint64_t memory_map_size;
 824
 825                 buf = channel_get_ring_buffer(&client_config, rb_chan,
 826                                 cpu, rb_chan->handle, &shm_fd, &wait_fd,
 827                                 &wakeup_fd, &memory_map_size);
 828                 lib_ring_buffer_switch(&client_config, buf,
 829                                 SWITCH_ACTIVE, rb_chan->handle);
 830         }
 831         return 0;
 832 }
 833
 834 static struct lttng_transport lttng_relay_transport = {
 835         .name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap",
 836         .ops = {
 837                 .struct_size = sizeof(struct lttng_ust_channel_buffer_ops),
 838                 .priv = __LTTNG_COMPOUND_LITERAL(struct lttng_ust_channel_buffer_ops_private, {
 839                         .pub = &lttng_relay_transport.ops,
 840                         .channel_create = _channel_create,
 841                         .channel_destroy = lttng_channel_destroy,
 842                         .packet_avail_size = NULL,      /* Would be racy anyway */
 843                         .is_finalized = lttng_is_finalized,
 844                         .is_disabled = lttng_is_disabled,
 845                         .flush_buffer = lttng_flush_buffer,
 846                 }),
 847                 .event_reserve = lttng_event_reserve,
 848                 .event_commit = lttng_event_commit,
 849                 .event_write = lttng_event_write,
 850                 .event_strcpy = lttng_event_strcpy,
 851                 .event_pstrcpy_pad = lttng_event_pstrcpy_pad,
 852         },
 853         .client_config = &client_config,
 854 };
 855
 856 void RING_BUFFER_MODE_TEMPLATE_INIT(void)
 857 {
 858         DBG("LTT : ltt ring buffer client \"%s\" init\n",
 859                 "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap");
 860         lttng_transport_register(&lttng_relay_transport);
 861 }
 862
 863 void RING_BUFFER_MODE_TEMPLATE_EXIT(void)
 864 {
 865         DBG("LTT : ltt ring buffer client \"%s\" exit\n",
 866                 "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap");
 867         lttng_transport_unregister(&lttng_relay_transport);
 868 }