liblttng-ust/lttng-ring-buffer-client.h

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * LTTng lib ring buffer client template.
   7  */
   8
   9 #include <stddef.h>
  10 #include <stdint.h>
  11
  12 #include <ust-events-internal.h>
  13 #include "ust-bitfield.h"
  14 #include "ust-compat.h"
  15 #include "clock.h"
  16 #include "context-internal.h"
  17 #include "lttng-tracer.h"
  18 #include "../libringbuffer/frontend_types.h"
  19
  20 #define LTTNG_COMPACT_EVENT_BITS       5
  21 #define LTTNG_COMPACT_TSC_BITS         27
  22
  23 enum app_ctx_mode {
  24         APP_CTX_DISABLED,
  25         APP_CTX_ENABLED,
  26 };
  27
  28 /*
  29  * Keep the natural field alignment for _each field_ within this structure if
  30  * you ever add/remove a field from this header. Packed attribute is not used
  31  * because gcc generates poor code on at least powerpc and mips. Don't ever
  32  * let gcc add padding between the structure elements.
  33  */
  34
  35 struct packet_header {
  36         /* Trace packet header */
  37         uint32_t magic;                 /*
  38                                          * Trace magic number.
  39                                          * contains endianness information.
  40                                          */
  41         uint8_t uuid[LTTNG_UST_UUID_LEN];
  42         uint32_t stream_id;
  43         uint64_t stream_instance_id;
  44
  45         struct {
  46                 /* Stream packet context */
  47                 uint64_t timestamp_begin;       /* Cycle count at subbuffer start */
  48                 uint64_t timestamp_end;         /* Cycle count at subbuffer end */
  49                 uint64_t content_size;          /* Size of data in subbuffer */
  50                 uint64_t packet_size;           /* Subbuffer size (include padding) */
  51                 uint64_t packet_seq_num;        /* Packet sequence number */
  52                 unsigned long events_discarded; /*
  53                                                  * Events lost in this subbuffer since
  54                                                  * the beginning of the trace.
  55                                                  * (may overflow)
  56                                                  */
  57                 uint32_t cpu_id;                /* CPU id associated with stream */
  58                 uint8_t header_end;             /* End of header */
  59         } ctx;
  60 };
  61
  62 struct lttng_client_ctx {
  63         size_t packet_context_len;
  64         size_t event_context_len;
  65 };
  66
  67 static inline uint64_t lib_ring_buffer_clock_read(struct channel *chan)
  68 {
  69         return trace_clock_read64();
  70 }
  71
  72 static inline
  73 size_t ctx_get_aligned_size(size_t offset, struct lttng_ctx *ctx,
  74                 size_t ctx_len)
  75 {
  76         size_t orig_offset = offset;
  77
  78         if (caa_likely(!ctx))
  79                 return 0;
  80         offset += lib_ring_buffer_align(offset, ctx->largest_align);
  81         offset += ctx_len;
  82         return offset - orig_offset;
  83 }
  84
  85 static inline
  86 void ctx_get_struct_size(struct lttng_ctx *ctx, size_t *ctx_len,
  87                 enum app_ctx_mode mode)
  88 {
  89         int i;
  90         size_t offset = 0;
  91
  92         if (caa_likely(!ctx)) {
  93                 *ctx_len = 0;
  94                 return;
  95         }
  96         for (i = 0; i < ctx->nr_fields; i++) {
  97                 if (mode == APP_CTX_ENABLED) {
  98                         offset += ctx->fields[i].get_size(&ctx->fields[i], offset);
  99                 } else {
 100                         if (lttng_context_is_app(ctx->fields[i].event_field.name)) {
 101                                 /*
 102                                  * Before UST 2.8, we cannot use the
 103                                  * application context, because we
 104                                  * cannot trust that the handler used
 105                                  * for get_size is the same used for
 106                                  * ctx_record, which would result in
 107                                  * corrupted traces when tracing
 108                                  * concurrently with application context
 109                                  * register/unregister.
 110                                  */
 111                                 offset += lttng_ust_dummy_get_size(&ctx->fields[i], offset);
 112                         } else {
 113                                 offset += ctx->fields[i].get_size(&ctx->fields[i], offset);
 114                         }
 115                 }
 116         }
 117         *ctx_len = offset;
 118 }
 119
 120 static inline
 121 void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx,
 122                 struct lttng_channel *chan,
 123                 struct lttng_ctx *ctx,
 124                 enum app_ctx_mode mode)
 125 {
 126         int i;
 127
 128         if (caa_likely(!ctx))
 129                 return;
 130         lib_ring_buffer_align_ctx(bufctx, ctx->largest_align);
 131         for (i = 0; i < ctx->nr_fields; i++) {
 132                 if (mode == APP_CTX_ENABLED) {
 133                         ctx->fields[i].record(&ctx->fields[i], bufctx, chan);
 134                 } else {
 135                         if (lttng_context_is_app(ctx->fields[i].event_field.name)) {
 136                                 /*
 137                                  * Before UST 2.8, we cannot use the
 138                                  * application context, because we
 139                                  * cannot trust that the handler used
 140                                  * for get_size is the same used for
 141                                  * ctx_record, which would result in
 142                                  * corrupted traces when tracing
 143                                  * concurrently with application context
 144                                  * register/unregister.
 145                                  */
 146                                 lttng_ust_dummy_record(&ctx->fields[i], bufctx, chan);
 147                         } else {
 148                                 ctx->fields[i].record(&ctx->fields[i], bufctx, chan);
 149                         }
 150                 }
 151         }
 152 }
 153
 154 /*
 155  * record_header_size - Calculate the header size and padding necessary.
 156  * @config: ring buffer instance configuration
 157  * @chan: channel
 158  * @offset: offset in the write buffer
 159  * @pre_header_padding: padding to add before the header (output)
 160  * @ctx: reservation context
 161  *
 162  * Returns the event header size (including padding).
 163  *
 164  * The payload must itself determine its own alignment from the biggest type it
 165  * contains.
 166  */
 167 static __inline__
 168 size_t record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
 169                                  struct channel *chan, size_t offset,
 170                                  size_t *pre_header_padding,
 171                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 172                                  struct lttng_client_ctx *client_ctx)
 173 {
 174         struct lttng_channel *lttng_chan = channel_get_private(chan);
 175         struct lttng_stack_ctx *lttng_ctx = ctx->priv;
 176         size_t orig_offset = offset;
 177         size_t padding;
 178
 179         switch (lttng_chan->header_type) {
 180         case 1: /* compact */
 181                 padding = lib_ring_buffer_align(offset, lttng_alignof(uint32_t));
 182                 offset += padding;
 183                 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 184                         offset += sizeof(uint32_t);     /* id and timestamp */
 185                 } else {
 186                         /* Minimum space taken by LTTNG_COMPACT_EVENT_BITS id */
 187                         offset += (LTTNG_COMPACT_EVENT_BITS + CHAR_BIT - 1) / CHAR_BIT;
 188                         /* Align extended struct on largest member */
 189                         offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
 190                         offset += sizeof(uint32_t);     /* id */
 191                         offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
 192                         offset += sizeof(uint64_t);     /* timestamp */
 193                 }
 194                 break;
 195         case 2: /* large */
 196                 padding = lib_ring_buffer_align(offset, lttng_alignof(uint16_t));
 197                 offset += padding;
 198                 offset += sizeof(uint16_t);
 199                 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 200                         offset += lib_ring_buffer_align(offset, lttng_alignof(uint32_t));
 201                         offset += sizeof(uint32_t);     /* timestamp */
 202                 } else {
 203                         /* Align extended struct on largest member */
 204                         offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
 205                         offset += sizeof(uint32_t);     /* id */
 206                         offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
 207                         offset += sizeof(uint64_t);     /* timestamp */
 208                 }
 209                 break;
 210         default:
 211                 padding = 0;
 212                 WARN_ON_ONCE(1);
 213         }
 214         offset += ctx_get_aligned_size(offset, lttng_ctx->chan_ctx,
 215                         client_ctx->packet_context_len);
 216         offset += ctx_get_aligned_size(offset, lttng_ctx->event_ctx,
 217                         client_ctx->event_context_len);
 218         *pre_header_padding = padding;
 219         return offset - orig_offset;
 220 }
 221
 222 #include "../libringbuffer/api.h"
 223 #include "lttng-rb-clients.h"
 224
 225 static
 226 void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
 227                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 228                                  uint32_t event_id);
 229
 230 /*
 231  * lttng_write_event_header
 232  *
 233  * Writes the event header to the offset (already aligned on 32-bits).
 234  *
 235  * @config: ring buffer instance configuration
 236  * @ctx: reservation context
 237  * @event_id: event ID
 238  */
 239 static __inline__
 240 void lttng_write_event_header(const struct lttng_ust_lib_ring_buffer_config *config,
 241                             struct lttng_ust_lib_ring_buffer_ctx *ctx,
 242                             uint32_t event_id)
 243 {
 244         struct lttng_channel *lttng_chan = channel_get_private(ctx->chan);
 245         struct lttng_stack_ctx *lttng_ctx = ctx->priv;
 246
 247         if (caa_unlikely(ctx->rflags))
 248                 goto slow_path;
 249
 250         switch (lttng_chan->header_type) {
 251         case 1: /* compact */
 252         {
 253                 uint32_t id_time = 0;
 254
 255                 bt_bitfield_write(&id_time, uint32_t,
 256                                 0,
 257                                 LTTNG_COMPACT_EVENT_BITS,
 258                                 event_id);
 259                 bt_bitfield_write(&id_time, uint32_t,
 260                                 LTTNG_COMPACT_EVENT_BITS,
 261                                 LTTNG_COMPACT_TSC_BITS,
 262                                 ctx->tsc);
 263                 lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
 264                 break;
 265         }
 266         case 2: /* large */
 267         {
 268                 uint32_t timestamp = (uint32_t) ctx->tsc;
 269                 uint16_t id = event_id;
 270
 271                 lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 272                 lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint32_t));
 273                 lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 274                 break;
 275         }
 276         default:
 277                 WARN_ON_ONCE(1);
 278         }
 279
 280         ctx_record(ctx, lttng_chan, lttng_ctx->chan_ctx, APP_CTX_ENABLED);
 281         ctx_record(ctx, lttng_chan, lttng_ctx->event_ctx, APP_CTX_ENABLED);
 282         lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
 283
 284         return;
 285
 286 slow_path:
 287         lttng_write_event_header_slow(config, ctx, event_id);
 288 }
 289
 290 static
 291 void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config,
 292                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 293                                  uint32_t event_id)
 294 {
 295         struct lttng_channel *lttng_chan = channel_get_private(ctx->chan);
 296         struct lttng_stack_ctx *lttng_ctx = ctx->priv;
 297
 298         switch (lttng_chan->header_type) {
 299         case 1: /* compact */
 300                 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 301                         uint32_t id_time = 0;
 302
 303                         bt_bitfield_write(&id_time, uint32_t,
 304                                         0,
 305                                         LTTNG_COMPACT_EVENT_BITS,
 306                                         event_id);
 307                         bt_bitfield_write(&id_time, uint32_t,
 308                                         LTTNG_COMPACT_EVENT_BITS,
 309                                         LTTNG_COMPACT_TSC_BITS,
 310                                         ctx->tsc);
 311                         lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time));
 312                 } else {
 313                         uint8_t id = 0;
 314                         uint64_t timestamp = ctx->tsc;
 315
 316                         bt_bitfield_write(&id, uint8_t,
 317                                         0,
 318                                         LTTNG_COMPACT_EVENT_BITS,
 319                                         31);
 320                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 321                         /* Align extended struct on largest member */
 322                         lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
 323                         lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
 324                         lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
 325                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 326                 }
 327                 break;
 328         case 2: /* large */
 329         {
 330                 if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) {
 331                         uint32_t timestamp = (uint32_t) ctx->tsc;
 332                         uint16_t id = event_id;
 333
 334                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 335                         lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint32_t));
 336                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 337                 } else {
 338                         uint16_t id = 65535;
 339                         uint64_t timestamp = ctx->tsc;
 340
 341                         lib_ring_buffer_write(config, ctx, &id, sizeof(id));
 342                         /* Align extended struct on largest member */
 343                         lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
 344                         lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id));
 345                         lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t));
 346                         lib_ring_buffer_write(config, ctx, &timestamp, sizeof(timestamp));
 347                 }
 348                 break;
 349         }
 350         default:
 351                 WARN_ON_ONCE(1);
 352         }
 353         ctx_record(ctx, lttng_chan, lttng_ctx->chan_ctx, APP_CTX_ENABLED);
 354         ctx_record(ctx, lttng_chan, lttng_ctx->event_ctx, APP_CTX_ENABLED);
 355         lib_ring_buffer_align_ctx(ctx, ctx->largest_align);
 356 }
 357
 358 static const struct lttng_ust_lib_ring_buffer_config client_config;
 359
 360 static uint64_t client_ring_buffer_clock_read(struct channel *chan)
 361 {
 362         return lib_ring_buffer_clock_read(chan);
 363 }
 364
 365 static
 366 size_t client_record_header_size(const struct lttng_ust_lib_ring_buffer_config *config,
 367                                  struct channel *chan, size_t offset,
 368                                  size_t *pre_header_padding,
 369                                  struct lttng_ust_lib_ring_buffer_ctx *ctx,
 370                                  void *client_ctx)
 371 {
 372         return record_header_size(config, chan, offset,
 373                                   pre_header_padding, ctx, client_ctx);
 374 }
 375
 376 /**
 377  * client_packet_header_size - called on buffer-switch to a new sub-buffer
 378  *
 379  * Return header size without padding after the structure. Don't use packed
 380  * structure because gcc generates inefficient code on some architectures
 381  * (powerpc, mips..)
 382  */
 383 static size_t client_packet_header_size(void)
 384 {
 385         return offsetof(struct packet_header, ctx.header_end);
 386 }
 387
 388 static void client_buffer_begin(struct lttng_ust_lib_ring_buffer *buf, uint64_t tsc,
 389                                 unsigned int subbuf_idx,
 390                                 struct lttng_ust_shm_handle *handle)
 391 {
 392         struct channel *chan = shmp(handle, buf->backend.chan);
 393         struct packet_header *header =
 394                 (struct packet_header *)
 395                         lib_ring_buffer_offset_address(&buf->backend,
 396                                 subbuf_idx * chan->backend.subbuf_size,
 397                                 handle);
 398         struct lttng_channel *lttng_chan = channel_get_private(chan);
 399         uint64_t cnt = shmp_index(handle, buf->backend.buf_cnt, subbuf_idx)->seq_cnt;
 400
 401         assert(header);
 402         if (!header)
 403                 return;
 404         header->magic = CTF_MAGIC_NUMBER;
 405         memcpy(header->uuid, lttng_chan->uuid, sizeof(lttng_chan->uuid));
 406         header->stream_id = lttng_chan->id;
 407         header->stream_instance_id = buf->backend.cpu;
 408         header->ctx.timestamp_begin = tsc;
 409         header->ctx.timestamp_end = 0;
 410         header->ctx.content_size = ~0ULL; /* for debugging */
 411         header->ctx.packet_size = ~0ULL;
 412         header->ctx.packet_seq_num = chan->backend.num_subbuf * cnt + subbuf_idx;
 413         header->ctx.events_discarded = 0;
 414         header->ctx.cpu_id = buf->backend.cpu;
 415 }
 416
 417 /*
 418  * offset is assumed to never be 0 here : never deliver a completely empty
 419  * subbuffer. data_size is between 1 and subbuf_size.
 420  */
 421 static void client_buffer_end(struct lttng_ust_lib_ring_buffer *buf, uint64_t tsc,
 422                               unsigned int subbuf_idx, unsigned long data_size,
 423                               struct lttng_ust_shm_handle *handle)
 424 {
 425         struct channel *chan = shmp(handle, buf->backend.chan);
 426         struct packet_header *header =
 427                 (struct packet_header *)
 428                         lib_ring_buffer_offset_address(&buf->backend,
 429                                 subbuf_idx * chan->backend.subbuf_size,
 430                                 handle);
 431         unsigned long records_lost = 0;
 432
 433         assert(header);
 434         if (!header)
 435                 return;
 436         header->ctx.timestamp_end = tsc;
 437         header->ctx.content_size =
 438                 (uint64_t) data_size * CHAR_BIT;                /* in bits */
 439         header->ctx.packet_size =
 440                 (uint64_t) LTTNG_UST_PAGE_ALIGN(data_size) * CHAR_BIT;  /* in bits */
 441
 442         records_lost += lib_ring_buffer_get_records_lost_full(&client_config, buf);
 443         records_lost += lib_ring_buffer_get_records_lost_wrap(&client_config, buf);
 444         records_lost += lib_ring_buffer_get_records_lost_big(&client_config, buf);
 445         header->ctx.events_discarded = records_lost;
 446 }
 447
 448 static int client_buffer_create(struct lttng_ust_lib_ring_buffer *buf, void *priv,
 449                                 int cpu, const char *name, struct lttng_ust_shm_handle *handle)
 450 {
 451         return 0;
 452 }
 453
 454 static void client_buffer_finalize(struct lttng_ust_lib_ring_buffer *buf, void *priv, int cpu, struct lttng_ust_shm_handle *handle)
 455 {
 456 }
 457
 458 static void client_content_size_field(const struct lttng_ust_lib_ring_buffer_config *config,
 459                                       size_t *offset, size_t *length)
 460 {
 461         *offset = offsetof(struct packet_header, ctx.content_size);
 462         *length = sizeof(((struct packet_header *) NULL)->ctx.content_size);
 463 }
 464
 465 static void client_packet_size_field(const struct lttng_ust_lib_ring_buffer_config *config,
 466                                       size_t *offset, size_t *length)
 467 {
 468         *offset = offsetof(struct packet_header, ctx.packet_size);
 469         *length = sizeof(((struct packet_header *) NULL)->ctx.packet_size);
 470 }
 471
 472 static struct packet_header *client_packet_header(struct lttng_ust_lib_ring_buffer *buf,
 473                 struct lttng_ust_shm_handle *handle)
 474 {
 475         return lib_ring_buffer_read_offset_address(&buf->backend, 0, handle);
 476 }
 477
 478 static int client_timestamp_begin(struct lttng_ust_lib_ring_buffer *buf,
 479                 struct lttng_ust_shm_handle *handle,
 480                 uint64_t *timestamp_begin)
 481 {
 482         struct packet_header *header;
 483
 484         header = client_packet_header(buf, handle);
 485         if (!header)
 486                 return -1;
 487         *timestamp_begin = header->ctx.timestamp_begin;
 488         return 0;
 489 }
 490
 491 static int client_timestamp_end(struct lttng_ust_lib_ring_buffer *buf,
 492                 struct lttng_ust_shm_handle *handle,
 493                 uint64_t *timestamp_end)
 494 {
 495         struct packet_header *header;
 496
 497         header = client_packet_header(buf, handle);
 498         if (!header)
 499                 return -1;
 500         *timestamp_end = header->ctx.timestamp_end;
 501         return 0;
 502 }
 503
 504 static int client_events_discarded(struct lttng_ust_lib_ring_buffer *buf,
 505                 struct lttng_ust_shm_handle *handle,
 506                 uint64_t *events_discarded)
 507 {
 508         struct packet_header *header;
 509
 510         header = client_packet_header(buf, handle);
 511         if (!header)
 512                 return -1;
 513         *events_discarded = header->ctx.events_discarded;
 514         return 0;
 515 }
 516
 517 static int client_content_size(struct lttng_ust_lib_ring_buffer *buf,
 518                 struct lttng_ust_shm_handle *handle,
 519                 uint64_t *content_size)
 520 {
 521         struct packet_header *header;
 522
 523         header = client_packet_header(buf, handle);
 524         if (!header)
 525                 return -1;
 526         *content_size = header->ctx.content_size;
 527         return 0;
 528 }
 529
 530 static int client_packet_size(struct lttng_ust_lib_ring_buffer *buf,
 531                 struct lttng_ust_shm_handle *handle,
 532                 uint64_t *packet_size)
 533 {
 534         struct packet_header *header;
 535
 536         header = client_packet_header(buf, handle);
 537         if (!header)
 538                 return -1;
 539         *packet_size = header->ctx.packet_size;
 540         return 0;
 541 }
 542
 543 static int client_stream_id(struct lttng_ust_lib_ring_buffer *buf,
 544                 struct lttng_ust_shm_handle *handle,
 545                 uint64_t *stream_id)
 546 {
 547         struct channel *chan = shmp(handle, buf->backend.chan);
 548         struct lttng_channel *lttng_chan = channel_get_private(chan);
 549
 550         *stream_id = lttng_chan->id;
 551
 552         return 0;
 553 }
 554
 555 static int client_current_timestamp(struct lttng_ust_lib_ring_buffer *buf,
 556                 struct lttng_ust_shm_handle *handle,
 557                 uint64_t *ts)
 558 {
 559         struct channel *chan;
 560
 561         chan = shmp(handle, handle->chan);
 562         *ts = client_ring_buffer_clock_read(chan);
 563
 564         return 0;
 565 }
 566
 567 static int client_sequence_number(struct lttng_ust_lib_ring_buffer *buf,
 568                 struct lttng_ust_shm_handle *handle,
 569                 uint64_t *seq)
 570 {
 571         struct packet_header *header;
 572
 573         header = client_packet_header(buf, handle);
 574         if (!header)
 575                 return -1;
 576         *seq = header->ctx.packet_seq_num;
 577         return 0;
 578 }
 579
 580 static int client_instance_id(struct lttng_ust_lib_ring_buffer *buf,
 581                 struct lttng_ust_shm_handle *handle,
 582                 uint64_t *id)
 583 {
 584         *id = buf->backend.cpu;
 585
 586         return 0;
 587 }
 588
 589 static const
 590 struct lttng_ust_client_lib_ring_buffer_client_cb client_cb = {
 591         .parent = {
 592                 .ring_buffer_clock_read = client_ring_buffer_clock_read,
 593                 .record_header_size = client_record_header_size,
 594                 .subbuffer_header_size = client_packet_header_size,
 595                 .buffer_begin = client_buffer_begin,
 596                 .buffer_end = client_buffer_end,
 597                 .buffer_create = client_buffer_create,
 598                 .buffer_finalize = client_buffer_finalize,
 599                 .content_size_field = client_content_size_field,
 600                 .packet_size_field = client_packet_size_field,
 601         },
 602         .timestamp_begin = client_timestamp_begin,
 603         .timestamp_end = client_timestamp_end,
 604         .events_discarded = client_events_discarded,
 605         .content_size = client_content_size,
 606         .packet_size = client_packet_size,
 607         .stream_id = client_stream_id,
 608         .current_timestamp = client_current_timestamp,
 609         .sequence_number = client_sequence_number,
 610         .instance_id = client_instance_id,
 611 };
 612
 613 static const struct lttng_ust_lib_ring_buffer_config client_config = {
 614         .cb.ring_buffer_clock_read = client_ring_buffer_clock_read,
 615         .cb.record_header_size = client_record_header_size,
 616         .cb.subbuffer_header_size = client_packet_header_size,
 617         .cb.buffer_begin = client_buffer_begin,
 618         .cb.buffer_end = client_buffer_end,
 619         .cb.buffer_create = client_buffer_create,
 620         .cb.buffer_finalize = client_buffer_finalize,
 621         .cb.content_size_field = client_content_size_field,
 622         .cb.packet_size_field = client_packet_size_field,
 623
 624         .tsc_bits = LTTNG_COMPACT_TSC_BITS,
 625         .alloc = RING_BUFFER_ALLOC_PER_CPU,
 626         .sync = RING_BUFFER_SYNC_GLOBAL,
 627         .mode = RING_BUFFER_MODE_TEMPLATE,
 628         .backend = RING_BUFFER_PAGE,
 629         .output = RING_BUFFER_MMAP,
 630         .oops = RING_BUFFER_OOPS_CONSISTENCY,
 631         .ipi = RING_BUFFER_NO_IPI_BARRIER,
 632         .wakeup = LTTNG_CLIENT_WAKEUP,
 633         .client_type = LTTNG_CLIENT_TYPE,
 634
 635         .cb_ptr = &client_cb.parent,
 636 };
 637
 638 static
 639 struct lttng_channel *_channel_create(const char *name,
 640                                 void *buf_addr,
 641                                 size_t subbuf_size, size_t num_subbuf,
 642                                 unsigned int switch_timer_interval,
 643                                 unsigned int read_timer_interval,
 644                                 unsigned char *uuid,
 645                                 uint32_t chan_id,
 646                                 const int *stream_fds, int nr_stream_fds,
 647                                 int64_t blocking_timeout)
 648 {
 649         struct lttng_channel chan_priv_init;
 650         struct lttng_ust_shm_handle *handle;
 651         struct lttng_channel *lttng_chan;
 652         void *priv;
 653
 654         memset(&chan_priv_init, 0, sizeof(chan_priv_init));
 655         memcpy(chan_priv_init.uuid, uuid, LTTNG_UST_UUID_LEN);
 656         chan_priv_init.id = chan_id;
 657         handle = channel_create(&client_config, name,
 658                         &priv, __alignof__(struct lttng_channel),
 659                         sizeof(struct lttng_channel),
 660                         &chan_priv_init,
 661                         buf_addr, subbuf_size, num_subbuf,
 662                         switch_timer_interval, read_timer_interval,
 663                         stream_fds, nr_stream_fds, blocking_timeout);
 664         if (!handle)
 665                 return NULL;
 666         lttng_chan = priv;
 667         lttng_chan->handle = handle;
 668         lttng_chan->chan = shmp(handle, handle->chan);
 669         return lttng_chan;
 670 }
 671
 672 static
 673 void lttng_channel_destroy(struct lttng_channel *chan)
 674 {
 675         channel_destroy(chan->chan, chan->handle, 1);
 676 }
 677
 678 static
 679 int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx,
 680                       uint32_t event_id)
 681 {
 682         struct lttng_channel *lttng_chan = channel_get_private(ctx->chan);
 683         struct lttng_stack_ctx *lttng_ctx = ctx->priv;
 684         struct lttng_client_ctx client_ctx;
 685         int ret, cpu;
 686
 687         /* Compute internal size of context structures. */
 688         ctx_get_struct_size(lttng_ctx->chan_ctx, &client_ctx.packet_context_len,
 689                         APP_CTX_ENABLED);
 690         ctx_get_struct_size(lttng_ctx->event_ctx, &client_ctx.event_context_len,
 691                         APP_CTX_ENABLED);
 692
 693         cpu = lib_ring_buffer_get_cpu(&client_config);
 694         if (cpu < 0)
 695                 return -EPERM;
 696         ctx->cpu = cpu;
 697
 698         switch (lttng_chan->header_type) {
 699         case 1: /* compact */
 700                 if (event_id > 30)
 701                         ctx->rflags |= LTTNG_RFLAG_EXTENDED;
 702                 break;
 703         case 2: /* large */
 704                 if (event_id > 65534)
 705                         ctx->rflags |= LTTNG_RFLAG_EXTENDED;
 706                 break;
 707         default:
 708                 WARN_ON_ONCE(1);
 709         }
 710
 711         ret = lib_ring_buffer_reserve(&client_config, ctx, &client_ctx);
 712         if (caa_unlikely(ret))
 713                 goto put;
 714         if (lib_ring_buffer_backend_get_pages(&client_config, ctx,
 715                         &ctx->backend_pages)) {
 716                 ret = -EPERM;
 717                 goto put;
 718         }
 719         lttng_write_event_header(&client_config, ctx, event_id);
 720         return 0;
 721 put:
 722         lib_ring_buffer_put_cpu(&client_config);
 723         return ret;
 724 }
 725
 726 static
 727 void lttng_event_commit(struct lttng_ust_lib_ring_buffer_ctx *ctx)
 728 {
 729         lib_ring_buffer_commit(&client_config, ctx);
 730         lib_ring_buffer_put_cpu(&client_config);
 731 }
 732
 733 static
 734 void lttng_event_write(struct lttng_ust_lib_ring_buffer_ctx *ctx, const void *src,
 735                      size_t len)
 736 {
 737         lib_ring_buffer_write(&client_config, ctx, src, len);
 738 }
 739
 740 static
 741 void lttng_event_strcpy(struct lttng_ust_lib_ring_buffer_ctx *ctx, const char *src,
 742                      size_t len)
 743 {
 744         lib_ring_buffer_strcpy(&client_config, ctx, src, len, '#');
 745 }
 746
 747 #if 0
 748 static
 749 wait_queue_head_t *lttng_get_reader_wait_queue(struct channel *chan)
 750 {
 751         return &chan->read_wait;
 752 }
 753
 754 static
 755 wait_queue_head_t *lttng_get_hp_wait_queue(struct channel *chan)
 756 {
 757         return &chan->hp_wait;
 758 }
 759 #endif //0
 760
 761 static
 762 int lttng_is_finalized(struct channel *chan)
 763 {
 764         return lib_ring_buffer_channel_is_finalized(chan);
 765 }
 766
 767 static
 768 int lttng_is_disabled(struct channel *chan)
 769 {
 770         return lib_ring_buffer_channel_is_disabled(chan);
 771 }
 772
 773 static
 774 int lttng_flush_buffer(struct channel *chan, struct lttng_ust_shm_handle *handle)
 775 {
 776         struct lttng_ust_lib_ring_buffer *buf;
 777         int cpu;
 778
 779         for_each_channel_cpu(cpu, chan) {
 780                 int shm_fd, wait_fd, wakeup_fd;
 781                 uint64_t memory_map_size;
 782
 783                 buf = channel_get_ring_buffer(&client_config, chan,
 784                                 cpu, handle, &shm_fd, &wait_fd,
 785                                 &wakeup_fd, &memory_map_size);
 786                 lib_ring_buffer_switch(&client_config, buf,
 787                                 SWITCH_ACTIVE, handle);
 788         }
 789         return 0;
 790 }
 791
 792 static struct lttng_transport lttng_relay_transport = {
 793         .name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap",
 794         .ops = {
 795                 .struct_size = sizeof(struct lttng_ust_channel_ops),
 796                 .channel_create = _channel_create,
 797                 .channel_destroy = lttng_channel_destroy,
 798                 .event_reserve = lttng_event_reserve,
 799                 .event_commit = lttng_event_commit,
 800                 .event_write = lttng_event_write,
 801                 .packet_avail_size = NULL,      /* Would be racy anyway */
 802                 .is_finalized = lttng_is_finalized,
 803                 .is_disabled = lttng_is_disabled,
 804                 .flush_buffer = lttng_flush_buffer,
 805                 .event_strcpy = lttng_event_strcpy,
 806         },
 807         .client_config = &client_config,
 808 };
 809
 810 void RING_BUFFER_MODE_TEMPLATE_INIT(void)
 811 {
 812         DBG("LTT : ltt ring buffer client \"%s\" init\n",
 813                 "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap");
 814         lttng_transport_register(&lttng_relay_transport);
 815 }
 816
 817 void RING_BUFFER_MODE_TEMPLATE_EXIT(void)
 818 {
 819         DBG("LTT : ltt ring buffer client \"%s\" exit\n",
 820                 "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap");
 821         lttng_transport_unregister(&lttng_relay_transport);
 822 }