X-Git-Url: http://git.liburcu.org/?a=blobdiff_plain;f=liblttng-ust%2Flttng-ring-buffer-client.h;h=fb7ba25521fce4eddb5519629218b738b651fd3a;hb=8acb8bdfec7f1827ca762900e0fcbaa8f187742f;hp=9cc862cf759166b1a4f57eafad4dc98aa72f5ac1;hpb=2b7080aa8c2481a3a316c98702884c902a01bde5;p=lttng-ust.git diff --git a/liblttng-ust/lttng-ring-buffer-client.h b/liblttng-ust/lttng-ring-buffer-client.h index 9cc862cf..fb7ba255 100644 --- a/liblttng-ust/lttng-ring-buffer-client.h +++ b/liblttng-ust/lttng-ring-buffer-client.h @@ -6,16 +6,19 @@ * LTTng lib ring buffer client template. */ +#include #include #include #include +#include #include "ust-bitfield.h" #include "ust-compat.h" #include "clock.h" #include "context-internal.h" #include "lttng-tracer.h" #include "../libringbuffer/frontend_types.h" +#include #define LTTNG_COMPACT_EVENT_BITS 5 #define LTTNG_COMPACT_TSC_BITS 27 @@ -62,28 +65,45 @@ struct packet_header { struct lttng_client_ctx { size_t packet_context_len; size_t event_context_len; + struct lttng_ust_ctx *chan_ctx; + struct lttng_ust_ctx *event_ctx; }; -static inline uint64_t lib_ring_buffer_clock_read(struct channel *chan) +/* + * Indexed by lib_ring_buffer_nesting_count(). + */ +typedef struct lttng_ust_lib_ring_buffer_ctx_private private_ctx_stack_t[LIB_RING_BUFFER_MAX_NESTING]; +static DEFINE_URCU_TLS(private_ctx_stack_t, private_ctx_stack); + +/* + * Force a read (imply TLS fixup for dlopen) of TLS variables. + */ +static +void lttng_fixup_rb_client_tls(void) +{ + asm volatile ("" : : "m" (URCU_TLS(private_ctx_stack))); +} + +static inline uint64_t lib_ring_buffer_clock_read(struct lttng_ust_lib_ring_buffer_channel *chan) { return trace_clock_read64(); } static inline -size_t ctx_get_aligned_size(size_t offset, struct lttng_ctx *ctx, +size_t ctx_get_aligned_size(size_t offset, struct lttng_ust_ctx *ctx, size_t ctx_len) { size_t orig_offset = offset; if (caa_likely(!ctx)) return 0; - offset += lib_ring_buffer_align(offset, ctx->largest_align); + offset += lttng_ust_lib_ring_buffer_align(offset, ctx->largest_align); offset += ctx_len; return offset - orig_offset; } static inline -void ctx_get_struct_size(struct lttng_ctx *ctx, size_t *ctx_len, +void ctx_get_struct_size(struct lttng_ust_ctx *ctx, size_t *ctx_len, enum app_ctx_mode mode) { int i; @@ -95,9 +115,9 @@ void ctx_get_struct_size(struct lttng_ctx *ctx, size_t *ctx_len, } for (i = 0; i < ctx->nr_fields; i++) { if (mode == APP_CTX_ENABLED) { - offset += ctx->fields[i].get_size(&ctx->fields[i], offset); + offset += ctx->fields[i]->get_size(ctx->fields[i], offset); } else { - if (lttng_context_is_app(ctx->fields[i].event_field.name)) { + if (lttng_context_is_app(ctx->fields[i]->event_field->name)) { /* * Before UST 2.8, we cannot use the * application context, because we @@ -108,9 +128,9 @@ void ctx_get_struct_size(struct lttng_ctx *ctx, size_t *ctx_len, * concurrently with application context * register/unregister. */ - offset += lttng_ust_dummy_get_size(&ctx->fields[i], offset); + offset += lttng_ust_dummy_get_size(ctx->fields[i], offset); } else { - offset += ctx->fields[i].get_size(&ctx->fields[i], offset); + offset += ctx->fields[i]->get_size(ctx->fields[i], offset); } } } @@ -119,20 +139,20 @@ void ctx_get_struct_size(struct lttng_ctx *ctx, size_t *ctx_len, static inline void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx, - struct lttng_channel *chan, - struct lttng_ctx *ctx, + struct lttng_ust_channel_buffer *chan, + struct lttng_ust_ctx *ctx, enum app_ctx_mode mode) { int i; if (caa_likely(!ctx)) return; - lib_ring_buffer_align_ctx(bufctx, ctx->largest_align); + lttng_ust_lib_ring_buffer_align_ctx(bufctx, ctx->largest_align); for (i = 0; i < ctx->nr_fields; i++) { if (mode == APP_CTX_ENABLED) { - ctx->fields[i].record(&ctx->fields[i], bufctx, chan); + ctx->fields[i]->record(ctx->fields[i], bufctx, chan); } else { - if (lttng_context_is_app(ctx->fields[i].event_field.name)) { + if (lttng_context_is_app(ctx->fields[i]->event_field->name)) { /* * Before UST 2.8, we cannot use the * application context, because we @@ -143,9 +163,9 @@ void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx, * concurrently with application context * register/unregister. */ - lttng_ust_dummy_record(&ctx->fields[i], bufctx, chan); + lttng_ust_dummy_record(ctx->fields[i], bufctx, chan); } else { - ctx->fields[i].record(&ctx->fields[i], bufctx, chan); + ctx->fields[i]->record(ctx->fields[i], bufctx, chan); } } } @@ -166,44 +186,44 @@ void ctx_record(struct lttng_ust_lib_ring_buffer_ctx *bufctx, */ static __inline__ size_t record_header_size(const struct lttng_ust_lib_ring_buffer_config *config, - struct channel *chan, size_t offset, + struct lttng_ust_lib_ring_buffer_channel *chan, + size_t offset, size_t *pre_header_padding, struct lttng_ust_lib_ring_buffer_ctx *ctx, struct lttng_client_ctx *client_ctx) { - struct lttng_channel *lttng_chan = channel_get_private(chan); - struct lttng_stack_ctx *lttng_ctx = ctx->priv; + struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan); size_t orig_offset = offset; size_t padding; - switch (lttng_chan->header_type) { + switch (lttng_chan->priv->header_type) { case 1: /* compact */ - padding = lib_ring_buffer_align(offset, lttng_alignof(uint32_t)); + padding = lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint32_t)); offset += padding; - if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { + if (!(ctx->priv->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { offset += sizeof(uint32_t); /* id and timestamp */ } else { /* Minimum space taken by LTTNG_COMPACT_EVENT_BITS id */ offset += (LTTNG_COMPACT_EVENT_BITS + CHAR_BIT - 1) / CHAR_BIT; /* Align extended struct on largest member */ - offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t)); + offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t)); offset += sizeof(uint32_t); /* id */ - offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t)); + offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t)); offset += sizeof(uint64_t); /* timestamp */ } break; case 2: /* large */ - padding = lib_ring_buffer_align(offset, lttng_alignof(uint16_t)); + padding = lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint16_t)); offset += padding; offset += sizeof(uint16_t); - if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { - offset += lib_ring_buffer_align(offset, lttng_alignof(uint32_t)); + if (!(ctx->priv->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { + offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint32_t)); offset += sizeof(uint32_t); /* timestamp */ } else { /* Align extended struct on largest member */ - offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t)); + offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t)); offset += sizeof(uint32_t); /* id */ - offset += lib_ring_buffer_align(offset, lttng_alignof(uint64_t)); + offset += lttng_ust_lib_ring_buffer_align(offset, lttng_ust_rb_alignof(uint64_t)); offset += sizeof(uint64_t); /* timestamp */ } break; @@ -211,9 +231,9 @@ size_t record_header_size(const struct lttng_ust_lib_ring_buffer_config *config, padding = 0; WARN_ON_ONCE(1); } - offset += ctx_get_aligned_size(offset, lttng_ctx->chan_ctx, + offset += ctx_get_aligned_size(offset, client_ctx->chan_ctx, client_ctx->packet_context_len); - offset += ctx_get_aligned_size(offset, lttng_ctx->event_ctx, + offset += ctx_get_aligned_size(offset, client_ctx->event_ctx, client_ctx->event_context_len); *pre_header_padding = padding; return offset - orig_offset; @@ -225,6 +245,7 @@ size_t record_header_size(const struct lttng_ust_lib_ring_buffer_config *config, static void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer_ctx *ctx, + struct lttng_client_ctx *client_ctx, uint32_t event_id); /* @@ -239,15 +260,15 @@ void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config static __inline__ void lttng_write_event_header(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer_ctx *ctx, + struct lttng_client_ctx *client_ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); - struct lttng_stack_ctx *lttng_ctx = ctx->priv; + struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(ctx->priv->chan); - if (caa_unlikely(ctx->rflags)) + if (caa_unlikely(ctx->priv->rflags)) goto slow_path; - switch (lttng_chan->header_type) { + switch (lttng_chan->priv->header_type) { case 1: /* compact */ { uint32_t id_time = 0; @@ -259,17 +280,17 @@ void lttng_write_event_header(const struct lttng_ust_lib_ring_buffer_config *con bt_bitfield_write(&id_time, uint32_t, LTTNG_COMPACT_EVENT_BITS, LTTNG_COMPACT_TSC_BITS, - ctx->tsc); + ctx->priv->tsc); lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time)); break; } case 2: /* large */ { - uint32_t timestamp = (uint32_t) ctx->tsc; + uint32_t timestamp = (uint32_t) ctx->priv->tsc; uint16_t id = event_id; lib_ring_buffer_write(config, ctx, &id, sizeof(id)); - lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint32_t)); + lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint32_t)); lib_ring_buffer_write(config, ctx, ×tamp, sizeof(timestamp)); break; } @@ -277,27 +298,28 @@ void lttng_write_event_header(const struct lttng_ust_lib_ring_buffer_config *con WARN_ON_ONCE(1); } - ctx_record(ctx, lttng_chan, lttng_ctx->chan_ctx, APP_CTX_ENABLED); - ctx_record(ctx, lttng_chan, lttng_ctx->event_ctx, APP_CTX_ENABLED); - lib_ring_buffer_align_ctx(ctx, ctx->largest_align); + ctx_record(ctx, lttng_chan, client_ctx->chan_ctx, APP_CTX_ENABLED); + ctx_record(ctx, lttng_chan, client_ctx->event_ctx, APP_CTX_ENABLED); + lttng_ust_lib_ring_buffer_align_ctx(ctx, ctx->largest_align); return; slow_path: - lttng_write_event_header_slow(config, ctx, event_id); + lttng_write_event_header_slow(config, ctx, client_ctx, event_id); } static void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer_ctx *ctx, + struct lttng_client_ctx *client_ctx, uint32_t event_id) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); - struct lttng_stack_ctx *lttng_ctx = ctx->priv; + struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv; + struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(ctx->priv->chan); - switch (lttng_chan->header_type) { + switch (lttng_chan->priv->header_type) { case 1: /* compact */ - if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { + if (!(ctx_private->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { uint32_t id_time = 0; bt_bitfield_write(&id_time, uint32_t, @@ -307,11 +329,11 @@ void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config bt_bitfield_write(&id_time, uint32_t, LTTNG_COMPACT_EVENT_BITS, LTTNG_COMPACT_TSC_BITS, - ctx->tsc); + ctx_private->tsc); lib_ring_buffer_write(config, ctx, &id_time, sizeof(id_time)); } else { uint8_t id = 0; - uint64_t timestamp = ctx->tsc; + uint64_t timestamp = ctx_private->tsc; bt_bitfield_write(&id, uint8_t, 0, @@ -319,30 +341,30 @@ void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config 31); lib_ring_buffer_write(config, ctx, &id, sizeof(id)); /* Align extended struct on largest member */ - lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t)); + lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t)); lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id)); - lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t)); + lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t)); lib_ring_buffer_write(config, ctx, ×tamp, sizeof(timestamp)); } break; case 2: /* large */ { - if (!(ctx->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { - uint32_t timestamp = (uint32_t) ctx->tsc; + if (!(ctx_private->rflags & (RING_BUFFER_RFLAG_FULL_TSC | LTTNG_RFLAG_EXTENDED))) { + uint32_t timestamp = (uint32_t) ctx_private->tsc; uint16_t id = event_id; lib_ring_buffer_write(config, ctx, &id, sizeof(id)); - lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint32_t)); + lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint32_t)); lib_ring_buffer_write(config, ctx, ×tamp, sizeof(timestamp)); } else { uint16_t id = 65535; - uint64_t timestamp = ctx->tsc; + uint64_t timestamp = ctx_private->tsc; lib_ring_buffer_write(config, ctx, &id, sizeof(id)); /* Align extended struct on largest member */ - lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t)); + lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t)); lib_ring_buffer_write(config, ctx, &event_id, sizeof(event_id)); - lib_ring_buffer_align_ctx(ctx, lttng_alignof(uint64_t)); + lttng_ust_lib_ring_buffer_align_ctx(ctx, lttng_ust_rb_alignof(uint64_t)); lib_ring_buffer_write(config, ctx, ×tamp, sizeof(timestamp)); } break; @@ -350,21 +372,22 @@ void lttng_write_event_header_slow(const struct lttng_ust_lib_ring_buffer_config default: WARN_ON_ONCE(1); } - ctx_record(ctx, lttng_chan, lttng_ctx->chan_ctx, APP_CTX_ENABLED); - ctx_record(ctx, lttng_chan, lttng_ctx->event_ctx, APP_CTX_ENABLED); - lib_ring_buffer_align_ctx(ctx, ctx->largest_align); + ctx_record(ctx, lttng_chan, client_ctx->chan_ctx, APP_CTX_ENABLED); + ctx_record(ctx, lttng_chan, client_ctx->event_ctx, APP_CTX_ENABLED); + lttng_ust_lib_ring_buffer_align_ctx(ctx, ctx->largest_align); } static const struct lttng_ust_lib_ring_buffer_config client_config; -static uint64_t client_ring_buffer_clock_read(struct channel *chan) +static uint64_t client_ring_buffer_clock_read(struct lttng_ust_lib_ring_buffer_channel *chan) { return lib_ring_buffer_clock_read(chan); } static size_t client_record_header_size(const struct lttng_ust_lib_ring_buffer_config *config, - struct channel *chan, size_t offset, + struct lttng_ust_lib_ring_buffer_channel *chan, + size_t offset, size_t *pre_header_padding, struct lttng_ust_lib_ring_buffer_ctx *ctx, void *client_ctx) @@ -389,21 +412,21 @@ static void client_buffer_begin(struct lttng_ust_lib_ring_buffer *buf, uint64_t unsigned int subbuf_idx, struct lttng_ust_shm_handle *handle) { - struct channel *chan = shmp(handle, buf->backend.chan); + struct lttng_ust_lib_ring_buffer_channel *chan = shmp(handle, buf->backend.chan); struct packet_header *header = (struct packet_header *) lib_ring_buffer_offset_address(&buf->backend, subbuf_idx * chan->backend.subbuf_size, handle); - struct lttng_channel *lttng_chan = channel_get_private(chan); + struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan); uint64_t cnt = shmp_index(handle, buf->backend.buf_cnt, subbuf_idx)->seq_cnt; assert(header); if (!header) return; header->magic = CTF_MAGIC_NUMBER; - memcpy(header->uuid, lttng_chan->uuid, sizeof(lttng_chan->uuid)); - header->stream_id = lttng_chan->id; + memcpy(header->uuid, lttng_chan->priv->uuid, sizeof(lttng_chan->priv->uuid)); + header->stream_id = lttng_chan->priv->id; header->stream_instance_id = buf->backend.cpu; header->ctx.timestamp_begin = tsc; header->ctx.timestamp_end = 0; @@ -422,7 +445,7 @@ static void client_buffer_end(struct lttng_ust_lib_ring_buffer *buf, uint64_t ts unsigned int subbuf_idx, unsigned long data_size, struct lttng_ust_shm_handle *handle) { - struct channel *chan = shmp(handle, buf->backend.chan); + struct lttng_ust_lib_ring_buffer_channel *chan = shmp(handle, buf->backend.chan); struct packet_header *header = (struct packet_header *) lib_ring_buffer_offset_address(&buf->backend, @@ -476,9 +499,10 @@ static struct packet_header *client_packet_header(struct lttng_ust_lib_ring_buff } static int client_timestamp_begin(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *timestamp_begin) { + struct lttng_ust_shm_handle *handle = chan->handle; struct packet_header *header; header = client_packet_header(buf, handle); @@ -489,9 +513,10 @@ static int client_timestamp_begin(struct lttng_ust_lib_ring_buffer *buf, } static int client_timestamp_end(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *timestamp_end) { + struct lttng_ust_shm_handle *handle = chan->handle; struct packet_header *header; header = client_packet_header(buf, handle); @@ -502,9 +527,10 @@ static int client_timestamp_end(struct lttng_ust_lib_ring_buffer *buf, } static int client_events_discarded(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *events_discarded) { + struct lttng_ust_shm_handle *handle = chan->handle; struct packet_header *header; header = client_packet_header(buf, handle); @@ -515,9 +541,10 @@ static int client_events_discarded(struct lttng_ust_lib_ring_buffer *buf, } static int client_content_size(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *content_size) { + struct lttng_ust_shm_handle *handle = chan->handle; struct packet_header *header; header = client_packet_header(buf, handle); @@ -528,9 +555,10 @@ static int client_content_size(struct lttng_ust_lib_ring_buffer *buf, } static int client_packet_size(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *packet_size) { + struct lttng_ust_shm_handle *handle = chan->handle; struct packet_header *header; header = client_packet_header(buf, handle); @@ -541,33 +569,30 @@ static int client_packet_size(struct lttng_ust_lib_ring_buffer *buf, } static int client_stream_id(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *stream_id) { - struct channel *chan = shmp(handle, buf->backend.chan); - struct lttng_channel *lttng_chan = channel_get_private(chan); + struct lttng_ust_channel_buffer *lttng_chan = channel_get_private(chan); - *stream_id = lttng_chan->id; + *stream_id = lttng_chan->priv->id; return 0; } static int client_current_timestamp(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *ts) { - struct channel *chan; - - chan = shmp(handle, handle->chan); *ts = client_ring_buffer_clock_read(chan); return 0; } static int client_sequence_number(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *seq) { + struct lttng_ust_shm_handle *handle = chan->handle; struct packet_header *header; header = client_packet_header(buf, handle); @@ -578,7 +603,7 @@ static int client_sequence_number(struct lttng_ust_lib_ring_buffer *buf, } static int client_instance_id(struct lttng_ust_lib_ring_buffer *buf, - struct lttng_ust_shm_handle *handle, + struct lttng_ust_lib_ring_buffer_channel *chan, uint64_t *id) { *id = buf->backend.cpu; @@ -636,7 +661,7 @@ static const struct lttng_ust_lib_ring_buffer_config client_config = { }; static -struct lttng_channel *_channel_create(const char *name, +struct lttng_ust_channel_buffer *_channel_create(const char *name, void *buf_addr, size_t subbuf_size, size_t num_subbuf, unsigned int switch_timer_interval, @@ -646,63 +671,82 @@ struct lttng_channel *_channel_create(const char *name, const int *stream_fds, int nr_stream_fds, int64_t blocking_timeout) { - struct lttng_channel chan_priv_init; + struct lttng_ust_abi_channel_config chan_priv_init; struct lttng_ust_shm_handle *handle; - struct lttng_channel *lttng_chan; - void *priv; + struct lttng_ust_channel_buffer *lttng_chan_buf; + + lttng_chan_buf = lttng_ust_alloc_channel_buffer(); + if (!lttng_chan_buf) + return NULL; + memcpy(lttng_chan_buf->priv->uuid, uuid, LTTNG_UST_UUID_LEN); + lttng_chan_buf->priv->id = chan_id; memset(&chan_priv_init, 0, sizeof(chan_priv_init)); memcpy(chan_priv_init.uuid, uuid, LTTNG_UST_UUID_LEN); chan_priv_init.id = chan_id; + handle = channel_create(&client_config, name, - &priv, __alignof__(struct lttng_channel), - sizeof(struct lttng_channel), + __alignof__(struct lttng_ust_abi_channel_config), + sizeof(struct lttng_ust_abi_channel_config), &chan_priv_init, - buf_addr, subbuf_size, num_subbuf, + lttng_chan_buf, buf_addr, subbuf_size, num_subbuf, switch_timer_interval, read_timer_interval, stream_fds, nr_stream_fds, blocking_timeout); if (!handle) - return NULL; - lttng_chan = priv; - lttng_chan->handle = handle; - lttng_chan->chan = shmp(handle, handle->chan); - return lttng_chan; + goto error; + lttng_chan_buf->priv->rb_chan = shmp(handle, handle->chan); + return lttng_chan_buf; + +error: + lttng_ust_free_channel_common(lttng_chan_buf->parent); + return NULL; } static -void lttng_channel_destroy(struct lttng_channel *chan) +void lttng_channel_destroy(struct lttng_ust_channel_buffer *lttng_chan_buf) { - channel_destroy(chan->chan, chan->handle, 1); + channel_destroy(lttng_chan_buf->priv->rb_chan, lttng_chan_buf->priv->rb_chan->handle, 1); + lttng_ust_free_channel_common(lttng_chan_buf->parent); } static -int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx, - uint32_t event_id) +int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx) { - struct lttng_channel *lttng_chan = channel_get_private(ctx->chan); - struct lttng_stack_ctx *lttng_ctx = ctx->priv; + struct lttng_ust_event_recorder *event_recorder = ctx->client_priv; + struct lttng_ust_channel_buffer *lttng_chan = event_recorder->chan; struct lttng_client_ctx client_ctx; - int ret, cpu; + int ret, nesting; + struct lttng_ust_lib_ring_buffer_ctx_private *private_ctx; + uint32_t event_id; + event_id = event_recorder->priv->id; + client_ctx.chan_ctx = lttng_ust_rcu_dereference(lttng_chan->priv->ctx); + client_ctx.event_ctx = lttng_ust_rcu_dereference(event_recorder->priv->ctx); /* Compute internal size of context structures. */ - ctx_get_struct_size(lttng_ctx->chan_ctx, &client_ctx.packet_context_len, + ctx_get_struct_size(client_ctx.chan_ctx, &client_ctx.packet_context_len, APP_CTX_ENABLED); - ctx_get_struct_size(lttng_ctx->event_ctx, &client_ctx.event_context_len, + ctx_get_struct_size(client_ctx.event_ctx, &client_ctx.event_context_len, APP_CTX_ENABLED); - cpu = lib_ring_buffer_get_cpu(&client_config); - if (cpu < 0) + nesting = lib_ring_buffer_nesting_inc(&client_config); + if (nesting < 0) return -EPERM; - ctx->cpu = cpu; - switch (lttng_chan->header_type) { + private_ctx = &URCU_TLS(private_ctx_stack[nesting]); + memset(private_ctx, 0, sizeof(*private_ctx)); + private_ctx->pub = ctx; + private_ctx->chan = lttng_chan->priv->rb_chan; + + ctx->priv = private_ctx; + + switch (lttng_chan->priv->header_type) { case 1: /* compact */ if (event_id > 30) - ctx->rflags |= LTTNG_RFLAG_EXTENDED; + private_ctx->rflags |= LTTNG_RFLAG_EXTENDED; break; case 2: /* large */ if (event_id > 65534) - ctx->rflags |= LTTNG_RFLAG_EXTENDED; + private_ctx->rflags |= LTTNG_RFLAG_EXTENDED; break; default: WARN_ON_ONCE(1); @@ -712,14 +756,14 @@ int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx, if (caa_unlikely(ret)) goto put; if (lib_ring_buffer_backend_get_pages(&client_config, ctx, - &ctx->backend_pages)) { + &private_ctx->backend_pages)) { ret = -EPERM; goto put; } - lttng_write_event_header(&client_config, ctx, event_id); + lttng_write_event_header(&client_config, ctx, &client_ctx, event_id); return 0; put: - lib_ring_buffer_put_cpu(&client_config); + lib_ring_buffer_nesting_dec(&client_config); return ret; } @@ -727,64 +771,63 @@ static void lttng_event_commit(struct lttng_ust_lib_ring_buffer_ctx *ctx) { lib_ring_buffer_commit(&client_config, ctx); - lib_ring_buffer_put_cpu(&client_config); + lib_ring_buffer_nesting_dec(&client_config); } static -void lttng_event_write(struct lttng_ust_lib_ring_buffer_ctx *ctx, const void *src, - size_t len) +void lttng_event_write(struct lttng_ust_lib_ring_buffer_ctx *ctx, + const void *src, size_t len, size_t alignment) { + lttng_ust_lib_ring_buffer_align_ctx(ctx, alignment); lib_ring_buffer_write(&client_config, ctx, src, len); } static -void lttng_event_strcpy(struct lttng_ust_lib_ring_buffer_ctx *ctx, const char *src, - size_t len) +void lttng_event_strcpy(struct lttng_ust_lib_ring_buffer_ctx *ctx, + const char *src, size_t len) { lib_ring_buffer_strcpy(&client_config, ctx, src, len, '#'); } -#if 0 static -wait_queue_head_t *lttng_get_reader_wait_queue(struct channel *chan) +void lttng_event_pstrcpy_pad(struct lttng_ust_lib_ring_buffer_ctx *ctx, + const char *src, size_t len) { - return &chan->read_wait; + lib_ring_buffer_pstrcpy(&client_config, ctx, src, len, '\0'); } static -wait_queue_head_t *lttng_get_hp_wait_queue(struct channel *chan) +int lttng_is_finalized(struct lttng_ust_channel_buffer *chan) { - return &chan->hp_wait; -} -#endif //0 + struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan; -static -int lttng_is_finalized(struct channel *chan) -{ - return lib_ring_buffer_channel_is_finalized(chan); + return lib_ring_buffer_channel_is_finalized(rb_chan); } static -int lttng_is_disabled(struct channel *chan) +int lttng_is_disabled(struct lttng_ust_channel_buffer *chan) { - return lib_ring_buffer_channel_is_disabled(chan); + struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan; + + return lib_ring_buffer_channel_is_disabled(rb_chan); } static -int lttng_flush_buffer(struct channel *chan, struct lttng_ust_shm_handle *handle) +int lttng_flush_buffer(struct lttng_ust_channel_buffer *chan) { + struct lttng_ust_lib_ring_buffer_channel *rb_chan = chan->priv->rb_chan; struct lttng_ust_lib_ring_buffer *buf; int cpu; - for_each_channel_cpu(cpu, chan) { + for_each_channel_cpu(cpu, rb_chan) { int shm_fd, wait_fd, wakeup_fd; uint64_t memory_map_size; - buf = channel_get_ring_buffer(&client_config, chan, - cpu, handle, &shm_fd, &wait_fd, + buf = channel_get_ring_buffer(&client_config, rb_chan, + cpu, rb_chan->handle, &shm_fd, &wait_fd, &wakeup_fd, &memory_map_size); lib_ring_buffer_switch(&client_config, buf, - SWITCH_ACTIVE, handle); + SWITCH_ACTIVE, rb_chan->handle); } return 0; } @@ -792,16 +835,21 @@ int lttng_flush_buffer(struct channel *chan, struct lttng_ust_shm_handle *handle static struct lttng_transport lttng_relay_transport = { .name = "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap", .ops = { - .channel_create = _channel_create, - .channel_destroy = lttng_channel_destroy, + .struct_size = sizeof(struct lttng_ust_channel_buffer_ops), + .priv = __LTTNG_COMPOUND_LITERAL(struct lttng_ust_channel_buffer_ops_private, { + .pub = <tng_relay_transport.ops, + .channel_create = _channel_create, + .channel_destroy = lttng_channel_destroy, + .packet_avail_size = NULL, /* Would be racy anyway */ + .is_finalized = lttng_is_finalized, + .is_disabled = lttng_is_disabled, + .flush_buffer = lttng_flush_buffer, + }), .event_reserve = lttng_event_reserve, .event_commit = lttng_event_commit, .event_write = lttng_event_write, - .packet_avail_size = NULL, /* Would be racy anyway */ - .is_finalized = lttng_is_finalized, - .is_disabled = lttng_is_disabled, - .flush_buffer = lttng_flush_buffer, .event_strcpy = lttng_event_strcpy, + .event_pstrcpy_pad = lttng_event_pstrcpy_pad, }, .client_config = &client_config, }; @@ -810,6 +858,7 @@ void RING_BUFFER_MODE_TEMPLATE_INIT(void) { DBG("LTT : ltt ring buffer client \"%s\" init\n", "relay-" RING_BUFFER_MODE_TEMPLATE_STRING "-mmap"); + lttng_fixup_rb_client_tls(); lttng_transport_register(<tng_relay_transport); }