X-Git-Url: http://git.liburcu.org/?a=blobdiff_plain;f=libringbuffer%2Ffrontend_api.h;h=ac5e31bf1c4f0ee95b0b74066b8fb9566a16672e;hb=6ba6fd60507f8e045bdc4f1be14e9d99c6a15f7f;hp=5ddb8ac460951f0bae59a3f098cf6bca510a3799;hpb=f52a5702dbeb8125ca373ba363b29cbff3042cc1;p=lttng-ust.git diff --git a/libringbuffer/frontend_api.h b/libringbuffer/frontend_api.h index 5ddb8ac4..ac5e31bf 100644 --- a/libringbuffer/frontend_api.h +++ b/libringbuffer/frontend_api.h @@ -1,46 +1,29 @@ -#ifndef _LTTNG_RING_BUFFER_FRONTEND_API_H -#define _LTTNG_RING_BUFFER_FRONTEND_API_H - /* - * libringbuffer/frontend_api.h + * SPDX-License-Identifier: LGPL-2.1-only * * Copyright (C) 2005-2012 Mathieu Desnoyers * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; only - * version 2.1 of the License. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Ring Buffer Library Synchronization Header (buffer write API). - * - * Author: - * Mathieu Desnoyers - * * See ring_buffer_frontend.c for more information on wait-free * algorithms. * See frontend.h for channel allocation and read-side API. */ -#include "frontend.h" -#include +#ifndef _LTTNG_RING_BUFFER_FRONTEND_API_H +#define _LTTNG_RING_BUFFER_FRONTEND_API_H + +#include + #include +#include "frontend.h" + /** - * lib_ring_buffer_get_cpu - Precedes ring buffer reserve/commit. + * lib_ring_buffer_nesting_inc - Ring buffer recursive use protection. * - * Keeps a ring buffer nesting count as supplementary safety net to - * ensure tracer client code will never trigger an endless recursion. - * Returns the processor ID on success, -EPERM on failure (nesting count - * too high). + * The rint buffer buffer nesting count is a safety net to ensure tracer + * client code will never trigger an endless recursion. + * Returns a nesting level >= 0 on success, -EPERM on failure (nesting + * count too high). * * asm volatile and "memory" clobber prevent the compiler from moving * instructions out of the ring buffer nesting count. This is required to ensure @@ -49,27 +32,31 @@ * section. */ static inline -int lib_ring_buffer_get_cpu(const struct lttng_ust_lib_ring_buffer_config *config) +int lib_ring_buffer_nesting_inc( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused))) { - int cpu, nesting; + int nesting; - cpu = lttng_ust_get_cpu(); nesting = ++URCU_TLS(lib_ring_buffer_nesting); cmm_barrier(); - - if (caa_unlikely(nesting > 4)) { + if (caa_unlikely(nesting >= LIB_RING_BUFFER_MAX_NESTING)) { WARN_ON_ONCE(1); URCU_TLS(lib_ring_buffer_nesting)--; return -EPERM; - } else - return cpu; + } + return nesting - 1; } -/** - * lib_ring_buffer_put_cpu - Follows ring buffer reserve/commit. - */ static inline -void lib_ring_buffer_put_cpu(const struct lttng_ust_lib_ring_buffer_config *config) +int lib_ring_buffer_nesting_count( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused))) +{ + return URCU_TLS(lib_ring_buffer_nesting); +} + +static inline +void lib_ring_buffer_nesting_dec( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused))) { cmm_barrier(); URCU_TLS(lib_ring_buffer_nesting)--; /* TLS */ @@ -84,16 +71,18 @@ void lib_ring_buffer_put_cpu(const struct lttng_ust_lib_ring_buffer_config *conf static inline int lib_ring_buffer_try_reserve(const struct lttng_ust_lib_ring_buffer_config *config, struct lttng_ust_lib_ring_buffer_ctx *ctx, + void *client_ctx, unsigned long *o_begin, unsigned long *o_end, unsigned long *o_old, size_t *before_hdr_pad) { - struct channel *chan = ctx->chan; - struct lttng_ust_lib_ring_buffer *buf = ctx->buf; + struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv; + struct lttng_ust_lib_ring_buffer_channel *chan = ctx_private->chan; + struct lttng_ust_lib_ring_buffer *buf = ctx_private->buf; *o_begin = v_read(config, &buf->offset); *o_old = *o_begin; - ctx->tsc = lib_ring_buffer_clock_read(chan); - if ((int64_t) ctx->tsc == -EIO) + ctx_private->tsc = lib_ring_buffer_clock_read(chan); + if ((int64_t) ctx_private->tsc == -EIO) return 1; /* @@ -103,18 +92,18 @@ int lib_ring_buffer_try_reserve(const struct lttng_ust_lib_ring_buffer_config *c */ //prefetch(&buf->commit_hot[subbuf_index(*o_begin, chan)]); - if (last_tsc_overflow(config, buf, ctx->tsc)) - ctx->rflags |= RING_BUFFER_RFLAG_FULL_TSC; + if (last_tsc_overflow(config, buf, ctx_private->tsc)) + ctx_private->rflags |= RING_BUFFER_RFLAG_FULL_TSC; if (caa_unlikely(subbuf_offset(*o_begin, chan) == 0)) return 1; - ctx->slot_size = record_header_size(config, chan, *o_begin, - before_hdr_pad, ctx); - ctx->slot_size += - lib_ring_buffer_align(*o_begin + ctx->slot_size, + ctx_private->slot_size = record_header_size(config, chan, *o_begin, + before_hdr_pad, ctx, client_ctx); + ctx_private->slot_size += + lttng_ust_lib_ring_buffer_align(*o_begin + ctx_private->slot_size, ctx->largest_align) + ctx->data_size; - if (caa_unlikely((subbuf_offset(*o_begin, chan) + ctx->slot_size) + if (caa_unlikely((subbuf_offset(*o_begin, chan) + ctx_private->slot_size) > chan->backend.subbuf_size)) return 1; @@ -122,7 +111,7 @@ int lib_ring_buffer_try_reserve(const struct lttng_ust_lib_ring_buffer_config *c * Record fits in the current buffer and we are not on a switch * boundary. It's safe to write. */ - *o_end = *o_begin + ctx->slot_size; + *o_end = *o_begin + ctx_private->slot_size; if (caa_unlikely((subbuf_offset(*o_end, chan)) == 0)) /* @@ -152,10 +141,12 @@ int lib_ring_buffer_try_reserve(const struct lttng_ust_lib_ring_buffer_config *c static inline int lib_ring_buffer_reserve(const struct lttng_ust_lib_ring_buffer_config *config, - struct lttng_ust_lib_ring_buffer_ctx *ctx) + struct lttng_ust_lib_ring_buffer_ctx *ctx, + void *client_ctx) { - struct channel *chan = ctx->chan; - struct lttng_ust_shm_handle *handle = ctx->handle; + struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv; + struct lttng_ust_lib_ring_buffer_channel *chan = ctx_private->chan; + struct lttng_ust_shm_handle *handle = chan->handle; struct lttng_ust_lib_ring_buffer *buf; unsigned long o_begin, o_end, o_old; size_t before_hdr_pad = 0; @@ -163,22 +154,26 @@ int lib_ring_buffer_reserve(const struct lttng_ust_lib_ring_buffer_config *confi if (caa_unlikely(uatomic_read(&chan->record_disabled))) return -EAGAIN; - if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) - buf = shmp(handle, chan->backend.buf[ctx->cpu].shmp); - else + if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) { + ctx_private->reserve_cpu = lttng_ust_get_cpu(); + buf = shmp(handle, chan->backend.buf[ctx_private->reserve_cpu].shmp); + } else { buf = shmp(handle, chan->backend.buf[0].shmp); + } + if (caa_unlikely(!buf)) + return -EIO; if (caa_unlikely(uatomic_read(&buf->record_disabled))) return -EAGAIN; - ctx->buf = buf; + ctx_private->buf = buf; /* * Perform retryable operations. */ - if (caa_unlikely(lib_ring_buffer_try_reserve(config, ctx, &o_begin, + if (caa_unlikely(lib_ring_buffer_try_reserve(config, ctx, client_ctx, &o_begin, &o_end, &o_old, &before_hdr_pad))) goto slow_path; - if (caa_unlikely(v_cmpxchg(config, &ctx->buf->offset, o_old, o_end) + if (caa_unlikely(v_cmpxchg(config, &buf->offset, o_old, o_end) != o_old)) goto slow_path; @@ -188,24 +183,24 @@ int lib_ring_buffer_reserve(const struct lttng_ust_lib_ring_buffer_config *confi * record headers, never the opposite (missing a full TSC record header * when it would be needed). */ - save_last_tsc(config, ctx->buf, ctx->tsc); + save_last_tsc(config, buf, ctx_private->tsc); /* * Push the reader if necessary */ - lib_ring_buffer_reserve_push_reader(ctx->buf, chan, o_end - 1); + lib_ring_buffer_reserve_push_reader(buf, chan, o_end - 1); /* * Clear noref flag for this subbuffer. */ - lib_ring_buffer_clear_noref(config, &ctx->buf->backend, + lib_ring_buffer_clear_noref(config, &buf->backend, subbuf_index(o_end - 1, chan), handle); - ctx->pre_offset = o_begin; - ctx->buf_offset = o_begin + before_hdr_pad; + ctx_private->pre_offset = o_begin; + ctx_private->buf_offset = o_begin + before_hdr_pad; return 0; slow_path: - return lib_ring_buffer_reserve_slow(ctx); + return lib_ring_buffer_reserve_slow(ctx, client_ctx); } /** @@ -223,9 +218,10 @@ slow_path: * disabled, for RING_BUFFER_SYNC_PER_CPU configuration. */ static inline -void lib_ring_buffer_switch(const struct lttng_ust_lib_ring_buffer_config *config, - struct lttng_ust_lib_ring_buffer *buf, enum switch_mode mode, - struct lttng_ust_shm_handle *handle) +void lib_ring_buffer_switch( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)), + struct lttng_ust_lib_ring_buffer *buf, enum switch_mode mode, + struct lttng_ust_shm_handle *handle) { lib_ring_buffer_switch_slow(buf, mode, handle); } @@ -244,17 +240,23 @@ static inline void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *config, const struct lttng_ust_lib_ring_buffer_ctx *ctx) { - struct channel *chan = ctx->chan; - struct lttng_ust_shm_handle *handle = ctx->handle; - struct lttng_ust_lib_ring_buffer *buf = ctx->buf; - unsigned long offset_end = ctx->buf_offset; + struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv; + struct lttng_ust_lib_ring_buffer_channel *chan = ctx_private->chan; + struct lttng_ust_shm_handle *handle = chan->handle; + struct lttng_ust_lib_ring_buffer *buf = ctx_private->buf; + unsigned long offset_end = ctx_private->buf_offset; unsigned long endidx = subbuf_index(offset_end - 1, chan); unsigned long commit_count; + struct commit_counters_hot *cc_hot = shmp_index(handle, + buf->commit_hot, endidx); + + if (caa_unlikely(!cc_hot)) + return; /* * Must count record before incrementing the commit count. */ - subbuffer_count_record(config, &buf->backend, endidx, handle); + subbuffer_count_record(config, ctx, &buf->backend, endidx, handle); /* * Order all writes to buffer before the commit count update that will @@ -262,7 +264,7 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi */ cmm_smp_wmb(); - v_add(config, ctx->slot_size, &shmp_index(handle, buf->commit_hot, endidx)->cc); + v_add(config, ctx_private->slot_size, &cc_hot->cc); /* * commit count read can race with concurrent OOO commit count updates. @@ -282,16 +284,16 @@ void lib_ring_buffer_commit(const struct lttng_ust_lib_ring_buffer_config *confi * count reaches back the reserve offset for a specific sub-buffer, * which is completely independent of the order. */ - commit_count = v_read(config, &shmp_index(handle, buf->commit_hot, endidx)->cc); + commit_count = v_read(config, &cc_hot->cc); lib_ring_buffer_check_deliver(config, buf, chan, offset_end - 1, - commit_count, endidx, handle, ctx->tsc); + commit_count, endidx, handle, ctx_private->tsc); /* * Update used size at each commit. It's needed only for extracting * ring_buffer buffers from vmcore, after crash. */ - lib_ring_buffer_write_commit_counter(config, buf, chan, endidx, - offset_end, commit_count, handle); + lib_ring_buffer_write_commit_counter(config, buf, chan, + offset_end, commit_count, handle, cc_hot); } /** @@ -308,8 +310,9 @@ static inline int lib_ring_buffer_try_discard_reserve(const struct lttng_ust_lib_ring_buffer_config *config, const struct lttng_ust_lib_ring_buffer_ctx *ctx) { - struct lttng_ust_lib_ring_buffer *buf = ctx->buf; - unsigned long end_offset = ctx->pre_offset + ctx->slot_size; + struct lttng_ust_lib_ring_buffer_ctx_private *ctx_private = ctx->priv; + struct lttng_ust_lib_ring_buffer *buf = ctx_private->buf; + unsigned long end_offset = ctx_private->pre_offset + ctx_private->slot_size; /* * We need to ensure that if the cmpxchg succeeds and discards the @@ -325,7 +328,7 @@ int lib_ring_buffer_try_discard_reserve(const struct lttng_ust_lib_ring_buffer_c */ save_last_tsc(config, buf, 0ULL); - if (caa_likely(v_cmpxchg(config, &buf->offset, end_offset, ctx->pre_offset) + if (caa_likely(v_cmpxchg(config, &buf->offset, end_offset, ctx_private->pre_offset) != end_offset)) return -EPERM; else @@ -333,29 +336,33 @@ int lib_ring_buffer_try_discard_reserve(const struct lttng_ust_lib_ring_buffer_c } static inline -void channel_record_disable(const struct lttng_ust_lib_ring_buffer_config *config, - struct channel *chan) +void channel_record_disable( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)), + struct lttng_ust_lib_ring_buffer_channel *chan) { uatomic_inc(&chan->record_disabled); } static inline -void channel_record_enable(const struct lttng_ust_lib_ring_buffer_config *config, - struct channel *chan) +void channel_record_enable( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)), + struct lttng_ust_lib_ring_buffer_channel *chan) { uatomic_dec(&chan->record_disabled); } static inline -void lib_ring_buffer_record_disable(const struct lttng_ust_lib_ring_buffer_config *config, - struct lttng_ust_lib_ring_buffer *buf) +void lib_ring_buffer_record_disable( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)), + struct lttng_ust_lib_ring_buffer *buf) { uatomic_inc(&buf->record_disabled); } static inline -void lib_ring_buffer_record_enable(const struct lttng_ust_lib_ring_buffer_config *config, - struct lttng_ust_lib_ring_buffer *buf) +void lib_ring_buffer_record_enable( + const struct lttng_ust_lib_ring_buffer_config *config __attribute__((unused)), + struct lttng_ust_lib_ring_buffer *buf) { uatomic_dec(&buf->record_disabled); }