From a3492932cffa2c9dfbc9416792b20ce763708fc1 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Sun, 18 Sep 2016 22:52:39 -0400 Subject: [PATCH] Performance: cache the backend pages pointer in context Getting the backend pages pointer requires walking through the ring buffer backend tables through multiple shmp operations. Cache the current value so it can be re-used for all backend write operations writing fields for the same event. Signed-off-by: Mathieu Desnoyers --- include/lttng/ringbuffer-config.h | 1 + liblttng-ust/lttng-ring-buffer-client.h | 8 +++ .../lttng-ring-buffer-metadata-client.h | 13 +++- libringbuffer/backend.h | 69 ++++++++----------- libringbuffer/backend_internal.h | 47 +++++++++++++ 5 files changed, 96 insertions(+), 42 deletions(-) diff --git a/include/lttng/ringbuffer-config.h b/include/lttng/ringbuffer-config.h index c95d35cb..9ed9a34d 100644 --- a/include/lttng/ringbuffer-config.h +++ b/include/lttng/ringbuffer-config.h @@ -271,6 +271,7 @@ struct lttng_ust_lib_ring_buffer_ctx { * ctx_len is 0. Use the value of ctx_len to find out which of the * following fields may be used. */ + struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages; }; /** diff --git a/liblttng-ust/lttng-ring-buffer-client.h b/liblttng-ust/lttng-ring-buffer-client.h index e5640b2e..c5867dc1 100644 --- a/liblttng-ust/lttng-ring-buffer-client.h +++ b/liblttng-ust/lttng-ring-buffer-client.h @@ -717,6 +717,14 @@ int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx, ret = lib_ring_buffer_reserve(&client_config, ctx); if (ret) goto put; + if (caa_likely(ctx->ctx_len + >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) { + if (lib_ring_buffer_backend_get_pages(&client_config, ctx, + &ctx->backend_pages)) { + ret = -EPERM; + goto put; + } + } lttng_write_event_header(&client_config, ctx, event_id); return 0; put: diff --git a/liblttng-ust/lttng-ring-buffer-metadata-client.h b/liblttng-ust/lttng-ring-buffer-metadata-client.h index f3c4a67c..af5162f6 100644 --- a/liblttng-ust/lttng-ring-buffer-metadata-client.h +++ b/liblttng-ust/lttng-ring-buffer-metadata-client.h @@ -240,7 +240,18 @@ void lttng_channel_destroy(struct lttng_channel *chan) static int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx, uint32_t event_id) { - return lib_ring_buffer_reserve(&client_config, ctx); + int ret; + + ret = lib_ring_buffer_reserve(&client_config, ctx); + if (ret) + return ret; + if (caa_likely(ctx->ctx_len + >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) { + if (lib_ring_buffer_backend_get_pages(&client_config, ctx, + &ctx->backend_pages)) + return -EPERM; + } + return 0; } static diff --git a/libringbuffer/backend.h b/libringbuffer/backend.h index 2dec0038..1364d383 100644 --- a/libringbuffer/backend.h +++ b/libringbuffer/backend.h @@ -76,36 +76,24 @@ void lib_ring_buffer_write(const struct lttng_ust_lib_ring_buffer_config *config struct lttng_ust_lib_ring_buffer_ctx *ctx, const void *src, size_t len) { - struct lttng_ust_lib_ring_buffer_backend *bufb = &ctx->buf->backend; struct channel_backend *chanb = &ctx->chan->backend; struct lttng_ust_shm_handle *handle = ctx->handle; - size_t sbidx; size_t offset = ctx->buf_offset; - struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages; - unsigned long sb_bindex, id; struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages; void *p; if (caa_unlikely(!len)) return; - offset &= chanb->buf_size - 1; - sbidx = offset >> chanb->subbuf_size_order; - id = shmp_index(handle, bufb->buf_wsb, sbidx)->id; - sb_bindex = subbuffer_id_get_index(config, id); - rpages = shmp_index(handle, bufb->array, sb_bindex); - if (caa_unlikely(!rpages)) - return; - CHAN_WARN_ON(ctx->chan, - config->mode == RING_BUFFER_OVERWRITE - && subbuffer_id_is_noref(config, id)); /* * Underlying layer should never ask for writes across * subbuffers. */ - CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - backend_pages = shmp(handle, rpages->shmp); - if (caa_unlikely(!backend_pages)) - return; + CHAN_WARN_ON(chanb, (offset & (chanb->buf_size - 1)) + len > chanb->buf_size); + backend_pages = lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + if (caa_unlikely(!backend_pages)) { + if (lib_ring_buffer_backend_get_pages(config, ctx, &backend_pages)) + return; + } p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1)); if (caa_unlikely(!p)) return; @@ -159,47 +147,46 @@ void lib_ring_buffer_strcpy(const struct lttng_ust_lib_ring_buffer_config *confi struct lttng_ust_lib_ring_buffer_ctx *ctx, const char *src, size_t len, int pad) { - struct lttng_ust_lib_ring_buffer_backend *bufb = &ctx->buf->backend; struct channel_backend *chanb = &ctx->chan->backend; struct lttng_ust_shm_handle *handle = ctx->handle; - size_t sbidx, count; + size_t count; size_t offset = ctx->buf_offset; - struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages; - unsigned long sb_bindex, id; + struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages; + void *p; if (caa_unlikely(!len)) return; - offset &= chanb->buf_size - 1; - sbidx = offset >> chanb->subbuf_size_order; - id = shmp_index(handle, bufb->buf_wsb, sbidx)->id; - sb_bindex = subbuffer_id_get_index(config, id); - rpages = shmp_index(handle, bufb->array, sb_bindex); - CHAN_WARN_ON(ctx->chan, - config->mode == RING_BUFFER_OVERWRITE - && subbuffer_id_is_noref(config, id)); /* * Underlying layer should never ask for writes across * subbuffers. */ - CHAN_WARN_ON(chanb, offset >= chanb->buf_size); - count = lib_ring_buffer_do_strcpy(config, - shmp_index(handle, shmp(handle, rpages->shmp)->p, - offset & (chanb->subbuf_size - 1)), - src, len - 1); + CHAN_WARN_ON(chanb, (offset & (chanb->buf_size - 1)) + len > chanb->buf_size); + backend_pages = lib_ring_buffer_get_backend_pages_from_ctx(config, ctx); + if (caa_unlikely(!backend_pages)) { + if (lib_ring_buffer_backend_get_pages(config, ctx, &backend_pages)) + return; + } + p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1)); + if (caa_unlikely(!p)) + return; + + count = lib_ring_buffer_do_strcpy(config, p, src, len - 1); offset += count; /* Padding */ if (caa_unlikely(count < len - 1)) { size_t pad_len = len - 1 - count; - lib_ring_buffer_do_memset(shmp_index(handle, shmp(handle, rpages->shmp)->p, - offset & (chanb->subbuf_size - 1)), - pad, pad_len); + p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1)); + if (caa_unlikely(!p)) + return; + lib_ring_buffer_do_memset(p, pad, pad_len); offset += pad_len; } /* Final '\0' */ - lib_ring_buffer_do_memset(shmp_index(handle, shmp(handle, rpages->shmp)->p, - offset & (chanb->subbuf_size - 1)), - '\0', 1); + p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1)); + if (caa_unlikely(!p)) + return; + lib_ring_buffer_do_memset(p, '\0', 1); ctx->buf_offset += len; } diff --git a/libringbuffer/backend_internal.h b/libringbuffer/backend_internal.h index 9edd2e59..90088b89 100644 --- a/libringbuffer/backend_internal.h +++ b/libringbuffer/backend_internal.h @@ -196,6 +196,53 @@ int subbuffer_id_check_index(const struct lttng_ust_lib_ring_buffer_config *conf return 0; } +static inline +int lib_ring_buffer_backend_get_pages(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_ctx *ctx, + struct lttng_ust_lib_ring_buffer_backend_pages **backend_pages) +{ + struct lttng_ust_lib_ring_buffer_backend *bufb = &ctx->buf->backend; + struct channel_backend *chanb = &ctx->chan->backend; + struct lttng_ust_shm_handle *handle = ctx->handle; + size_t sbidx; + size_t offset = ctx->buf_offset; + struct lttng_ust_lib_ring_buffer_backend_subbuffer *wsb; + struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages; + unsigned long sb_bindex, id; + struct lttng_ust_lib_ring_buffer_backend_pages *_backend_pages; + + offset &= chanb->buf_size - 1; + sbidx = offset >> chanb->subbuf_size_order; + wsb = shmp_index(handle, bufb->buf_wsb, sbidx); + if (caa_unlikely(!wsb)) + return -1; + id = wsb->id; + sb_bindex = subbuffer_id_get_index(config, id); + rpages = shmp_index(handle, bufb->array, sb_bindex); + if (caa_unlikely(!rpages)) + return -1; + CHAN_WARN_ON(ctx->chan, + config->mode == RING_BUFFER_OVERWRITE + && subbuffer_id_is_noref(config, id)); + _backend_pages = shmp(handle, rpages->shmp); + if (caa_unlikely(!_backend_pages)) + return -1; + *backend_pages = _backend_pages; + return 0; +} + +/* Get backend pages from cache. */ +static inline +struct lttng_ust_lib_ring_buffer_backend_pages * + lib_ring_buffer_get_backend_pages_from_ctx(const struct lttng_ust_lib_ring_buffer_config *config, + struct lttng_ust_lib_ring_buffer_ctx *ctx) +{ + if (caa_unlikely(ctx->ctx_len + < sizeof(struct lttng_ust_lib_ring_buffer_ctx))) + return NULL; + return ctx->backend_pages; +} + /* * The ring buffer can count events recorded and overwritten per buffer, * but it is disabled by default due to its performance overhead. -- 2.34.1