Performance: cache the backend pages pointer in context

Getting the backend pages pointer requires walking through the ring
buffer backend tables through multiple shmp operations. Cache the
current value so it can be re-used for all backend write operations
writing fields for the same event.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>

diff --git a/include/lttng/ringbuffer-config.h b/include/lttng/ringbuffer-config.h
index c95d35cb348d820e69adba599ad3f90b9583b130..9ed9a34d34125a8b7bb8fc360768da1eedcafa64 100644 (file)
--- a/include/lttng/ringbuffer-config.h
+++ b/include/lttng/ringbuffer-config.h
@@ -271,6 +271,7 @@ struct lttng_ust_lib_ring_buffer_ctx {
         * ctx_len is 0. Use the value of ctx_len to find out which of the
         * following fields may be used.
         */
+       struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
 };
 
 /**

diff --git a/liblttng-ust/lttng-ring-buffer-client.h b/liblttng-ust/lttng-ring-buffer-client.h
index e5640b2ea0e4e7096e8729e975f75141b0892692..c5867dc1b697f2434a5bf17e74a98de85878661c 100644 (file)
--- a/liblttng-ust/lttng-ring-buffer-client.h
+++ b/liblttng-ust/lttng-ring-buffer-client.h
@@ -717,6 +717,14 @@ int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx,
        ret = lib_ring_buffer_reserve(&client_config, ctx);
        if (ret)
                goto put;
+       if (caa_likely(ctx->ctx_len
+                       >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) {
+               if (lib_ring_buffer_backend_get_pages(&client_config, ctx,
+                               &ctx->backend_pages)) {
+                       ret = -EPERM;
+                       goto put;
+               }
+       }
        lttng_write_event_header(&client_config, ctx, event_id);
        return 0;
 put:

diff --git a/liblttng-ust/lttng-ring-buffer-metadata-client.h b/liblttng-ust/lttng-ring-buffer-metadata-client.h
index f3c4a67c8843ea0e9514ff994e04ed16ef979c08..af5162f6e8cd23d944f92555c95146205016da22 100644 (file)
--- a/liblttng-ust/lttng-ring-buffer-metadata-client.h
+++ b/liblttng-ust/lttng-ring-buffer-metadata-client.h
@@ -240,7 +240,18 @@ void lttng_channel_destroy(struct lttng_channel *chan)
 static
 int lttng_event_reserve(struct lttng_ust_lib_ring_buffer_ctx *ctx, uint32_t event_id)
 {
-       return lib_ring_buffer_reserve(&client_config, ctx);
+       int ret;
+
+       ret = lib_ring_buffer_reserve(&client_config, ctx);
+       if (ret)
+               return ret;
+       if (caa_likely(ctx->ctx_len
+                       >= sizeof(struct lttng_ust_lib_ring_buffer_ctx))) {
+               if (lib_ring_buffer_backend_get_pages(&client_config, ctx,
+                               &ctx->backend_pages))
+                       return -EPERM;
+       }
+       return 0;
 }
 
 static

diff --git a/libringbuffer/backend.h b/libringbuffer/backend.h
index 2dec003883c7536ce3912b94b4a9987cc5d3b777..1364d383f4006b7c9db851aeb0813d8e55cbd2b6 100644 (file)
--- a/libringbuffer/backend.h
+++ b/libringbuffer/backend.h
@@ -76,36 +76,24 @@ void lib_ring_buffer_write(const struct lttng_ust_lib_ring_buffer_config *config
                           struct lttng_ust_lib_ring_buffer_ctx *ctx,
                           const void *src, size_t len)
 {
-       struct lttng_ust_lib_ring_buffer_backend *bufb = &ctx->buf->backend;
        struct channel_backend *chanb = &ctx->chan->backend;
        struct lttng_ust_shm_handle *handle = ctx->handle;
-       size_t sbidx;
        size_t offset = ctx->buf_offset;
-       struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
-       unsigned long sb_bindex, id;
        struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
        void *p;
 
        if (caa_unlikely(!len))
                return;
-       offset &= chanb->buf_size - 1;
-       sbidx = offset >> chanb->subbuf_size_order;
-       id = shmp_index(handle, bufb->buf_wsb, sbidx)->id;
-       sb_bindex = subbuffer_id_get_index(config, id);
-       rpages = shmp_index(handle, bufb->array, sb_bindex);
-       if (caa_unlikely(!rpages))
-               return;
-       CHAN_WARN_ON(ctx->chan,
-                    config->mode == RING_BUFFER_OVERWRITE
-                    && subbuffer_id_is_noref(config, id));
        /*
         * Underlying layer should never ask for writes across
         * subbuffers.
         */
-       CHAN_WARN_ON(chanb, offset >= chanb->buf_size);
-       backend_pages = shmp(handle, rpages->shmp);
-       if (caa_unlikely(!backend_pages))
-               return;
+       CHAN_WARN_ON(chanb, (offset & (chanb->buf_size - 1)) + len > chanb->buf_size);
+       backend_pages = lib_ring_buffer_get_backend_pages_from_ctx(config, ctx);
+       if (caa_unlikely(!backend_pages)) {
+               if (lib_ring_buffer_backend_get_pages(config, ctx, &backend_pages))
+                       return;
+       }
        p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
        if (caa_unlikely(!p))
                return;
@@ -159,47 +147,46 @@ void lib_ring_buffer_strcpy(const struct lttng_ust_lib_ring_buffer_config *confi
                           struct lttng_ust_lib_ring_buffer_ctx *ctx,
                           const char *src, size_t len, int pad)
 {
-       struct lttng_ust_lib_ring_buffer_backend *bufb = &ctx->buf->backend;
        struct channel_backend *chanb = &ctx->chan->backend;
        struct lttng_ust_shm_handle *handle = ctx->handle;
-       size_t sbidx, count;
+       size_t count;
        size_t offset = ctx->buf_offset;
-       struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
-       unsigned long sb_bindex, id;
+       struct lttng_ust_lib_ring_buffer_backend_pages *backend_pages;
+       void *p;
 
        if (caa_unlikely(!len))
                return;
-       offset &= chanb->buf_size - 1;
-       sbidx = offset >> chanb->subbuf_size_order;
-       id = shmp_index(handle, bufb->buf_wsb, sbidx)->id;
-       sb_bindex = subbuffer_id_get_index(config, id);
-       rpages = shmp_index(handle, bufb->array, sb_bindex);
-       CHAN_WARN_ON(ctx->chan,
-                    config->mode == RING_BUFFER_OVERWRITE
-                    && subbuffer_id_is_noref(config, id));
        /*
         * Underlying layer should never ask for writes across
         * subbuffers.
         */
-       CHAN_WARN_ON(chanb, offset >= chanb->buf_size);
-       count = lib_ring_buffer_do_strcpy(config,
-                       shmp_index(handle, shmp(handle, rpages->shmp)->p,
-                               offset & (chanb->subbuf_size - 1)),
-                       src, len - 1);
+       CHAN_WARN_ON(chanb, (offset & (chanb->buf_size - 1)) + len > chanb->buf_size);
+       backend_pages = lib_ring_buffer_get_backend_pages_from_ctx(config, ctx);
+       if (caa_unlikely(!backend_pages)) {
+               if (lib_ring_buffer_backend_get_pages(config, ctx, &backend_pages))
+                       return;
+       }
+       p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
+       if (caa_unlikely(!p))
+               return;
+
+       count = lib_ring_buffer_do_strcpy(config, p, src, len - 1);
        offset += count;
        /* Padding */
        if (caa_unlikely(count < len - 1)) {
                size_t pad_len = len - 1 - count;
 
-               lib_ring_buffer_do_memset(shmp_index(handle, shmp(handle, rpages->shmp)->p,
-                               offset & (chanb->subbuf_size - 1)),
-                       pad, pad_len);
+               p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
+               if (caa_unlikely(!p))
+                       return;
+               lib_ring_buffer_do_memset(p, pad, pad_len);
                offset += pad_len;
        }
        /* Final '\0' */
-       lib_ring_buffer_do_memset(shmp_index(handle, shmp(handle, rpages->shmp)->p,
-                               offset & (chanb->subbuf_size - 1)),
-                       '\0', 1);
+       p = shmp_index(handle, backend_pages->p, offset & (chanb->subbuf_size - 1));
+       if (caa_unlikely(!p))
+               return;
+       lib_ring_buffer_do_memset(p, '\0', 1);
        ctx->buf_offset += len;
 }
 

diff --git a/libringbuffer/backend_internal.h b/libringbuffer/backend_internal.h
index 9edd2e59f8b348bc5f8352e589d3499fcb0f476d..90088b89c7a8ffbfb81eac3efedad3cbefbb6f8e 100644 (file)
--- a/libringbuffer/backend_internal.h
+++ b/libringbuffer/backend_internal.h
@@ -196,6 +196,53 @@ int subbuffer_id_check_index(const struct lttng_ust_lib_ring_buffer_config *conf
                return 0;
 }
 
+static inline
+int lib_ring_buffer_backend_get_pages(const struct lttng_ust_lib_ring_buffer_config *config,
+                       struct lttng_ust_lib_ring_buffer_ctx *ctx,
+                       struct lttng_ust_lib_ring_buffer_backend_pages **backend_pages)
+{
+       struct lttng_ust_lib_ring_buffer_backend *bufb = &ctx->buf->backend;
+       struct channel_backend *chanb = &ctx->chan->backend;
+       struct lttng_ust_shm_handle *handle = ctx->handle;
+       size_t sbidx;
+       size_t offset = ctx->buf_offset;
+       struct lttng_ust_lib_ring_buffer_backend_subbuffer *wsb;
+       struct lttng_ust_lib_ring_buffer_backend_pages_shmp *rpages;
+       unsigned long sb_bindex, id;
+       struct lttng_ust_lib_ring_buffer_backend_pages *_backend_pages;
+
+       offset &= chanb->buf_size - 1;
+       sbidx = offset >> chanb->subbuf_size_order;
+       wsb = shmp_index(handle, bufb->buf_wsb, sbidx);
+       if (caa_unlikely(!wsb))
+               return -1;
+       id = wsb->id;
+       sb_bindex = subbuffer_id_get_index(config, id);
+       rpages = shmp_index(handle, bufb->array, sb_bindex);
+       if (caa_unlikely(!rpages))
+               return -1;
+       CHAN_WARN_ON(ctx->chan,
+                    config->mode == RING_BUFFER_OVERWRITE
+                    && subbuffer_id_is_noref(config, id));
+       _backend_pages = shmp(handle, rpages->shmp);
+       if (caa_unlikely(!_backend_pages))
+               return -1;
+       *backend_pages = _backend_pages;
+       return 0;
+}
+
+/* Get backend pages from cache. */
+static inline
+struct lttng_ust_lib_ring_buffer_backend_pages *
+       lib_ring_buffer_get_backend_pages_from_ctx(const struct lttng_ust_lib_ring_buffer_config *config,
+               struct lttng_ust_lib_ring_buffer_ctx *ctx)
+{
+       if (caa_unlikely(ctx->ctx_len
+                       < sizeof(struct lttng_ust_lib_ring_buffer_ctx)))
+               return NULL;
+       return ctx->backend_pages;
+}
+
 /*
  * The ring buffer can count events recorded and overwritten per buffer,
  * but it is disabled by default due to its performance overhead.