libringbuffer/frontend_types.h

   1 /*
   2  * SPDX-License-Identifier: LGPL-2.1-only
   3  *
   4  * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   5  *
   6  * Ring Buffer Library Synchronization Header (types).
   7  *
   8  * See ring_buffer_frontend.c for more information on wait-free algorithms.
   9  */
  10
  11 #ifndef _LTTNG_RING_BUFFER_FRONTEND_TYPES_H
  12 #define _LTTNG_RING_BUFFER_FRONTEND_TYPES_H
  13
  14 #include <stdint.h>
  15 #include <string.h>
  16 #include <time.h>       /* for timer_t */
  17
  18 #include <urcu/list.h>
  19 #include <urcu/uatomic.h>
  20
  21 #include <lttng/ringbuffer-context.h>
  22 #include "ringbuffer-config.h"
  23 #include <usterr-signal-safe.h>
  24 #include "backend_types.h"
  25 #include "shm_internal.h"
  26 #include "shm_types.h"
  27 #include "vatomic.h"
  28
  29 /*
  30  * A switch is done during tracing or as a final flush after tracing (so it
  31  * won't write in the new sub-buffer).
  32  */
  33 enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };
  34
  35 /* channel: collection of per-cpu ring buffers. */
  36 #define RB_CHANNEL_PADDING              32
  37 struct lttng_ust_lib_ring_buffer_channel {
  38         int record_disabled;
  39         unsigned long commit_count_mask;        /*
  40                                                  * Commit count mask, removing
  41                                                  * the MSBs corresponding to
  42                                                  * bits used to represent the
  43                                                  * subbuffer index.
  44                                                  */
  45
  46         unsigned long switch_timer_interval;    /* Buffer flush (us) */
  47         timer_t switch_timer;
  48         int switch_timer_enabled;
  49
  50         unsigned long read_timer_interval;      /* Reader wakeup (us) */
  51         timer_t read_timer;
  52         int read_timer_enabled;
  53
  54         int finalized;                          /* Has channel been finalized */
  55         size_t priv_data_offset;                /* Offset of private data channel config */
  56         unsigned int nr_streams;                /* Number of streams */
  57         struct lttng_ust_shm_handle *handle;
  58         /* Extended options. */
  59         union {
  60                 struct {
  61                         int32_t blocking_timeout_ms;
  62                         void *priv;             /* Private data pointer. */
  63                 } s;
  64                 char padding[RB_CHANNEL_PADDING];
  65         } u;
  66         /*
  67          * Associated backend contains a variable-length array. Needs to
  68          * be last member.
  69          */
  70         struct channel_backend backend;         /* Associated backend */
  71 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
  72
  73 /* Per-subbuffer commit counters used on the hot path */
  74 #define RB_COMMIT_COUNT_HOT_PADDING     16
  75 struct commit_counters_hot {
  76         union v_atomic cc;              /* Commit counter */
  77         union v_atomic seq;             /* Consecutive commits */
  78         char padding[RB_COMMIT_COUNT_HOT_PADDING];
  79 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
  80
  81 /* Per-subbuffer commit counters used only on cold paths */
  82 #define RB_COMMIT_COUNT_COLD_PADDING    24
  83 struct commit_counters_cold {
  84         union v_atomic cc_sb;           /* Incremented _once_ at sb switch */
  85         char padding[RB_COMMIT_COUNT_COLD_PADDING];
  86 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
  87
  88 /* ring buffer state */
  89 #define RB_CRASH_DUMP_ABI_LEN           256
  90 #define RB_RING_BUFFER_PADDING          60
  91
  92 #define RB_CRASH_DUMP_ABI_MAGIC_LEN     16
  93
  94 /*
  95  * The 128-bit magic number is xor'd in the process data so it does not
  96  * cause a false positive when searching for buffers by scanning memory.
  97  * The actual magic number is:
  98  *   0x17, 0x7B, 0xF1, 0x77, 0xBF, 0x17, 0x7B, 0xF1,
  99  *   0x77, 0xBF, 0x17, 0x7B, 0xF1, 0x77, 0xBF, 0x17,
 100  */
 101 #define RB_CRASH_DUMP_ABI_MAGIC_XOR                                     \
 102         {                                                               \
 103                 0x17 ^ 0xFF, 0x7B ^ 0xFF, 0xF1 ^ 0xFF, 0x77 ^ 0xFF,     \
 104                 0xBF ^ 0xFF, 0x17 ^ 0xFF, 0x7B ^ 0xFF, 0xF1 ^ 0xFF,     \
 105                 0x77 ^ 0xFF, 0xBF ^ 0xFF, 0x17 ^ 0xFF, 0x7B ^ 0xFF,     \
 106                 0xF1 ^ 0xFF, 0x77 ^ 0xFF, 0xBF ^ 0xFF, 0x17 ^ 0xFF,     \
 107         }
 108
 109 #define RB_CRASH_ENDIAN                 0x1234
 110
 111 #define RB_CRASH_DUMP_ABI_MAJOR         0
 112 #define RB_CRASH_DUMP_ABI_MINOR         0
 113
 114 enum lttng_crash_type {
 115         LTTNG_CRASH_TYPE_UST = 0,
 116         LTTNG_CRASH_TYPE_KERNEL = 1,
 117 };
 118
 119 struct lttng_crash_abi {
 120         uint8_t magic[RB_CRASH_DUMP_ABI_MAGIC_LEN];
 121         uint64_t mmap_length;   /* Overall lenght of crash record */
 122         uint16_t endian;        /*
 123                                  * { 0x12, 0x34 }: big endian
 124                                  * { 0x34, 0x12 }: little endian
 125                                  */
 126         uint16_t major;         /* Major number. */
 127         uint16_t minor;         /* Minor number. */
 128         uint8_t word_size;      /* Word size (bytes). */
 129         uint8_t layout_type;    /* enum lttng_crash_type */
 130
 131         struct {
 132                 uint32_t prod_offset;
 133                 uint32_t consumed_offset;
 134                 uint32_t commit_hot_array;
 135                 uint32_t commit_hot_seq;
 136                 uint32_t buf_wsb_array;
 137                 uint32_t buf_wsb_id;
 138                 uint32_t sb_array;
 139                 uint32_t sb_array_shmp_offset;
 140                 uint32_t sb_backend_p_offset;
 141                 uint32_t content_size;
 142                 uint32_t packet_size;
 143         } __attribute__((packed)) offset;
 144         struct {
 145                 uint8_t prod_offset;
 146                 uint8_t consumed_offset;
 147                 uint8_t commit_hot_seq;
 148                 uint8_t buf_wsb_id;
 149                 uint8_t sb_array_shmp_offset;
 150                 uint8_t sb_backend_p_offset;
 151                 uint8_t content_size;
 152                 uint8_t packet_size;
 153         } __attribute__((packed)) length;
 154         struct {
 155                 uint32_t commit_hot_array;
 156                 uint32_t buf_wsb_array;
 157                 uint32_t sb_array;
 158         } __attribute__((packed)) stride;
 159
 160         uint64_t buf_size;      /* Size of the buffer */
 161         uint64_t subbuf_size;   /* Sub-buffer size */
 162         uint64_t num_subbuf;    /* Number of sub-buffers for writer */
 163         uint32_t mode;          /* Buffer mode: 0: overwrite, 1: discard */
 164 } __attribute__((packed));
 165
 166 struct lttng_ust_lib_ring_buffer {
 167         /* First 32 bytes are for the buffer crash dump ABI */
 168         struct lttng_crash_abi crash_abi;
 169
 170         /* 32 bytes cache-hot cacheline */
 171         union v_atomic __attribute__((aligned(32))) offset;
 172                                         /* Current offset in the buffer */
 173         DECLARE_SHMP(struct commit_counters_hot, commit_hot);
 174                                         /* Commit count per sub-buffer */
 175         long consumed;                  /*
 176                                          * Current offset in the buffer
 177                                          * standard atomic access (shared)
 178                                          */
 179         int record_disabled;
 180         /* End of cache-hot 32 bytes cacheline */
 181
 182         union v_atomic last_tsc;        /*
 183                                          * Last timestamp written in the buffer.
 184                                          */
 185
 186         struct lttng_ust_lib_ring_buffer_backend backend;
 187                                         /* Associated backend */
 188
 189         DECLARE_SHMP(struct commit_counters_cold, commit_cold);
 190                                         /* Commit count per sub-buffer */
 191         DECLARE_SHMP(uint64_t, ts_end); /*
 192                                          * timestamp_end per sub-buffer.
 193                                          * Time is sampled by the
 194                                          * switch_*_end() callbacks
 195                                          * which are the last space
 196                                          * reservation performed in the
 197                                          * sub-buffer before it can be
 198                                          * fully committed and
 199                                          * delivered. This time value is
 200                                          * then read by the deliver
 201                                          * callback, performed by the
 202                                          * last commit before the buffer
 203                                          * becomes readable.
 204                                          */
 205         long active_readers;            /*
 206                                          * Active readers count
 207                                          * standard atomic access (shared)
 208                                          */
 209                                         /* Dropped records */
 210         union v_atomic records_lost_full;       /* Buffer full */
 211         union v_atomic records_lost_wrap;       /* Nested wrap-around */
 212         union v_atomic records_lost_big;        /* Events too big */
 213         union v_atomic records_count;   /* Number of records written */
 214         union v_atomic records_overrun; /* Number of overwritten records */
 215         //wait_queue_head_t read_wait;  /* reader buffer-level wait queue */
 216         int finalized;                  /* buffer has been finalized */
 217         unsigned long get_subbuf_consumed;      /* Read-side consumed */
 218         unsigned long prod_snapshot;    /* Producer count snapshot */
 219         unsigned long cons_snapshot;    /* Consumer count snapshot */
 220         unsigned int get_subbuf:1;      /* Sub-buffer being held by reader */
 221         /* shmp pointer to self */
 222         DECLARE_SHMP(struct lttng_ust_lib_ring_buffer, self);
 223         char padding[RB_RING_BUFFER_PADDING];
 224 } __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
 225
 226 static inline
 227 void *channel_get_private_config(struct lttng_ust_lib_ring_buffer_channel *chan)
 228 {
 229         return ((char *) chan) + chan->priv_data_offset;
 230 }
 231
 232 static inline
 233 void *channel_get_private(struct lttng_ust_lib_ring_buffer_channel *chan)
 234 {
 235         return chan->u.s.priv;
 236 }
 237
 238 static inline
 239 void channel_set_private(struct lttng_ust_lib_ring_buffer_channel *chan, void *priv)
 240 {
 241         chan->u.s.priv = priv;
 242 }
 243
 244 #ifndef __rb_same_type
 245 #define __rb_same_type(a, b)    __builtin_types_compatible_p(typeof(a), typeof(b))
 246 #endif
 247
 248 /*
 249  * Issue warnings and disable channels upon internal error.
 250  * Can receive struct lttng_ust_lib_ring_buffer or struct lttng_ust_lib_ring_buffer_backend
 251  * parameters.
 252  */
 253 #define CHAN_WARN_ON(c, cond)                                           \
 254         ({                                                              \
 255                 struct lttng_ust_lib_ring_buffer_channel *__chan;       \
 256                 int _____ret = caa_unlikely(cond);                              \
 257                 if (_____ret) {                                         \
 258                         if (__rb_same_type(*(c), struct channel_backend))       \
 259                                 __chan = caa_container_of((void *) (c), \
 260                                         struct lttng_ust_lib_ring_buffer_channel, \
 261                                         backend);                       \
 262                         else if (__rb_same_type(*(c),                   \
 263                                         struct lttng_ust_lib_ring_buffer_channel)) \
 264                                 __chan = (void *) (c);                  \
 265                         else                                            \
 266                                 BUG_ON(1);                              \
 267                         uatomic_inc(&__chan->record_disabled);          \
 268                         WARN_ON(1);                                     \
 269                 }                                                       \
 270                 _____ret = _____ret; /* For clang "unused result". */   \
 271         })
 272
 273 #endif /* _LTTNG_RING_BUFFER_FRONTEND_TYPES_H */