lib/ringbuffer/frontend_types.h

   1 #ifndef _LIB_RING_BUFFER_FRONTEND_TYPES_H
   2 #define _LIB_RING_BUFFER_FRONTEND_TYPES_H
   3
   4 /*
   5  * lib/ringbuffer/frontend_types.h
   6  *
   7  * Ring Buffer Library Synchronization Header (types).
   8  *
   9  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; only
  14  * version 2.1 of the License.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  24  *
  25  * Author:
  26  *      Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  27  *
  28  * See ring_buffer_frontend.c for more information on wait-free algorithms.
  29  */
  30
  31 #include <linux/kref.h>
  32 #include <wrapper/ringbuffer/config.h>
  33 #include <wrapper/ringbuffer/backend_types.h>
  34 #include <wrapper/spinlock.h>
  35 #include <lib/prio_heap/lttng_prio_heap.h>      /* For per-CPU read-side iterator */
  36
  37 /*
  38  * A switch is done during tracing or as a final flush after tracing (so it
  39  * won't write in the new sub-buffer).
  40  */
  41 enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };
  42
  43 /* channel-level read-side iterator */
  44 struct channel_iter {
  45         /* Prio heap of buffers. Lowest timestamps at the top. */
  46         struct lttng_ptr_heap heap;     /* Heap of struct lib_ring_buffer ptrs */
  47         struct list_head empty_head;    /* Empty buffers linked-list head */
  48         int read_open;                  /* Opened for reading ? */
  49         u64 last_qs;                    /* Last quiescent state timestamp */
  50         u64 last_timestamp;             /* Last timestamp (for WARN_ON) */
  51         int last_cpu;                   /* Last timestamp cpu */
  52         /*
  53          * read() file operation state.
  54          */
  55         unsigned long len_left;
  56 };
  57
  58 /* channel: collection of per-cpu ring buffers. */
  59 struct channel {
  60         atomic_t record_disabled;
  61         unsigned long commit_count_mask;        /*
  62                                                  * Commit count mask, removing
  63                                                  * the MSBs corresponding to
  64                                                  * bits used to represent the
  65                                                  * subbuffer index.
  66                                                  */
  67
  68         struct channel_backend backend;         /* Associated backend */
  69
  70         unsigned long switch_timer_interval;    /* Buffer flush (jiffies) */
  71         unsigned long read_timer_interval;      /* Reader wakeup (jiffies) */
  72         struct notifier_block cpu_hp_notifier;  /* CPU hotplug notifier */
  73         struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
  74         struct notifier_block hp_iter_notifier; /* hotplug iterator notifier */
  75         unsigned int cpu_hp_enable:1;           /* Enable CPU hotplug notif. */
  76         unsigned int hp_iter_enable:1;          /* Enable hp iter notif. */
  77         wait_queue_head_t read_wait;            /* reader wait queue */
  78         wait_queue_head_t hp_wait;              /* CPU hotplug wait queue */
  79         int finalized;                          /* Has channel been finalized */
  80         struct channel_iter iter;               /* Channel read-side iterator */
  81         struct kref ref;                        /* Reference count */
  82 };
  83
  84 /* Per-subbuffer commit counters used on the hot path */
  85 struct commit_counters_hot {
  86         union v_atomic cc;              /* Commit counter */
  87         union v_atomic seq;             /* Consecutive commits */
  88 };
  89
  90 /* Per-subbuffer commit counters used only on cold paths */
  91 struct commit_counters_cold {
  92         union v_atomic cc_sb;           /* Incremented _once_ at sb switch */
  93 };
  94
  95 /* Per-buffer read iterator */
  96 struct lib_ring_buffer_iter {
  97         u64 timestamp;                  /* Current record timestamp */
  98         size_t header_len;              /* Current record header length */
  99         size_t payload_len;             /* Current record payload length */
 100
 101         struct list_head empty_node;    /* Linked list of empty buffers */
 102         unsigned long consumed, read_offset, data_size;
 103         enum {
 104                 ITER_GET_SUBBUF = 0,
 105                 ITER_TEST_RECORD,
 106                 ITER_NEXT_RECORD,
 107                 ITER_PUT_SUBBUF,
 108         } state;
 109         unsigned int allocated:1;
 110         unsigned int read_open:1;       /* Opened for reading ? */
 111 };
 112
 113 /* ring buffer state */
 114 struct lib_ring_buffer {
 115         /* First 32 bytes cache-hot cacheline */
 116         union v_atomic offset;          /* Current offset in the buffer */
 117         struct commit_counters_hot *commit_hot;
 118                                         /* Commit count per sub-buffer */
 119         atomic_long_t consumed;         /*
 120                                          * Current offset in the buffer
 121                                          * standard atomic access (shared)
 122                                          */
 123         atomic_t record_disabled;
 124         /* End of first 32 bytes cacheline */
 125         union v_atomic last_tsc;        /*
 126                                          * Last timestamp written in the buffer.
 127                                          */
 128
 129         struct lib_ring_buffer_backend backend; /* Associated backend */
 130
 131         struct commit_counters_cold *commit_cold;
 132                                         /* Commit count per sub-buffer */
 133         atomic_long_t active_readers;   /*
 134                                          * Active readers count
 135                                          * standard atomic access (shared)
 136                                          */
 137                                         /* Dropped records */
 138         union v_atomic records_lost_full;       /* Buffer full */
 139         union v_atomic records_lost_wrap;       /* Nested wrap-around */
 140         union v_atomic records_lost_big;        /* Events too big */
 141         union v_atomic records_count;   /* Number of records written */
 142         union v_atomic records_overrun; /* Number of overwritten records */
 143         wait_queue_head_t read_wait;    /* reader buffer-level wait queue */
 144         wait_queue_head_t write_wait;   /* writer buffer-level wait queue (for metadata only) */
 145         int finalized;                  /* buffer has been finalized */
 146         struct timer_list switch_timer; /* timer for periodical switch */
 147         struct timer_list read_timer;   /* timer for read poll */
 148         raw_spinlock_t raw_tick_nohz_spinlock;  /* nohz entry lock/trylock */
 149         struct lib_ring_buffer_iter iter;       /* read-side iterator */
 150         unsigned long get_subbuf_consumed;      /* Read-side consumed */
 151         unsigned long prod_snapshot;    /* Producer count snapshot */
 152         unsigned long cons_snapshot;    /* Consumer count snapshot */
 153         unsigned int get_subbuf:1,      /* Sub-buffer being held by reader */
 154                 switch_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */
 155                 read_timer_enabled:1;   /* Protected by ring_buffer_nohz_lock */
 156 };
 157
 158 static inline
 159 void *channel_get_private(struct channel *chan)
 160 {
 161         return chan->backend.priv;
 162 }
 163
 164 /*
 165  * Issue warnings and disable channels upon internal error.
 166  * Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend
 167  * parameters.
 168  */
 169 #define CHAN_WARN_ON(c, cond)                                           \
 170         ({                                                              \
 171                 struct channel *__chan;                                 \
 172                 int _____ret = unlikely(cond);                          \
 173                 if (_____ret) {                                         \
 174                         if (__same_type(*(c), struct channel_backend))  \
 175                                 __chan = container_of((void *) (c),     \
 176                                                         struct channel, \
 177                                                         backend);       \
 178                         else if (__same_type(*(c), struct channel))     \
 179                                 __chan = (void *) (c);                  \
 180                         else                                            \
 181                                 BUG_ON(1);                              \
 182                         atomic_inc(&__chan->record_disabled);           \
 183                         WARN_ON(1);                                     \
 184                 }                                                       \
 185                 _____ret;                                               \
 186         })
 187
 188 #endif /* _LIB_RING_BUFFER_FRONTEND_TYPES_H */