lib/ringbuffer/frontend_types.h

   1 #ifndef _LINUX_RING_BUFFER_FRONTEND_TYPES_H
   2 #define _LINUX_RING_BUFFER_FRONTEND_TYPES_H
   3
   4 /*
   5  * linux/ringbuffer/frontend_types.h
   6  *
   7  * (C) Copyright 2005-2010 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   8  *
   9  * Ring Buffer Library Synchronization Header (types).
  10  *
  11  * Author:
  12  *      Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  13  *
  14  * See ring_buffer_frontend.c for more information on wait-free algorithms.
  15  *
  16  * Dual LGPL v2.1/GPL v2 license.
  17  */
  18
  19 #include <linux/kref.h>
  20 #include "../../wrapper/ringbuffer/config.h"
  21 #include "../../wrapper/ringbuffer/backend_types.h"
  22 #include "../../wrapper/spinlock.h"
  23 #include "../../lib/prio_heap/lttng_prio_heap.h"        /* For per-CPU read-side iterator */
  24
  25 /*
  26  * A switch is done during tracing or as a final flush after tracing (so it
  27  * won't write in the new sub-buffer).
  28  */
  29 enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };
  30
  31 /* channel-level read-side iterator */
  32 struct channel_iter {
  33         /* Prio heap of buffers. Lowest timestamps at the top. */
  34         struct lttng_ptr_heap heap;     /* Heap of struct lib_ring_buffer ptrs */
  35         struct list_head empty_head;    /* Empty buffers linked-list head */
  36         int read_open;                  /* Opened for reading ? */
  37         u64 last_qs;                    /* Last quiescent state timestamp */
  38         u64 last_timestamp;             /* Last timestamp (for WARN_ON) */
  39         int last_cpu;                   /* Last timestamp cpu */
  40         /*
  41          * read() file operation state.
  42          */
  43         unsigned long len_left;
  44 };
  45
  46 /* channel: collection of per-cpu ring buffers. */
  47 struct channel {
  48         atomic_t record_disabled;
  49         unsigned long commit_count_mask;        /*
  50                                                  * Commit count mask, removing
  51                                                  * the MSBs corresponding to
  52                                                  * bits used to represent the
  53                                                  * subbuffer index.
  54                                                  */
  55
  56         struct channel_backend backend;         /* Associated backend */
  57
  58         unsigned long switch_timer_interval;    /* Buffer flush (jiffies) */
  59         unsigned long read_timer_interval;      /* Reader wakeup (jiffies) */
  60         struct notifier_block cpu_hp_notifier;  /* CPU hotplug notifier */
  61         struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
  62         struct notifier_block hp_iter_notifier; /* hotplug iterator notifier */
  63         int cpu_hp_enable:1;                    /* Enable CPU hotplug notif. */
  64         int hp_iter_enable:1;                   /* Enable hp iter notif. */
  65         wait_queue_head_t read_wait;            /* reader wait queue */
  66         wait_queue_head_t hp_wait;              /* CPU hotplug wait queue */
  67         int finalized;                          /* Has channel been finalized */
  68         struct channel_iter iter;               /* Channel read-side iterator */
  69         struct kref ref;                        /* Reference count */
  70 };
  71
  72 /* Per-subbuffer commit counters used on the hot path */
  73 struct commit_counters_hot {
  74         union v_atomic cc;              /* Commit counter */
  75         union v_atomic seq;             /* Consecutive commits */
  76 };
  77
  78 /* Per-subbuffer commit counters used only on cold paths */
  79 struct commit_counters_cold {
  80         union v_atomic cc_sb;           /* Incremented _once_ at sb switch */
  81 };
  82
  83 /* Per-buffer read iterator */
  84 struct lib_ring_buffer_iter {
  85         u64 timestamp;                  /* Current record timestamp */
  86         size_t header_len;              /* Current record header length */
  87         size_t payload_len;             /* Current record payload length */
  88
  89         struct list_head empty_node;    /* Linked list of empty buffers */
  90         unsigned long consumed, read_offset, data_size;
  91         enum {
  92                 ITER_GET_SUBBUF = 0,
  93                 ITER_TEST_RECORD,
  94                 ITER_NEXT_RECORD,
  95                 ITER_PUT_SUBBUF,
  96         } state;
  97         int allocated:1;
  98         int read_open:1;                /* Opened for reading ? */
  99 };
 100
 101 /* ring buffer state */
 102 struct lib_ring_buffer {
 103         /* First 32 bytes cache-hot cacheline */
 104         union v_atomic offset;          /* Current offset in the buffer */
 105         struct commit_counters_hot *commit_hot;
 106                                         /* Commit count per sub-buffer */
 107         atomic_long_t consumed;         /*
 108                                          * Current offset in the buffer
 109                                          * standard atomic access (shared)
 110                                          */
 111         atomic_t record_disabled;
 112         /* End of first 32 bytes cacheline */
 113         union v_atomic last_tsc;        /*
 114                                          * Last timestamp written in the buffer.
 115                                          */
 116
 117         struct lib_ring_buffer_backend backend; /* Associated backend */
 118
 119         struct commit_counters_cold *commit_cold;
 120                                         /* Commit count per sub-buffer */
 121         atomic_long_t active_readers;   /*
 122                                          * Active readers count
 123                                          * standard atomic access (shared)
 124                                          */
 125                                         /* Dropped records */
 126         union v_atomic records_lost_full;       /* Buffer full */
 127         union v_atomic records_lost_wrap;       /* Nested wrap-around */
 128         union v_atomic records_lost_big;        /* Events too big */
 129         union v_atomic records_count;   /* Number of records written */
 130         union v_atomic records_overrun; /* Number of overwritten records */
 131         wait_queue_head_t read_wait;    /* reader buffer-level wait queue */
 132         int finalized;                  /* buffer has been finalized */
 133         struct timer_list switch_timer; /* timer for periodical switch */
 134         struct timer_list read_timer;   /* timer for read poll */
 135         raw_spinlock_t raw_tick_nohz_spinlock;  /* nohz entry lock/trylock */
 136         struct lib_ring_buffer_iter iter;       /* read-side iterator */
 137         unsigned long get_subbuf_consumed;      /* Read-side consumed */
 138         unsigned long prod_snapshot;    /* Producer count snapshot */
 139         unsigned long cons_snapshot;    /* Consumer count snapshot */
 140         int get_subbuf:1;               /* Sub-buffer being held by reader */
 141         int switch_timer_enabled:1;     /* Protected by ring_buffer_nohz_lock */
 142         int read_timer_enabled:1;       /* Protected by ring_buffer_nohz_lock */
 143 };
 144
 145 static inline
 146 void *channel_get_private(struct channel *chan)
 147 {
 148         return chan->backend.priv;
 149 }
 150
 151 /*
 152  * Issue warnings and disable channels upon internal error.
 153  * Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend
 154  * parameters.
 155  */
 156 #define CHAN_WARN_ON(c, cond)                                           \
 157         ({                                                              \
 158                 struct channel *__chan;                                 \
 159                 int _____ret = unlikely(cond);                          \
 160                 if (_____ret) {                                         \
 161                         if (__same_type(*(c), struct channel_backend))  \
 162                                 __chan = container_of((void *) (c),     \
 163                                                         struct channel, \
 164                                                         backend);       \
 165                         else if (__same_type(*(c), struct channel))     \
 166                                 __chan = (void *) (c);                  \
 167                         else                                            \
 168                                 BUG_ON(1);                              \
 169                         atomic_inc(&__chan->record_disabled);           \
 170                         WARN_ON(1);                                     \
 171                 }                                                       \
 172                 _____ret;                                               \
 173         })
 174
 175 #endif /* _LINUX_RING_BUFFER_FRONTEND_TYPES_H */