[lttng-modules.git] / lib / ringbuffer / frontend_types.h

/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
 *
 * lib/ringbuffer/frontend_types.h
 *
 * Ring Buffer Library Synchronization Header (types).
 *
 * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * See ring_buffer_frontend.c for more information on wait-free algorithms.
 */

#ifndef _LIB_RING_BUFFER_FRONTEND_TYPES_H
#define _LIB_RING_BUFFER_FRONTEND_TYPES_H

#include <linux/kref.h>
#include <wrapper/ringbuffer/config.h>
#include <wrapper/ringbuffer/backend_types.h>
#include <lib/prio_heap/lttng_prio_heap.h>	/* For per-CPU read-side iterator */
#include <lttng-cpuhotplug.h>

/*
 * A switch is done during tracing or as a final flush after tracing (so it
 * won't write in the new sub-buffer).
 */
enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };

/* channel-level read-side iterator */
struct channel_iter {
	/* Prio heap of buffers. Lowest timestamps at the top. */
	struct lttng_ptr_heap heap;	/* Heap of struct lib_ring_buffer ptrs */
	struct list_head empty_head;	/* Empty buffers linked-list head */
	int read_open;			/* Opened for reading ? */
	u64 last_qs;			/* Last quiescent state timestamp */
	u64 last_timestamp;		/* Last timestamp (for WARN_ON) */
	int last_cpu;			/* Last timestamp cpu */
	/*
	 * read() file operation state.
	 */
	unsigned long len_left;
};

/* channel: collection of per-cpu ring buffers. */
struct channel {
	atomic_t record_disabled;
	unsigned long commit_count_mask;	/*
						 * Commit count mask, removing
						 * the MSBs corresponding to
						 * bits used to represent the
						 * subbuffer index.
						 */

	struct channel_backend backend;		/* Associated backend */

	unsigned long switch_timer_interval;	/* Buffer flush (jiffies) */
	unsigned long read_timer_interval;	/* Reader wakeup (jiffies) */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
	struct lttng_cpuhp_node cpuhp_prepare;
	struct lttng_cpuhp_node cpuhp_online;
	struct lttng_cpuhp_node cpuhp_iter_online;
#else
	struct notifier_block cpu_hp_notifier;	/* CPU hotplug notifier */
	struct notifier_block hp_iter_notifier;	/* hotplug iterator notifier */
	unsigned int cpu_hp_enable:1;		/* Enable CPU hotplug notif. */
	unsigned int hp_iter_enable:1;		/* Enable hp iter notif. */
#endif
	struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
	wait_queue_head_t read_wait;		/* reader wait queue */
	wait_queue_head_t hp_wait;		/* CPU hotplug wait queue */
	int finalized;				/* Has channel been finalized */
	struct channel_iter iter;		/* Channel read-side iterator */
	struct kref ref;			/* Reference count */
};

/* Per-subbuffer commit counters used on the hot path */
struct commit_counters_hot {
	union v_atomic cc;		/* Commit counter */
	union v_atomic seq;		/* Consecutive commits */
};

/* Per-subbuffer commit counters used only on cold paths */
struct commit_counters_cold {
	union v_atomic cc_sb;		/* Incremented _once_ at sb switch */
};

/* Per-buffer read iterator */
struct lib_ring_buffer_iter {
	u64 timestamp;			/* Current record timestamp */
	size_t header_len;		/* Current record header length */
	size_t payload_len;		/* Current record payload length */

	struct list_head empty_node;	/* Linked list of empty buffers */
	unsigned long consumed, read_offset, data_size;
	enum {
		ITER_GET_SUBBUF = 0,
		ITER_TEST_RECORD,
		ITER_NEXT_RECORD,
		ITER_PUT_SUBBUF,
	} state;
	unsigned int allocated:1;
	unsigned int read_open:1;	/* Opened for reading ? */
};

/* ring buffer state */
struct lib_ring_buffer {
	/* First 32 bytes cache-hot cacheline */
	union v_atomic offset;		/* Current offset in the buffer */
	struct commit_counters_hot *commit_hot;
					/* Commit count per sub-buffer */
	atomic_long_t consumed;		/*
					 * Current offset in the buffer
					 * standard atomic access (shared)
					 */
	atomic_t record_disabled;
	/* End of first 32 bytes cacheline */
	union v_atomic last_tsc;	/*
					 * Last timestamp written in the buffer.
					 */

	struct lib_ring_buffer_backend backend;	/* Associated backend */

	struct commit_counters_cold *commit_cold;
					/* Commit count per sub-buffer */
	u64 *ts_end;			/*
					 * timestamp_end per sub-buffer.
					 * Time is sampled by the
					 * switch_*_end() callbacks which
					 * are the last space reservation
					 * performed in the sub-buffer
					 * before it can be fully
					 * committed and delivered. This
					 * time value is then read by
					 * the deliver callback,
					 * performed by the last commit
					 * before the buffer becomes
					 * readable.
					 */
	atomic_long_t active_readers;	/*
					 * Active readers count
					 * standard atomic access (shared)
					 */
					/* Dropped records */
	union v_atomic records_lost_full;	/* Buffer full */
	union v_atomic records_lost_wrap;	/* Nested wrap-around */
	union v_atomic records_lost_big;	/* Events too big */
	union v_atomic records_count;	/* Number of records written */
	union v_atomic records_overrun;	/* Number of overwritten records */
	wait_queue_head_t read_wait;	/* reader buffer-level wait queue */
	wait_queue_head_t write_wait;	/* writer buffer-level wait queue (for metadata only) */
	int finalized;			/* buffer has been finalized */
	struct timer_list switch_timer;	/* timer for periodical switch */
	struct timer_list read_timer;	/* timer for read poll */
	raw_spinlock_t raw_tick_nohz_spinlock;	/* nohz entry lock/trylock */
	struct lib_ring_buffer_iter iter;	/* read-side iterator */
	unsigned long get_subbuf_consumed;	/* Read-side consumed */
	unsigned long prod_snapshot;	/* Producer count snapshot */
	unsigned long cons_snapshot;	/* Consumer count snapshot */
	unsigned int get_subbuf:1,	/* Sub-buffer being held by reader */
		switch_timer_enabled:1,	/* Protected by ring_buffer_nohz_lock */
		read_timer_enabled:1,	/* Protected by ring_buffer_nohz_lock */
		quiescent:1;
};

static inline
void *channel_get_private(struct channel *chan)
{
	return chan->backend.priv;
}

void lib_ring_buffer_lost_event_too_big(struct channel *chan);

/*
 * Issue warnings and disable channels upon internal error.
 * Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend
 * parameters.
 */
#define CHAN_WARN_ON(c, cond)						\
	({								\
		struct channel *__chan;					\
		int _____ret = unlikely(cond);				\
		if (_____ret) {						\
			if (__same_type(*(c), struct channel_backend))	\
				__chan = container_of((void *) (c),	\
							struct channel, \
							backend);	\
			else if (__same_type(*(c), struct channel))	\
				__chan = (void *) (c);			\
			else						\
				BUG_ON(1);				\
			atomic_inc(&__chan->record_disabled);		\
			WARN_ON(1);					\
		}							\
		_____ret;						\
	})

#endif /* _LIB_RING_BUFFER_FRONTEND_TYPES_H */
Commit	Line	Data
b7cdc182	1	/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
9f36eaed	2	*
886d51a3	3	* lib/ringbuffer/frontend_types.h
f3bc08c5 MD	4	*
	5	* Ring Buffer Library Synchronization Header (types).
	6	*
886d51a3 MD	7	* Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
886d51a3 MD	8	*
f3bc08c5	9	* See ring_buffer_frontend.c for more information on wait-free algorithms.
f3bc08c5 MD	10	*/
f3bc08c5 MD	11
9f36eaed MJ	12	#ifndef _LIB_RING_BUFFER_FRONTEND_TYPES_H
	13	#define _LIB_RING_BUFFER_FRONTEND_TYPES_H
	14
f40270ad	15	#include <linux/kref.h>
5671a661 MD	16	#include <wrapper/ringbuffer/config.h>
5671a661 MD	17	#include <wrapper/ringbuffer/backend_types.h>
5671a661	18	#include <lib/prio_heap/lttng_prio_heap.h> /* For per-CPU read-side iterator */
1e367326	19	#include <lttng-cpuhotplug.h>
f3bc08c5 MD	20
	21	/*
	22	* A switch is done during tracing or as a final flush after tracing (so it
	23	* won't write in the new sub-buffer).
	24	*/
	25	enum switch_mode { SWITCH_ACTIVE, SWITCH_FLUSH };
	26
	27	/* channel-level read-side iterator */
	28	struct channel_iter {
	29	/* Prio heap of buffers. Lowest timestamps at the top. */
a88db018	30	struct lttng_ptr_heap heap; /* Heap of struct lib_ring_buffer ptrs */
f3bc08c5 MD	31	struct list_head empty_head; /* Empty buffers linked-list head */
	32	int read_open; /* Opened for reading ? */
	33	u64 last_qs; /* Last quiescent state timestamp */
	34	u64 last_timestamp; /* Last timestamp (for WARN_ON) */
	35	int last_cpu; /* Last timestamp cpu */
	36	/*
	37	* read() file operation state.
	38	*/
	39	unsigned long len_left;
	40	};
	41
	42	/* channel: collection of per-cpu ring buffers. */
	43	struct channel {
	44	atomic_t record_disabled;
	45	unsigned long commit_count_mask; /*
	46	* Commit count mask, removing
	47	* the MSBs corresponding to
	48	* bits used to represent the
	49	* subbuffer index.
	50	*/
	51
	52	struct channel_backend backend; /* Associated backend */
	53
	54	unsigned long switch_timer_interval; /* Buffer flush (jiffies) */
	55	unsigned long read_timer_interval; /* Reader wakeup (jiffies) */
1e367326 MD	56	#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0))
	57	struct lttng_cpuhp_node cpuhp_prepare;
	58	struct lttng_cpuhp_node cpuhp_online;
	59	struct lttng_cpuhp_node cpuhp_iter_online;
	60	#else
f3bc08c5	61	struct notifier_block cpu_hp_notifier; /* CPU hotplug notifier */
f3bc08c5	62	struct notifier_block hp_iter_notifier; /* hotplug iterator notifier */
9cccf98a MD	63	unsigned int cpu_hp_enable:1; /* Enable CPU hotplug notif. */
9cccf98a MD	64	unsigned int hp_iter_enable:1; /* Enable hp iter notif. */
1e367326 MD	65	#endif
1e367326 MD	66	struct notifier_block tick_nohz_notifier; /* CPU nohz notifier */
f3bc08c5	67	wait_queue_head_t read_wait; /* reader wait queue */
24cedcfe MD	68	wait_queue_head_t hp_wait; /* CPU hotplug wait queue */
24cedcfe MD	69	int finalized; /* Has channel been finalized */
f3bc08c5	70	struct channel_iter iter; /* Channel read-side iterator */
f40270ad	71	struct kref ref; /* Reference count */
f3bc08c5 MD	72	};
	73
	74	/* Per-subbuffer commit counters used on the hot path */
	75	struct commit_counters_hot {
	76	union v_atomic cc; /* Commit counter */
	77	union v_atomic seq; /* Consecutive commits */
	78	};
	79
	80	/* Per-subbuffer commit counters used only on cold paths */
	81	struct commit_counters_cold {
	82	union v_atomic cc_sb; /* Incremented _once_ at sb switch */
	83	};
	84
	85	/* Per-buffer read iterator */
	86	struct lib_ring_buffer_iter {
	87	u64 timestamp; /* Current record timestamp */
	88	size_t header_len; /* Current record header length */
	89	size_t payload_len; /* Current record payload length */
	90
	91	struct list_head empty_node; /* Linked list of empty buffers */
	92	unsigned long consumed, read_offset, data_size;
	93	enum {
	94	ITER_GET_SUBBUF = 0,
	95	ITER_TEST_RECORD,
	96	ITER_NEXT_RECORD,
	97	ITER_PUT_SUBBUF,
	98	} state;
9cccf98a MD	99	unsigned int allocated:1;
9cccf98a MD	100	unsigned int read_open:1; /* Opened for reading ? */
f3bc08c5 MD	101	};
	102
	103	/* ring buffer state */
	104	struct lib_ring_buffer {
	105	/* First 32 bytes cache-hot cacheline */
	106	union v_atomic offset; /* Current offset in the buffer */
	107	struct commit_counters_hot *commit_hot;
	108	/* Commit count per sub-buffer */
	109	atomic_long_t consumed; /*
	110	* Current offset in the buffer
	111	* standard atomic access (shared)
	112	*/
	113	atomic_t record_disabled;
	114	/* End of first 32 bytes cacheline */
	115	union v_atomic last_tsc; /*
	116	* Last timestamp written in the buffer.
	117	*/
	118
	119	struct lib_ring_buffer_backend backend; /* Associated backend */
	120
	121	struct commit_counters_cold *commit_cold;
	122	/* Commit count per sub-buffer */
2485a430 MD	123	u64 ts_end; /
	124	* timestamp_end per sub-buffer.
	125	* Time is sampled by the
	126	* switch_*_end() callbacks which
	127	* are the last space reservation
	128	* performed in the sub-buffer
	129	* before it can be fully
	130	* committed and delivered. This
	131	* time value is then read by
	132	* the deliver callback,
	133	* performed by the last commit
	134	* before the buffer becomes
	135	* readable.
	136	*/
f3bc08c5 MD	137	atomic_long_t active_readers; /*
	138	* Active readers count
	139	* standard atomic access (shared)
	140	*/
	141	/* Dropped records */
	142	union v_atomic records_lost_full; /* Buffer full */
	143	union v_atomic records_lost_wrap; /* Nested wrap-around */
	144	union v_atomic records_lost_big; /* Events too big */
	145	union v_atomic records_count; /* Number of records written */
	146	union v_atomic records_overrun; /* Number of overwritten records */
	147	wait_queue_head_t read_wait; /* reader buffer-level wait queue */
71c1d843	148	wait_queue_head_t write_wait; /* writer buffer-level wait queue (for metadata only) */
f3bc08c5 MD	149	int finalized; /* buffer has been finalized */
	150	struct timer_list switch_timer; /* timer for periodical switch */
	151	struct timer_list read_timer; /* timer for read poll */
	152	raw_spinlock_t raw_tick_nohz_spinlock; /* nohz entry lock/trylock */
	153	struct lib_ring_buffer_iter iter; /* read-side iterator */
	154	unsigned long get_subbuf_consumed; /* Read-side consumed */
	155	unsigned long prod_snapshot; /* Producer count snapshot */
	156	unsigned long cons_snapshot; /* Consumer count snapshot */
9cccf98a MD	157	unsigned int get_subbuf:1, /* Sub-buffer being held by reader */
9cccf98a MD	158	switch_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */
64af2437 MD	159	read_timer_enabled:1, /* Protected by ring_buffer_nohz_lock */
64af2437 MD	160	quiescent:1;
f3bc08c5 MD	161	};
f3bc08c5 MD	162
9115fbdc MD	163	static inline
	164	void channel_get_private(struct channel chan)
	165	{
	166	return chan->backend.priv;
	167	}
	168
d7e74017 MD	169	void lib_ring_buffer_lost_event_too_big(struct channel *chan);
d7e74017 MD	170
f3bc08c5 MD	171	/*
	172	* Issue warnings and disable channels upon internal error.
	173	* Can receive struct lib_ring_buffer or struct lib_ring_buffer_backend
	174	* parameters.
	175	*/
	176	#define CHAN_WARN_ON(c, cond) \
	177	({ \
	178	struct channel *__chan; \
	179	int _____ret = unlikely(cond); \
	180	if (_____ret) { \
	181	if (__same_type(*(c), struct channel_backend)) \
	182	__chan = container_of((void *) (c), \
	183	struct channel, \
	184	backend); \
	185	else if (__same_type(*(c), struct channel)) \
	186	__chan = (void *) (c); \
	187	else \
	188	BUG_ON(1); \
	189	atomic_inc(&__chan->record_disabled); \
	190	WARN_ON(1); \
	191	} \
	192	_____ret; \
	193	})
	194
886d51a3	195	#endif /* _LIB_RING_BUFFER_FRONTEND_TYPES_H */