[lttng-modules.git] / lib / ringbuffer / frontend.h

/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
 *
 * lib/ringbuffer/frontend.h
 *
 * Ring Buffer Library Synchronization Header (API).
 *
 * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * See ring_buffer_frontend.c for more information on wait-free algorithms.
 */

#ifndef _LIB_RING_BUFFER_FRONTEND_H
#define _LIB_RING_BUFFER_FRONTEND_H

#include <linux/pipe_fs_i.h>
#include <linux/rcupdate.h>
#include <linux/cpumask.h>
#include <linux/module.h>
#include <linux/bitops.h>
#include <linux/splice.h>
#include <linux/string.h>
#include <linux/timer.h>
#include <linux/sched.h>
#include <linux/cache.h>
#include <linux/time.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/stat.h>
#include <linux/cpu.h>
#include <linux/fs.h>

#include <asm/atomic.h>
#include <asm/local.h>

/* Internal helpers */
#include <wrapper/ringbuffer/frontend_internal.h>

/* Max ring buffer nesting count, see lib_ring_buffer_get_cpu(). */
#define RING_BUFFER_MAX_NESTING 4

/* Buffer creation/removal and setup operations */

/*
 * switch_timer_interval is the time interval (in us) to fill sub-buffers with
 * padding to let readers get those sub-buffers.  Used for live streaming.
 *
 * read_timer_interval is the time interval (in us) to wake up pending readers.
 *
 * buf_addr is a pointer the the beginning of the preallocated buffer contiguous
 * address mapping. It is used only by RING_BUFFER_STATIC configuration. It can
 * be set to NULL for other backends.
 */

extern
struct channel *channel_create(const struct lib_ring_buffer_config *config,
			       const char *name, void *priv,
			       void *buf_addr,
			       size_t subbuf_size, size_t num_subbuf,
			       unsigned int switch_timer_interval,
			       unsigned int read_timer_interval);

/*
 * channel_destroy returns the private data pointer. It finalizes all channel's
 * buffers, waits for readers to release all references, and destroys the
 * channel.
 */
extern
void *channel_destroy(struct channel *chan);


/* Buffer read operations */

/*
 * Iteration on channel cpumask needs to issue a read barrier to match the write
 * barrier in cpu hotplug. It orders the cpumask read before read of per-cpu
 * buffer data. The per-cpu buffer is never removed by cpu hotplug; teardown is
 * only performed at channel destruction.
 */
#define for_each_channel_cpu(cpu, chan)					\
	for ((cpu) = -1;						\
		({ (cpu) = cpumask_next(cpu, (chan)->backend.cpumask);	\
		   smp_read_barrier_depends(); (cpu) < nr_cpu_ids; });)

extern struct lib_ring_buffer *channel_get_ring_buffer(
				const struct lib_ring_buffer_config *config,
				struct channel *chan, int cpu);
extern int lib_ring_buffer_open_read(struct lib_ring_buffer *buf);
extern void lib_ring_buffer_release_read(struct lib_ring_buffer *buf);

/*
 * Read sequence: snapshot, many get_subbuf/put_subbuf, move_consumer.
 */
extern int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf,
				    unsigned long *consumed,
				    unsigned long *produced);
extern int lib_ring_buffer_snapshot_sample_positions(
				    struct lib_ring_buffer *buf,
				    unsigned long *consumed,
				    unsigned long *produced);
extern void lib_ring_buffer_move_consumer(struct lib_ring_buffer *buf,
					  unsigned long consumed_new);

extern int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf,
				      unsigned long consumed);
extern void lib_ring_buffer_put_subbuf(struct lib_ring_buffer *buf);

void lib_ring_buffer_set_quiescent_channel(struct channel *chan);
void lib_ring_buffer_clear_quiescent_channel(struct channel *chan);

/*
 * lib_ring_buffer_get_next_subbuf/lib_ring_buffer_put_next_subbuf are helpers
 * to read sub-buffers sequentially.
 */
static inline int lib_ring_buffer_get_next_subbuf(struct lib_ring_buffer *buf)
{
	int ret;

	ret = lib_ring_buffer_snapshot(buf, &buf->cons_snapshot,
				       &buf->prod_snapshot);
	if (ret)
		return ret;
	ret = lib_ring_buffer_get_subbuf(buf, buf->cons_snapshot);
	return ret;
}

static inline void lib_ring_buffer_put_next_subbuf(struct lib_ring_buffer *buf)
{
	lib_ring_buffer_put_subbuf(buf);
	lib_ring_buffer_move_consumer(buf, subbuf_align(buf->cons_snapshot,
						    buf->backend.chan));
}

extern void channel_reset(struct channel *chan);
extern void lib_ring_buffer_reset(struct lib_ring_buffer *buf);

static inline
unsigned long lib_ring_buffer_get_offset(const struct lib_ring_buffer_config *config,
					 struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->offset);
}

static inline
unsigned long lib_ring_buffer_get_consumed(const struct lib_ring_buffer_config *config,
					   struct lib_ring_buffer *buf)
{
	return atomic_long_read(&buf->consumed);
}

/*
 * Must call lib_ring_buffer_is_finalized before reading counters (memory
 * ordering enforced with respect to trace teardown).
 */
static inline
int lib_ring_buffer_is_finalized(const struct lib_ring_buffer_config *config,
				 struct lib_ring_buffer *buf)
{
	int finalized = READ_ONCE(buf->finalized);
	/*
	 * Read finalized before counters.
	 */
	smp_rmb();
	return finalized;
}

static inline
int lib_ring_buffer_channel_is_finalized(const struct channel *chan)
{
	return chan->finalized;
}

static inline
int lib_ring_buffer_channel_is_disabled(const struct channel *chan)
{
	return atomic_read(&chan->record_disabled);
}

static inline
unsigned long lib_ring_buffer_get_read_data_size(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return subbuffer_get_read_data_size(config, &buf->backend);
}

static inline
unsigned long lib_ring_buffer_get_records_count(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->records_count);
}

static inline
unsigned long lib_ring_buffer_get_records_overrun(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->records_overrun);
}

static inline
unsigned long lib_ring_buffer_get_records_lost_full(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->records_lost_full);
}

static inline
unsigned long lib_ring_buffer_get_records_lost_wrap(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->records_lost_wrap);
}

static inline
unsigned long lib_ring_buffer_get_records_lost_big(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->records_lost_big);
}

static inline
unsigned long lib_ring_buffer_get_records_read(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	return v_read(config, &buf->backend.records_read);
}

#endif /* _LIB_RING_BUFFER_FRONTEND_H */
Commit	Line	Data
b7cdc182	1	/* SPDX-License-Identifier: (GPL-2.0-only OR LGPL-2.1-only)
9f36eaed	2	*
886d51a3	3	* lib/ringbuffer/frontend.h
f3bc08c5 MD	4	*
	5	* Ring Buffer Library Synchronization Header (API).
	6	*
886d51a3 MD	7	* Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
886d51a3 MD	8	*
f3bc08c5	9	* See ring_buffer_frontend.c for more information on wait-free algorithms.
f3bc08c5 MD	10	*/
f3bc08c5 MD	11
9f36eaed MJ	12	#ifndef _LIB_RING_BUFFER_FRONTEND_H
	13	#define _LIB_RING_BUFFER_FRONTEND_H
	14
f3bc08c5 MD	15	#include <linux/pipe_fs_i.h>
	16	#include <linux/rcupdate.h>
	17	#include <linux/cpumask.h>
	18	#include <linux/module.h>
	19	#include <linux/bitops.h>
	20	#include <linux/splice.h>
	21	#include <linux/string.h>
	22	#include <linux/timer.h>
	23	#include <linux/sched.h>
	24	#include <linux/cache.h>
	25	#include <linux/time.h>
	26	#include <linux/slab.h>
	27	#include <linux/init.h>
	28	#include <linux/stat.h>
	29	#include <linux/cpu.h>
	30	#include <linux/fs.h>
	31
	32	#include <asm/atomic.h>
	33	#include <asm/local.h>
	34
	35	/* Internal helpers */
5671a661	36	#include <wrapper/ringbuffer/frontend_internal.h>
f3bc08c5	37
b055caba FG	38	/* Max ring buffer nesting count, see lib_ring_buffer_get_cpu(). */
	39	#define RING_BUFFER_MAX_NESTING 4
	40
f3bc08c5 MD	41	/* Buffer creation/removal and setup operations */
	42
	43	/*
	44	* switch_timer_interval is the time interval (in us) to fill sub-buffers with
	45	* padding to let readers get those sub-buffers. Used for live streaming.
	46	*
	47	* read_timer_interval is the time interval (in us) to wake up pending readers.
	48	*
	49	* buf_addr is a pointer the the beginning of the preallocated buffer contiguous
	50	* address mapping. It is used only by RING_BUFFER_STATIC configuration. It can
	51	* be set to NULL for other backends.
	52	*/
	53
	54	extern
	55	struct channel channel_create(const struct lib_ring_buffer_config config,
	56	const char name, void priv,
	57	void *buf_addr,
	58	size_t subbuf_size, size_t num_subbuf,
	59	unsigned int switch_timer_interval,
	60	unsigned int read_timer_interval);
	61
	62	/*
	63	* channel_destroy returns the private data pointer. It finalizes all channel's
	64	* buffers, waits for readers to release all references, and destroys the
	65	* channel.
	66	*/
	67	extern
	68	void channel_destroy(struct channel chan);
	69
	70
	71	/* Buffer read operations */
	72
	73	/*
	74	* Iteration on channel cpumask needs to issue a read barrier to match the write
	75	* barrier in cpu hotplug. It orders the cpumask read before read of per-cpu
	76	* buffer data. The per-cpu buffer is never removed by cpu hotplug; teardown is
	77	* only performed at channel destruction.
	78	*/
	79	#define for_each_channel_cpu(cpu, chan) \
	80	for ((cpu) = -1; \
	81	({ (cpu) = cpumask_next(cpu, (chan)->backend.cpumask); \
	82	smp_read_barrier_depends(); (cpu) < nr_cpu_ids; });)
	83
	84	extern struct lib_ring_buffer *channel_get_ring_buffer(
	85	const struct lib_ring_buffer_config *config,
	86	struct channel *chan, int cpu);
	87	extern int lib_ring_buffer_open_read(struct lib_ring_buffer *buf);
	88	extern void lib_ring_buffer_release_read(struct lib_ring_buffer *buf);
	89
	90	/*
	91	* Read sequence: snapshot, many get_subbuf/put_subbuf, move_consumer.
	92	*/
	93	extern int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf,
	94	unsigned long *consumed,
	95	unsigned long *produced);
4dce5a48 JG	96	extern int lib_ring_buffer_snapshot_sample_positions(
	97	struct lib_ring_buffer *buf,
	98	unsigned long *consumed,
	99	unsigned long *produced);
f3bc08c5 MD	100	extern void lib_ring_buffer_move_consumer(struct lib_ring_buffer *buf,
	101	unsigned long consumed_new);
	102
	103	extern int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf,
	104	unsigned long consumed);
	105	extern void lib_ring_buffer_put_subbuf(struct lib_ring_buffer *buf);
	106
64af2437 MD	107	void lib_ring_buffer_set_quiescent_channel(struct channel *chan);
	108	void lib_ring_buffer_clear_quiescent_channel(struct channel *chan);
	109
f3bc08c5 MD	110	/*
	111	* lib_ring_buffer_get_next_subbuf/lib_ring_buffer_put_next_subbuf are helpers
	112	* to read sub-buffers sequentially.
	113	*/
	114	static inline int lib_ring_buffer_get_next_subbuf(struct lib_ring_buffer *buf)
	115	{
	116	int ret;
	117
	118	ret = lib_ring_buffer_snapshot(buf, &buf->cons_snapshot,
	119	&buf->prod_snapshot);
	120	if (ret)
	121	return ret;
	122	ret = lib_ring_buffer_get_subbuf(buf, buf->cons_snapshot);
	123	return ret;
	124	}
	125
	126	static inline void lib_ring_buffer_put_next_subbuf(struct lib_ring_buffer *buf)
	127	{
	128	lib_ring_buffer_put_subbuf(buf);
	129	lib_ring_buffer_move_consumer(buf, subbuf_align(buf->cons_snapshot,
	130	buf->backend.chan));
	131	}
	132
	133	extern void channel_reset(struct channel *chan);
	134	extern void lib_ring_buffer_reset(struct lib_ring_buffer *buf);
	135
	136	static inline
	137	unsigned long lib_ring_buffer_get_offset(const struct lib_ring_buffer_config *config,
	138	struct lib_ring_buffer *buf)
	139	{
	140	return v_read(config, &buf->offset);
	141	}
	142
	143	static inline
	144	unsigned long lib_ring_buffer_get_consumed(const struct lib_ring_buffer_config *config,
	145	struct lib_ring_buffer *buf)
	146	{
	147	return atomic_long_read(&buf->consumed);
	148	}
	149
	150	/*
	151	* Must call lib_ring_buffer_is_finalized before reading counters (memory
	152	* ordering enforced with respect to trace teardown).
	153	*/
	154	static inline
	155	int lib_ring_buffer_is_finalized(const struct lib_ring_buffer_config *config,
	156	struct lib_ring_buffer *buf)
	157	{
a8f2d0c7	158	int finalized = READ_ONCE(buf->finalized);
f3bc08c5 MD	159	/*
	160	* Read finalized before counters.
	161	*/
	162	smp_rmb();
	163	return finalized;
	164	}
	165
24cedcfe MD	166	static inline
	167	int lib_ring_buffer_channel_is_finalized(const struct channel *chan)
	168	{
	169	return chan->finalized;
	170	}
	171
254ec7bc MD	172	static inline
	173	int lib_ring_buffer_channel_is_disabled(const struct channel *chan)
	174	{
	175	return atomic_read(&chan->record_disabled);
	176	}
	177
f3bc08c5 MD	178	static inline
	179	unsigned long lib_ring_buffer_get_read_data_size(
	180	const struct lib_ring_buffer_config *config,
	181	struct lib_ring_buffer *buf)
	182	{
	183	return subbuffer_get_read_data_size(config, &buf->backend);
	184	}
	185
	186	static inline
	187	unsigned long lib_ring_buffer_get_records_count(
	188	const struct lib_ring_buffer_config *config,
	189	struct lib_ring_buffer *buf)
	190	{
	191	return v_read(config, &buf->records_count);
	192	}
	193
	194	static inline
	195	unsigned long lib_ring_buffer_get_records_overrun(
	196	const struct lib_ring_buffer_config *config,
	197	struct lib_ring_buffer *buf)
	198	{
	199	return v_read(config, &buf->records_overrun);
	200	}
	201
	202	static inline
	203	unsigned long lib_ring_buffer_get_records_lost_full(
	204	const struct lib_ring_buffer_config *config,
	205	struct lib_ring_buffer *buf)
	206	{
	207	return v_read(config, &buf->records_lost_full);
	208	}
	209
	210	static inline
	211	unsigned long lib_ring_buffer_get_records_lost_wrap(
	212	const struct lib_ring_buffer_config *config,
	213	struct lib_ring_buffer *buf)
	214	{
	215	return v_read(config, &buf->records_lost_wrap);
	216	}
	217
	218	static inline
	219	unsigned long lib_ring_buffer_get_records_lost_big(
	220	const struct lib_ring_buffer_config *config,
	221	struct lib_ring_buffer *buf)
	222	{
	223	return v_read(config, &buf->records_lost_big);
	224	}
	225
	226	static inline
	227	unsigned long lib_ring_buffer_get_records_read(
	228	const struct lib_ring_buffer_config *config,
	229	struct lib_ring_buffer *buf)
	230	{
	231	return v_read(config, &buf->backend.records_read);
	232	}
	233
886d51a3	234	#endif /* _LIB_RING_BUFFER_FRONTEND_H */