[lttng-modules.git] / lib / ringbuffer / backend.h

#ifndef _LIB_RING_BUFFER_BACKEND_H
#define _LIB_RING_BUFFER_BACKEND_H

/*
 * lib/ringbuffer/backend.h
 *
 * Ring buffer backend (API).
 *
 * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
 * the reader in flight recorder mode.
 */

#include <linux/types.h>
#include <linux/sched.h>
#include <linux/timer.h>
#include <linux/wait.h>
#include <linux/poll.h>
#include <linux/list.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/uaccess.h>

/* Internal helpers */
#include "../../wrapper/ringbuffer/backend_internal.h"
#include "../../wrapper/ringbuffer/frontend_internal.h"

/* Ring buffer backend API */

/* Ring buffer backend access (read/write) */

extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
				   size_t offset, void *dest, size_t len);

extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
					  size_t offset, void __user *dest,
					  size_t len);

extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
				     size_t offset, void *dest, size_t len);

extern struct page **
lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
			      void ***virt);

/*
 * Return the address where a given offset is located.
 * Should be used to get the current subbuffer header pointer. Given we know
 * it's never on a page boundary, it's safe to write directly to this address,
 * as long as the write is never bigger than a page size.
 */
extern void *
lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
			       size_t offset);
extern void *
lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
				    size_t offset);

/**
 * lib_ring_buffer_write - write data to a buffer backend
 * @config : ring buffer instance configuration
 * @ctx: ring buffer context. (input arguments only)
 * @src : source pointer to copy from
 * @len : length of data to copy
 *
 * This function copies "len" bytes of data from a source pointer to a buffer
 * backend, at the current context offset. This is more or less a buffer
 * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
 * if copy is crossing a page boundary.
 */
static inline
void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
			   struct lib_ring_buffer_ctx *ctx,
			   const void *src, size_t len)
{
	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;

	if (unlikely(!len))
		return;
	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));
	if (likely(pagecpy == len))
		lib_ring_buffer_do_copy(config,
					rpages->p[index].virt
					    + (offset & ~PAGE_MASK),
					src, len);
	else
		_lib_ring_buffer_write(bufb, offset, src, len, 0);
	ctx->buf_offset += len;
}

/**
 * lib_ring_buffer_memset - write len bytes of c to a buffer backend
 * @config : ring buffer instance configuration
 * @bufb : ring buffer backend
 * @offset : offset within the buffer
 * @c : the byte to copy
 * @len : number of bytes to copy
 *
 * This function writes "len" bytes of "c" to a buffer backend, at a specific
 * offset. This is more or less a buffer backend-specific memset() operation.
 * Calls the slow path (_ring_buffer_memset) if write is crossing a page
 * boundary.
 */
static inline
void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
			    struct lib_ring_buffer_ctx *ctx, int c, size_t len)
{

	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;

	if (unlikely(!len))
		return;
	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));
	if (likely(pagecpy == len))
		lib_ring_buffer_do_memset(rpages->p[index].virt
					  + (offset & ~PAGE_MASK),
					  c, len);
	else
		_lib_ring_buffer_memset(bufb, offset, c, len, 0);
	ctx->buf_offset += len;
}

/**
 * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
 * @config : ring buffer instance configuration
 * @ctx: ring buffer context. (input arguments only)
 * @src : userspace source pointer to copy from
 * @len : length of data to copy
 *
 * This function copies "len" bytes of data from a userspace pointer to a
 * buffer backend, at the current context offset. This is more or less a buffer
 * backend-specific memcpy() operation. Calls the slow path
 * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
 * Disable the page fault handler to ensure we never try to take the mmap_sem.
 */
static inline
void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config,
				    struct lib_ring_buffer_ctx *ctx,
				    const void __user *src, size_t len)
{
	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	struct channel_backend *chanb = &ctx->chan->backend;
	size_t sbidx, index;
	size_t offset = ctx->buf_offset;
	ssize_t pagecpy;
	struct lib_ring_buffer_backend_pages *rpages;
	unsigned long sb_bindex, id;
	unsigned long ret;
	mm_segment_t old_fs = get_fs();

	if (unlikely(!len))
		return;
	offset &= chanb->buf_size - 1;
	sbidx = offset >> chanb->subbuf_size_order;
	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	id = bufb->buf_wsb[sbidx].id;
	sb_bindex = subbuffer_id_get_index(config, id);
	rpages = bufb->array[sb_bindex];
	CHAN_WARN_ON(ctx->chan,
		     config->mode == RING_BUFFER_OVERWRITE
		     && subbuffer_id_is_noref(config, id));

	set_fs(KERNEL_DS);
	pagefault_disable();
	if (unlikely(!access_ok(VERIFY_READ, src, len)))
		goto fill_buffer;

	if (likely(pagecpy == len)) {
		ret = lib_ring_buffer_do_copy_from_user_inatomic(
			rpages->p[index].virt + (offset & ~PAGE_MASK),
			src, len);
		if (unlikely(ret > 0)) {
			len -= (pagecpy - ret);
			offset += (pagecpy - ret);
			goto fill_buffer;
		}
	} else {
		_lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0);
	}
	pagefault_enable();
	set_fs(old_fs);
	ctx->buf_offset += len;

	return;

fill_buffer:
	pagefault_enable();
	set_fs(old_fs);
	/*
	 * In the error path we call the slow path version to avoid
	 * the pollution of static inline code.
	 */
	_lib_ring_buffer_memset(bufb, offset, 0, len, 0);
}

/*
 * This accessor counts the number of unread records in a buffer.
 * It only provides a consistent value if no reads not writes are performed
 * concurrently.
 */
static inline
unsigned long lib_ring_buffer_get_records_unread(
				const struct lib_ring_buffer_config *config,
				struct lib_ring_buffer *buf)
{
	struct lib_ring_buffer_backend *bufb = &buf->backend;
	struct lib_ring_buffer_backend_pages *pages;
	unsigned long records_unread = 0, sb_bindex, id;
	unsigned int i;

	for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
		id = bufb->buf_wsb[i].id;
		sb_bindex = subbuffer_id_get_index(config, id);
		pages = bufb->array[sb_bindex];
		records_unread += v_read(config, &pages->records_unread);
	}
	if (config->mode == RING_BUFFER_OVERWRITE) {
		id = bufb->buf_rsb.id;
		sb_bindex = subbuffer_id_get_index(config, id);
		pages = bufb->array[sb_bindex];
		records_unread += v_read(config, &pages->records_unread);
	}
	return records_unread;
}

ssize_t lib_ring_buffer_file_splice_read(struct file *in, loff_t *ppos,
					 struct pipe_inode_info *pipe,
					 size_t len, unsigned int flags);
loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin);

#endif /* _LIB_RING_BUFFER_BACKEND_H */
Commit	Line	Data
886d51a3 MD	1	#ifndef _LIB_RING_BUFFER_BACKEND_H
886d51a3 MD	2	#define _LIB_RING_BUFFER_BACKEND_H
f3bc08c5 MD	3
f3bc08c5 MD	4	/*
886d51a3	5	* lib/ringbuffer/backend.h
f3bc08c5 MD	6	*
	7	* Ring buffer backend (API).
	8	*
886d51a3 MD	9	* Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; only
	14	* version 2.1 of the License.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
f3bc08c5 MD	24	*
	25	* Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
	26	* the reader in flight recorder mode.
	27	*/
	28
	29	#include <linux/types.h>
	30	#include <linux/sched.h>
	31	#include <linux/timer.h>
	32	#include <linux/wait.h>
	33	#include <linux/poll.h>
	34	#include <linux/list.h>
	35	#include <linux/fs.h>
	36	#include <linux/mm.h>
7b8ea3a5	37	#include <linux/uaccess.h>
f3bc08c5 MD	38
	39	/* Internal helpers */
	40	#include "../../wrapper/ringbuffer/backend_internal.h"
	41	#include "../../wrapper/ringbuffer/frontend_internal.h"
	42
	43	/* Ring buffer backend API */
	44
	45	/* Ring buffer backend access (read/write) */
	46
	47	extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
	48	size_t offset, void *dest, size_t len);
	49
	50	extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
	51	size_t offset, void __user *dest,
	52	size_t len);
	53
	54	extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
	55	size_t offset, void *dest, size_t len);
	56
	57	extern struct page **
	58	lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
	59	void ***virt);
	60
	61	/*
	62	* Return the address where a given offset is located.
	63	* Should be used to get the current subbuffer header pointer. Given we know
	64	* it's never on a page boundary, it's safe to write directly to this address,
	65	* as long as the write is never bigger than a page size.
	66	*/
	67	extern void *
	68	lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
	69	size_t offset);
	70	extern void *
	71	lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
	72	size_t offset);
	73
	74	/**
	75	* lib_ring_buffer_write - write data to a buffer backend
	76	* @config : ring buffer instance configuration
	77	* @ctx: ring buffer context. (input arguments only)
	78	* @src : source pointer to copy from
	79	* @len : length of data to copy
	80	*
	81	* This function copies "len" bytes of data from a source pointer to a buffer
	82	* backend, at the current context offset. This is more or less a buffer
	83	* backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
	84	* if copy is crossing a page boundary.
	85	*/
	86	static inline
	87	void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
	88	struct lib_ring_buffer_ctx *ctx,
	89	const void *src, size_t len)
	90	{
	91	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	92	struct channel_backend *chanb = &ctx->chan->backend;
	93	size_t sbidx, index;
	94	size_t offset = ctx->buf_offset;
	95	ssize_t pagecpy;
	96	struct lib_ring_buffer_backend_pages *rpages;
	97	unsigned long sb_bindex, id;
	98
d2bd780e MD	99	if (unlikely(!len))
d2bd780e MD	100	return;
f3bc08c5 MD	101	offset &= chanb->buf_size - 1;
	102	sbidx = offset >> chanb->subbuf_size_order;
	103	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	104	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	105	id = bufb->buf_wsb[sbidx].id;
	106	sb_bindex = subbuffer_id_get_index(config, id);
	107	rpages = bufb->array[sb_bindex];
	108	CHAN_WARN_ON(ctx->chan,
	109	config->mode == RING_BUFFER_OVERWRITE
	110	&& subbuffer_id_is_noref(config, id));
	111	if (likely(pagecpy == len))
	112	lib_ring_buffer_do_copy(config,
	113	rpages->p[index].virt
	114	+ (offset & ~PAGE_MASK),
	115	src, len);
	116	else
	117	_lib_ring_buffer_write(bufb, offset, src, len, 0);
	118	ctx->buf_offset += len;
	119	}
	120
4ea00e4f JD	121	/**
	122	* lib_ring_buffer_memset - write len bytes of c to a buffer backend
	123	* @config : ring buffer instance configuration
	124	* @bufb : ring buffer backend
	125	* @offset : offset within the buffer
	126	* @c : the byte to copy
	127	* @len : number of bytes to copy
	128	*
	129	* This function writes "len" bytes of "c" to a buffer backend, at a specific
	130	* offset. This is more or less a buffer backend-specific memset() operation.
	131	* Calls the slow path (_ring_buffer_memset) if write is crossing a page
	132	* boundary.
	133	*/
	134	static inline
	135	void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
	136	struct lib_ring_buffer_ctx *ctx, int c, size_t len)
	137	{
	138
	139	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	140	struct channel_backend *chanb = &ctx->chan->backend;
	141	size_t sbidx, index;
	142	size_t offset = ctx->buf_offset;
	143	ssize_t pagecpy;
	144	struct lib_ring_buffer_backend_pages *rpages;
	145	unsigned long sb_bindex, id;
	146
d2bd780e MD	147	if (unlikely(!len))
d2bd780e MD	148	return;
4ea00e4f JD	149	offset &= chanb->buf_size - 1;
	150	sbidx = offset >> chanb->subbuf_size_order;
	151	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	152	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	153	id = bufb->buf_wsb[sbidx].id;
	154	sb_bindex = subbuffer_id_get_index(config, id);
	155	rpages = bufb->array[sb_bindex];
	156	CHAN_WARN_ON(ctx->chan,
	157	config->mode == RING_BUFFER_OVERWRITE
	158	&& subbuffer_id_is_noref(config, id));
	159	if (likely(pagecpy == len))
	160	lib_ring_buffer_do_memset(rpages->p[index].virt
	161	+ (offset & ~PAGE_MASK),
	162	c, len);
	163	else
	164	_lib_ring_buffer_memset(bufb, offset, c, len, 0);
	165	ctx->buf_offset += len;
	166	}
	167
	168	/**
7b8ea3a5	169	* lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
4ea00e4f JD	170	* @config : ring buffer instance configuration
	171	* @ctx: ring buffer context. (input arguments only)
	172	* @src : userspace source pointer to copy from
	173	* @len : length of data to copy
	174	*
	175	* This function copies "len" bytes of data from a userspace pointer to a
	176	* buffer backend, at the current context offset. This is more or less a buffer
	177	* backend-specific memcpy() operation. Calls the slow path
7b8ea3a5 MD	178	* (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
7b8ea3a5 MD	179	* Disable the page fault handler to ensure we never try to take the mmap_sem.
4ea00e4f JD	180	*/
4ea00e4f JD	181	static inline
7b8ea3a5	182	void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config,
4ea00e4f JD	183	struct lib_ring_buffer_ctx *ctx,
	184	const void __user *src, size_t len)
	185	{
	186	struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
	187	struct channel_backend *chanb = &ctx->chan->backend;
	188	size_t sbidx, index;
	189	size_t offset = ctx->buf_offset;
	190	ssize_t pagecpy;
	191	struct lib_ring_buffer_backend_pages *rpages;
	192	unsigned long sb_bindex, id;
	193	unsigned long ret;
7b8ea3a5	194	mm_segment_t old_fs = get_fs();
4ea00e4f	195
d2bd780e MD	196	if (unlikely(!len))
d2bd780e MD	197	return;
4ea00e4f JD	198	offset &= chanb->buf_size - 1;
	199	sbidx = offset >> chanb->subbuf_size_order;
	200	index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
	201	pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
	202	id = bufb->buf_wsb[sbidx].id;
	203	sb_bindex = subbuffer_id_get_index(config, id);
	204	rpages = bufb->array[sb_bindex];
	205	CHAN_WARN_ON(ctx->chan,
	206	config->mode == RING_BUFFER_OVERWRITE
	207	&& subbuffer_id_is_noref(config, id));
	208
7b8ea3a5 MD	209	set_fs(KERNEL_DS);
7b8ea3a5 MD	210	pagefault_disable();
4ea00e4f JD	211	if (unlikely(!access_ok(VERIFY_READ, src, len)))
	212	goto fill_buffer;
	213
	214	if (likely(pagecpy == len)) {
7b8ea3a5	215	ret = lib_ring_buffer_do_copy_from_user_inatomic(
4ea00e4f JD	216	rpages->p[index].virt + (offset & ~PAGE_MASK),
	217	src, len);
	218	if (unlikely(ret > 0)) {
	219	len -= (pagecpy - ret);
	220	offset += (pagecpy - ret);
	221	goto fill_buffer;
	222	}
	223	} else {
7b8ea3a5	224	_lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0);
4ea00e4f	225	}
7b8ea3a5 MD	226	pagefault_enable();
7b8ea3a5 MD	227	set_fs(old_fs);
4ea00e4f JD	228	ctx->buf_offset += len;
	229
	230	return;
	231
	232	fill_buffer:
7b8ea3a5 MD	233	pagefault_enable();
7b8ea3a5 MD	234	set_fs(old_fs);
4ea00e4f JD	235	/*
	236	* In the error path we call the slow path version to avoid
	237	* the pollution of static inline code.
	238	*/
	239	_lib_ring_buffer_memset(bufb, offset, 0, len, 0);
	240	}
	241
f3bc08c5 MD	242	/*
	243	* This accessor counts the number of unread records in a buffer.
	244	* It only provides a consistent value if no reads not writes are performed
	245	* concurrently.
	246	*/
	247	static inline
	248	unsigned long lib_ring_buffer_get_records_unread(
	249	const struct lib_ring_buffer_config *config,
	250	struct lib_ring_buffer *buf)
	251	{
	252	struct lib_ring_buffer_backend *bufb = &buf->backend;
	253	struct lib_ring_buffer_backend_pages *pages;
	254	unsigned long records_unread = 0, sb_bindex, id;
	255	unsigned int i;
	256
	257	for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
	258	id = bufb->buf_wsb[i].id;
	259	sb_bindex = subbuffer_id_get_index(config, id);
	260	pages = bufb->array[sb_bindex];
	261	records_unread += v_read(config, &pages->records_unread);
	262	}
	263	if (config->mode == RING_BUFFER_OVERWRITE) {
	264	id = bufb->buf_rsb.id;
	265	sb_bindex = subbuffer_id_get_index(config, id);
	266	pages = bufb->array[sb_bindex];
	267	records_unread += v_read(config, &pages->records_unread);
	268	}
	269	return records_unread;
	270	}
	271
	272	ssize_t lib_ring_buffer_file_splice_read(struct file in, loff_t ppos,
	273	struct pipe_inode_info *pipe,
	274	size_t len, unsigned int flags);
	275	loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin);
	276
886d51a3	277	#endif /* _LIB_RING_BUFFER_BACKEND_H */