[lttng-modules.git] / src / lib / ringbuffer / ring_buffer_splice.c

/* SPDX-License-Identifier: GPL-2.0-only
 *
 * ring_buffer_splice.c
 *
 * Copyright (C) 2002-2005 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp
 * Copyright (C) 1999-2005 - Karim Yaghmour <karim@opersys.com>
 * Copyright (C) 2008-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * Re-using code from kernel/relay.c, which is why it is licensed under
 * the GPL-2.0.
 */

#include <linux/module.h>
#include <linux/fs.h>
#include <linux/splice.h>
#include <lttng/kernel-version.h>

#include <ringbuffer/backend.h>
#include <ringbuffer/frontend.h>
#include <ringbuffer/vfs.h>

#ifdef DEBUG
#define printk_dbg(fmt, args...) printk(fmt, args)
#else
#define printk_dbg(fmt, args...)			\
do {							\
	/* do nothing but check printf format */	\
	if (0)						\
		printk(fmt, ## args);			\
} while (0)
#endif

loff_t vfs_lib_ring_buffer_no_llseek(struct file *file, loff_t offset,
		int origin)
{
	return -ESPIPE;
}
EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_no_llseek);

/*
 * Release pages from the buffer so splice pipe_to_file can move them.
 * Called after the pipe has been populated with buffer pages.
 */
static void lib_ring_buffer_pipe_buf_release(struct pipe_inode_info *pipe,
					     struct pipe_buffer *pbuf)
{
	__free_page(pbuf->page);
}

#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))
static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	.release = lib_ring_buffer_pipe_buf_release,
	.try_steal = generic_pipe_buf_try_steal,
	.get = generic_pipe_buf_get
};
#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,1,0))
static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	.confirm = generic_pipe_buf_confirm,
	.release = lib_ring_buffer_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get
};
#else
static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = lib_ring_buffer_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get
};
#endif

/*
 * Page release operation after splice pipe_to_file ends.
 */
static void lib_ring_buffer_page_release(struct splice_pipe_desc *spd,
					 unsigned int i)
{
	__free_page(spd->pages[i]);
}

/*
 *	subbuf_splice_actor - splice up to one subbuf's worth of data
 */
static int subbuf_splice_actor(struct file *in,
			       loff_t *ppos,
			       struct pipe_inode_info *pipe,
			       size_t len,
			       unsigned int flags,
			       struct lttng_kernel_ring_buffer *buf)
{
	struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
	const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
	unsigned int poff, subbuf_pages, nr_pages;
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
	struct splice_pipe_desc spd = {
		.pages = pages,
		.nr_pages = 0,
		.partial = partial,
#if (LTTNG_LINUX_VERSION_CODE < LTTNG_KERNEL_VERSION(4,12,0))
		.flags = flags,
#endif
		.ops = &ring_buffer_pipe_buf_ops,
		.spd_release = lib_ring_buffer_page_release,
	};
	unsigned long consumed_old, roffset;
	unsigned long bytes_avail;

	/*
	 * Check that a GET_SUBBUF ioctl has been done before.
	 */
	WARN_ON(atomic_long_read(&buf->active_readers) != 1);
	consumed_old = lib_ring_buffer_get_consumed(config, buf);
	consumed_old += *ppos;

	/*
	 * Adjust read len, if longer than what is available.
	 * Max read size is 1 subbuffer due to get_subbuf/put_subbuf for
	 * protection.
	 */
	bytes_avail = chan->backend.subbuf_size;
	WARN_ON(bytes_avail > chan->backend.buf_size);
	len = min_t(size_t, len, bytes_avail);
	subbuf_pages = bytes_avail >> PAGE_SHIFT;
	nr_pages = min_t(unsigned int, subbuf_pages, PIPE_DEF_BUFFERS);
	roffset = consumed_old & PAGE_MASK;
	poff = consumed_old & ~PAGE_MASK;
	printk_dbg(KERN_DEBUG "LTTng: SPLICE actor len %zu pos %zd write_pos %ld\n",
		   len, (ssize_t)*ppos, lib_ring_buffer_get_offset(config, buf));

	for (; spd.nr_pages < nr_pages; spd.nr_pages++) {
		unsigned int this_len;
		unsigned long *pfnp, new_pfn;
		struct page *new_page;
		void **virt;

		if (!len)
			break;
		printk_dbg(KERN_DEBUG "LTTng: SPLICE actor loop len %zu roffset %ld\n",
			   len, roffset);

		/*
		 * We have to replace the page we are moving into the splice
		 * pipe.
		 */
		new_page = alloc_pages_node(cpu_to_node(max(buf->backend.cpu,
							    0)),
					    GFP_KERNEL | __GFP_ZERO, 0);
		if (!new_page)
			break;
		new_pfn = page_to_pfn(new_page);
		this_len = PAGE_SIZE - poff;
		pfnp = lib_ring_buffer_read_get_pfn(&buf->backend, roffset, &virt);
		spd.pages[spd.nr_pages] = pfn_to_page(*pfnp);
		*pfnp = new_pfn;
		*virt = page_address(new_page);
		spd.partial[spd.nr_pages].offset = poff;
		spd.partial[spd.nr_pages].len = this_len;

		poff = 0;
		roffset += PAGE_SIZE;
		len -= this_len;
	}

	if (!spd.nr_pages)
		return 0;

	return splice_to_pipe(pipe, &spd);
}

ssize_t lib_ring_buffer_splice_read(struct file *in, loff_t *ppos,
				    struct pipe_inode_info *pipe, size_t len,
				    unsigned int flags,
				    struct lttng_kernel_ring_buffer *buf)
{
	struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
	const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
	ssize_t spliced;
	int ret;

	if (config->output != RING_BUFFER_SPLICE)
		return -EINVAL;

	/*
	 * We require ppos and length to be page-aligned for performance reasons
	 * (no page copy). Size is known using the ioctl
	 * RING_BUFFER_GET_PADDED_SUBBUF_SIZE, which is page-size padded.
	 * We fail when the ppos or len passed is not page-sized, because splice
	 * is not allowed to copy more than the length passed as parameter (so
	 * the ABI does not let us silently copy more than requested to include
	 * padding).
	 */
	if (*ppos != PAGE_ALIGN(*ppos) || len != PAGE_ALIGN(len))
		return -EINVAL;

	ret = 0;
	spliced = 0;

	printk_dbg(KERN_DEBUG "LTTng: SPLICE read len %zu pos %zd\n", len,
		   (ssize_t)*ppos);
	while (len && !spliced) {
		ret = subbuf_splice_actor(in, ppos, pipe, len, flags, buf);
		printk_dbg(KERN_DEBUG "LTTng: SPLICE read loop ret %d\n", ret);
		if (ret < 0)
			break;
		else if (!ret) {
			if (flags & SPLICE_F_NONBLOCK)
				ret = -EAGAIN;
			break;
		}

		*ppos += ret;
		if (ret > len)
			len = 0;
		else
			len -= ret;
		spliced += ret;
	}

	if (spliced)
		return spliced;

	return ret;
}
EXPORT_SYMBOL_GPL(lib_ring_buffer_splice_read);

ssize_t vfs_lib_ring_buffer_splice_read(struct file *in, loff_t *ppos,
				    struct pipe_inode_info *pipe, size_t len,
				    unsigned int flags)
{
	struct lttng_kernel_ring_buffer *buf = in->private_data;

	return lib_ring_buffer_splice_read(in, ppos, pipe, len, flags, buf);
}
EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_splice_read);
Commit	Line	Data
b7cdc182	1	/* SPDX-License-Identifier: GPL-2.0-only
9f36eaed	2	*
f3bc08c5 MD	3	* ring_buffer_splice.c
	4	*
	5	* Copyright (C) 2002-2005 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp
	6	* Copyright (C) 1999-2005 - Karim Yaghmour <karim@opersys.com>
886d51a3	7	* Copyright (C) 2008-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
f3bc08c5	8	*
886d51a3	9	* Re-using code from kernel/relay.c, which is why it is licensed under
9f36eaed	10	* the GPL-2.0.
f3bc08c5 MD	11	*/
	12
	13	#include <linux/module.h>
	14	#include <linux/fs.h>
dfdb34fd	15	#include <linux/splice.h>
5f4c791e	16	#include <lttng/kernel-version.h>
f3bc08c5	17
24591303 MD	18	#include <ringbuffer/backend.h>
	19	#include <ringbuffer/frontend.h>
	20	#include <ringbuffer/vfs.h>
f3bc08c5	21
5a15f70c	22	#ifdef DEBUG
f3bc08c5 MD	23	#define printk_dbg(fmt, args...) printk(fmt, args)
f3bc08c5 MD	24	#else
5a15f70c MJ	25	#define printk_dbg(fmt, args...) \
	26	do { \
	27	/* do nothing but check printf format */ \
	28	if (0) \
	29	printk(fmt, ## args); \
	30	} while (0)
f3bc08c5 MD	31	#endif
f3bc08c5 MD	32
d83004aa JD	33	loff_t vfs_lib_ring_buffer_no_llseek(struct file *file, loff_t offset,
d83004aa JD	34	int origin)
f3bc08c5 MD	35	{
	36	return -ESPIPE;
	37	}
d83004aa	38	EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_no_llseek);
f3bc08c5 MD	39
	40	/*
	41	* Release pages from the buffer so splice pipe_to_file can move them.
	42	* Called after the pipe has been populated with buffer pages.
	43	*/
	44	static void lib_ring_buffer_pipe_buf_release(struct pipe_inode_info *pipe,
	45	struct pipe_buffer *pbuf)
	46	{
	47	__free_page(pbuf->page);
	48	}
	49
5f4c791e	50	#if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,8,0))
4352d508 MJ	51	static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	52	.release = lib_ring_buffer_pipe_buf_release,
	53	.try_steal = generic_pipe_buf_try_steal,
	54	.get = generic_pipe_buf_get
	55	};
5f4c791e	56	#elif (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,1,0))
4352d508 MJ	57	static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	58	.confirm = generic_pipe_buf_confirm,
	59	.release = lib_ring_buffer_pipe_buf_release,
	60	.steal = generic_pipe_buf_steal,
	61	.get = generic_pipe_buf_get
	62	};
4352d508	63	#else
f3bc08c5 MD	64	static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
f3bc08c5 MD	65	.can_merge = 0,
f3bc08c5 MD	66	.confirm = generic_pipe_buf_confirm,
	67	.release = lib_ring_buffer_pipe_buf_release,
	68	.steal = generic_pipe_buf_steal,
4352d508	69	.get = generic_pipe_buf_get
f3bc08c5	70	};
4352d508	71	#endif
f3bc08c5 MD	72
	73	/*
	74	* Page release operation after splice pipe_to_file ends.
	75	*/
	76	static void lib_ring_buffer_page_release(struct splice_pipe_desc *spd,
	77	unsigned int i)
	78	{
	79	__free_page(spd->pages[i]);
	80	}
	81
	82	/*
	83	* subbuf_splice_actor - splice up to one subbuf's worth of data
	84	*/
	85	static int subbuf_splice_actor(struct file *in,
	86	loff_t *ppos,
	87	struct pipe_inode_info *pipe,
	88	size_t len,
d83004aa	89	unsigned int flags,
e20c0fec	90	struct lttng_kernel_ring_buffer *buf)
f3bc08c5	91	{
860c213b	92	struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
e20c0fec	93	const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
f3bc08c5 MD	94	unsigned int poff, subbuf_pages, nr_pages;
	95	struct page *pages[PIPE_DEF_BUFFERS];
	96	struct partial_page partial[PIPE_DEF_BUFFERS];
	97	struct splice_pipe_desc spd = {
	98	.pages = pages,
	99	.nr_pages = 0,
	100	.partial = partial,
5f4c791e	101	#if (LTTNG_LINUX_VERSION_CODE < LTTNG_KERNEL_VERSION(4,12,0))
f3bc08c5	102	.flags = flags,
e2833bed	103	#endif
f3bc08c5 MD	104	.ops = &ring_buffer_pipe_buf_ops,
	105	.spd_release = lib_ring_buffer_page_release,
	106	};
88dfd899	107	unsigned long consumed_old, roffset;
f3bc08c5 MD	108	unsigned long bytes_avail;
	109
	110	/*
	111	* Check that a GET_SUBBUF ioctl has been done before.
	112	*/
	113	WARN_ON(atomic_long_read(&buf->active_readers) != 1);
	114	consumed_old = lib_ring_buffer_get_consumed(config, buf);
	115	consumed_old += *ppos;
f3bc08c5 MD	116
	117	/*
	118	* Adjust read len, if longer than what is available.
	119	* Max read size is 1 subbuffer due to get_subbuf/put_subbuf for
	120	* protection.
	121	*/
	122	bytes_avail = chan->backend.subbuf_size;
	123	WARN_ON(bytes_avail > chan->backend.buf_size);
	124	len = min_t(size_t, len, bytes_avail);
	125	subbuf_pages = bytes_avail >> PAGE_SHIFT;
	126	nr_pages = min_t(unsigned int, subbuf_pages, PIPE_DEF_BUFFERS);
	127	roffset = consumed_old & PAGE_MASK;
	128	poff = consumed_old & ~PAGE_MASK;
5a15f70c	129	printk_dbg(KERN_DEBUG "LTTng: SPLICE actor len %zu pos %zd write_pos %ld\n",
f3bc08c5 MD	130	len, (ssize_t)*ppos, lib_ring_buffer_get_offset(config, buf));
	131
	132	for (; spd.nr_pages < nr_pages; spd.nr_pages++) {
	133	unsigned int this_len;
0112cb7b MD	134	unsigned long *pfnp, new_pfn;
0112cb7b MD	135	struct page *new_page;
f3bc08c5 MD	136	void **virt;
	137
	138	if (!len)
	139	break;
5a15f70c	140	printk_dbg(KERN_DEBUG "LTTng: SPLICE actor loop len %zu roffset %ld\n",
f3bc08c5 MD	141	len, roffset);
	142
	143	/*
	144	* We have to replace the page we are moving into the splice
	145	* pipe.
	146	*/
	147	new_page = alloc_pages_node(cpu_to_node(max(buf->backend.cpu,
	148	0)),
	149	GFP_KERNEL \| __GFP_ZERO, 0);
	150	if (!new_page)
	151	break;
0112cb7b	152	new_pfn = page_to_pfn(new_page);
f3bc08c5	153	this_len = PAGE_SIZE - poff;
0112cb7b MD	154	pfnp = lib_ring_buffer_read_get_pfn(&buf->backend, roffset, &virt);
	155	spd.pages[spd.nr_pages] = pfn_to_page(*pfnp);
	156	*pfnp = new_pfn;
f3bc08c5 MD	157	*virt = page_address(new_page);
	158	spd.partial[spd.nr_pages].offset = poff;
	159	spd.partial[spd.nr_pages].len = this_len;
	160
	161	poff = 0;
	162	roffset += PAGE_SIZE;
	163	len -= this_len;
	164	}
	165
	166	if (!spd.nr_pages)
	167	return 0;
	168
dfdb34fd	169	return splice_to_pipe(pipe, &spd);
f3bc08c5 MD	170	}
	171
	172	ssize_t lib_ring_buffer_splice_read(struct file in, loff_t ppos,
	173	struct pipe_inode_info *pipe, size_t len,
d83004aa	174	unsigned int flags,
e20c0fec	175	struct lttng_kernel_ring_buffer *buf)
f3bc08c5	176	{
860c213b	177	struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
e20c0fec	178	const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
f3bc08c5 MD	179	ssize_t spliced;
	180	int ret;
	181
	182	if (config->output != RING_BUFFER_SPLICE)
	183	return -EINVAL;
	184
	185	/*
	186	* We require ppos and length to be page-aligned for performance reasons
	187	* (no page copy). Size is known using the ioctl
	188	* RING_BUFFER_GET_PADDED_SUBBUF_SIZE, which is page-size padded.
	189	* We fail when the ppos or len passed is not page-sized, because splice
	190	* is not allowed to copy more than the length passed as parameter (so
	191	* the ABI does not let us silently copy more than requested to include
	192	* padding).
	193	*/
	194	if (ppos != PAGE_ALIGN(ppos) \|\| len != PAGE_ALIGN(len))
	195	return -EINVAL;
	196
	197	ret = 0;
	198	spliced = 0;
	199
5a15f70c	200	printk_dbg(KERN_DEBUG "LTTng: SPLICE read len %zu pos %zd\n", len,
f3bc08c5 MD	201	(ssize_t)*ppos);
f3bc08c5 MD	202	while (len && !spliced) {
d83004aa	203	ret = subbuf_splice_actor(in, ppos, pipe, len, flags, buf);
5a15f70c	204	printk_dbg(KERN_DEBUG "LTTng: SPLICE read loop ret %d\n", ret);
f3bc08c5 MD	205	if (ret < 0)
	206	break;
	207	else if (!ret) {
	208	if (flags & SPLICE_F_NONBLOCK)
	209	ret = -EAGAIN;
	210	break;
	211	}
	212
	213	*ppos += ret;
	214	if (ret > len)
	215	len = 0;
	216	else
	217	len -= ret;
	218	spliced += ret;
	219	}
	220
	221	if (spliced)
	222	return spliced;
	223
	224	return ret;
	225	}
	226	EXPORT_SYMBOL_GPL(lib_ring_buffer_splice_read);
d83004aa JD	227
	228	ssize_t vfs_lib_ring_buffer_splice_read(struct file in, loff_t ppos,
	229	struct pipe_inode_info *pipe, size_t len,
	230	unsigned int flags)
	231	{
e20c0fec	232	struct lttng_kernel_ring_buffer *buf = in->private_data;
d83004aa JD	233
	234	return lib_ring_buffer_splice_read(in, ppos, pipe, len, flags, buf);
	235	}
	236	EXPORT_SYMBOL_GPL(vfs_lib_ring_buffer_splice_read);