[lttng-modules.git] / lib / ringbuffer / ring_buffer_splice.c

/*
 * ring_buffer_splice.c
 *
 * Copyright (C) 2002-2005 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp
 * Copyright (C) 1999-2005 - Karim Yaghmour <karim@opersys.com>
 * Copyright (C) 2008-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; only
 * version 2.1 of the License.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * Re-using code from kernel/relay.c, which is why it is licensed under
 * the GPLv2.
 */

#include <linux/module.h>
#include <linux/fs.h>

#include "../../wrapper/splice.h"
#include "../../wrapper/ringbuffer/backend.h"
#include "../../wrapper/ringbuffer/frontend.h"
#include "../../wrapper/ringbuffer/vfs.h"

#if 0
#define printk_dbg(fmt, args...) printk(fmt, args)
#else
#define printk_dbg(fmt, args...)
#endif

loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin)
{
	return -ESPIPE;
}

/*
 * Release pages from the buffer so splice pipe_to_file can move them.
 * Called after the pipe has been populated with buffer pages.
 */
static void lib_ring_buffer_pipe_buf_release(struct pipe_inode_info *pipe,
					     struct pipe_buffer *pbuf)
{
	__free_page(pbuf->page);
}

static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	.can_merge = 0,
	.map = generic_pipe_buf_map,
	.unmap = generic_pipe_buf_unmap,
	.confirm = generic_pipe_buf_confirm,
	.release = lib_ring_buffer_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};

/*
 * Page release operation after splice pipe_to_file ends.
 */
static void lib_ring_buffer_page_release(struct splice_pipe_desc *spd,
					 unsigned int i)
{
	__free_page(spd->pages[i]);
}

/*
 *	subbuf_splice_actor - splice up to one subbuf's worth of data
 */
static int subbuf_splice_actor(struct file *in,
			       loff_t *ppos,
			       struct pipe_inode_info *pipe,
			       size_t len,
			       unsigned int flags)
{
	struct lib_ring_buffer *buf = in->private_data;
	struct channel *chan = buf->backend.chan;
	const struct lib_ring_buffer_config *config = &chan->backend.config;
	unsigned int poff, subbuf_pages, nr_pages;
	struct page *pages[PIPE_DEF_BUFFERS];
	struct partial_page partial[PIPE_DEF_BUFFERS];
	struct splice_pipe_desc spd = {
		.pages = pages,
		.nr_pages = 0,
		.partial = partial,
		.flags = flags,
		.ops = &ring_buffer_pipe_buf_ops,
		.spd_release = lib_ring_buffer_page_release,
	};
	unsigned long consumed_old, roffset;
	unsigned long bytes_avail;

	/*
	 * Check that a GET_SUBBUF ioctl has been done before.
	 */
	WARN_ON(atomic_long_read(&buf->active_readers) != 1);
	consumed_old = lib_ring_buffer_get_consumed(config, buf);
	consumed_old += *ppos;

	/*
	 * Adjust read len, if longer than what is available.
	 * Max read size is 1 subbuffer due to get_subbuf/put_subbuf for
	 * protection.
	 */
	bytes_avail = chan->backend.subbuf_size;
	WARN_ON(bytes_avail > chan->backend.buf_size);
	len = min_t(size_t, len, bytes_avail);
	subbuf_pages = bytes_avail >> PAGE_SHIFT;
	nr_pages = min_t(unsigned int, subbuf_pages, PIPE_DEF_BUFFERS);
	roffset = consumed_old & PAGE_MASK;
	poff = consumed_old & ~PAGE_MASK;
	printk_dbg(KERN_DEBUG "SPLICE actor len %zu pos %zd write_pos %ld\n",
		   len, (ssize_t)*ppos, lib_ring_buffer_get_offset(config, buf));

	for (; spd.nr_pages < nr_pages; spd.nr_pages++) {
		unsigned int this_len;
		struct page **page, *new_page;
		void **virt;

		if (!len)
			break;
		printk_dbg(KERN_DEBUG "SPLICE actor loop len %zu roffset %ld\n",
			   len, roffset);

		/*
		 * We have to replace the page we are moving into the splice
		 * pipe.
		 */
		new_page = alloc_pages_node(cpu_to_node(max(buf->backend.cpu,
							    0)),
					    GFP_KERNEL | __GFP_ZERO, 0);
		if (!new_page)
			break;

		this_len = PAGE_SIZE - poff;
		page = lib_ring_buffer_read_get_page(&buf->backend, roffset, &virt);
		spd.pages[spd.nr_pages] = *page;
		*page = new_page;
		*virt = page_address(new_page);
		spd.partial[spd.nr_pages].offset = poff;
		spd.partial[spd.nr_pages].len = this_len;

		poff = 0;
		roffset += PAGE_SIZE;
		len -= this_len;
	}

	if (!spd.nr_pages)
		return 0;

	return wrapper_splice_to_pipe(pipe, &spd);
}

ssize_t lib_ring_buffer_splice_read(struct file *in, loff_t *ppos,
				    struct pipe_inode_info *pipe, size_t len,
				    unsigned int flags)
{
	struct lib_ring_buffer *buf = in->private_data;
	struct channel *chan = buf->backend.chan;
	const struct lib_ring_buffer_config *config = &chan->backend.config;
	ssize_t spliced;
	int ret;

	if (config->output != RING_BUFFER_SPLICE)
		return -EINVAL;

	/*
	 * We require ppos and length to be page-aligned for performance reasons
	 * (no page copy). Size is known using the ioctl
	 * RING_BUFFER_GET_PADDED_SUBBUF_SIZE, which is page-size padded.
	 * We fail when the ppos or len passed is not page-sized, because splice
	 * is not allowed to copy more than the length passed as parameter (so
	 * the ABI does not let us silently copy more than requested to include
	 * padding).
	 */
	if (*ppos != PAGE_ALIGN(*ppos) || len != PAGE_ALIGN(len))
		return -EINVAL;

	ret = 0;
	spliced = 0;

	printk_dbg(KERN_DEBUG "SPLICE read len %zu pos %zd\n", len,
		   (ssize_t)*ppos);
	while (len && !spliced) {
		ret = subbuf_splice_actor(in, ppos, pipe, len, flags);
		printk_dbg(KERN_DEBUG "SPLICE read loop ret %d\n", ret);
		if (ret < 0)
			break;
		else if (!ret) {
			if (flags & SPLICE_F_NONBLOCK)
				ret = -EAGAIN;
			break;
		}

		*ppos += ret;
		if (ret > len)
			len = 0;
		else
			len -= ret;
		spliced += ret;
	}

	if (spliced)
		return spliced;

	return ret;
}
EXPORT_SYMBOL_GPL(lib_ring_buffer_splice_read);
Commit	Line	Data
f3bc08c5 MD	1	/*
	2	* ring_buffer_splice.c
	3	*
	4	* Copyright (C) 2002-2005 - Tom Zanussi <zanussi@us.ibm.com>, IBM Corp
	5	* Copyright (C) 1999-2005 - Karim Yaghmour <karim@opersys.com>
886d51a3	6	* Copyright (C) 2008-2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
f3bc08c5	7	*
886d51a3 MD	8	* This library is free software; you can redistribute it and/or
	9	* modify it under the terms of the GNU Lesser General Public
	10	* License as published by the Free Software Foundation; only
	11	* version 2.1 of the License.
f3bc08c5	12	*
886d51a3 MD	13	* This library is distributed in the hope that it will be useful,
	14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	* Lesser General Public License for more details.
	17	*
	18	* You should have received a copy of the GNU Lesser General Public
	19	* License along with this library; if not, write to the Free Software
	20	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	*
	22	* Re-using code from kernel/relay.c, which is why it is licensed under
	23	* the GPLv2.
f3bc08c5 MD	24	*/
	25
	26	#include <linux/module.h>
	27	#include <linux/fs.h>
	28
90225db5	29	#include "../../wrapper/splice.h"
f3bc08c5 MD	30	#include "../../wrapper/ringbuffer/backend.h"
	31	#include "../../wrapper/ringbuffer/frontend.h"
	32	#include "../../wrapper/ringbuffer/vfs.h"
	33
	34	#if 0
	35	#define printk_dbg(fmt, args...) printk(fmt, args)
	36	#else
	37	#define printk_dbg(fmt, args...)
	38	#endif
	39
	40	loff_t lib_ring_buffer_no_llseek(struct file *file, loff_t offset, int origin)
	41	{
	42	return -ESPIPE;
	43	}
	44
	45	/*
	46	* Release pages from the buffer so splice pipe_to_file can move them.
	47	* Called after the pipe has been populated with buffer pages.
	48	*/
	49	static void lib_ring_buffer_pipe_buf_release(struct pipe_inode_info *pipe,
	50	struct pipe_buffer *pbuf)
	51	{
	52	__free_page(pbuf->page);
	53	}
	54
	55	static const struct pipe_buf_operations ring_buffer_pipe_buf_ops = {
	56	.can_merge = 0,
	57	.map = generic_pipe_buf_map,
	58	.unmap = generic_pipe_buf_unmap,
	59	.confirm = generic_pipe_buf_confirm,
	60	.release = lib_ring_buffer_pipe_buf_release,
	61	.steal = generic_pipe_buf_steal,
	62	.get = generic_pipe_buf_get,
	63	};
	64
	65	/*
	66	* Page release operation after splice pipe_to_file ends.
	67	*/
	68	static void lib_ring_buffer_page_release(struct splice_pipe_desc *spd,
	69	unsigned int i)
	70	{
	71	__free_page(spd->pages[i]);
	72	}
	73
	74	/*
	75	* subbuf_splice_actor - splice up to one subbuf's worth of data
	76	*/
	77	static int subbuf_splice_actor(struct file *in,
	78	loff_t *ppos,
	79	struct pipe_inode_info *pipe,
	80	size_t len,
	81	unsigned int flags)
	82	{
	83	struct lib_ring_buffer *buf = in->private_data;
	84	struct channel *chan = buf->backend.chan;
5a8fd222	85	const struct lib_ring_buffer_config *config = &chan->backend.config;
f3bc08c5 MD	86	unsigned int poff, subbuf_pages, nr_pages;
	87	struct page *pages[PIPE_DEF_BUFFERS];
	88	struct partial_page partial[PIPE_DEF_BUFFERS];
	89	struct splice_pipe_desc spd = {
	90	.pages = pages,
	91	.nr_pages = 0,
	92	.partial = partial,
	93	.flags = flags,
	94	.ops = &ring_buffer_pipe_buf_ops,
	95	.spd_release = lib_ring_buffer_page_release,
	96	};
88dfd899	97	unsigned long consumed_old, roffset;
f3bc08c5 MD	98	unsigned long bytes_avail;
	99
	100	/*
	101	* Check that a GET_SUBBUF ioctl has been done before.
	102	*/
	103	WARN_ON(atomic_long_read(&buf->active_readers) != 1);
	104	consumed_old = lib_ring_buffer_get_consumed(config, buf);
	105	consumed_old += *ppos;
f3bc08c5 MD	106
	107	/*
	108	* Adjust read len, if longer than what is available.
	109	* Max read size is 1 subbuffer due to get_subbuf/put_subbuf for
	110	* protection.
	111	*/
	112	bytes_avail = chan->backend.subbuf_size;
	113	WARN_ON(bytes_avail > chan->backend.buf_size);
	114	len = min_t(size_t, len, bytes_avail);
	115	subbuf_pages = bytes_avail >> PAGE_SHIFT;
	116	nr_pages = min_t(unsigned int, subbuf_pages, PIPE_DEF_BUFFERS);
	117	roffset = consumed_old & PAGE_MASK;
	118	poff = consumed_old & ~PAGE_MASK;
	119	printk_dbg(KERN_DEBUG "SPLICE actor len %zu pos %zd write_pos %ld\n",
	120	len, (ssize_t)*ppos, lib_ring_buffer_get_offset(config, buf));
	121
	122	for (; spd.nr_pages < nr_pages; spd.nr_pages++) {
	123	unsigned int this_len;
	124	struct page *page, new_page;
	125	void **virt;
	126
	127	if (!len)
	128	break;
	129	printk_dbg(KERN_DEBUG "SPLICE actor loop len %zu roffset %ld\n",
	130	len, roffset);
	131
	132	/*
	133	* We have to replace the page we are moving into the splice
	134	* pipe.
	135	*/
	136	new_page = alloc_pages_node(cpu_to_node(max(buf->backend.cpu,
	137	0)),
	138	GFP_KERNEL \| __GFP_ZERO, 0);
	139	if (!new_page)
	140	break;
	141
	142	this_len = PAGE_SIZE - poff;
	143	page = lib_ring_buffer_read_get_page(&buf->backend, roffset, &virt);
	144	spd.pages[spd.nr_pages] = *page;
	145	*page = new_page;
	146	*virt = page_address(new_page);
	147	spd.partial[spd.nr_pages].offset = poff;
	148	spd.partial[spd.nr_pages].len = this_len;
	149
	150	poff = 0;
	151	roffset += PAGE_SIZE;
	152	len -= this_len;
	153	}
	154
	155	if (!spd.nr_pages)
	156	return 0;
	157
90225db5	158	return wrapper_splice_to_pipe(pipe, &spd);
f3bc08c5 MD	159	}
	160
	161	ssize_t lib_ring_buffer_splice_read(struct file in, loff_t ppos,
	162	struct pipe_inode_info *pipe, size_t len,
	163	unsigned int flags)
	164	{
	165	struct lib_ring_buffer *buf = in->private_data;
	166	struct channel *chan = buf->backend.chan;
5a8fd222	167	const struct lib_ring_buffer_config *config = &chan->backend.config;
f3bc08c5 MD	168	ssize_t spliced;
	169	int ret;
	170
	171	if (config->output != RING_BUFFER_SPLICE)
	172	return -EINVAL;
	173
	174	/*
	175	* We require ppos and length to be page-aligned for performance reasons
	176	* (no page copy). Size is known using the ioctl
	177	* RING_BUFFER_GET_PADDED_SUBBUF_SIZE, which is page-size padded.
	178	* We fail when the ppos or len passed is not page-sized, because splice
	179	* is not allowed to copy more than the length passed as parameter (so
	180	* the ABI does not let us silently copy more than requested to include
	181	* padding).
	182	*/
	183	if (ppos != PAGE_ALIGN(ppos) \|\| len != PAGE_ALIGN(len))
	184	return -EINVAL;
	185
	186	ret = 0;
	187	spliced = 0;
	188
	189	printk_dbg(KERN_DEBUG "SPLICE read len %zu pos %zd\n", len,
	190	(ssize_t)*ppos);
	191	while (len && !spliced) {
	192	ret = subbuf_splice_actor(in, ppos, pipe, len, flags);
	193	printk_dbg(KERN_DEBUG "SPLICE read loop ret %d\n", ret);
	194	if (ret < 0)
	195	break;
	196	else if (!ret) {
	197	if (flags & SPLICE_F_NONBLOCK)
	198	ret = -EAGAIN;
	199	break;
	200	}
	201
	202	*ppos += ret;
	203	if (ret > len)
	204	len = 0;
	205	else
	206	len -= ret;
	207	spliced += ret;
	208	}
	209
	210	if (spliced)
	211	return spliced;
	212
	213	return ret;
	214	}
	215	EXPORT_SYMBOL_GPL(lib_ring_buffer_splice_read);