From aa5d4c0dc4ba7a1f48a8a5e7e19888addc4548f0 Mon Sep 17 00:00:00 2001 From: Julien Desfossez Date: Fri, 23 Feb 2018 11:37:10 -0500 Subject: [PATCH] Create a memory pool for temporary tracepoint probes storage This memory pool is created when the lttng-tracer module is loaded. It allocates 4 buffers of 4k on each CPU. These buffers are designed to allow tracepoint probes to temporarily store data that does not fit on the stack (during the code_pre and code_post phases). The memory is freed when the lttng-tracer module is unloaded. This removes the need for dynamic allocation during the execution of tracepoint probes, which does not behave well on PREEMPT_RT kernel, even when invoked with the GFP_ATOMIC | GFP_NOWAIT flags. Signed-off-by: Julien Desfossez Signed-off-by: Mathieu Desnoyers --- Makefile | 3 +- lttng-abi.c | 9 +++ lttng-tp-mempool.c | 172 +++++++++++++++++++++++++++++++++++++++++++++ lttng-tp-mempool.h | 63 +++++++++++++++++ 4 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 lttng-tp-mempool.c create mode 100644 lttng-tp-mempool.h diff --git a/Makefile b/Makefile index f65e4632..6b230e98 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,8 @@ ifneq ($(KERNELRELEASE),) lttng-filter.o lttng-filter-interpreter.o \ lttng-filter-specialize.o \ lttng-filter-validator.o \ - probes/lttng-probe-user.o + probes/lttng-probe-user.o \ + lttng-tp-mempool.o ifneq ($(CONFIG_HAVE_SYSCALL_TRACEPOINTS),) lttng-tracer-objs += lttng-syscalls.o diff --git a/lttng-abi.c b/lttng-abi.c index 77e5e985..a57e5e9f 100644 --- a/lttng-abi.c +++ b/lttng-abi.c @@ -56,6 +56,7 @@ #include #include #include +#include #include /* @@ -1727,6 +1728,12 @@ int __init lttng_abi_init(void) wrapper_vmalloc_sync_all(); lttng_clock_ref(); + + ret = lttng_tp_mempool_init(); + if (ret) { + goto error; + } + lttng_proc_dentry = proc_create_data("lttng", S_IRUSR | S_IWUSR, NULL, <tng_fops, NULL); @@ -1739,6 +1746,7 @@ int __init lttng_abi_init(void) return 0; error: + lttng_tp_mempool_destroy(); lttng_clock_unref(); return ret; } @@ -1746,6 +1754,7 @@ error: /* No __exit annotation because used by init error path too. */ void lttng_abi_exit(void) { + lttng_tp_mempool_destroy(); lttng_clock_unref(); if (lttng_proc_dentry) remove_proc_entry("lttng", NULL); diff --git a/lttng-tp-mempool.c b/lttng-tp-mempool.c new file mode 100644 index 00000000..d984bd45 --- /dev/null +++ b/lttng-tp-mempool.c @@ -0,0 +1,172 @@ +/* + * lttng-tp-mempool.c + * + * Copyright (C) 2018 Julien Desfossez + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; only + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include + +#include + +struct lttng_tp_buf_entry { + int cpu; /* To make sure we return the entry to the right pool. */ + char buf[LTTNG_TP_MEMPOOL_BUF_SIZE]; + struct list_head list; +}; + +/* + * No exclusive access strategy for now, this memory pool is currently only + * used from a non-preemptible context, and the interrupt tracepoint probes do + * not use this facility. + */ +struct per_cpu_buf { + struct list_head free_list; /* Free struct lttng_tp_buf_entry. */ +}; + +static struct per_cpu_buf __percpu *pool; /* Per-cpu buffer. */ + +int lttng_tp_mempool_init(void) +{ + int ret, cpu; + + /* The pool is only supposed to be allocated once. */ + if (pool) { + WARN_ON_ONCE(1); + ret = -1; + goto end; + } + + pool = alloc_percpu(struct per_cpu_buf); + if (!pool) { + ret = -ENOMEM; + goto end; + } + + for_each_possible_cpu(cpu) { + struct per_cpu_buf *cpu_buf = per_cpu_ptr(pool, cpu); + + INIT_LIST_HEAD(&cpu_buf->free_list); + } + + for_each_possible_cpu(cpu) { + int i; + struct per_cpu_buf *cpu_buf = per_cpu_ptr(pool, cpu); + + for (i = 0; i < LTTNG_TP_MEMPOOL_NR_BUF_PER_CPU; i++) { + struct lttng_tp_buf_entry *entry; + + entry = kzalloc(sizeof(struct lttng_tp_buf_entry), + GFP_KERNEL); + if (!entry) { + ret = -ENOMEM; + goto error_free_pool; + } + entry->cpu = cpu; + list_add_tail(&entry->list, &cpu_buf->free_list); + } + } + + ret = 0; + goto end; + +error_free_pool: + lttng_tp_mempool_destroy(); +end: + return ret; +} + +void lttng_tp_mempool_destroy(void) +{ + int cpu; + + if (!pool) { + return; + } + + for_each_possible_cpu(cpu) { + struct per_cpu_buf *cpu_buf = per_cpu_ptr(pool, cpu); + struct lttng_tp_buf_entry *entry, *tmp; + int i = 0; + + list_for_each_entry_safe(entry, tmp, &cpu_buf->free_list, list) { + list_del(&entry->list); + kfree(entry); + i++; + } + if (i < LTTNG_TP_MEMPOOL_NR_BUF_PER_CPU) { + printk(KERN_WARNING "Leak detected in tp-mempool\n"); + } + } + free_percpu(pool); + pool = NULL; +} + +void *lttng_tp_mempool_alloc(size_t size) +{ + void *ret; + struct lttng_tp_buf_entry *entry; + struct per_cpu_buf *cpu_buf; + int cpu = smp_processor_id(); + + if (size > LTTNG_TP_MEMPOOL_BUF_SIZE) { + ret = NULL; + goto end; + } + + cpu_buf = per_cpu_ptr(pool, cpu); + if (list_empty(&cpu_buf->free_list)) { + ret = NULL; + goto end; + } + + entry = list_first_entry(&cpu_buf->free_list, struct lttng_tp_buf_entry, list); + /* Remove the entry from the free list. */ + list_del(&entry->list); + + memset(entry->buf, 0, LTTNG_TP_MEMPOOL_BUF_SIZE); + + ret = (void *) entry->buf; + +end: + return ret; +} + +void lttng_tp_mempool_free(void *ptr) +{ + struct lttng_tp_buf_entry *entry; + struct per_cpu_buf *cpu_buf; + + if (!ptr) { + goto end; + } + + entry = container_of(ptr, struct lttng_tp_buf_entry, buf); + if (!entry) { + goto end; + } + + cpu_buf = per_cpu_ptr(pool, entry->cpu); + if (!cpu_buf) { + goto end; + } + /* Add it to the free list. */ + list_add_tail(&entry->list, &cpu_buf->free_list); + +end: + return; +} diff --git a/lttng-tp-mempool.h b/lttng-tp-mempool.h new file mode 100644 index 00000000..7ce41129 --- /dev/null +++ b/lttng-tp-mempool.h @@ -0,0 +1,63 @@ +#ifndef LTTNG_TP_MEMPOOL_H +#define LTTNG_TP_MEMPOOL_H + +/* + * lttng-tp-mempool.h + * + * Copyright (C) 2018 Julien Desfossez + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; only + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#define LTTNG_TP_MEMPOOL_NR_BUF_PER_CPU 4 +#define LTTNG_TP_MEMPOOL_BUF_SIZE 4096 + +/* + * Initialize the pool, only performed once. The pool is a set of + * LTTNG_TP_MEMPOOL_NR_BUF_PER_CPU buffers of size LTTNG_TP_MEMPOOL_BUF_SIZE + * per-cpu. + * + * Returns 0 on success, a negative value on error. + */ +int lttng_tp_mempool_init(void); + +/* + * Destroy the pool and free all the memory allocated. + */ +void lttng_tp_mempool_destroy(void); + +/* + * Ask for a buffer on the current cpu. + * + * The pool is per-cpu, but there is no exclusive access guarantee on the + * per-cpu free-list, the caller needs to ensure it cannot get preempted or + * interrupted while performing the allocation. + * + * The maximum size that can be allocated is LTTNG_TP_MEMPOOL_BUF_SIZE, and the + * maximum number of buffers allocated simultaneously on the same CPU is + * LTTNG_TP_MEMPOOL_NR_BUF_PER_CPU. + * + * Return a pointer to a buffer on success, NULL on error. + */ +void *lttng_tp_mempool_alloc(size_t size); + +/* + * Release the memory reserved. Same concurrency limitations as the allocation. + */ +void lttng_tp_mempool_free(void *ptr); + +#endif /* LTTNG_TP_MEMPOOL_H */ -- 2.34.1