From 7a09dcb7df9caab5c867f39f53009e860b98accb Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 23 Apr 2015 16:56:33 -0400 Subject: [PATCH] Fix: add lttng rcu wrapper to allow tracing RCU Signed-off-by: Mathieu Desnoyers --- lttng-tracker-pid.c | 3 +- probes/lttng-events.h | 9 +++-- wrapper/list.h | 24 ------------ wrapper/rcu.h | 90 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 29 deletions(-) create mode 100644 wrapper/rcu.h diff --git a/lttng-tracker-pid.c b/lttng-tracker-pid.c index eeee9d55..55bf63c9 100644 --- a/lttng-tracker-pid.c +++ b/lttng-tracker-pid.c @@ -29,6 +29,7 @@ #include #include "wrapper/tracepoint.h" +#include "wrapper/rcu.h" #include "wrapper/list.h" #include "lttng-events.h" @@ -59,7 +60,7 @@ bool lttng_pid_tracker_lookup(struct lttng_pid_tracker *lpf, int pid) uint32_t hash = hash_32(pid, 32); head = &lpf->pid_hash[hash & (LTTNG_PID_TABLE_SIZE - 1)]; - lttng_hlist_for_each_entry_rcu_notrace(e, head, hlist) { + lttng_hlist_for_each_entry_rcu(e, head, hlist) { if (pid == e->pid) return 1; /* Found */ } diff --git a/probes/lttng-events.h b/probes/lttng-events.h index 29b44af7..58f4e120 100644 --- a/probes/lttng-events.h +++ b/probes/lttng-events.h @@ -27,6 +27,7 @@ #include "lttng-probe-user.h" #include "../wrapper/vmalloc.h" /* for wrapper_vmalloc_sync_all() */ #include "../wrapper/ringbuffer/frontend_types.h" +#include "../wrapper/rcu.h" #include "../lttng-events.h" #include "../lttng-tracer-core.h" @@ -728,7 +729,7 @@ static void __event_probe__##_name(void *__data, _proto) \ return; \ if (unlikely(!ACCESS_ONCE(__event->enabled))) \ return; \ - __lpf = rcu_dereference(__session->pid_tracker); \ + __lpf = lttng_rcu_dereference(__session->pid_tracker); \ if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid))) \ return; \ _code \ @@ -738,7 +739,7 @@ static void __event_probe__##_name(void *__data, _proto) \ \ __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \ tp_locvar, _args); \ - list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \ + lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->filter(bc_runtime, \ __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \ __filter_record = 1; \ @@ -787,7 +788,7 @@ static void __event_probe__##_name(void *__data) \ return; \ if (unlikely(!ACCESS_ONCE(__event->enabled))) \ return; \ - __lpf = rcu_dereference(__session->pid_tracker); \ + __lpf = lttng_rcu_dereference(__session->pid_tracker); \ if (__lpf && likely(!lttng_pid_tracker_lookup(__lpf, current->pid))) \ return; \ _code \ @@ -797,7 +798,7 @@ static void __event_probe__##_name(void *__data) \ \ __event_prepare_filter_stack__##_name(__stackvar.__filter_stack_data, \ tp_locvar); \ - list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \ + lttng_list_for_each_entry_rcu(bc_runtime, &__event->bytecode_runtime_head, node) { \ if (unlikely(bc_runtime->filter(bc_runtime, \ __stackvar.__filter_stack_data) & LTTNG_FILTER_RECORD_FLAG)) \ __filter_record = 1; \ diff --git a/wrapper/list.h b/wrapper/list.h index c816684d..0d79e512 100644 --- a/wrapper/list.h +++ b/wrapper/list.h @@ -57,28 +57,4 @@ pos && ({ n = pos->member.next; 1; }); \ pos = lttng_hlist_entry_safe(n, typeof(*pos), member)) -#ifndef rcu_dereference_raw_notrace -#define rcu_dereference_raw_notrace(p) rcu_dereference_raw(p) -#endif - -/** - * lttng_hlist_for_each_entry_rcu_notrace - iterate over rcu list of given type (for tracing) - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the hlist_node within the struct. - * - * This list-traversal primitive may safely run concurrently with - * the _rcu list-mutation primitives such as hlist_add_head_rcu() - * as long as the traversal is guarded by rcu_read_lock(). - * - * This is the same as hlist_for_each_entry_rcu() except that it does - * not do any RCU debugging or tracing. - */ -#define lttng_hlist_for_each_entry_rcu_notrace(pos, head, member) \ - for (pos = lttng_hlist_entry_safe (rcu_dereference_raw_notrace(hlist_first_rcu(head)),\ - typeof(*(pos)), member); \ - pos; \ - pos = lttng_hlist_entry_safe(rcu_dereference_raw_notrace(hlist_next_rcu(\ - &(pos)->member)), typeof(*(pos)), member)) - #endif /* _LTTNG_WRAPPER_LIST_H */ diff --git a/wrapper/rcu.h b/wrapper/rcu.h new file mode 100644 index 00000000..d96d42d1 --- /dev/null +++ b/wrapper/rcu.h @@ -0,0 +1,90 @@ +#ifndef _LTTNG_WRAPPER_RCU_H +#define _LTTNG_WRAPPER_RCU_H + +/* + * wrapper/rcu.h + * + * wrapper around linux/rcupdate.h and linux/rculist.h. + * + * Copyright (C) 2014 Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; only + * version 2.1 of the License. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include +#include +#include "list.h" + +#ifndef rcu_dereference_raw_notrace +#define rcu_dereference_raw_notrace(p) rcu_dereference_raw(p) +#endif + +#define lttng_rcu_dereference(p) rcu_dereference_raw_notrace(p) + +/** + * lttng_list_entry_rcu - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_head within the struct. + * + * This primitive may safely run concurrently with the _rcu list-mutation + * primitives such as list_add_rcu() as long as it's guarded by + * rcu_read_lock_sched(). + * Can be used while tracing RCU. + */ +#define lttng_list_entry_rcu(ptr, type, member) \ +({ \ + typeof(*ptr) __rcu *__ptr = (typeof(*ptr) __rcu __force *)ptr; \ + container_of((typeof(ptr))lttng_rcu_dereference(__ptr), type, member); \ +}) + +/** + * lttng_list_for_each_entry_rcu - iterate over rcu list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_head within the struct. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as list_add_rcu() + * as long as the traversal is guarded by rcu_read_lock_sched(). + * Can be used while tracing RCU. + */ +#define lttng_list_for_each_entry_rcu(pos, head, member) \ + for (pos = lttng_list_entry_rcu((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = lttng_list_entry_rcu(pos->member.next, typeof(*pos), member)) + +/** + * lttng_hlist_for_each_entry_rcu - iterate over rcu list of given type (for tracing) + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as hlist_add_head_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + * + * This is the same as hlist_for_each_entry_rcu() except that it does + * not do any RCU debugging or tracing. + */ +#define lttng_hlist_for_each_entry_rcu(pos, head, member) \ + for (pos = lttng_hlist_entry_safe (lttng_rcu_dereference(hlist_first_rcu(head)), \ + typeof(*(pos)), member); \ + pos; \ + pos = lttng_hlist_entry_safe(lttng_rcu_dereference(hlist_next_rcu( \ + &(pos)->member)), typeof(*(pos)), member)) + +#endif /* _LTTNG_WRAPPER_RCU_H */ -- 2.34.1