From: Michael Jeanson Date: Tue, 25 Aug 2020 14:56:29 +0000 (-0400) Subject: fix: removal of [smp_]read_barrier_depends (v5.9) X-Git-Tag: v2.13.0-rc1~175 X-Git-Url: http://git.liburcu.org/?p=lttng-modules.git;a=commitdiff_plain;h=585e5dcc4bf017c03b86dc84371f9c6170e12785 fix: removal of [smp_]read_barrier_depends (v5.9) See upstream commits: commit 76ebbe78f7390aee075a7f3768af197ded1bdfbb Author: Will Deacon Date: Tue Oct 24 11:22:47 2017 +0100 locking/barriers: Add implicit smp_read_barrier_depends() to READ_ONCE() In preparation for the removal of lockless_dereference(), which is the same as READ_ONCE() on all architectures other than Alpha, add an implicit smp_read_barrier_depends() to READ_ONCE() so that it can be used to head dependency chains on all architectures. commit 76ebbe78f7390aee075a7f3768af197ded1bdfbb Author: Will Deacon Date: Tue Oct 24 11:22:47 2017 +0100 locking/barriers: Add implicit smp_read_barrier_depends() to READ_ONCE() In preparation for the removal of lockless_dereference(), which is the same as READ_ONCE() on all architectures other than Alpha, add an implicit smp_read_barrier_depends() to READ_ONCE() so that it can be used to head dependency chains on all architectures. Change-Id: Ife8880bd9378dca2972da8838f40fc35ccdfaaac Signed-off-by: Michael Jeanson Signed-off-by: Mathieu Desnoyers --- diff --git a/include/instrumentation/events/i2c.h b/include/instrumentation/events/i2c.h index 312f6733..ad0f80bf 100644 --- a/include/instrumentation/events/i2c.h +++ b/include/instrumentation/events/i2c.h @@ -23,7 +23,7 @@ LTTNG_TRACEPOINT_EVENT_CODE(i2c_write, TP_code_pre( tp_locvar->extract_sensitive_payload = - READ_ONCE(extract_sensitive_payload); + LTTNG_READ_ONCE(extract_sensitive_payload); ), TP_FIELDS( @@ -78,7 +78,7 @@ LTTNG_TRACEPOINT_EVENT_CODE(i2c_reply, TP_code_pre( tp_locvar->extract_sensitive_payload = - READ_ONCE(extract_sensitive_payload); + LTTNG_READ_ONCE(extract_sensitive_payload); ), TP_FIELDS( diff --git a/include/lttng/tracepoint-event-impl.h b/include/lttng/tracepoint-event-impl.h index b5094619..f27a45c0 100644 --- a/include/lttng/tracepoint-event-impl.h +++ b/include/lttng/tracepoint-event-impl.h @@ -1153,11 +1153,11 @@ static void __event_probe__##_name(void *__data, _proto) \ \ if (!_TP_SESSION_CHECK(session, __session)) \ return; \ - if (unlikely(!READ_ONCE(__session->active))) \ + if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ return; \ - if (unlikely(!READ_ONCE(__chan->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ return; \ - if (unlikely(!READ_ONCE(__event->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \ return; \ __lf = lttng_rcu_dereference(__session->pid_tracker.p); \ if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \ @@ -1248,11 +1248,11 @@ static void __event_probe__##_name(void *__data) \ \ if (!_TP_SESSION_CHECK(session, __session)) \ return; \ - if (unlikely(!READ_ONCE(__session->active))) \ + if (unlikely(!LTTNG_READ_ONCE(__session->active))) \ return; \ - if (unlikely(!READ_ONCE(__chan->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__chan->enabled))) \ return; \ - if (unlikely(!READ_ONCE(__event->enabled))) \ + if (unlikely(!LTTNG_READ_ONCE(__event->enabled))) \ return; \ __lf = lttng_rcu_dereference(__session->pid_tracker.p); \ if (__lf && likely(!lttng_id_tracker_lookup(__lf, current->tgid))) \ diff --git a/include/ringbuffer/backend.h b/include/ringbuffer/backend.h index 1f499c90..67592546 100644 --- a/include/ringbuffer/backend.h +++ b/include/ringbuffer/backend.h @@ -156,7 +156,7 @@ size_t lib_ring_buffer_do_strcpy(const struct lib_ring_buffer_config *config, * Only read source character once, in case it is * modified concurrently. */ - c = READ_ONCE(src[count]); + c = LTTNG_READ_ONCE(src[count]); if (!c) break; lib_ring_buffer_do_copy(config, &dest[count], &c, 1); diff --git a/include/ringbuffer/backend_internal.h b/include/ringbuffer/backend_internal.h index aab408fb..c76b25a8 100644 --- a/include/ringbuffer/backend_internal.h +++ b/include/ringbuffer/backend_internal.h @@ -368,7 +368,7 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config, * Performing a volatile access to read the sb_pages, because we want to * read a coherent version of the pointer and the associated noref flag. */ - id = READ_ONCE(bufb->buf_wsb[idx].id); + id = LTTNG_READ_ONCE(bufb->buf_wsb[idx].id); for (;;) { /* This check is called on the fast path for each record. */ if (likely(!subbuffer_id_is_noref(config, id))) { diff --git a/include/ringbuffer/frontend.h b/include/ringbuffer/frontend.h index 41fec2a0..1707d713 100644 --- a/include/ringbuffer/frontend.h +++ b/include/ringbuffer/frontend.h @@ -79,7 +79,7 @@ void *channel_destroy(struct channel *chan); #define for_each_channel_cpu(cpu, chan) \ for ((cpu) = -1; \ ({ (cpu) = cpumask_next(cpu, (chan)->backend.cpumask); \ - smp_read_barrier_depends(); (cpu) < nr_cpu_ids; });) + smp_rmb(); (cpu) < nr_cpu_ids; });) extern struct lib_ring_buffer *channel_get_ring_buffer( const struct lib_ring_buffer_config *config, @@ -155,7 +155,7 @@ static inline int lib_ring_buffer_is_finalized(const struct lib_ring_buffer_config *config, struct lib_ring_buffer *buf) { - int finalized = READ_ONCE(buf->finalized); + int finalized = LTTNG_READ_ONCE(buf->finalized); /* * Read finalized before counters. */ diff --git a/include/wrapper/compiler.h b/include/wrapper/compiler.h index 38974c06..4efe0cfa 100644 --- a/include/wrapper/compiler.h +++ b/include/wrapper/compiler.h @@ -9,6 +9,7 @@ #define _LTTNG_WRAPPER_COMPILER_H #include +#include /* * Don't allow compiling with buggy compiler. @@ -39,6 +40,23 @@ # define WRITE_ONCE(x, val) ({ ACCESS_ONCE(x) = val; }) #endif +/* + * In v4.15 a smp read barrier was added to READ_ONCE to replace + * lockless_dereference(), replicate this behavior on prior kernels + * and remove calls to smp_read_barrier_depends which was dropped + * in v5.9. + */ +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,15,0)) +#define LTTNG_READ_ONCE(x) READ_ONCE(x) +#else +#define LTTNG_READ_ONCE(x) \ +({ \ + typeof(x) __val = READ_ONCE(x); \ + smp_read_barrier_depends(); \ + __val; \ +}) +#endif + #define __LTTNG_COMPOUND_LITERAL(type, ...) (type[]) { __VA_ARGS__ } #endif /* _LTTNG_WRAPPER_COMPILER_H */ diff --git a/include/wrapper/trace-clock.h b/include/wrapper/trace-clock.h index 7cef08df..fba3b9fe 100644 --- a/include/wrapper/trace-clock.h +++ b/include/wrapper/trace-clock.h @@ -160,33 +160,30 @@ static inline void put_trace_clock(void) static inline u64 trace_clock_read64(void) { - struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock); + struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock); if (likely(!ltc)) { return trace_clock_read64_monotonic(); } else { - read_barrier_depends(); /* load ltc before content */ return ltc->read64(); } } static inline u64 trace_clock_freq(void) { - struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock); + struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock); if (!ltc) { return trace_clock_freq_monotonic(); } else { - read_barrier_depends(); /* load ltc before content */ return ltc->freq(); } } static inline int trace_clock_uuid(char *uuid) { - struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock); + struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock); - read_barrier_depends(); /* load ltc before content */ /* Use default UUID cb when NULL */ if (!ltc || !ltc->uuid) { return trace_clock_uuid_monotonic(uuid); @@ -197,24 +194,22 @@ static inline int trace_clock_uuid(char *uuid) static inline const char *trace_clock_name(void) { - struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock); + struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock); if (!ltc) { return trace_clock_name_monotonic(); } else { - read_barrier_depends(); /* load ltc before content */ return ltc->name(); } } static inline const char *trace_clock_description(void) { - struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock); + struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock); if (!ltc) { return trace_clock_description_monotonic(); } else { - read_barrier_depends(); /* load ltc before content */ return ltc->description(); } } diff --git a/src/lib/ringbuffer/ring_buffer_frontend.c b/src/lib/ringbuffer/ring_buffer_frontend.c index d3dfcb1b..cc5ac836 100644 --- a/src/lib/ringbuffer/ring_buffer_frontend.c +++ b/src/lib/ringbuffer/ring_buffer_frontend.c @@ -1074,7 +1074,7 @@ int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf, int finalized; retry: - finalized = READ_ONCE(buf->finalized); + finalized = LTTNG_READ_ONCE(buf->finalized); /* * Read finalized before counters. */ @@ -1245,7 +1245,7 @@ int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf, return -EBUSY; } retry: - finalized = READ_ONCE(buf->finalized); + finalized = LTTNG_READ_ONCE(buf->finalized); /* * Read finalized before counters. */ diff --git a/src/lib/ringbuffer/ring_buffer_iterator.c b/src/lib/ringbuffer/ring_buffer_iterator.c index 64fc5086..a136a8fa 100644 --- a/src/lib/ringbuffer/ring_buffer_iterator.c +++ b/src/lib/ringbuffer/ring_buffer_iterator.c @@ -46,7 +46,7 @@ restart: switch (iter->state) { case ITER_GET_SUBBUF: ret = lib_ring_buffer_get_next_subbuf(buf); - if (ret && !READ_ONCE(buf->finalized) + if (ret && !LTTNG_READ_ONCE(buf->finalized) && config->alloc == RING_BUFFER_ALLOC_GLOBAL) { /* * Use "pull" scheme for global buffers. The reader diff --git a/src/lttng-events.c b/src/lttng-events.c index 631dfcb6..3986217f 100644 --- a/src/lttng-events.c +++ b/src/lttng-events.c @@ -1723,7 +1723,7 @@ int lttng_metadata_printf(struct lttng_session *session, size_t len; va_list ap; - WARN_ON_ONCE(!READ_ONCE(session->active)); + WARN_ON_ONCE(!LTTNG_READ_ONCE(session->active)); va_start(ap, fmt); str = kvasprintf(GFP_KERNEL, fmt, ap); @@ -2347,7 +2347,7 @@ int _lttng_event_metadata_statedump(struct lttng_session *session, { int ret = 0; - if (event->metadata_dumped || !READ_ONCE(session->active)) + if (event->metadata_dumped || !LTTNG_READ_ONCE(session->active)) return 0; if (chan->channel_type == METADATA_CHANNEL) return 0; @@ -2419,7 +2419,7 @@ int _lttng_channel_metadata_statedump(struct lttng_session *session, { int ret = 0; - if (chan->metadata_dumped || !READ_ONCE(session->active)) + if (chan->metadata_dumped || !LTTNG_READ_ONCE(session->active)) return 0; if (chan->channel_type == METADATA_CHANNEL) @@ -2646,7 +2646,7 @@ int _lttng_session_metadata_statedump(struct lttng_session *session) struct lttng_event *event; int ret = 0; - if (!READ_ONCE(session->active)) + if (!LTTNG_READ_ONCE(session->active)) return 0; lttng_metadata_begin(session); diff --git a/src/probes/lttng-kprobes.c b/src/probes/lttng-kprobes.c index a2474d0d..e127d72f 100644 --- a/src/probes/lttng-kprobes.c +++ b/src/probes/lttng-kprobes.c @@ -31,11 +31,11 @@ int lttng_kprobes_handler_pre(struct kprobe *p, struct pt_regs *regs) int ret; unsigned long data = (unsigned long) p->addr; - if (unlikely(!READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) return 0; - if (unlikely(!READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) return 0; - if (unlikely(!READ_ONCE(event->enabled))) + if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx, sizeof(data), diff --git a/src/probes/lttng-kretprobes.c b/src/probes/lttng-kretprobes.c index 00675937..8cde4edd 100644 --- a/src/probes/lttng-kretprobes.c +++ b/src/probes/lttng-kretprobes.c @@ -51,11 +51,11 @@ int _lttng_kretprobes_handler(struct kretprobe_instance *krpi, unsigned long parent_ip; } payload; - if (unlikely(!READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) return 0; - if (unlikely(!READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) return 0; - if (unlikely(!READ_ONCE(event->enabled))) + if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; payload.ip = (unsigned long) krpi->rp->kp.addr; diff --git a/src/probes/lttng-uprobes.c b/src/probes/lttng-uprobes.c index be6a50fc..435ff5ac 100644 --- a/src/probes/lttng-uprobes.c +++ b/src/probes/lttng-uprobes.c @@ -40,11 +40,11 @@ int lttng_uprobes_handler_pre(struct uprobe_consumer *uc, struct pt_regs *regs) unsigned long ip; } payload; - if (unlikely(!READ_ONCE(chan->session->active))) + if (unlikely(!LTTNG_READ_ONCE(chan->session->active))) return 0; - if (unlikely(!READ_ONCE(chan->enabled))) + if (unlikely(!LTTNG_READ_ONCE(chan->enabled))) return 0; - if (unlikely(!READ_ONCE(event->enabled))) + if (unlikely(!LTTNG_READ_ONCE(event->enabled))) return 0; lib_ring_buffer_ctx_init(&ctx, chan->chan, <tng_probe_ctx,