-/*
- * Copyright (C) 2011 Mathieu Desnoyers (mathieu.desnoyers@efficios.com)
+/* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
+ *
+ * wrapper/trace-clock.h
*
* Contains LTTng trace clock mapping to LTTng trace clock or mainline monotonic
* clock. This wrapper depends on CONFIG_HIGH_RES_TIMERS=y.
*
- * Dual LGPL v2.1/GPL v2 license.
+ * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
*/
#ifndef _LTTNG_TRACE_CLOCK_H
#include <linux/ktime.h>
#include <linux/time.h>
#include <linux/hrtimer.h>
+#include <linux/percpu.h>
+#include <linux/version.h>
+#include <asm/local.h>
+#include <lttng/kernel-version.h>
+#include <lttng/clock.h>
+#include <wrapper/compiler.h>
+#include <wrapper/percpu-defs.h>
+#include <wrapper/random.h>
+#include <blacklist/timekeeping.h>
+
+extern struct lttng_trace_clock *lttng_trace_clock;
+
+/*
+ * Upstream Linux commit 27727df240c7 ("Avoid taking lock in NMI path with
+ * CONFIG_DEBUG_TIMEKEEPING") introduces a buggy ktime_get_mono_fast_ns().
+ * This is fixed by patch "timekeeping: Fix __ktime_get_fast_ns() regression".
+ */
+#if (LTTNG_KERNEL_RANGE(4,8,0, 4,8,2) \
+ || LTTNG_KERNEL_RANGE(4,7,4, 4,7,8) \
+ || LTTNG_KERNEL_RANGE(4,4,20, 4,4,25) \
+ || LTTNG_KERNEL_RANGE(4,1,32, 4,1,35))
+#define LTTNG_CLOCK_NMI_SAFE_BROKEN
+#endif
+
+/*
+ * We need clock values to be monotonically increasing per-cpu, which is
+ * not strictly guaranteed by ktime_get_mono_fast_ns(). It is
+ * straightforward to do on architectures with a 64-bit cmpxchg(), but
+ * not so on architectures without 64-bit cmpxchg. For now, only enable
+ * this feature on 64-bit architectures.
+ */
+
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0) \
+ && BITS_PER_LONG == 64 \
+ && !defined(LTTNG_CLOCK_NMI_SAFE_BROKEN))
+#define LTTNG_USE_NMI_SAFE_CLOCK
+#endif
+
+#ifdef LTTNG_USE_NMI_SAFE_CLOCK
+
+DECLARE_PER_CPU(u64, lttng_last_tsc);
+
+/*
+ * Sometimes called with preemption enabled. Can be interrupted.
+ */
+static inline u64 trace_clock_monotonic_wrapper(void)
+{
+ u64 now, last, result;
+ u64 *last_tsc_ptr;
+
+ /* Use fast nmi-safe monotonic clock provided by the Linux kernel. */
+ preempt_disable();
+ last_tsc_ptr = lttng_this_cpu_ptr(<tng_last_tsc);
+ last = *last_tsc_ptr;
+ /*
+ * Read "last" before "now". It is not strictly required, but it ensures
+ * that an interrupt coming in won't artificially trigger a case where
+ * "now" < "last". This kind of situation should only happen if the
+ * mono_fast time source goes slightly backwards.
+ */
+ barrier();
+ now = ktime_get_mono_fast_ns();
+ if (U64_MAX / 2 < now - last)
+ now = last;
+ result = cmpxchg64_local(last_tsc_ptr, last, now);
+ preempt_enable();
+ if (result == last) {
+ /* Update done. */
+ return now;
+ } else {
+ /*
+ * Update not done, due to concurrent update. We can use
+ * "result", since it has been sampled concurrently with our
+ * time read, so it should not be far from "now".
+ */
+ return result;
+ }
+}
+#else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
static inline u64 trace_clock_monotonic_wrapper(void)
{
ktime_t ktime;
return (u64) -EIO;
ktime = ktime_get();
- return (u64) ktime.tv64;
+ return ktime_to_ns(ktime);
}
+#endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
-static inline u32 trace_clock_read32(void)
+static inline u64 trace_clock_read64_monotonic(void)
{
- return (u32) trace_clock_monotonic_wrapper();
+ return (u64) trace_clock_monotonic_wrapper();
}
-static inline u64 trace_clock_read64(void)
+static inline u64 trace_clock_freq_monotonic(void)
{
- return (u64) trace_clock_monotonic_wrapper();
+ return (u64) NSEC_PER_SEC;
}
-static inline u64 trace_clock_frequency(void)
+static inline int trace_clock_uuid_monotonic(char *uuid)
{
- return (u64)NSEC_PER_SEC;
+ return wrapper_get_bootid(uuid);
}
-static inline u32 trace_clock_freq_scale(void)
+static inline const char *trace_clock_name_monotonic(void)
{
- return 1;
+ return "monotonic";
}
-static inline int get_trace_clock(void)
+static inline const char *trace_clock_description_monotonic(void)
{
- printk(KERN_WARNING "LTTng: Using mainline kernel monotonic clock.\n");
- printk(KERN_WARNING " * NMIs will not be traced,\n");
- printk(KERN_WARNING " * expect significant performance degradation compared to the\n");
- printk(KERN_WARNING " LTTng trace clocks.\n");
- printk(KERN_WARNING "Integration of the LTTng 0.x trace clocks into LTTng 2.0 is planned\n");
- printk(KERN_WARNING "in a near future.\n");
-
-#warning "LTTng: Using mainline kernel monotonic clock. NMIs will not be traced, and expect significant performance degradation compared to the LTTng trace clocks. Integration of the LTTng 0.x trace clocks into LTTng 2.0 is planned in a near future."
+ return "Monotonic Clock";
+}
+#ifdef LTTNG_USE_NMI_SAFE_CLOCK
+static inline int get_trace_clock(void)
+{
+ printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic fast clock, which is NMI-safe.\n");
+ return 0;
+}
+#else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
+static inline int get_trace_clock(void)
+{
+ printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic clock. NMIs will not be traced.\n");
return 0;
}
+#endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
static inline void put_trace_clock(void)
{
}
+static inline u64 trace_clock_read64(void)
+{
+ struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
+
+ if (likely(!ltc)) {
+ return trace_clock_read64_monotonic();
+ } else {
+ read_barrier_depends(); /* load ltc before content */
+ return ltc->read64();
+ }
+}
+
+static inline u64 trace_clock_freq(void)
+{
+ struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
+
+ if (!ltc) {
+ return trace_clock_freq_monotonic();
+ } else {
+ read_barrier_depends(); /* load ltc before content */
+ return ltc->freq();
+ }
+}
+
+static inline int trace_clock_uuid(char *uuid)
+{
+ struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
+
+ read_barrier_depends(); /* load ltc before content */
+ /* Use default UUID cb when NULL */
+ if (!ltc || !ltc->uuid) {
+ return trace_clock_uuid_monotonic(uuid);
+ } else {
+ return ltc->uuid(uuid);
+ }
+}
+
+static inline const char *trace_clock_name(void)
+{
+ struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
+
+ if (!ltc) {
+ return trace_clock_name_monotonic();
+ } else {
+ read_barrier_depends(); /* load ltc before content */
+ return ltc->name();
+ }
+}
+
+static inline const char *trace_clock_description(void)
+{
+ struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
+
+ if (!ltc) {
+ return trace_clock_description_monotonic();
+ } else {
+ read_barrier_depends(); /* load ltc before content */
+ return ltc->description();
+ }
+}
+
#endif /* CONFIG_HAVE_TRACE_CLOCK */
#endif /* _LTTNG_TRACE_CLOCK_H */
projects / lttng-modules.git / blobdiff