include/wrapper/trace-clock.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * wrapper/trace-clock.h
   4  *
   5  * Contains LTTng trace clock mapping to LTTng trace clock or mainline monotonic
   6  * clock. This wrapper depends on CONFIG_HIGH_RES_TIMERS=y.
   7  *
   8  * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   9  */
  10
  11 #ifndef _LTTNG_TRACE_CLOCK_H
  12 #define _LTTNG_TRACE_CLOCK_H
  13
  14 #ifdef CONFIG_HAVE_TRACE_CLOCK
  15 #include <linux/trace-clock.h>
  16 #else /* CONFIG_HAVE_TRACE_CLOCK */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/ktime.h>
  20 #include <linux/time.h>
  21 #include <linux/hrtimer.h>
  22 #include <linux/percpu.h>
  23 #include <linux/percpu-defs.h>
  24
  25 #include <lttng/kernel-version.h>
  26 #include <asm/local.h>
  27 #include <lttng/kernel-version.h>
  28 #include <lttng/clock.h>
  29 #include <wrapper/compiler.h>
  30 #include <wrapper/random.h>
  31 #include <blacklist/timekeeping.h>
  32
  33 extern struct lttng_trace_clock *lttng_trace_clock;
  34
  35 /*
  36  * Upstream Linux commit 27727df240c7 ("Avoid taking lock in NMI path with
  37  * CONFIG_DEBUG_TIMEKEEPING") introduces a buggy ktime_get_mono_fast_ns().
  38  * This is fixed by patch "timekeeping: Fix __ktime_get_fast_ns() regression".
  39  */
  40 #if (LTTNG_KERNEL_RANGE(4,8,0, 4,8,2) \
  41         || LTTNG_KERNEL_RANGE(4,7,4, 4,7,8) \
  42         || LTTNG_KERNEL_RANGE(4,4,20, 4,4,25) \
  43         || LTTNG_KERNEL_RANGE(4,1,32, 4,1,35))
  44 #define LTTNG_CLOCK_NMI_SAFE_BROKEN
  45 #endif
  46
  47 /*
  48  * We need clock values to be monotonically increasing per-cpu, which is
  49  * not strictly guaranteed by ktime_get_mono_fast_ns(). It is
  50  * straightforward to do on architectures with a 64-bit cmpxchg(), but
  51  * not so on architectures without 64-bit cmpxchg. For now, only enable
  52  * this feature on 64-bit architectures.
  53  */
  54
  55 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,17,0) \
  56         && BITS_PER_LONG == 64 \
  57         && !defined(LTTNG_CLOCK_NMI_SAFE_BROKEN))
  58 #define LTTNG_USE_NMI_SAFE_CLOCK
  59 #endif
  60
  61 #ifdef LTTNG_USE_NMI_SAFE_CLOCK
  62
  63 DECLARE_PER_CPU(u64, lttng_last_tsc);
  64
  65 /*
  66  * Sometimes called with preemption enabled. Can be interrupted.
  67  */
  68 static inline u64 trace_clock_monotonic_wrapper(void)
  69 {
  70         u64 now, last, result;
  71         u64 *last_tsc_ptr;
  72
  73         /* Use fast nmi-safe monotonic clock provided by the Linux kernel. */
  74         preempt_disable();
  75         last_tsc_ptr = this_cpu_ptr(&lttng_last_tsc);
  76         last = *last_tsc_ptr;
  77         /*
  78          * Read "last" before "now". It is not strictly required, but it ensures
  79          * that an interrupt coming in won't artificially trigger a case where
  80          * "now" < "last". This kind of situation should only happen if the
  81          * mono_fast time source goes slightly backwards.
  82          */
  83         barrier();
  84         now = ktime_get_mono_fast_ns();
  85         if (U64_MAX / 2 < now - last)
  86                 now = last;
  87         result = cmpxchg64_local(last_tsc_ptr, last, now);
  88         preempt_enable();
  89         if (result == last) {
  90                 /* Update done. */
  91                 return now;
  92         } else {
  93                 /*
  94                  * Update not done, due to concurrent update. We can use
  95                  * "result", since it has been sampled concurrently with our
  96                  * time read, so it should not be far from "now".
  97                  */
  98                 return result;
  99         }
 100 }
 101
 102 #else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 103 static inline u64 trace_clock_monotonic_wrapper(void)
 104 {
 105         ktime_t ktime;
 106
 107         /*
 108          * Refuse to trace from NMIs with this wrapper, because an NMI could
 109          * nest over the xtime write seqlock and deadlock.
 110          */
 111         if (in_nmi())
 112                 return (u64) -EIO;
 113
 114         ktime = ktime_get();
 115         return ktime_to_ns(ktime);
 116 }
 117 #endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 118
 119 static inline u64 trace_clock_read64_monotonic(void)
 120 {
 121         return (u64) trace_clock_monotonic_wrapper();
 122 }
 123
 124 static inline u64 trace_clock_freq_monotonic(void)
 125 {
 126         return (u64) NSEC_PER_SEC;
 127 }
 128
 129 static inline int trace_clock_uuid_monotonic(char *uuid)
 130 {
 131         return wrapper_get_bootid(uuid);
 132 }
 133
 134 static inline const char *trace_clock_name_monotonic(void)
 135 {
 136         return "monotonic";
 137 }
 138
 139 static inline const char *trace_clock_description_monotonic(void)
 140 {
 141         return "Monotonic Clock";
 142 }
 143
 144 #ifdef LTTNG_USE_NMI_SAFE_CLOCK
 145 static inline int get_trace_clock(void)
 146 {
 147         printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic fast clock, which is NMI-safe.\n");
 148         return 0;
 149 }
 150 #else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 151 static inline int get_trace_clock(void)
 152 {
 153         printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic clock. NMIs will not be traced.\n");
 154         return 0;
 155 }
 156 #endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 157
 158 static inline void put_trace_clock(void)
 159 {
 160 }
 161
 162 static inline u64 trace_clock_read64(void)
 163 {
 164         struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock);
 165
 166         if (likely(!ltc)) {
 167                 return trace_clock_read64_monotonic();
 168         } else {
 169                 return ltc->read64();
 170         }
 171 }
 172
 173 static inline u64 trace_clock_freq(void)
 174 {
 175         struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock);
 176
 177         if (!ltc) {
 178                 return trace_clock_freq_monotonic();
 179         } else {
 180                 return ltc->freq();
 181         }
 182 }
 183
 184 static inline int trace_clock_uuid(char *uuid)
 185 {
 186         struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock);
 187
 188         /* Use default UUID cb when NULL */
 189         if (!ltc || !ltc->uuid) {
 190                 return trace_clock_uuid_monotonic(uuid);
 191         } else {
 192                 return ltc->uuid(uuid);
 193         }
 194 }
 195
 196 static inline const char *trace_clock_name(void)
 197 {
 198         struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock);
 199
 200         if (!ltc) {
 201                 return trace_clock_name_monotonic();
 202         } else {
 203                 return ltc->name();
 204         }
 205 }
 206
 207 static inline const char *trace_clock_description(void)
 208 {
 209         struct lttng_trace_clock *ltc = LTTNG_READ_ONCE(lttng_trace_clock);
 210
 211         if (!ltc) {
 212                 return trace_clock_description_monotonic();
 213         } else {
 214                 return ltc->description();
 215         }
 216 }
 217
 218 #endif /* CONFIG_HAVE_TRACE_CLOCK */
 219
 220 #endif /* _LTTNG_TRACE_CLOCK_H */