wrapper/trace-clock.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * wrapper/trace-clock.h
   4  *
   5  * Contains LTTng trace clock mapping to LTTng trace clock or mainline monotonic
   6  * clock. This wrapper depends on CONFIG_HIGH_RES_TIMERS=y.
   7  *
   8  * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   9  */
  10
  11 #ifndef _LTTNG_TRACE_CLOCK_H
  12 #define _LTTNG_TRACE_CLOCK_H
  13
  14 #ifdef CONFIG_HAVE_TRACE_CLOCK
  15 #include <linux/trace-clock.h>
  16 #else /* CONFIG_HAVE_TRACE_CLOCK */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/ktime.h>
  20 #include <linux/time.h>
  21 #include <linux/hrtimer.h>
  22 #include <linux/percpu.h>
  23 #include <linux/version.h>
  24 #include <linux/percpu-defs.h>
  25 #include <asm/local.h>
  26 #include <lttng-kernel-version.h>
  27 #include <lttng-clock.h>
  28 #include <wrapper/compiler.h>
  29 #include <wrapper/random.h>
  30 #include <blacklist/timekeeping.h>
  31
  32 extern struct lttng_trace_clock *lttng_trace_clock;
  33
  34 /*
  35  * We need clock values to be monotonically increasing per-cpu, which is
  36  * not strictly guaranteed by ktime_get_mono_fast_ns(). It is
  37  * straightforward to do on architectures with a 64-bit cmpxchg(), but
  38  * not so on architectures without 64-bit cmpxchg. For now, only enable
  39  * this feature on 64-bit architectures.
  40  */
  41
  42 #if BITS_PER_LONG == 64
  43 #define LTTNG_USE_NMI_SAFE_CLOCK
  44 #endif
  45
  46 #ifdef LTTNG_USE_NMI_SAFE_CLOCK
  47
  48 DECLARE_PER_CPU(u64, lttng_last_tsc);
  49
  50 /*
  51  * Sometimes called with preemption enabled. Can be interrupted.
  52  */
  53 static inline u64 trace_clock_monotonic_wrapper(void)
  54 {
  55         u64 now, last, result;
  56         u64 *last_tsc_ptr;
  57
  58         /* Use fast nmi-safe monotonic clock provided by the Linux kernel. */
  59         preempt_disable();
  60         last_tsc_ptr = this_cpu_ptr(&lttng_last_tsc);
  61         last = *last_tsc_ptr;
  62         /*
  63          * Read "last" before "now". It is not strictly required, but it ensures
  64          * that an interrupt coming in won't artificially trigger a case where
  65          * "now" < "last". This kind of situation should only happen if the
  66          * mono_fast time source goes slightly backwards.
  67          */
  68         barrier();
  69         now = ktime_get_mono_fast_ns();
  70         if (U64_MAX / 2 < now - last)
  71                 now = last;
  72         result = cmpxchg64_local(last_tsc_ptr, last, now);
  73         preempt_enable();
  74         if (result == last) {
  75                 /* Update done. */
  76                 return now;
  77         } else {
  78                 /*
  79                  * Update not done, due to concurrent update. We can use
  80                  * "result", since it has been sampled concurrently with our
  81                  * time read, so it should not be far from "now".
  82                  */
  83                 return result;
  84         }
  85 }
  86
  87 #else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
  88 static inline u64 trace_clock_monotonic_wrapper(void)
  89 {
  90         ktime_t ktime;
  91
  92         /*
  93          * Refuse to trace from NMIs with this wrapper, because an NMI could
  94          * nest over the xtime write seqlock and deadlock.
  95          */
  96         if (in_nmi())
  97                 return (u64) -EIO;
  98
  99         ktime = ktime_get();
 100         return ktime_to_ns(ktime);
 101 }
 102 #endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 103
 104 static inline u64 trace_clock_read64_monotonic(void)
 105 {
 106         return (u64) trace_clock_monotonic_wrapper();
 107 }
 108
 109 static inline u64 trace_clock_freq_monotonic(void)
 110 {
 111         return (u64) NSEC_PER_SEC;
 112 }
 113
 114 static inline int trace_clock_uuid_monotonic(char *uuid)
 115 {
 116         return wrapper_get_bootid(uuid);
 117 }
 118
 119 static inline const char *trace_clock_name_monotonic(void)
 120 {
 121         return "monotonic";
 122 }
 123
 124 static inline const char *trace_clock_description_monotonic(void)
 125 {
 126         return "Monotonic Clock";
 127 }
 128
 129 #ifdef LTTNG_USE_NMI_SAFE_CLOCK
 130 static inline int get_trace_clock(void)
 131 {
 132         printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic fast clock, which is NMI-safe.\n");
 133         return 0;
 134 }
 135 #else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 136 static inline int get_trace_clock(void)
 137 {
 138         printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic clock. NMIs will not be traced.\n");
 139         return 0;
 140 }
 141 #endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 142
 143 static inline void put_trace_clock(void)
 144 {
 145 }
 146
 147 static inline u64 trace_clock_read64(void)
 148 {
 149         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 150
 151         if (likely(!ltc)) {
 152                 return trace_clock_read64_monotonic();
 153         } else {
 154                 read_barrier_depends(); /* load ltc before content */
 155                 return ltc->read64();
 156         }
 157 }
 158
 159 static inline u64 trace_clock_freq(void)
 160 {
 161         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 162
 163         if (!ltc) {
 164                 return trace_clock_freq_monotonic();
 165         } else {
 166                 read_barrier_depends(); /* load ltc before content */
 167                 return ltc->freq();
 168         }
 169 }
 170
 171 static inline int trace_clock_uuid(char *uuid)
 172 {
 173         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 174
 175         read_barrier_depends(); /* load ltc before content */
 176         /* Use default UUID cb when NULL */
 177         if (!ltc || !ltc->uuid) {
 178                 return trace_clock_uuid_monotonic(uuid);
 179         } else {
 180                 return ltc->uuid(uuid);
 181         }
 182 }
 183
 184 static inline const char *trace_clock_name(void)
 185 {
 186         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 187
 188         if (!ltc) {
 189                 return trace_clock_name_monotonic();
 190         } else {
 191                 read_barrier_depends(); /* load ltc before content */
 192                 return ltc->name();
 193         }
 194 }
 195
 196 static inline const char *trace_clock_description(void)
 197 {
 198         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 199
 200         if (!ltc) {
 201                 return trace_clock_description_monotonic();
 202         } else {
 203                 read_barrier_depends(); /* load ltc before content */
 204                 return ltc->description();
 205         }
 206 }
 207
 208 #endif /* CONFIG_HAVE_TRACE_CLOCK */
 209
 210 #endif /* _LTTNG_TRACE_CLOCK_H */