wrapper/trace-clock.h

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * wrapper/trace-clock.h
   4  *
   5  * Contains LTTng trace clock mapping to LTTng trace clock or mainline monotonic
   6  * clock. This wrapper depends on CONFIG_HIGH_RES_TIMERS=y.
   7  *
   8  * Copyright (C) 2011-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   9  */
  10
  11 #ifndef _LTTNG_TRACE_CLOCK_H
  12 #define _LTTNG_TRACE_CLOCK_H
  13
  14 #ifdef CONFIG_HAVE_TRACE_CLOCK
  15 #include <linux/trace-clock.h>
  16 #else /* CONFIG_HAVE_TRACE_CLOCK */
  17
  18 #include <linux/hardirq.h>
  19 #include <linux/ktime.h>
  20 #include <linux/time.h>
  21 #include <linux/hrtimer.h>
  22 #include <linux/percpu.h>
  23 #include <linux/version.h>
  24 #include <linux/percpu-defs.h>
  25 #include <asm/local.h>
  26 #include <lttng-kernel-version.h>
  27 #include <lttng-clock.h>
  28 #include <wrapper/compiler.h>
  29 #include <wrapper/random.h>
  30
  31 extern struct lttng_trace_clock *lttng_trace_clock;
  32
  33 /*
  34  * We need clock values to be monotonically increasing per-cpu, which is
  35  * not strictly guaranteed by ktime_get_mono_fast_ns(). It is
  36  * straightforward to do on architectures with a 64-bit cmpxchg(), but
  37  * not so on architectures without 64-bit cmpxchg. For now, only enable
  38  * this feature on 64-bit architectures.
  39  */
  40
  41 #if BITS_PER_LONG == 64
  42 #define LTTNG_USE_NMI_SAFE_CLOCK
  43 #endif
  44
  45 #ifdef LTTNG_USE_NMI_SAFE_CLOCK
  46
  47 DECLARE_PER_CPU(u64, lttng_last_tsc);
  48
  49 /*
  50  * Sometimes called with preemption enabled. Can be interrupted.
  51  */
  52 static inline u64 trace_clock_monotonic_wrapper(void)
  53 {
  54         u64 now, last, result;
  55         u64 *last_tsc_ptr;
  56
  57         /* Use fast nmi-safe monotonic clock provided by the Linux kernel. */
  58         preempt_disable();
  59         last_tsc_ptr = this_cpu_ptr(&lttng_last_tsc);
  60         last = *last_tsc_ptr;
  61         /*
  62          * Read "last" before "now". It is not strictly required, but it ensures
  63          * that an interrupt coming in won't artificially trigger a case where
  64          * "now" < "last". This kind of situation should only happen if the
  65          * mono_fast time source goes slightly backwards.
  66          */
  67         barrier();
  68         now = ktime_get_mono_fast_ns();
  69         if (U64_MAX / 2 < now - last)
  70                 now = last;
  71         result = cmpxchg64_local(last_tsc_ptr, last, now);
  72         preempt_enable();
  73         if (result == last) {
  74                 /* Update done. */
  75                 return now;
  76         } else {
  77                 /*
  78                  * Update not done, due to concurrent update. We can use
  79                  * "result", since it has been sampled concurrently with our
  80                  * time read, so it should not be far from "now".
  81                  */
  82                 return result;
  83         }
  84 }
  85
  86 #else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
  87 static inline u64 trace_clock_monotonic_wrapper(void)
  88 {
  89         ktime_t ktime;
  90
  91         /*
  92          * Refuse to trace from NMIs with this wrapper, because an NMI could
  93          * nest over the xtime write seqlock and deadlock.
  94          */
  95         if (in_nmi())
  96                 return (u64) -EIO;
  97
  98         ktime = ktime_get();
  99         return ktime_to_ns(ktime);
 100 }
 101 #endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 102
 103 static inline u64 trace_clock_read64_monotonic(void)
 104 {
 105         return (u64) trace_clock_monotonic_wrapper();
 106 }
 107
 108 static inline u64 trace_clock_freq_monotonic(void)
 109 {
 110         return (u64) NSEC_PER_SEC;
 111 }
 112
 113 static inline int trace_clock_uuid_monotonic(char *uuid)
 114 {
 115         return wrapper_get_bootid(uuid);
 116 }
 117
 118 static inline const char *trace_clock_name_monotonic(void)
 119 {
 120         return "monotonic";
 121 }
 122
 123 static inline const char *trace_clock_description_monotonic(void)
 124 {
 125         return "Monotonic Clock";
 126 }
 127
 128 #ifdef LTTNG_USE_NMI_SAFE_CLOCK
 129 static inline int get_trace_clock(void)
 130 {
 131         printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic fast clock, which is NMI-safe.\n");
 132         return 0;
 133 }
 134 #else /* #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 135 static inline int get_trace_clock(void)
 136 {
 137         printk_once(KERN_WARNING "LTTng: Using mainline kernel monotonic clock. NMIs will not be traced.\n");
 138         return 0;
 139 }
 140 #endif /* #else #ifdef LTTNG_USE_NMI_SAFE_CLOCK */
 141
 142 static inline void put_trace_clock(void)
 143 {
 144 }
 145
 146 static inline u64 trace_clock_read64(void)
 147 {
 148         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 149
 150         if (likely(!ltc)) {
 151                 return trace_clock_read64_monotonic();
 152         } else {
 153                 read_barrier_depends(); /* load ltc before content */
 154                 return ltc->read64();
 155         }
 156 }
 157
 158 static inline u64 trace_clock_freq(void)
 159 {
 160         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 161
 162         if (!ltc) {
 163                 return trace_clock_freq_monotonic();
 164         } else {
 165                 read_barrier_depends(); /* load ltc before content */
 166                 return ltc->freq();
 167         }
 168 }
 169
 170 static inline int trace_clock_uuid(char *uuid)
 171 {
 172         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 173
 174         read_barrier_depends(); /* load ltc before content */
 175         /* Use default UUID cb when NULL */
 176         if (!ltc || !ltc->uuid) {
 177                 return trace_clock_uuid_monotonic(uuid);
 178         } else {
 179                 return ltc->uuid(uuid);
 180         }
 181 }
 182
 183 static inline const char *trace_clock_name(void)
 184 {
 185         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 186
 187         if (!ltc) {
 188                 return trace_clock_name_monotonic();
 189         } else {
 190                 read_barrier_depends(); /* load ltc before content */
 191                 return ltc->name();
 192         }
 193 }
 194
 195 static inline const char *trace_clock_description(void)
 196 {
 197         struct lttng_trace_clock *ltc = READ_ONCE(lttng_trace_clock);
 198
 199         if (!ltc) {
 200                 return trace_clock_description_monotonic();
 201         } else {
 202                 read_barrier_depends(); /* load ltc before content */
 203                 return ltc->description();
 204         }
 205 }
 206
 207 #endif /* CONFIG_HAVE_TRACE_CLOCK */
 208
 209 #endif /* _LTTNG_TRACE_CLOCK_H */