include/urcu/static/urcu-qsbr.h

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 #ifndef _URCU_QSBR_STATIC_H
   7 #define _URCU_QSBR_STATIC_H
   8
   9 /*
  10  * Userspace RCU QSBR header.
  11  *
  12  * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
  13  * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
  14  *
  15  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  16  */
  17
  18 #include <stdlib.h>
  19 #include <pthread.h>
  20 #include <limits.h>
  21 #include <unistd.h>
  22 #include <stdint.h>
  23
  24 #include <urcu/debug.h>
  25 #include <urcu/compiler.h>
  26 #include <urcu/arch.h>
  27 #include <urcu/system.h>
  28 #include <urcu/uatomic.h>
  29 #include <urcu/list.h>
  30 #include <urcu/futex.h>
  31 #include <urcu/tls-compat.h>
  32 #include <urcu/static/urcu-common.h>
  33
  34 #ifdef __cplusplus
  35 extern "C" {
  36 #endif
  37
  38 /*
  39  * This code section can only be included in LGPL 2.1 compatible source code.
  40  * See below for the function call wrappers which can be used in code meant to
  41  * be only linked with the Userspace RCU library. This comes with a small
  42  * performance degradation on the read-side due to the added function calls.
  43  * This is required to permit relinking with newer versions of the library.
  44  */
  45
  46 #define URCU_QSBR_GP_ONLINE             (1UL << 0)
  47 #define URCU_QSBR_GP_CTR                (1UL << 1)
  48
  49 extern struct urcu_gp urcu_qsbr_gp;
  50
  51 struct urcu_qsbr_reader {
  52         /* Data used by both reader and synchronize_rcu() */
  53         unsigned long ctr;
  54         /* Data used for registry */
  55         struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
  56         int waiting;
  57         pthread_t tid;
  58         /* Reader registered flag, for internal checks. */
  59         unsigned int registered:1;
  60 };
  61
  62 extern DECLARE_URCU_TLS(struct urcu_qsbr_reader, urcu_qsbr_reader);
  63
  64 /*
  65  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  66  */
  67 static inline void urcu_qsbr_wake_up_gp(void)
  68 {
  69         if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(urcu_qsbr_reader).waiting))) {
  70                 _CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader).waiting, 0);
  71                 cmm_smp_mb();
  72                 if (uatomic_read(&urcu_qsbr_gp.futex) != -1)
  73                         return;
  74                 uatomic_set(&urcu_qsbr_gp.futex, 0);
  75                 /*
  76                  * Ignoring return value until we can make this function
  77                  * return something (because urcu_die() is not publicly
  78                  * exposed).
  79                  */
  80                 (void) futex_noasync(&urcu_qsbr_gp.futex, FUTEX_WAKE, 1,
  81                                 NULL, NULL, 0);
  82         }
  83 }
  84
  85 static inline enum urcu_state urcu_qsbr_reader_state(unsigned long *ctr)
  86 {
  87         unsigned long v;
  88
  89         v = CMM_LOAD_SHARED(*ctr);
  90         if (!v)
  91                 return URCU_READER_INACTIVE;
  92         if (v == urcu_qsbr_gp.ctr)
  93                 return URCU_READER_ACTIVE_CURRENT;
  94         return URCU_READER_ACTIVE_OLD;
  95 }
  96
  97 /*
  98  * Enter an RCU read-side critical section.
  99  *
 100  * This function is less than 10 lines long.  The intent is that this
 101  * function meets the 10-line criterion for LGPL, allowing this function
 102  * to be invoked directly from non-LGPL code.
 103  */
 104 static inline void _urcu_qsbr_read_lock(void)
 105 {
 106         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).ctr);
 107 }
 108
 109 /*
 110  * Exit an RCU read-side critical section.
 111  *
 112  * This function is less than 10 lines long.  The intent is that this
 113  * function meets the 10-line criterion for LGPL, allowing this function
 114  * to be invoked directly from non-LGPL code.
 115  */
 116 static inline void _urcu_qsbr_read_unlock(void)
 117 {
 118         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).ctr);
 119 }
 120
 121 /*
 122  * Returns whether within a RCU read-side critical section.
 123  *
 124  * This function is less than 10 lines long.  The intent is that this
 125  * function meets the 10-line criterion for LGPL, allowing this function
 126  * to be invoked directly from non-LGPL code.
 127  */
 128 static inline int _urcu_qsbr_read_ongoing(void)
 129 {
 130         return URCU_TLS(urcu_qsbr_reader).ctr;
 131 }
 132
 133 /*
 134  * This is a helper function for _rcu_quiescent_state().
 135  * The first cmm_smp_mb() ensures memory accesses in the prior read-side
 136  * critical sections are not reordered with store to
 137  * URCU_TLS(urcu_qsbr_reader).ctr, and ensures that mutexes held within an
 138  * offline section that would happen to end with this
 139  * urcu_qsbr_quiescent_state() call are not reordered with
 140  * store to URCU_TLS(urcu_qsbr_reader).ctr.
 141  */
 142 static inline void _urcu_qsbr_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
 143 {
 144         cmm_smp_mb();
 145         _CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader).ctr, gp_ctr);
 146         cmm_smp_mb();   /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
 147         urcu_qsbr_wake_up_gp();
 148         cmm_smp_mb();
 149 }
 150
 151 /*
 152  * Inform RCU of a quiescent state.
 153  *
 154  * This function is less than 10 lines long.  The intent is that this
 155  * function meets the 10-line criterion for LGPL, allowing this function
 156  * to be invoked directly from non-LGPL code.
 157  *
 158  * We skip the memory barriers and gp store if our local ctr already
 159  * matches the global urcu_qsbr_gp.ctr value: this is OK because a prior
 160  * _rcu_quiescent_state() or _rcu_thread_online() already updated it
 161  * within our thread, so we have no quiescent state to report.
 162  */
 163 static inline void _urcu_qsbr_quiescent_state(void)
 164 {
 165         unsigned long gp_ctr;
 166
 167         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
 168         if ((gp_ctr = CMM_LOAD_SHARED(urcu_qsbr_gp.ctr)) == URCU_TLS(urcu_qsbr_reader).ctr)
 169                 return;
 170         _urcu_qsbr_quiescent_state_update_and_wakeup(gp_ctr);
 171 }
 172
 173 /*
 174  * Take a thread offline, prohibiting it from entering further RCU
 175  * read-side critical sections.
 176  *
 177  * This function is less than 10 lines long.  The intent is that this
 178  * function meets the 10-line criterion for LGPL, allowing this function
 179  * to be invoked directly from non-LGPL code.
 180  */
 181 static inline void _urcu_qsbr_thread_offline(void)
 182 {
 183         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
 184         cmm_smp_mb();
 185         CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader).ctr, 0);
 186         cmm_smp_mb();   /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
 187         urcu_qsbr_wake_up_gp();
 188         cmm_barrier();  /* Ensure the compiler does not reorder us with mutex */
 189 }
 190
 191 /*
 192  * Bring a thread online, allowing it to once again enter RCU
 193  * read-side critical sections.
 194  *
 195  * This function is less than 10 lines long.  The intent is that this
 196  * function meets the 10-line criterion for LGPL, allowing this function
 197  * to be invoked directly from non-LGPL code.
 198  */
 199 static inline void _urcu_qsbr_thread_online(void)
 200 {
 201         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
 202         cmm_barrier();  /* Ensure the compiler does not reorder us with mutex */
 203         _CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader).ctr, CMM_LOAD_SHARED(urcu_qsbr_gp.ctr));
 204         cmm_smp_mb();
 205 }
 206
 207 #ifdef __cplusplus
 208 }
 209 #endif
 210
 211 #endif /* _URCU_QSBR_STATIC_H */