include/urcu/static/urcu-qsbr.h

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 #ifndef _URCU_QSBR_STATIC_H
   7 #define _URCU_QSBR_STATIC_H
   8
   9 /*
  10  * Userspace RCU QSBR header.
  11  *
  12  * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
  13  * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
  14  *
  15  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  16  */
  17
  18 #include <stdlib.h>
  19 #include <pthread.h>
  20 #include <limits.h>
  21 #include <unistd.h>
  22 #include <stdint.h>
  23
  24 #include <urcu/annotate.h>
  25 #include <urcu/debug.h>
  26 #include <urcu/compiler.h>
  27 #include <urcu/arch.h>
  28 #include <urcu/system.h>
  29 #include <urcu/uatomic.h>
  30 #include <urcu/list.h>
  31 #include <urcu/futex.h>
  32 #include <urcu/tls-compat.h>
  33 #include <urcu/static/urcu-common.h>
  34
  35 #ifdef __cplusplus
  36 extern "C" {
  37 #endif
  38
  39 /*
  40  * This code section can only be included in LGPL 2.1 compatible source code.
  41  * See below for the function call wrappers which can be used in code meant to
  42  * be only linked with the Userspace RCU library. This comes with a small
  43  * performance degradation on the read-side due to the added function calls.
  44  * This is required to permit relinking with newer versions of the library.
  45  */
  46
  47 #define URCU_QSBR_GP_ONLINE             (1UL << 0)
  48 #define URCU_QSBR_GP_CTR                (1UL << 1)
  49
  50 extern struct urcu_gp urcu_qsbr_gp;
  51
  52 struct urcu_qsbr_reader {
  53         /* Data used by both reader and synchronize_rcu() */
  54         unsigned long ctr;
  55         /* Data used for registry */
  56         struct cds_list_head node __attribute__((aligned(CAA_CACHE_LINE_SIZE)));
  57         int waiting;
  58         pthread_t tid;
  59         /* Reader registered flag, for internal checks. */
  60         unsigned int registered:1;
  61 };
  62
  63 extern DECLARE_URCU_TLS(struct urcu_qsbr_reader, urcu_qsbr_reader);
  64
  65 /*
  66  * Wake-up waiting synchronize_rcu(). Called from many concurrent threads.
  67  */
  68 static inline void urcu_qsbr_wake_up_gp(void)
  69 {
  70         if (caa_unlikely(_CMM_LOAD_SHARED(URCU_TLS(urcu_qsbr_reader).waiting))) {
  71                 _CMM_STORE_SHARED(URCU_TLS(urcu_qsbr_reader).waiting, 0);
  72                 cmm_smp_mb();
  73                 if (uatomic_read(&urcu_qsbr_gp.futex) != -1)
  74                         return;
  75                 uatomic_set(&urcu_qsbr_gp.futex, 0);
  76                 /*
  77                  * Ignoring return value until we can make this function
  78                  * return something (because urcu_die() is not publicly
  79                  * exposed).
  80                  */
  81                 (void) futex_noasync(&urcu_qsbr_gp.futex, FUTEX_WAKE, 1,
  82                                 NULL, NULL, 0);
  83         }
  84 }
  85
  86 static inline enum urcu_state urcu_qsbr_reader_state(unsigned long *ctr,
  87                                                 cmm_annotate_t *group)
  88 {
  89         unsigned long v;
  90
  91         v = uatomic_load(ctr, CMM_RELAXED);
  92         cmm_annotate_group_mem_acquire(group, ctr);
  93
  94         if (!v)
  95                 return URCU_READER_INACTIVE;
  96         if (v == urcu_qsbr_gp.ctr)
  97                 return URCU_READER_ACTIVE_CURRENT;
  98         return URCU_READER_ACTIVE_OLD;
  99 }
 100
 101 /*
 102  * Enter an RCU read-side critical section.
 103  *
 104  * This function is less than 10 lines long.  The intent is that this
 105  * function meets the 10-line criterion for LGPL, allowing this function
 106  * to be invoked directly from non-LGPL code.
 107  */
 108 static inline void _urcu_qsbr_read_lock(void)
 109 {
 110         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).ctr);
 111 }
 112
 113 /*
 114  * Exit an RCU read-side critical section.
 115  *
 116  * This function is less than 10 lines long.  The intent is that this
 117  * function meets the 10-line criterion for LGPL, allowing this function
 118  * to be invoked directly from non-LGPL code.
 119  */
 120 static inline void _urcu_qsbr_read_unlock(void)
 121 {
 122         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).ctr);
 123 }
 124
 125 /*
 126  * Returns whether within a RCU read-side critical section.
 127  *
 128  * This function is less than 10 lines long.  The intent is that this
 129  * function meets the 10-line criterion for LGPL, allowing this function
 130  * to be invoked directly from non-LGPL code.
 131  */
 132 static inline int _urcu_qsbr_read_ongoing(void)
 133 {
 134         return URCU_TLS(urcu_qsbr_reader).ctr;
 135 }
 136
 137 /*
 138  * This is a helper function for _rcu_quiescent_state().
 139  * The first cmm_smp_mb() ensures memory accesses in the prior read-side
 140  * critical sections are not reordered with store to
 141  * URCU_TLS(urcu_qsbr_reader).ctr, and ensures that mutexes held within an
 142  * offline section that would happen to end with this
 143  * urcu_qsbr_quiescent_state() call are not reordered with
 144  * store to URCU_TLS(urcu_qsbr_reader).ctr.
 145  */
 146 static inline void _urcu_qsbr_quiescent_state_update_and_wakeup(unsigned long gp_ctr)
 147 {
 148         uatomic_store(&URCU_TLS(urcu_qsbr_reader).ctr, gp_ctr, CMM_SEQ_CST);
 149
 150         /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
 151         urcu_qsbr_wake_up_gp();
 152         cmm_smp_mb();
 153 }
 154
 155 /*
 156  * Inform RCU of a quiescent state.
 157  *
 158  * This function is less than 10 lines long.  The intent is that this
 159  * function meets the 10-line criterion for LGPL, allowing this function
 160  * to be invoked directly from non-LGPL code.
 161  *
 162  * We skip the memory barriers and gp store if our local ctr already
 163  * matches the global urcu_qsbr_gp.ctr value: this is OK because a prior
 164  * _rcu_quiescent_state() or _rcu_thread_online() already updated it
 165  * within our thread, so we have no quiescent state to report.
 166  */
 167 static inline void _urcu_qsbr_quiescent_state(void)
 168 {
 169         unsigned long gp_ctr;
 170
 171         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
 172         gp_ctr = uatomic_load(&urcu_qsbr_gp.ctr, CMM_RELAXED);
 173         if (gp_ctr == URCU_TLS(urcu_qsbr_reader).ctr)
 174                 return;
 175         _urcu_qsbr_quiescent_state_update_and_wakeup(gp_ctr);
 176 }
 177
 178 /*
 179  * Take a thread offline, prohibiting it from entering further RCU
 180  * read-side critical sections.
 181  *
 182  * This function is less than 10 lines long.  The intent is that this
 183  * function meets the 10-line criterion for LGPL, allowing this function
 184  * to be invoked directly from non-LGPL code.
 185  */
 186 static inline void _urcu_qsbr_thread_offline(void)
 187 {
 188         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
 189         uatomic_store(&URCU_TLS(urcu_qsbr_reader).ctr, 0, CMM_SEQ_CST);
 190         /* write URCU_TLS(urcu_qsbr_reader).ctr before read futex */
 191         urcu_qsbr_wake_up_gp();
 192         cmm_barrier();  /* Ensure the compiler does not reorder us with mutex */
 193 }
 194
 195 /*
 196  * Bring a thread online, allowing it to once again enter RCU
 197  * read-side critical sections.
 198  *
 199  * This function is less than 10 lines long.  The intent is that this
 200  * function meets the 10-line criterion for LGPL, allowing this function
 201  * to be invoked directly from non-LGPL code.
 202  */
 203 static inline void _urcu_qsbr_thread_online(void)
 204 {
 205         unsigned long *pctr = &URCU_TLS(urcu_qsbr_reader).ctr;
 206         unsigned long ctr;
 207
 208         urcu_assert_debug(URCU_TLS(urcu_qsbr_reader).registered);
 209         cmm_barrier();  /* Ensure the compiler does not reorder us with mutex */
 210         ctr = uatomic_load(&urcu_qsbr_gp.ctr, CMM_RELAXED);
 211         cmm_annotate_mem_acquire(&urcu_qsbr_gp.ctr);
 212         uatomic_store(pctr, ctr, CMM_RELAXED);
 213         cmm_smp_mb();
 214 }
 215
 216 #ifdef __cplusplus
 217 }
 218 #endif
 219
 220 #endif /* _URCU_QSBR_STATIC_H */