include/urcu/static/urcu-memb.h

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 #ifndef _URCU_MEMB_STATIC_H
   7 #define _URCU_MEMB_STATIC_H
   8
   9 /*
  10  * Userspace RCU header.
  11  *
  12  * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
  13  * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
  14  *
  15  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  16  */
  17
  18 #include <stdlib.h>
  19 #include <pthread.h>
  20 #include <unistd.h>
  21 #include <stdint.h>
  22
  23 #include <urcu/annotate.h>
  24 #include <urcu/debug.h>
  25 #include <urcu/config.h>
  26 #include <urcu/compiler.h>
  27 #include <urcu/arch.h>
  28 #include <urcu/system.h>
  29 #include <urcu/uatomic.h>
  30 #include <urcu/list.h>
  31 #include <urcu/futex.h>
  32 #include <urcu/tls-compat.h>
  33 #include <urcu/static/urcu-common.h>
  34
  35 #ifdef __cplusplus
  36 extern "C" {
  37 #endif
  38
  39 /*
  40  * This code section can only be included in LGPL 2.1 compatible source code.
  41  * See below for the function call wrappers which can be used in code meant to
  42  * be only linked with the Userspace RCU library. This comes with a small
  43  * performance degradation on the read-side due to the added function calls.
  44  * This is required to permit relinking with newer versions of the library.
  45  */
  46
  47 /*
  48  * Slave barriers are only guaranteed to be ordered wrt master barriers.
  49  *
  50  * The pair ordering is detailed as (O: ordered, X: not ordered) :
  51  *               slave  master
  52  *        slave    X      O
  53  *        master   O      O
  54  */
  55
  56 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
  57 #define urcu_memb_has_sys_membarrier            1
  58 #else
  59 extern int urcu_memb_has_sys_membarrier;
  60 #endif
  61
  62 static inline void urcu_memb_smp_mb_slave(void)
  63 {
  64         if (caa_likely(urcu_memb_has_sys_membarrier))
  65                 cmm_barrier();
  66         else
  67                 cmm_smp_mb();
  68 }
  69
  70 extern struct urcu_gp urcu_memb_gp;
  71
  72 extern DECLARE_URCU_TLS(struct urcu_reader, urcu_memb_reader);
  73
  74 /*
  75  * Helper for _rcu_read_lock().  The format of urcu_memb_gp.ctr (as well as
  76  * the per-thread rcu_reader.ctr) has the lower-order bits containing a count of
  77  * _rcu_read_lock() nesting, and a single high-order URCU_BP_GP_CTR_PHASE bit
  78  * that contains either zero or one.  The smp_mb_slave() ensures that the accesses in
  79  * _rcu_read_lock() happen before the subsequent read-side critical section.
  80  */
  81 static inline void _urcu_memb_read_lock_update(unsigned long tmp)
  82 {
  83         unsigned long *ctr = &URCU_TLS(urcu_memb_reader).ctr;
  84
  85         if (caa_likely(!(tmp & URCU_GP_CTR_NEST_MASK))) {
  86                 unsigned long *pgctr = &urcu_memb_gp.ctr;
  87                 unsigned long gctr = uatomic_load(pgctr, CMM_RELAXED);
  88
  89                 /* Paired with following mb slave. */
  90                 cmm_annotate_mem_acquire(pgctr);
  91                 uatomic_store(ctr, gctr, CMM_RELAXED);
  92
  93                 urcu_memb_smp_mb_slave();
  94         } else {
  95                 uatomic_store(ctr, tmp + URCU_GP_COUNT, CMM_RELAXED);
  96         }
  97 }
  98
  99 /*
 100  * Enter an RCU read-side critical section.
 101  *
 102  * The first cmm_barrier() call ensures that the compiler does not reorder
 103  * the body of _rcu_read_lock() with a mutex.
 104  *
 105  * This function and its helper are both less than 10 lines long.  The
 106  * intent is that this function meets the 10-line criterion in LGPL,
 107  * allowing this function to be invoked directly from non-LGPL code.
 108  */
 109 static inline void _urcu_memb_read_lock(void)
 110 {
 111         unsigned long tmp;
 112
 113         urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
 114         cmm_barrier();
 115         tmp = URCU_TLS(urcu_memb_reader).ctr;
 116         urcu_assert_debug((tmp & URCU_GP_CTR_NEST_MASK) != URCU_GP_CTR_NEST_MASK);
 117         _urcu_memb_read_lock_update(tmp);
 118 }
 119
 120 /*
 121  * This is a helper function for _rcu_read_unlock().
 122  *
 123  * The first smp_mb_slave() call ensures that the critical section is
 124  * seen to precede the store to rcu_reader.ctr.
 125  * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr
 126  * before reading the update-side futex.
 127  */
 128 static inline void _urcu_memb_read_unlock_update_and_wakeup(unsigned long tmp)
 129 {
 130         unsigned long *ctr = &URCU_TLS(urcu_memb_reader).ctr;
 131
 132         if (caa_likely((tmp & URCU_GP_CTR_NEST_MASK) == URCU_GP_COUNT)) {
 133                 urcu_memb_smp_mb_slave();
 134                 cmm_annotate_mem_release(ctr);
 135                 uatomic_store(ctr, tmp - URCU_GP_COUNT, CMM_RELAXED);
 136                 urcu_memb_smp_mb_slave();
 137                 urcu_common_wake_up_gp(&urcu_memb_gp);
 138         } else {
 139                 uatomic_store(ctr, tmp - URCU_GP_COUNT, CMM_RELAXED);
 140         }
 141 }
 142
 143 /*
 144  * Exit an RCU read-side critical section.  Both this function and its
 145  * helper are smaller than 10 lines of code, and are intended to be
 146  * usable by non-LGPL code, as called out in LGPL.
 147  */
 148 static inline void _urcu_memb_read_unlock(void)
 149 {
 150         unsigned long tmp;
 151
 152         urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
 153         tmp = URCU_TLS(urcu_memb_reader).ctr;
 154         urcu_assert_debug(tmp & URCU_GP_CTR_NEST_MASK);
 155         _urcu_memb_read_unlock_update_and_wakeup(tmp);
 156         cmm_barrier();  /* Ensure the compiler does not reorder us with mutex */
 157 }
 158
 159 /*
 160  * Returns whether within a RCU read-side critical section.
 161  *
 162  * This function is less than 10 lines long.  The intent is that this
 163  * function meets the 10-line criterion for LGPL, allowing this function
 164  * to be invoked directly from non-LGPL code.
 165  */
 166 static inline int _urcu_memb_read_ongoing(void)
 167 {
 168         return URCU_TLS(urcu_memb_reader).ctr & URCU_GP_CTR_NEST_MASK;
 169 }
 170
 171 #ifdef __cplusplus
 172 }
 173 #endif
 174
 175 #endif /* _URCU_MEMB_STATIC_H */