include/urcu/static/urcu-memb.h

   1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
   3 //
   4 // SPDX-License-Identifier: LGPL-2.1-or-later
   5
   6 #ifndef _URCU_MEMB_STATIC_H
   7 #define _URCU_MEMB_STATIC_H
   8
   9 /*
  10  * Userspace RCU header.
  11  *
  12  * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
  13  * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
  14  *
  15  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  16  */
  17
  18 #include <stdlib.h>
  19 #include <pthread.h>
  20 #include <unistd.h>
  21 #include <stdint.h>
  22
  23 #include <urcu/debug.h>
  24 #include <urcu/config.h>
  25 #include <urcu/compiler.h>
  26 #include <urcu/arch.h>
  27 #include <urcu/system.h>
  28 #include <urcu/uatomic.h>
  29 #include <urcu/list.h>
  30 #include <urcu/futex.h>
  31 #include <urcu/tls-compat.h>
  32 #include <urcu/static/urcu-common.h>
  33
  34 #ifdef __cplusplus
  35 extern "C" {
  36 #endif
  37
  38 /*
  39  * This code section can only be included in LGPL 2.1 compatible source code.
  40  * See below for the function call wrappers which can be used in code meant to
  41  * be only linked with the Userspace RCU library. This comes with a small
  42  * performance degradation on the read-side due to the added function calls.
  43  * This is required to permit relinking with newer versions of the library.
  44  */
  45
  46 /*
  47  * Slave barriers are only guaranteed to be ordered wrt master barriers.
  48  *
  49  * The pair ordering is detailed as (O: ordered, X: not ordered) :
  50  *               slave  master
  51  *        slave    X      O
  52  *        master   O      O
  53  */
  54
  55 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
  56 #define urcu_memb_has_sys_membarrier            1
  57 #else
  58 extern int urcu_memb_has_sys_membarrier;
  59 #endif
  60
  61 static inline void urcu_memb_smp_mb_slave(void)
  62 {
  63         if (caa_likely(urcu_memb_has_sys_membarrier))
  64                 cmm_barrier();
  65         else
  66                 cmm_smp_mb();
  67 }
  68
  69 extern struct urcu_gp urcu_memb_gp;
  70
  71 extern DECLARE_URCU_TLS(struct urcu_reader, urcu_memb_reader);
  72
  73 /*
  74  * Helper for _rcu_read_lock().  The format of urcu_memb_gp.ctr (as well as
  75  * the per-thread rcu_reader.ctr) has the lower-order bits containing a count of
  76  * _rcu_read_lock() nesting, and a single high-order URCU_BP_GP_CTR_PHASE bit
  77  * that contains either zero or one.  The smp_mb_slave() ensures that the accesses in
  78  * _rcu_read_lock() happen before the subsequent read-side critical section.
  79  */
  80 static inline void _urcu_memb_read_lock_update(unsigned long tmp)
  81 {
  82         if (caa_likely(!(tmp & URCU_GP_CTR_NEST_MASK))) {
  83                 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, _CMM_LOAD_SHARED(urcu_memb_gp.ctr));
  84                 urcu_memb_smp_mb_slave();
  85         } else
  86                 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, tmp + URCU_GP_COUNT);
  87 }
  88
  89 /*
  90  * Enter an RCU read-side critical section.
  91  *
  92  * The first cmm_barrier() call ensures that the compiler does not reorder
  93  * the body of _rcu_read_lock() with a mutex.
  94  *
  95  * This function and its helper are both less than 10 lines long.  The
  96  * intent is that this function meets the 10-line criterion in LGPL,
  97  * allowing this function to be invoked directly from non-LGPL code.
  98  */
  99 static inline void _urcu_memb_read_lock(void)
 100 {
 101         unsigned long tmp;
 102
 103         urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
 104         cmm_barrier();
 105         tmp = URCU_TLS(urcu_memb_reader).ctr;
 106         urcu_assert_debug((tmp & URCU_GP_CTR_NEST_MASK) != URCU_GP_CTR_NEST_MASK);
 107         _urcu_memb_read_lock_update(tmp);
 108 }
 109
 110 /*
 111  * This is a helper function for _rcu_read_unlock().
 112  *
 113  * The first smp_mb_slave() call ensures that the critical section is
 114  * seen to precede the store to rcu_reader.ctr.
 115  * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr
 116  * before reading the update-side futex.
 117  */
 118 static inline void _urcu_memb_read_unlock_update_and_wakeup(unsigned long tmp)
 119 {
 120         if (caa_likely((tmp & URCU_GP_CTR_NEST_MASK) == URCU_GP_COUNT)) {
 121                 urcu_memb_smp_mb_slave();
 122                 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, tmp - URCU_GP_COUNT);
 123                 urcu_memb_smp_mb_slave();
 124                 urcu_common_wake_up_gp(&urcu_memb_gp);
 125         } else
 126                 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, tmp - URCU_GP_COUNT);
 127 }
 128
 129 /*
 130  * Exit an RCU read-side critical section.  Both this function and its
 131  * helper are smaller than 10 lines of code, and are intended to be
 132  * usable by non-LGPL code, as called out in LGPL.
 133  */
 134 static inline void _urcu_memb_read_unlock(void)
 135 {
 136         unsigned long tmp;
 137
 138         urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
 139         tmp = URCU_TLS(urcu_memb_reader).ctr;
 140         urcu_assert_debug(tmp & URCU_GP_CTR_NEST_MASK);
 141         _urcu_memb_read_unlock_update_and_wakeup(tmp);
 142         cmm_barrier();  /* Ensure the compiler does not reorder us with mutex */
 143 }
 144
 145 /*
 146  * Returns whether within a RCU read-side critical section.
 147  *
 148  * This function is less than 10 lines long.  The intent is that this
 149  * function meets the 10-line criterion for LGPL, allowing this function
 150  * to be invoked directly from non-LGPL code.
 151  */
 152 static inline int _urcu_memb_read_ongoing(void)
 153 {
 154         return URCU_TLS(urcu_memb_reader).ctr & URCU_GP_CTR_NEST_MASK;
 155 }
 156
 157 #ifdef __cplusplus
 158 }
 159 #endif
 160
 161 #endif /* _URCU_MEMB_STATIC_H */