22e15f623bc91db05455bf004ea36345cd1b8237
[urcu.git] / include / urcu / static / urcu-memb.h
1 // SPDX-FileCopyrightText: 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
2 // SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation.
3 //
4 // SPDX-License-Identifier: LGPL-2.1-or-later
5
6 #ifndef _URCU_MEMB_STATIC_H
7 #define _URCU_MEMB_STATIC_H
8
9 /*
10 * Userspace RCU header.
11 *
12 * TO BE INCLUDED ONLY IN CODE THAT IS TO BE RECOMPILED ON EACH LIBURCU
13 * RELEASE. See urcu.h for linking dynamically with the userspace rcu library.
14 *
15 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
16 */
17
18 #include <stdlib.h>
19 #include <pthread.h>
20 #include <unistd.h>
21 #include <stdint.h>
22
23 #include <urcu/debug.h>
24 #include <urcu/config.h>
25 #include <urcu/compiler.h>
26 #include <urcu/arch.h>
27 #include <urcu/system.h>
28 #include <urcu/uatomic.h>
29 #include <urcu/list.h>
30 #include <urcu/futex.h>
31 #include <urcu/tls-compat.h>
32 #include <urcu/static/urcu-common.h>
33
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37
38 /*
39 * This code section can only be included in LGPL 2.1 compatible source code.
40 * See below for the function call wrappers which can be used in code meant to
41 * be only linked with the Userspace RCU library. This comes with a small
42 * performance degradation on the read-side due to the added function calls.
43 * This is required to permit relinking with newer versions of the library.
44 */
45
46 /*
47 * Slave barriers are only guaranteed to be ordered wrt master barriers.
48 *
49 * The pair ordering is detailed as (O: ordered, X: not ordered) :
50 * slave master
51 * slave X O
52 * master O O
53 */
54
55 #ifdef CONFIG_RCU_FORCE_SYS_MEMBARRIER
56 #define urcu_memb_has_sys_membarrier 1
57 #else
58 extern int urcu_memb_has_sys_membarrier;
59 #endif
60
61 static inline void urcu_memb_smp_mb_slave(void)
62 {
63 if (caa_likely(urcu_memb_has_sys_membarrier))
64 cmm_barrier();
65 else
66 cmm_smp_mb();
67 }
68
69 extern struct urcu_gp urcu_memb_gp;
70
71 extern DECLARE_URCU_TLS(struct urcu_reader, urcu_memb_reader);
72
73 /*
74 * Helper for _rcu_read_lock(). The format of urcu_memb_gp.ctr (as well as
75 * the per-thread rcu_reader.ctr) has the lower-order bits containing a count of
76 * _rcu_read_lock() nesting, and a single high-order URCU_BP_GP_CTR_PHASE bit
77 * that contains either zero or one. The smp_mb_slave() ensures that the accesses in
78 * _rcu_read_lock() happen before the subsequent read-side critical section.
79 */
80 static inline void _urcu_memb_read_lock_update(unsigned long tmp)
81 {
82 if (caa_likely(!(tmp & URCU_GP_CTR_NEST_MASK))) {
83 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, _CMM_LOAD_SHARED(urcu_memb_gp.ctr));
84 urcu_memb_smp_mb_slave();
85 } else
86 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, tmp + URCU_GP_COUNT);
87 }
88
89 /*
90 * Enter an RCU read-side critical section.
91 *
92 * The first cmm_barrier() call ensures that the compiler does not reorder
93 * the body of _rcu_read_lock() with a mutex.
94 *
95 * This function and its helper are both less than 10 lines long. The
96 * intent is that this function meets the 10-line criterion in LGPL,
97 * allowing this function to be invoked directly from non-LGPL code.
98 */
99 static inline void _urcu_memb_read_lock(void)
100 {
101 unsigned long tmp;
102
103 urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
104 cmm_barrier();
105 tmp = URCU_TLS(urcu_memb_reader).ctr;
106 urcu_assert_debug((tmp & URCU_GP_CTR_NEST_MASK) != URCU_GP_CTR_NEST_MASK);
107 _urcu_memb_read_lock_update(tmp);
108 }
109
110 /*
111 * This is a helper function for _rcu_read_unlock().
112 *
113 * The first smp_mb_slave() call ensures that the critical section is
114 * seen to precede the store to rcu_reader.ctr.
115 * The second smp_mb_slave() call ensures that we write to rcu_reader.ctr
116 * before reading the update-side futex.
117 */
118 static inline void _urcu_memb_read_unlock_update_and_wakeup(unsigned long tmp)
119 {
120 if (caa_likely((tmp & URCU_GP_CTR_NEST_MASK) == URCU_GP_COUNT)) {
121 urcu_memb_smp_mb_slave();
122 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, tmp - URCU_GP_COUNT);
123 urcu_memb_smp_mb_slave();
124 urcu_common_wake_up_gp(&urcu_memb_gp);
125 } else
126 _CMM_STORE_SHARED(URCU_TLS(urcu_memb_reader).ctr, tmp - URCU_GP_COUNT);
127 }
128
129 /*
130 * Exit an RCU read-side critical section. Both this function and its
131 * helper are smaller than 10 lines of code, and are intended to be
132 * usable by non-LGPL code, as called out in LGPL.
133 */
134 static inline void _urcu_memb_read_unlock(void)
135 {
136 unsigned long tmp;
137
138 urcu_assert_debug(URCU_TLS(urcu_memb_reader).registered);
139 tmp = URCU_TLS(urcu_memb_reader).ctr;
140 urcu_assert_debug(tmp & URCU_GP_CTR_NEST_MASK);
141 _urcu_memb_read_unlock_update_and_wakeup(tmp);
142 cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */
143 }
144
145 /*
146 * Returns whether within a RCU read-side critical section.
147 *
148 * This function is less than 10 lines long. The intent is that this
149 * function meets the 10-line criterion for LGPL, allowing this function
150 * to be invoked directly from non-LGPL code.
151 */
152 static inline int _urcu_memb_read_ongoing(void)
153 {
154 return URCU_TLS(urcu_memb_reader).ctr & URCU_GP_CTR_NEST_MASK;
155 }
156
157 #ifdef __cplusplus
158 }
159 #endif
160
161 #endif /* _URCU_MEMB_STATIC_H */
This page took 0.032535 seconds and 4 git commands to generate.