Commit | Line | Data |
---|---|---|
786ee85b MD |
1 | #ifndef _URCU_DEFER_STATIC_H |
2 | #define _URCU_DEFER_STATIC_H | |
90075a50 MD |
3 | |
4 | /* | |
786ee85b | 5 | * urcu-defer-static.h |
90075a50 | 6 | * |
658225a7 | 7 | * Userspace RCU header - memory reclamation. |
90075a50 | 8 | * |
786ee85b | 9 | * TO BE INCLUDED ONLY IN LGPL-COMPATIBLE CODE. See urcu-defer.h for linking |
658225a7 | 10 | * dynamically with the userspace rcu reclamation library. |
90075a50 MD |
11 | * |
12 | * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> | |
13 | * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. | |
14 | * | |
15 | * This library is free software; you can redistribute it and/or | |
16 | * modify it under the terms of the GNU Lesser General Public | |
17 | * License as published by the Free Software Foundation; either | |
18 | * version 2.1 of the License, or (at your option) any later version. | |
19 | * | |
20 | * This library is distributed in the hope that it will be useful, | |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
23 | * Lesser General Public License for more details. | |
24 | * | |
25 | * You should have received a copy of the GNU Lesser General Public | |
26 | * License along with this library; if not, write to the Free Software | |
27 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
28 | * | |
29 | * IBM's contributions to this file may be relicensed under LGPLv2 or later. | |
30 | */ | |
31 | ||
32 | #include <stdlib.h> | |
33 | #include <pthread.h> | |
34 | ||
35 | #include <compiler.h> | |
36 | #include <arch.h> | |
37 | ||
38 | ||
39 | /* | |
786ee85b | 40 | * Number of entries in the per-thread defer queue. Must be power of 2. |
90075a50 | 41 | */ |
786ee85b MD |
42 | #define DEFER_QUEUE_SIZE (1 << 12) |
43 | #define DEFER_QUEUE_MASK (DEFER_QUEUE_SIZE - 1) | |
90075a50 | 44 | |
804b4375 MD |
45 | /* |
46 | * Typically, data is aligned at least on the architecture size. | |
47 | * Use lowest bit to indicate that the current callback is changing. | |
48 | * Assumes that (void *)-2L is not used often. Used to encode non-aligned | |
49 | * functions and non-aligned data using extra space. | |
50 | * We encode the (void *)-2L fct as: -2L, fct, data. | |
51 | * We encode the (void *)-2L data as: -2L, fct, data. | |
52 | * Here, DQ_FCT_MARK == ~DQ_FCT_BIT. Required for the test order. | |
53 | */ | |
54 | #define DQ_FCT_BIT (1 << 0) | |
55 | #define DQ_IS_FCT_BIT(x) ((unsigned long)(x) & DQ_FCT_BIT) | |
56 | #define DQ_SET_FCT_BIT(x) \ | |
57 | (x = (void *)((unsigned long)(x) | DQ_FCT_BIT)) | |
58 | #define DQ_CLEAR_FCT_BIT(x) \ | |
59 | (x = (void *)((unsigned long)(x) & ~DQ_FCT_BIT)) | |
60 | #define DQ_FCT_MARK ((void *)(~DQ_FCT_BIT)) | |
61 | ||
90075a50 MD |
62 | /* |
63 | * Identify a shared load. A smp_rmc() or smp_mc() should come before the load. | |
64 | */ | |
65 | #define _LOAD_SHARED(p) ACCESS_ONCE(p) | |
66 | ||
67 | /* | |
68 | * Load a data from shared memory, doing a cache flush if required. | |
69 | */ | |
70 | #define LOAD_SHARED(p) \ | |
71 | ({ \ | |
72 | smp_rmc(); \ | |
73 | _LOAD_SHARED(p); \ | |
74 | }) | |
75 | ||
76 | /* | |
77 | * Identify a shared store. A smp_wmc() or smp_mc() should follow the store. | |
78 | */ | |
79 | #define _STORE_SHARED(x, v) ({ ACCESS_ONCE(x) = (v); }) | |
80 | ||
81 | /* | |
82 | * Store v into x, where x is located in shared memory. Performs the required | |
83 | * cache flush after writing. Returns v. | |
84 | */ | |
85 | #define STORE_SHARED(x, v) \ | |
86 | ({ \ | |
87 | _STORE_SHARED(x, v); \ | |
88 | smp_wmc(); \ | |
89 | (v); \ | |
90 | }) | |
91 | ||
92 | /* | |
93 | * This code section can only be included in LGPL 2.1 compatible source code. | |
94 | * See below for the function call wrappers which can be used in code meant to | |
95 | * be only linked with the Userspace RCU library. This comes with a small | |
96 | * performance degradation on the read-side due to the added function calls. | |
97 | * This is required to permit relinking with newer versions of the library. | |
98 | */ | |
99 | ||
100 | #ifdef DEBUG_RCU | |
101 | #define rcu_assert(args...) assert(args) | |
102 | #else | |
103 | #define rcu_assert(args...) | |
104 | #endif | |
105 | ||
804b4375 MD |
106 | /* |
107 | * defer queue. | |
108 | * Contains pointers. Encoded to save space when same callback is often used. | |
109 | * When looking up the next item: | |
110 | * - if DQ_FCT_BIT is set, set the current callback to DQ_CLEAR_FCT_BIT(ptr) | |
111 | * - next element contains pointer to data. | |
112 | * - else if item == DQ_FCT_MARK | |
113 | * - set the current callback to next element ptr | |
114 | * - following next element contains pointer to data. | |
115 | * - else current element contains data | |
116 | */ | |
786ee85b | 117 | struct defer_queue { |
90075a50 | 118 | unsigned long head; /* add element at head */ |
804b4375 | 119 | void *last_fct_in; /* last fct pointer encoded */ |
90075a50 | 120 | unsigned long tail; /* next element to remove at tail */ |
804b4375 | 121 | void *last_fct_out; /* last fct pointer encoded */ |
90075a50 MD |
122 | void **q; |
123 | }; | |
124 | ||
786ee85b | 125 | extern struct defer_queue __thread defer_queue; |
90075a50 | 126 | |
786ee85b | 127 | extern void rcu_defer_barrier_thread(void); |
90075a50 MD |
128 | |
129 | /* | |
130 | * not signal-safe. | |
131 | */ | |
804b4375 | 132 | static inline void _rcu_defer_queue(void (*fct)(void *p), void *p) |
90075a50 MD |
133 | { |
134 | unsigned long head, tail; | |
135 | ||
136 | /* | |
137 | * Head is only modified by ourself. Tail can be modified by reclamation | |
138 | * thread. | |
139 | */ | |
786ee85b MD |
140 | head = defer_queue.head; |
141 | tail = LOAD_SHARED(defer_queue.tail); | |
90075a50 MD |
142 | |
143 | /* | |
144 | * If queue is full, empty it ourself. | |
804b4375 | 145 | * Worse-case: must allow 2 supplementary entries for fct pointer. |
90075a50 | 146 | */ |
804b4375 MD |
147 | if (unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) { |
148 | assert(head - tail <= DEFER_QUEUE_SIZE); | |
786ee85b MD |
149 | rcu_defer_barrier_thread(); |
150 | assert(head - LOAD_SHARED(defer_queue.tail) == 0); | |
90075a50 MD |
151 | } |
152 | ||
804b4375 | 153 | if (unlikely(defer_queue.last_fct_in != fct)) { |
804b4375 MD |
154 | defer_queue.last_fct_in = fct; |
155 | if (unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) { | |
156 | /* | |
157 | * If the function to encode is not aligned or the | |
158 | * marker, write DQ_FCT_MARK followed by the function | |
159 | * pointer. | |
160 | */ | |
161 | _STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], | |
162 | DQ_FCT_MARK); | |
163 | _STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], | |
164 | fct); | |
165 | } else { | |
166 | DQ_SET_FCT_BIT(fct); | |
167 | _STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], | |
168 | fct); | |
169 | } | |
170 | } else { | |
804b4375 MD |
171 | if (unlikely(DQ_IS_FCT_BIT(p) || p == DQ_FCT_MARK)) { |
172 | /* | |
173 | * If the data to encode is not aligned or the marker, | |
174 | * write DQ_FCT_MARK followed by the function pointer. | |
175 | */ | |
176 | _STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], | |
177 | DQ_FCT_MARK); | |
178 | _STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], | |
179 | fct); | |
180 | } | |
181 | } | |
182 | _STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], p); | |
9374f873 MD |
183 | smp_wmb(); /* Publish new pointer before head */ |
184 | /* Write q[] before head. */ | |
804b4375 | 185 | STORE_SHARED(defer_queue.head, head); |
90075a50 MD |
186 | } |
187 | ||
786ee85b | 188 | #endif /* _URCU_DEFER_STATIC_H */ |