Cleanup: remove unused value warning
[urcu.git] / urcu-bp.c
CommitLineData
fdee2e6d
MD
1/*
2 * urcu-bp.c
3 *
4 * Userspace RCU library, "bulletproof" version.
5 *
6982d6d7 6 * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
fdee2e6d
MD
7 * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
8 *
9 * This library is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public
11 * License as published by the Free Software Foundation; either
12 * version 2.1 of the License, or (at your option) any later version.
13 *
14 * This library is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with this library; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 *
23 * IBM's contributions to this file may be relicensed under LGPLv2 or later.
24 */
25
0617bf4c 26#define _GNU_SOURCE
fdee2e6d
MD
27#include <stdio.h>
28#include <pthread.h>
29#include <signal.h>
30#include <assert.h>
31#include <stdlib.h>
32#include <string.h>
33#include <errno.h>
34#include <poll.h>
35#include <unistd.h>
36#include <sys/mman.h>
37
38#include "urcu-bp-static.h"
39/* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */
40#include "urcu-bp.h"
41
42/* Sleep delay in us */
43#define RCU_SLEEP_DELAY 1000
44#define ARENA_INIT_ALLOC 16
45
02be5561 46void __attribute__((destructor)) rcu_bp_exit(void);
fdee2e6d 47
6abb4bd5 48static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER;
fdee2e6d
MD
49
50#ifdef DEBUG_YIELD
51unsigned int yield_active;
52unsigned int __thread rand_yield;
53#endif
54
55/*
56 * Global grace period counter.
02be5561 57 * Contains the current RCU_GP_CTR_PHASE.
fdee2e6d
MD
58 * Also has a RCU_GP_COUNT of 1, to accelerate the reader fast path.
59 * Written to only by writer with mutex taken. Read by both writer and readers.
60 */
02be5561 61long rcu_gp_ctr = RCU_GP_COUNT;
fdee2e6d
MD
62
63/*
64 * Pointer to registry elements. Written to only by each individual reader. Read
65 * by both the reader and the writers.
66 */
02be5561 67struct rcu_reader __thread *rcu_reader;
fdee2e6d 68
16aa9ee8 69static CDS_LIST_HEAD(registry);
fdee2e6d
MD
70
71struct registry_arena {
72 void *p;
73 size_t len;
74 size_t used;
75};
76
77static struct registry_arena registry_arena;
78
4cf1675f
MD
79/* Saved fork signal mask, protected by rcu_gp_lock */
80static sigset_t saved_fork_signal_mask;
81
fdee2e6d
MD
82static void rcu_gc_registry(void);
83
6abb4bd5 84static void mutex_lock(pthread_mutex_t *mutex)
fdee2e6d
MD
85{
86 int ret;
87
88#ifndef DISTRUST_SIGNALS_EXTREME
6abb4bd5 89 ret = pthread_mutex_lock(mutex);
fdee2e6d
MD
90 if (ret) {
91 perror("Error in pthread mutex lock");
92 exit(-1);
93 }
94#else /* #ifndef DISTRUST_SIGNALS_EXTREME */
6abb4bd5 95 while ((ret = pthread_mutex_trylock(mutex)) != 0) {
fdee2e6d
MD
96 if (ret != EBUSY && ret != EINTR) {
97 printf("ret = %d, errno = %d\n", ret, errno);
98 perror("Error in pthread mutex lock");
99 exit(-1);
100 }
fdee2e6d
MD
101 poll(NULL,0,10);
102 }
103#endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */
104}
105
6abb4bd5 106static void mutex_unlock(pthread_mutex_t *mutex)
fdee2e6d
MD
107{
108 int ret;
109
6abb4bd5 110 ret = pthread_mutex_unlock(mutex);
fdee2e6d
MD
111 if (ret) {
112 perror("Error in pthread mutex unlock");
113 exit(-1);
114 }
115}
116
2dfb8b5e 117void update_counter_and_wait(void)
fdee2e6d 118{
16aa9ee8 119 CDS_LIST_HEAD(qsreaders);
fdee2e6d 120 int wait_loops = 0;
02be5561 121 struct rcu_reader *index, *tmp;
fdee2e6d 122
32c15e4e 123 /* Switch parity: 0 -> 1, 1 -> 0 */
6cf3827c 124 CMM_STORE_SHARED(rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR_PHASE);
2dfb8b5e
MD
125
126 /*
127 * Must commit qparity update to memory before waiting for other parity
128 * quiescent state. Failure to do so could result in the writer waiting
129 * forever while new readers are always accessing data (no progress).
6cf3827c 130 * Ensured by CMM_STORE_SHARED and CMM_LOAD_SHARED.
2dfb8b5e
MD
131 */
132
133 /*
5481ddb3 134 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
2dfb8b5e
MD
135 * model easier to understand. It does not have a big performance impact
136 * anyway, given this is the write-side.
137 */
5481ddb3 138 cmm_smp_mb();
2dfb8b5e 139
fdee2e6d 140 /*
02be5561 141 * Wait for each thread rcu_reader.ctr count to become 0.
fdee2e6d
MD
142 */
143 for (;;) {
144 wait_loops++;
16aa9ee8 145 cds_list_for_each_entry_safe(index, tmp, &registry, node) {
fdee2e6d 146 if (!rcu_old_gp_ongoing(&index->ctr))
16aa9ee8 147 cds_list_move(&index->node, &qsreaders);
fdee2e6d
MD
148 }
149
16aa9ee8 150 if (cds_list_empty(&registry)) {
fdee2e6d
MD
151 break;
152 } else {
153 if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS)
154 usleep(RCU_SLEEP_DELAY);
155 else
06f22bdb 156 caa_cpu_relax();
fdee2e6d
MD
157 }
158 }
159 /* put back the reader list in the registry */
16aa9ee8 160 cds_list_splice(&qsreaders, &registry);
fdee2e6d
MD
161}
162
163void synchronize_rcu(void)
164{
165 sigset_t newmask, oldmask;
166 int ret;
167
168 ret = sigemptyset(&newmask);
169 assert(!ret);
170 ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
171 assert(!ret);
172
6abb4bd5 173 mutex_lock(&rcu_gp_lock);
fdee2e6d 174
16aa9ee8 175 if (cds_list_empty(&registry))
2dfb8b5e 176 goto out;
fdee2e6d
MD
177
178 /* All threads should read qparity before accessing data structure
2dfb8b5e 179 * where new ptr points to. */
fdee2e6d 180 /* Write new ptr before changing the qparity */
5481ddb3 181 cmm_smp_mb();
fdee2e6d 182
2dfb8b5e
MD
183 /* Remove old registry elements */
184 rcu_gc_registry();
fdee2e6d
MD
185
186 /*
187 * Wait for previous parity to be empty of readers.
188 */
2dfb8b5e 189 update_counter_and_wait(); /* 0 -> 1, wait readers in parity 0 */
fdee2e6d
MD
190
191 /*
5481ddb3 192 * Adding a cmm_smp_mb() which is _not_ formally required, but makes the
fdee2e6d
MD
193 * model easier to understand. It does not have a big performance impact
194 * anyway, given this is the write-side.
195 */
5481ddb3 196 cmm_smp_mb();
fdee2e6d 197
fdee2e6d 198 /*
2dfb8b5e 199 * Wait for previous parity to be empty of readers.
fdee2e6d 200 */
2dfb8b5e 201 update_counter_and_wait(); /* 1 -> 0, wait readers in parity 1 */
fdee2e6d
MD
202
203 /*
2dfb8b5e
MD
204 * Finish waiting for reader threads before letting the old ptr being
205 * freed.
fdee2e6d 206 */
5481ddb3 207 cmm_smp_mb();
2dfb8b5e 208out:
6abb4bd5 209 mutex_unlock(&rcu_gp_lock);
fdee2e6d
MD
210 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
211 assert(!ret);
212}
213
214/*
215 * library wrappers to be used by non-LGPL compatible source code.
216 */
217
218void rcu_read_lock(void)
219{
220 _rcu_read_lock();
221}
222
223void rcu_read_unlock(void)
224{
225 _rcu_read_unlock();
226}
227
228/*
229 * only grow for now.
230 */
231static void resize_arena(struct registry_arena *arena, size_t len)
232{
233 void *new_arena;
234
0617bf4c
MD
235 if (!arena->p)
236 new_arena = mmap(arena->p, len,
237 PROT_READ | PROT_WRITE,
238 MAP_ANONYMOUS | MAP_PRIVATE,
239 -1, 0);
240 else
241 new_arena = mremap(arena->p, arena->len,
242 len, MREMAP_MAYMOVE);
243 assert(new_arena != MAP_FAILED);
244
fdee2e6d
MD
245 /*
246 * re-used the same region ?
247 */
248 if (new_arena == arena->p)
249 return;
250
251 memcpy(new_arena, arena->p, arena->len);
252 bzero(new_arena + arena->len, len - arena->len);
253 arena->p = new_arena;
254}
255
256/* Called with signals off and mutex locked */
257static void add_thread(void)
258{
02be5561 259 struct rcu_reader *rcu_reader_reg;
fdee2e6d
MD
260
261 if (registry_arena.len
02be5561 262 < registry_arena.used + sizeof(struct rcu_reader))
fdee2e6d
MD
263 resize_arena(&registry_arena,
264 max(registry_arena.len << 1, ARENA_INIT_ALLOC));
265 /*
266 * Find a free spot.
267 */
02be5561
MD
268 for (rcu_reader_reg = registry_arena.p;
269 (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
270 rcu_reader_reg++) {
271 if (!rcu_reader_reg->alloc)
fdee2e6d
MD
272 break;
273 }
02be5561
MD
274 rcu_reader_reg->alloc = 1;
275 registry_arena.used += sizeof(struct rcu_reader);
fdee2e6d
MD
276
277 /* Add to registry */
02be5561
MD
278 rcu_reader_reg->tid = pthread_self();
279 assert(rcu_reader_reg->ctr == 0);
16aa9ee8 280 cds_list_add(&rcu_reader_reg->node, &registry);
02be5561 281 rcu_reader = rcu_reader_reg;
fdee2e6d
MD
282}
283
284/* Called with signals off and mutex locked */
285static void rcu_gc_registry(void)
286{
02be5561 287 struct rcu_reader *rcu_reader_reg;
fdee2e6d
MD
288 pthread_t tid;
289 int ret;
290
02be5561
MD
291 for (rcu_reader_reg = registry_arena.p;
292 (void *)rcu_reader_reg < registry_arena.p + registry_arena.len;
293 rcu_reader_reg++) {
294 if (!rcu_reader_reg->alloc)
fdee2e6d 295 continue;
02be5561 296 tid = rcu_reader_reg->tid;
fdee2e6d
MD
297 ret = pthread_kill(tid, 0);
298 assert(ret != EINVAL);
299 if (ret == ESRCH) {
16aa9ee8 300 cds_list_del(&rcu_reader_reg->node);
79266051 301 rcu_reader_reg->ctr = 0;
02be5561
MD
302 rcu_reader_reg->alloc = 0;
303 registry_arena.used -= sizeof(struct rcu_reader);
fdee2e6d
MD
304 }
305 }
306}
307
308/* Disable signals, take mutex, add to registry */
309void rcu_bp_register(void)
310{
311 sigset_t newmask, oldmask;
312 int ret;
313
314 ret = sigemptyset(&newmask);
315 assert(!ret);
316 ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
317 assert(!ret);
318
319 /*
320 * Check if a signal concurrently registered our thread since
321 * the check in rcu_read_lock(). */
02be5561 322 if (rcu_reader)
fdee2e6d
MD
323 goto end;
324
6abb4bd5 325 mutex_lock(&rcu_gp_lock);
fdee2e6d 326 add_thread();
6abb4bd5 327 mutex_unlock(&rcu_gp_lock);
fdee2e6d
MD
328end:
329 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
330 assert(!ret);
331}
332
02be5561 333void rcu_bp_exit()
fdee2e6d
MD
334{
335 munmap(registry_arena.p, registry_arena.len);
336}
4cf1675f
MD
337
338/*
339 * Holding the rcu_gp_lock across fork will make sure we fork() don't race with
340 * a concurrent thread executing with this same lock held. This ensures that the
341 * registry is in a coherent state in the child.
342 */
343void rcu_bp_before_fork(void)
344{
345 sigset_t newmask, oldmask;
346 int ret;
347
348 ret = sigemptyset(&newmask);
349 assert(!ret);
350 ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask);
351 assert(!ret);
352 mutex_lock(&rcu_gp_lock);
353 saved_fork_signal_mask = oldmask;
354}
355
356void rcu_bp_after_fork_parent(void)
357{
358 sigset_t oldmask;
359 int ret;
360
361 oldmask = saved_fork_signal_mask;
362 mutex_unlock(&rcu_gp_lock);
363 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
364 assert(!ret);
365}
366
367void rcu_bp_after_fork_child(void)
368{
369 sigset_t oldmask;
370 int ret;
371
372 rcu_gc_registry();
373 oldmask = saved_fork_signal_mask;
374 mutex_unlock(&rcu_gp_lock);
375 ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
376 assert(!ret);
377}
This page took 0.040088 seconds and 4 git commands to generate.