X-Git-Url: http://git.liburcu.org/?p=urcu.git;a=blobdiff_plain;f=src%2Furcu-bp.c;h=05efd97eb67a2c05c99a09dfb5c6e29e888ff81e;hp=33f280799208c66f0337e25acfeaf8adce213aa1;hb=HEAD;hpb=60dd72eaef48fe7fafd398d1c6577973e50828c9 diff --git a/src/urcu-bp.c b/src/urcu-bp.c index 33f2807..38f867e 100644 --- a/src/urcu-bp.c +++ b/src/urcu-bp.c @@ -1,25 +1,11 @@ +// SPDX-FileCopyrightText: 2009 Mathieu Desnoyers +// SPDX-FileCopyrightText: 2009 Paul E. McKenney, IBM Corporation. +// +// SPDX-License-Identifier: LGPL-2.1-or-later + /* - * urcu-bp.c - * * Userspace RCU library, "bulletproof" version. * - * Copyright (c) 2009 Mathieu Desnoyers - * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * * IBM's contributions to this file may be relicensed under LGPLv2 or later. */ @@ -28,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -37,6 +22,9 @@ #include #include +#include +#include +#include #include #include #include @@ -74,10 +62,12 @@ void *mremap_wrapper(void *old_address, size_t old_size, * This is not generic. */ static -void *mremap_wrapper(void *old_address, size_t old_size, - size_t new_size, int flags) +void *mremap_wrapper(void *old_address __attribute__((unused)), + size_t old_size __attribute__((unused)), + size_t new_size __attribute__((unused)), + int flags) { - assert(!(flags & MREMAP_MAYMOVE)); + urcu_posix_assert(!(flags & MREMAP_MAYMOVE)); return MAP_FAILED; } @@ -85,10 +75,7 @@ void *mremap_wrapper(void *old_address, size_t old_size, /* Sleep delay in ms */ #define RCU_SLEEP_DELAY_MS 10 -#define INIT_NR_THREADS 8 -#define ARENA_INIT_ALLOC \ - sizeof(struct registry_chunk) \ - + INIT_NR_THREADS * sizeof(struct urcu_bp_reader) +#define INIT_READER_COUNT 8 /* * Active attempts to check for reader Q.S. before calling sleep(). @@ -117,7 +104,10 @@ enum membarrier_cmd { static void __attribute__((constructor)) _urcu_bp_init(void); static -void __attribute__((destructor)) urcu_bp_exit(void); +void urcu_bp_exit(void); +static +void __attribute__((destructor)) urcu_bp_exit_destructor(void); +static void urcu_call_rcu_exit(void); #ifndef CONFIG_RCU_FORCE_SYS_MEMBARRIER int urcu_bp_has_sys_membarrier; @@ -145,22 +135,20 @@ static int initialized; static pthread_key_t urcu_bp_key; struct urcu_bp_gp urcu_bp_gp = { .ctr = URCU_BP_GP_COUNT }; -URCU_ATTR_ALIAS("urcu_bp_gp") extern struct urcu_bp_gp rcu_gp_bp; /* * Pointer to registry elements. Written to only by each individual reader. Read * by both the reader and the writers. */ DEFINE_URCU_TLS(struct urcu_bp_reader *, urcu_bp_reader); -DEFINE_URCU_TLS_ALIAS(struct urcu_bp_reader *, urcu_bp_reader, rcu_reader_bp); static CDS_LIST_HEAD(registry); struct registry_chunk { - size_t data_len; /* data length */ - size_t used; /* amount of data used */ + size_t capacity; /* capacity of this chunk (in elements) */ + size_t used; /* count of elements used */ struct cds_list_head node; /* chunk_list node */ - char data[]; + struct urcu_bp_reader readers[]; }; struct registry_arena { @@ -210,13 +198,21 @@ static void smp_mb_master(void) } } +/* Get the size of a chunk's allocation from its capacity (an element count). */ +static size_t chunk_allocation_size(size_t capacity) +{ + return (capacity * sizeof(struct urcu_bp_reader)) + + sizeof(struct registry_chunk); +} + /* * Always called with rcu_registry lock held. Releases this lock between * iterations and grabs it again. Holds the lock when it returns. */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, - struct cds_list_head *qsreaders) + struct cds_list_head *qsreaders, + cmm_annotate_t *group) { unsigned int wait_loops = 0; struct urcu_bp_reader *index, *tmp; @@ -231,7 +227,7 @@ static void wait_for_readers(struct cds_list_head *input_readers, wait_loops++; cds_list_for_each_entry_safe(index, tmp, input_readers, node) { - switch (urcu_bp_reader_state(&index->ctr)) { + switch (urcu_bp_reader_state(&index->ctr, group)) { case URCU_BP_READER_ACTIVE_CURRENT: if (cur_snap_readers) { cds_list_move(&index->node, @@ -270,15 +266,17 @@ static void wait_for_readers(struct cds_list_head *input_readers, void urcu_bp_synchronize_rcu(void) { + cmm_annotate_define(acquire_group); + cmm_annotate_define(release_group); CDS_LIST_HEAD(cur_snap_readers); CDS_LIST_HEAD(qsreaders); sigset_t newmask, oldmask; int ret; ret = sigfillset(&newmask); - assert(!ret); + urcu_posix_assert(!ret); ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); - assert(!ret); + urcu_posix_assert(!ret); mutex_lock(&rcu_gp_lock); @@ -291,13 +289,14 @@ void urcu_bp_synchronize_rcu(void) * where new ptr points to. */ /* Write new ptr before changing the qparity */ smp_mb_master(); + cmm_annotate_group_mb_release(&release_group); /* * Wait for readers to observe original parity or be quiescent. * wait_for_readers() can release and grab again rcu_registry_lock - * interally. + * internally. */ - wait_for_readers(®istry, &cur_snap_readers, &qsreaders); + wait_for_readers(®istry, &cur_snap_readers, &qsreaders, &acquire_group); /* * Adding a cmm_smp_mb() which is _not_ formally required, but makes the @@ -307,7 +306,8 @@ void urcu_bp_synchronize_rcu(void) cmm_smp_mb(); /* Switch parity: 0 -> 1, 1 -> 0 */ - CMM_STORE_SHARED(rcu_gp.ctr, rcu_gp.ctr ^ URCU_BP_GP_CTR_PHASE); + cmm_annotate_group_mem_release(&release_group, &rcu_gp.ctr); + uatomic_store(&rcu_gp.ctr, rcu_gp.ctr ^ URCU_BP_GP_CTR_PHASE, CMM_RELAXED); /* * Must commit qparity update to memory before waiting for other parity @@ -326,9 +326,9 @@ void urcu_bp_synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. * wait_for_readers() can release and grab again rcu_registry_lock - * interally. + * internally. */ - wait_for_readers(&cur_snap_readers, NULL, &qsreaders); + wait_for_readers(&cur_snap_readers, NULL, &qsreaders, &acquire_group); /* * Put quiescent reader list back into registry. @@ -340,13 +340,13 @@ void urcu_bp_synchronize_rcu(void) * freed. */ smp_mb_master(); + cmm_annotate_group_mb_acquire(&acquire_group); out: mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); - assert(!ret); + urcu_posix_assert(!ret); } -URCU_ATTR_ALIAS("urcu_bp_synchronize_rcu") void synchronize_rcu_bp(); /* * library wrappers to be used by non-LGPL compatible source code. @@ -356,19 +356,16 @@ void urcu_bp_read_lock(void) { _urcu_bp_read_lock(); } -URCU_ATTR_ALIAS("urcu_bp_read_lock") void rcu_read_lock_bp(); void urcu_bp_read_unlock(void) { _urcu_bp_read_unlock(); } -URCU_ATTR_ALIAS("urcu_bp_read_unlock") void rcu_read_unlock_bp(); int urcu_bp_read_ongoing(void) { return _urcu_bp_read_ongoing(); } -URCU_ATTR_ALIAS("urcu_bp_read_ongoing") int rcu_read_ongoing_bp(); /* * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk. @@ -382,24 +379,20 @@ static void expand_arena(struct registry_arena *arena) { struct registry_chunk *new_chunk, *last_chunk; - size_t old_chunk_len, new_chunk_len; + size_t old_chunk_size_bytes, new_chunk_size_bytes, new_capacity; /* No chunk. */ if (cds_list_empty(&arena->chunk_list)) { - assert(ARENA_INIT_ALLOC >= - sizeof(struct registry_chunk) - + sizeof(struct rcu_reader)); - new_chunk_len = ARENA_INIT_ALLOC; + new_chunk_size_bytes = chunk_allocation_size(INIT_READER_COUNT); new_chunk = (struct registry_chunk *) mmap(NULL, - new_chunk_len, + new_chunk_size_bytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (new_chunk == MAP_FAILED) abort(); - memset(new_chunk, 0, new_chunk_len); - new_chunk->data_len = - new_chunk_len - sizeof(struct registry_chunk); + memset(new_chunk, 0, new_chunk_size_bytes); + new_chunk->capacity = INIT_READER_COUNT; cds_list_add_tail(&new_chunk->node, &arena->chunk_list); return; /* We're done. */ } @@ -407,34 +400,32 @@ void expand_arena(struct registry_arena *arena) /* Try expanding last chunk. */ last_chunk = cds_list_entry(arena->chunk_list.prev, struct registry_chunk, node); - old_chunk_len = - last_chunk->data_len + sizeof(struct registry_chunk); - new_chunk_len = old_chunk_len << 1; + old_chunk_size_bytes = chunk_allocation_size(last_chunk->capacity); + new_capacity = last_chunk->capacity << 1; + new_chunk_size_bytes = chunk_allocation_size(new_capacity); /* Don't allow memory mapping to move, just expand. */ - new_chunk = mremap_wrapper(last_chunk, old_chunk_len, - new_chunk_len, 0); + new_chunk = mremap_wrapper(last_chunk, old_chunk_size_bytes, + new_chunk_size_bytes, 0); if (new_chunk != MAP_FAILED) { /* Should not have moved. */ assert(new_chunk == last_chunk); - memset((char *) last_chunk + old_chunk_len, 0, - new_chunk_len - old_chunk_len); - last_chunk->data_len = - new_chunk_len - sizeof(struct registry_chunk); + memset((char *) last_chunk + old_chunk_size_bytes, 0, + new_chunk_size_bytes - old_chunk_size_bytes); + last_chunk->capacity = new_capacity; return; /* We're done. */ } /* Remap did not succeed, we need to add a new chunk. */ new_chunk = (struct registry_chunk *) mmap(NULL, - new_chunk_len, + new_chunk_size_bytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (new_chunk == MAP_FAILED) abort(); - memset(new_chunk, 0, new_chunk_len); - new_chunk->data_len = - new_chunk_len - sizeof(struct registry_chunk); + memset(new_chunk, 0, new_chunk_size_bytes); + new_chunk->capacity = new_capacity; cds_list_add_tail(&new_chunk->node, &arena->chunk_list); } @@ -442,22 +433,23 @@ static struct rcu_reader *arena_alloc(struct registry_arena *arena) { struct registry_chunk *chunk; - struct rcu_reader *rcu_reader_reg; int expand_done = 0; /* Only allow to expand once per alloc */ - size_t len = sizeof(struct rcu_reader); retry: cds_list_for_each_entry(chunk, &arena->chunk_list, node) { - if (chunk->data_len - chunk->used < len) + size_t spot_idx; + + /* Skip fully used chunks. */ + if (chunk->used == chunk->capacity) { continue; - /* Find spot */ - for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0]; - rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len]; - rcu_reader_reg++) { - if (!rcu_reader_reg->alloc) { - rcu_reader_reg->alloc = 1; - chunk->used += len; - return rcu_reader_reg; + } + + /* Find a spot. */ + for (spot_idx = 0; spot_idx < chunk->capacity; spot_idx++) { + if (!chunk->readers[spot_idx].alloc) { + chunk->readers[spot_idx].alloc = 1; + chunk->used++; + return &chunk->readers[spot_idx]; } } } @@ -487,7 +479,7 @@ void add_thread(void) /* Add to registry */ rcu_reader_reg->tid = pthread_self(); - assert(rcu_reader_reg->ctr == 0); + urcu_posix_assert(rcu_reader_reg->ctr == 0); cds_list_add(&rcu_reader_reg->node, ®istry); /* * Reader threads are pointing to the reader registry. This is @@ -505,7 +497,7 @@ void cleanup_thread(struct registry_chunk *chunk, cds_list_del(&rcu_reader_reg->node); rcu_reader_reg->tid = 0; rcu_reader_reg->alloc = 0; - chunk->used -= sizeof(struct rcu_reader); + chunk->used--; } static @@ -514,9 +506,9 @@ struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg) struct registry_chunk *chunk; cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) { - if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0]) + if (rcu_reader_reg < (struct urcu_bp_reader *) &chunk->readers[0]) continue; - if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len]) + if (rcu_reader_reg >= (struct urcu_bp_reader *) &chunk->readers[chunk->capacity]) continue; return chunk; } @@ -564,7 +556,12 @@ end: if (ret) abort(); } -URCU_ATTR_ALIAS("urcu_bp_register") void rcu_bp_register(); + +void urcu_bp_register_thread(void) +{ + if (caa_unlikely(!URCU_TLS(urcu_bp_reader))) + urcu_bp_register(); /* If not yet registered. */ +} /* Disable signals, take mutex, remove from registry */ static @@ -660,8 +657,7 @@ void urcu_bp_exit(void) cds_list_for_each_entry_safe(chunk, tmp, ®istry_arena.chunk_list, node) { - munmap((void *) chunk, chunk->data_len - + sizeof(struct registry_chunk)); + munmap((void *) chunk, chunk_allocation_size(chunk->capacity)); } CDS_INIT_LIST_HEAD(®istry_arena.chunk_list); ret = pthread_key_delete(urcu_bp_key); @@ -671,6 +667,13 @@ void urcu_bp_exit(void) mutex_unlock(&init_lock); } +static +void urcu_bp_exit_destructor(void) +{ + urcu_call_rcu_exit(); + urcu_bp_exit(); +} + /* * Holding the rcu_gp_lock and rcu_registry_lock across fork will make * sure we fork() don't race with a concurrent thread executing with @@ -683,14 +686,13 @@ void urcu_bp_before_fork(void) int ret; ret = sigfillset(&newmask); - assert(!ret); + urcu_posix_assert(!ret); ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); - assert(!ret); + urcu_posix_assert(!ret); mutex_lock(&rcu_gp_lock); mutex_lock(&rcu_registry_lock); saved_fork_signal_mask = oldmask; } -URCU_ATTR_ALIAS("urcu_bp_before_fork") void rcu_bp_before_fork(); void urcu_bp_after_fork_parent(void) { @@ -701,10 +703,8 @@ void urcu_bp_after_fork_parent(void) mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); - assert(!ret); + urcu_posix_assert(!ret); } -URCU_ATTR_ALIAS("urcu_bp_after_fork_parent") -void rcu_bp_after_fork_parent(void); /* * Prune all entries from registry except our own thread. Fits the Linux @@ -714,17 +714,18 @@ static void urcu_bp_prune_registry(void) { struct registry_chunk *chunk; - struct urcu_bp_reader *rcu_reader_reg; cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) { - for (rcu_reader_reg = (struct urcu_bp_reader *) &chunk->data[0]; - rcu_reader_reg < (struct urcu_bp_reader *) &chunk->data[chunk->data_len]; - rcu_reader_reg++) { - if (!rcu_reader_reg->alloc) + size_t spot_idx; + + for (spot_idx = 0; spot_idx < chunk->capacity; spot_idx++) { + struct urcu_bp_reader *reader = &chunk->readers[spot_idx]; + + if (!reader->alloc) continue; - if (rcu_reader_reg->tid == pthread_self()) + if (reader->tid == pthread_self()) continue; - cleanup_thread(chunk, rcu_reader_reg); + cleanup_thread(chunk, reader); } } } @@ -739,17 +740,13 @@ void urcu_bp_after_fork_child(void) mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); - assert(!ret); + urcu_posix_assert(!ret); } -URCU_ATTR_ALIAS("urcu_bp_after_fork_child") -void rcu_bp_after_fork_child(void); void *urcu_bp_dereference_sym(void *p) { return _rcu_dereference(p); } -URCU_ATTR_ALIAS("urcu_bp_dereference_sym") -void *rcu_dereference_sym_bp(); void *urcu_bp_set_pointer_sym(void **p, void *v) { @@ -757,27 +754,21 @@ void *urcu_bp_set_pointer_sym(void **p, void *v) uatomic_set(p, v); return v; } -URCU_ATTR_ALIAS("urcu_bp_set_pointer_sym") -void *rcu_set_pointer_sym_bp(); void *urcu_bp_xchg_pointer_sym(void **p, void *v) { cmm_wmb(); return uatomic_xchg(p, v); } -URCU_ATTR_ALIAS("urcu_bp_xchg_pointer_sym") -void *rcu_xchg_pointer_sym_bp(); void *urcu_bp_cmpxchg_pointer_sym(void **p, void *old, void *_new) { cmm_wmb(); return uatomic_cmpxchg(p, old, _new); } -URCU_ATTR_ALIAS("urcu_bp_cmpxchg_pointer_sym") -void *rcu_cmpxchg_pointer_sym_bp(); DEFINE_RCU_FLAVOR(rcu_flavor); -DEFINE_RCU_FLAVOR_ALIAS(rcu_flavor, alias_rcu_flavor); #include "urcu-call-rcu-impl.h" #include "urcu-defer-impl.h" +#include "urcu-poll-impl.h"