X-Git-Url: http://git.liburcu.org/?a=blobdiff_plain;f=urcu-bp.c;h=3afaf38edac5eab727139c3b2307eed52e6efb22;hb=89451e1b67738953bb2a10fd1fba7b12e73c0440;hp=4c0ab54a28e71b47947905d6f5f42354de0135cd;hpb=1f689e13ea7e519b1afc001e9c55a7b1b60b599f;p=userspace-rcu.git diff --git a/urcu-bp.c b/urcu-bp.c index 4c0ab54..3afaf38 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -40,6 +40,9 @@ #include "urcu/map/urcu-bp.h" #include "urcu/static/urcu-bp.h" #include "urcu-pointer.h" +#include "urcu/tls-compat.h" + +#include "urcu-die.h" /* Do not #define _LGPL_SOURCE to ensure we can emit the wrapper symbols */ #undef _LGPL_SOURCE @@ -50,38 +53,38 @@ #define MAP_ANONYMOUS MAP_ANON #endif -#ifndef __linux__ +#ifdef __linux__ +static +void *mremap_wrapper(void *old_address, size_t old_size, + size_t new_size, int flags) +{ + return mremap(old_address, old_size, new_size, flags); +} +#else #define MREMAP_MAYMOVE 1 #define MREMAP_FIXED 2 /* - * mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping. + * mremap wrapper for non-Linux systems not allowing MAYMOVE. * This is not generic. */ -void *mremap(void *old_address, size_t old_size, size_t new_size, int flags) +static +void *mremap_wrapper(void *old_address, size_t old_size, + size_t new_size, int flags) { - void *new_address; - - assert(flags & MREMAP_MAYMOVE); - assert(!(flags & MREMAP_FIXED)); - new_address = mmap(old_address, new_size, - PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, - -1, 0); - if (new_address == MAP_FAILED) - return MAP_FAILED; - if (old_address) { - memcpy(new_address, old_address, old_size); - munmap(old_address, old_size); - } - return new_address; + assert(!(flags & MREMAP_MAYMOVE)); + + return MAP_FAILED; } #endif /* Sleep delay in us */ #define RCU_SLEEP_DELAY 1000 -#define ARENA_INIT_ALLOC 16 +#define INIT_NR_THREADS 8 +#define ARENA_INIT_ALLOC \ + sizeof(struct registry_chunk) \ + + INIT_NR_THREADS * sizeof(struct rcu_reader) /* * Active attempts to check for reader Q.S. before calling sleep(). @@ -94,7 +97,7 @@ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; #ifdef DEBUG_YIELD unsigned int yield_active; -unsigned int __thread rand_yield; +DEFINE_URCU_TLS(unsigned int, rand_yield); #endif /* @@ -109,17 +112,24 @@ long rcu_gp_ctr = RCU_GP_COUNT; * Pointer to registry elements. Written to only by each individual reader. Read * by both the reader and the writers. */ -struct rcu_reader __thread *rcu_reader; +DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader); static CDS_LIST_HEAD(registry); +struct registry_chunk { + size_t data_len; /* data length */ + size_t used; /* data used */ + struct cds_list_head node; /* chunk_list node */ + char data[]; +}; + struct registry_arena { - void *p; - size_t len; - size_t used; + struct cds_list_head chunk_list; }; -static struct registry_arena registry_arena; +static struct registry_arena registry_arena = { + .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list), +}; /* Saved fork signal mask, protected by rcu_gp_lock */ static sigset_t saved_fork_signal_mask; @@ -132,17 +142,12 @@ static void mutex_lock(pthread_mutex_t *mutex) #ifndef DISTRUST_SIGNALS_EXTREME ret = pthread_mutex_lock(mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } + if (ret) + urcu_die(ret); #else /* #ifndef DISTRUST_SIGNALS_EXTREME */ while ((ret = pthread_mutex_trylock(mutex)) != 0) { - if (ret != EBUSY && ret != EINTR) { - printf("ret = %d, errno = %d\n", ret, errno); - perror("Error in pthread mutex lock"); - exit(-1); - } + if (ret != EBUSY && ret != EINTR) + urcu_die(ret); poll(NULL,0,10); } #endif /* #else #ifndef DISTRUST_SIGNALS_EXTREME */ @@ -153,10 +158,8 @@ static void mutex_unlock(pthread_mutex_t *mutex) int ret; ret = pthread_mutex_unlock(mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } + if (ret) + urcu_die(ret); } void update_counter_and_wait(void) @@ -210,9 +213,9 @@ void synchronize_rcu(void) sigset_t newmask, oldmask; int ret; - ret = sigemptyset(&newmask); + ret = sigfillset(&newmask); assert(!ret); - ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); @@ -271,80 +274,150 @@ void rcu_read_unlock(void) } /* - * only grow for now. + * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk. + * Else, try expanding the last chunk. If this fails, allocate a new + * chunk twice as big as the last chunk. + * Memory used by chunks _never_ moves. A chunk could theoretically be + * freed when all "used" slots are released, but we don't do it at this + * point. */ -static void resize_arena(struct registry_arena *arena, size_t len) +static +void expand_arena(struct registry_arena *arena) { - void *new_arena; - - if (!arena->p) - new_arena = mmap(arena->p, len, - PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, - -1, 0); - else - new_arena = mremap(arena->p, arena->len, - len, MREMAP_MAYMOVE); - assert(new_arena != MAP_FAILED); + struct registry_chunk *new_chunk, *last_chunk; + size_t old_chunk_len, new_chunk_len; + + /* No chunk. */ + if (cds_list_empty(&arena->chunk_list)) { + assert(ARENA_INIT_ALLOC >= + sizeof(struct registry_chunk) + + sizeof(struct rcu_reader)); + new_chunk_len = ARENA_INIT_ALLOC; + new_chunk = mmap(NULL, new_chunk_len, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + if (new_chunk == MAP_FAILED) + abort(); + bzero(new_chunk, new_chunk_len); + new_chunk->data_len = + new_chunk_len - sizeof(struct registry_chunk); + cds_list_add_tail(&new_chunk->node, &arena->chunk_list); + return; /* We're done. */ + } - /* - * re-used the same region ? - */ - if (new_arena == arena->p) - return; + /* Try expanding last chunk. */ + last_chunk = cds_list_entry(arena->chunk_list.prev, + struct registry_chunk, node); + old_chunk_len = + last_chunk->data_len + sizeof(struct registry_chunk); + new_chunk_len = old_chunk_len << 1; + + /* Don't allow memory mapping to move, just expand. */ + new_chunk = mremap_wrapper(last_chunk, old_chunk_len, + new_chunk_len, 0); + if (new_chunk != MAP_FAILED) { + /* Should not have moved. */ + assert(new_chunk == last_chunk); + bzero((char *) last_chunk + old_chunk_len, + new_chunk_len - old_chunk_len); + last_chunk->data_len = + new_chunk_len - sizeof(struct registry_chunk); + return; /* We're done. */ + } - bzero(new_arena + arena->len, len - arena->len); - arena->p = new_arena; + /* Remap did not succeed, we need to add a new chunk. */ + new_chunk = mmap(NULL, new_chunk_len, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + if (new_chunk == MAP_FAILED) + abort(); + bzero(new_chunk, new_chunk_len); + new_chunk->data_len = + new_chunk_len - sizeof(struct registry_chunk); + cds_list_add_tail(&new_chunk->node, &arena->chunk_list); } -/* Called with signals off and mutex locked */ -static void add_thread(void) +static +struct rcu_reader *arena_alloc(struct registry_arena *arena) { + struct registry_chunk *chunk; struct rcu_reader *rcu_reader_reg; + int expand_done = 0; /* Only allow to expand once per alloc */ + size_t len = sizeof(struct rcu_reader); - if (registry_arena.len - < registry_arena.used + sizeof(struct rcu_reader)) - resize_arena(®istry_arena, - caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC)); - /* - * Find a free spot. - */ - for (rcu_reader_reg = registry_arena.p; - (void *)rcu_reader_reg < registry_arena.p + registry_arena.len; - rcu_reader_reg++) { - if (!rcu_reader_reg->alloc) - break; +retry: + cds_list_for_each_entry(chunk, &arena->chunk_list, node) { + if (chunk->data_len - chunk->used < len) + continue; + /* Find spot */ + for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0]; + rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len]; + rcu_reader_reg++) { + if (!rcu_reader_reg->alloc) { + rcu_reader_reg->alloc = 1; + chunk->used += len; + return rcu_reader_reg; + } + } + } + + if (!expand_done) { + expand_arena(arena); + expand_done = 1; + goto retry; } - rcu_reader_reg->alloc = 1; - registry_arena.used += sizeof(struct rcu_reader); + + return NULL; +} + +/* Called with signals off and mutex locked */ +static +void add_thread(void) +{ + struct rcu_reader *rcu_reader_reg; + + rcu_reader_reg = arena_alloc(®istry_arena); + if (!rcu_reader_reg) + abort(); /* Add to registry */ rcu_reader_reg->tid = pthread_self(); assert(rcu_reader_reg->ctr == 0); cds_list_add(&rcu_reader_reg->node, ®istry); - rcu_reader = rcu_reader_reg; + /* + * Reader threads are pointing to the reader registry. This is + * why its memory should never be relocated. + */ + URCU_TLS(rcu_reader) = rcu_reader_reg; } /* Called with signals off and mutex locked */ static void rcu_gc_registry(void) { + struct registry_chunk *chunk; struct rcu_reader *rcu_reader_reg; - pthread_t tid; - int ret; - for (rcu_reader_reg = registry_arena.p; - (void *)rcu_reader_reg < registry_arena.p + registry_arena.len; - rcu_reader_reg++) { - if (!rcu_reader_reg->alloc) - continue; - tid = rcu_reader_reg->tid; - ret = pthread_kill(tid, 0); - assert(ret != EINVAL); - if (ret == ESRCH) { - cds_list_del(&rcu_reader_reg->node); - rcu_reader_reg->ctr = 0; - rcu_reader_reg->alloc = 0; - registry_arena.used -= sizeof(struct rcu_reader); + cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) { + for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0]; + rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len]; + rcu_reader_reg++) { + pthread_t tid; + int ret; + + if (!rcu_reader_reg->alloc) + continue; + tid = rcu_reader_reg->tid; + ret = pthread_kill(tid, 0); + assert(ret != EINVAL); + if (ret == ESRCH) { + cds_list_del(&rcu_reader_reg->node); + rcu_reader_reg->ctr = 0; + rcu_reader_reg->alloc = 0; + chunk->used -= sizeof(struct rcu_reader); + } + } } } @@ -355,15 +428,15 @@ void rcu_bp_register(void) sigset_t newmask, oldmask; int ret; - ret = sigemptyset(&newmask); + ret = sigfillset(&newmask); assert(!ret); - ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); /* * Check if a signal concurrently registered our thread since * the check in rcu_read_lock(). */ - if (rcu_reader) + if (URCU_TLS(rcu_reader)) goto end; mutex_lock(&rcu_gp_lock); @@ -376,8 +449,12 @@ end: void rcu_bp_exit(void) { - if (registry_arena.p) - munmap(registry_arena.p, registry_arena.len); + struct registry_chunk *chunk, *tmp; + + cds_list_for_each_entry_safe(chunk, tmp, + ®istry_arena.chunk_list, node) { + munmap(chunk, chunk->data_len + sizeof(struct registry_chunk)); + } } /* @@ -390,9 +467,9 @@ void rcu_bp_before_fork(void) sigset_t newmask, oldmask; int ret; - ret = sigemptyset(&newmask); + ret = sigfillset(&newmask); assert(!ret); - ret = pthread_sigmask(SIG_SETMASK, &newmask, &oldmask); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); saved_fork_signal_mask = oldmask; @@ -421,5 +498,31 @@ void rcu_bp_after_fork_child(void) assert(!ret); } +void *rcu_dereference_sym_bp(void *p) +{ + return _rcu_dereference(p); +} + +void *rcu_set_pointer_sym_bp(void **p, void *v) +{ + cmm_wmb(); + uatomic_set(p, v); + return v; +} + +void *rcu_xchg_pointer_sym_bp(void **p, void *v) +{ + cmm_wmb(); + return uatomic_xchg(p, v); +} + +void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new) +{ + cmm_wmb(); + return uatomic_cmpxchg(p, old, _new); +} + +DEFINE_RCU_FLAVOR(rcu_flavor); + #include "urcu-call-rcu-impl.h" #include "urcu-defer-impl.h"