From: Mathieu Desnoyers Date: Wed, 2 Nov 2011 00:15:55 +0000 (-0400) Subject: Merge branch 'master' into urcu/ht-shrink X-Git-Tag: v0.7.0~43^2~65 X-Git-Url: https://git.liburcu.org/?p=urcu.git;a=commitdiff_plain;h=cc6b0c208a9707f8a64609a51167290caeace288;hp=7f52427be704778d53f85e17fb71629b50ada827 Merge branch 'master' into urcu/ht-shrink Fixed conflicts: tests/test_urcu_qsbr.c tests/test_urcu_qsbr_gc.c Signed-off-by: Mathieu Desnoyers --- diff --git a/tests/Makefile.am b/tests/Makefile.am index 7c638ed..05a8024 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -5,12 +5,12 @@ noinst_PROGRAMS = test_urcu test_urcu_dynamic_link test_urcu_timing \ test_urcu_signal test_urcu_signal_dynamic_link test_urcu_signal_timing \ test_rwlock_timing test_rwlock test_perthreadlock_timing \ test_perthreadlock test_urcu_yield test_urcu_signal_yield test_urcu_mb \ - test_qsbr_timing test_qsbr rcutorture_urcu rcutorture_urcu_signal \ - rcutorture_urcu_mb rcutorture_urcu_bp rcutorture_qsbr \ + test_urcu_qsbr_timing test_urcu_qsbr rcutorture_urcu rcutorture_urcu_signal \ + rcutorture_urcu_mb rcutorture_urcu_bp rcutorture_urcu_qsbr \ test_mutex test_looplen test_urcu_gc test_urcu_signal_gc \ test_urcu_lgc \ - test_urcu_mb_gc test_qsbr_gc test_qsbr_lgc test_urcu_signal_lgc \ - test_urcu_mb_lgc test_qsbr_dynamic_link test_urcu_defer \ + test_urcu_mb_gc test_urcu_qsbr_gc test_urcu_qsbr_lgc test_urcu_signal_lgc \ + test_urcu_mb_lgc test_urcu_qsbr_dynamic_link test_urcu_defer \ test_uatomic test_urcu_assign test_urcu_assign_dynamic_link \ test_urcu_bp test_urcu_bp_dynamic_link test_cycles_per_loop \ test_urcu_lfq test_urcu_wfq test_urcu_lfs test_urcu_wfs \ @@ -58,9 +58,9 @@ test_urcu_yield_SOURCES = test_urcu.c $(URCU) test_urcu_yield_CFLAGS = -DDEBUG_YIELD $(AM_CFLAGS) -test_qsbr_SOURCES = test_qsbr.c $(URCU_QSBR) +test_urcu_qsbr_SOURCES = test_urcu_qsbr.c $(URCU_QSBR) -test_qsbr_timing_SOURCES = test_qsbr_timing.c $(URCU_QSBR) +test_urcu_qsbr_timing_SOURCES = test_urcu_qsbr_timing.c $(URCU_QSBR) test_urcu_mb_SOURCES = test_urcu.c $(URCU_MB) @@ -98,9 +98,9 @@ rcutorture_urcu_mb_SOURCES = urcutorture.c rcutorture_urcu_mb_CFLAGS = -DRCU_MB $(AM_CFLAGS) rcutorture_urcu_mb_LDADD = $(URCU_MB_LIB) -rcutorture_qsbr_SOURCES = urcutorture.c -rcutorture_qsbr_CFLAGS = -DTORTURE_QSBR -DRCU_QSBR $(AM_CFLAGS) -rcutorture_qsbr_LDADD = $(URCU_QSBR_LIB) +rcutorture_urcu_qsbr_SOURCES = urcutorture.c +rcutorture_urcu_qsbr_CFLAGS = -DTORTURE_QSBR -DRCU_QSBR $(AM_CFLAGS) +rcutorture_urcu_qsbr_LDADD = $(URCU_QSBR_LIB) rcutorture_urcu_signal_SOURCES = urcutorture.c rcutorture_urcu_signal_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) @@ -122,10 +122,10 @@ test_urcu_signal_gc_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) test_urcu_mb_gc_SOURCES = test_urcu_gc.c $(URCU_MB) test_urcu_mb_gc_CFLAGS = -DRCU_MB $(AM_CFLAGS) -test_qsbr_gc_SOURCES = test_qsbr_gc.c $(URCU_QSBR) +test_urcu_qsbr_gc_SOURCES = test_urcu_qsbr_gc.c $(URCU_QSBR) -test_qsbr_lgc_SOURCES = test_qsbr_gc.c $(URCU_QSBR) -test_qsbr_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) +test_urcu_qsbr_lgc_SOURCES = test_urcu_qsbr_gc.c $(URCU_QSBR) +test_urcu_qsbr_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) test_urcu_lgc_SOURCES = test_urcu_gc.c $(URCU) test_urcu_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) @@ -136,8 +136,8 @@ test_urcu_signal_lgc_CFLAGS = -DRCU_SIGNAL -DTEST_LOCAL_GC $(AM_CFLAGS) test_urcu_mb_lgc_SOURCES = test_urcu_gc.c $(URCU_MB) test_urcu_mb_lgc_CFLAGS = -DTEST_LOCAL_GC -DRCU_MB $(AM_CFLAGS) -test_qsbr_dynamic_link_SOURCES = test_qsbr.c $(URCU_QSBR) -test_qsbr_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_qsbr_dynamic_link_SOURCES = test_urcu_qsbr.c $(URCU_QSBR) +test_urcu_qsbr_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) test_urcu_defer_SOURCES = test_urcu_defer.c $(URCU_DEFER) diff --git a/tests/runtests-batch.sh b/tests/runtests-batch.sh index 67e019b..4340147 100755 --- a/tests/runtests-batch.sh +++ b/tests/runtests-batch.sh @@ -1,6 +1,6 @@ #!/bin/sh -#for a in test_urcu_gc test_urcu_gc_mb test_qsbr_gc; do +#for a in test_urcu_gc test_urcu_gc_mb test_urcu_qsbr_gc; do for a in test_urcu_gc; do echo "./${a} $*" | tee -a runall.detail.log /usr/bin/time --append --output runall.detail.log ./${a} $* diff --git a/tests/runtests.sh b/tests/runtests.sh index 4b5f745..aace60b 100755 --- a/tests/runtests.sh +++ b/tests/runtests.sh @@ -1,8 +1,8 @@ #!/bin/sh -for a in test_urcu_gc test_urcu_signal_gc test_urcu_mb_gc test_qsbr_gc \ - test_urcu_lgc test_urcu_signal_lgc test_urcu_mb_lgc test_qsbr_lgc \ - test_urcu test_urcu_signal test_urcu_mb test_qsbr \ +for a in test_urcu_gc test_urcu_signal_gc test_urcu_mb_gc test_urcu_qsbr_gc \ + test_urcu_lgc test_urcu_signal_lgc test_urcu_mb_lgc test_urcu_qsbr_lgc \ + test_urcu test_urcu_signal test_urcu_mb test_urcu_qsbr \ test_rwlock test_perthreadlock test_mutex; do echo "./${a} $*" | tee -a runall.detail.log /usr/bin/time --append --output runall.detail.log ./${a} $* diff --git a/tests/test_mutex.c b/tests/test_mutex.c index 1194b02..3f84bbf 100644 --- a/tests/test_mutex.c +++ b/tests/test_mutex.c @@ -205,11 +205,11 @@ void *thr_reader(void *data) for (;;) { pthread_mutex_lock(&lock); assert(test_array.a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); pthread_mutex_unlock(&lock); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -238,13 +238,13 @@ void *thr_writer(void *data) pthread_mutex_lock(&lock); test_array.a = 0; test_array.a = 8; - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); pthread_mutex_unlock(&lock); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_perthreadlock.c b/tests/test_perthreadlock.c index 6c7114c..fa9c89a 100644 --- a/tests/test_perthreadlock.c +++ b/tests/test_perthreadlock.c @@ -209,11 +209,11 @@ void *thr_reader(void *data) for (;;) { pthread_mutex_lock(&per_thread_lock[tidx].lock); assert(test_array.a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); pthread_mutex_unlock(&per_thread_lock[tidx].lock); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -245,15 +245,15 @@ void *thr_writer(void *data) } test_array.a = 0; test_array.a = 8; - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); for (tidx = (long)nr_readers - 1; tidx >= 0; tidx--) { pthread_mutex_unlock(&per_thread_lock[tidx].lock); } nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_qsbr.c b/tests/test_qsbr.c deleted file mode 100644 index 1c5a696..0000000 --- a/tests/test_qsbr.c +++ /dev/null @@ -1,465 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifdef __linux__ -#include -#endif - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#if defined(_syscall0) -_syscall0(pid_t, gettid) -#elif defined(__NR_gettid) -static inline pid_t gettid(void) -{ - return syscall(__NR_gettid); -} -#else -#warning "use pid as tid" -static inline pid_t gettid(void) -{ - return getpid(); -} -#endif - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include "urcu-qsbr.h" - -struct test_array { - int a; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static struct test_array *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long l) -{ - while(l-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - -static void set_affinity(void) -{ - cpu_set_t mask; - int cpu; - int ret; - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static unsigned long long __thread nr_writes; -static unsigned long long __thread nr_reads; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -/* - * malloc/free are reusing memory areas too quickly, which does not let us - * test races appropriately. Use a large circular array for allocations. - * ARRAY_SIZE is larger than nr_writers, and we keep the mutex across - * both alloc and free, which insures we never run over our tail. - */ -#define ARRAY_SIZE (1048576 * nr_writers) -#define ARRAY_POISON 0xDEADBEEF -static int array_index; -static struct test_array *test_array; - -static struct test_array *test_array_alloc(void) -{ - struct test_array *ret; - int index; - - index = array_index % ARRAY_SIZE; - assert(test_array[index].a == ARRAY_POISON || - test_array[index].a == 0); - ret = &test_array[index]; - array_index++; - if (array_index == ARRAY_SIZE) - array_index = 0; - return ret; -} - -static void test_array_free(struct test_array *ptr) -{ - if (!ptr) - return; - ptr->a = ARRAY_POISON; -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_array *local_ptr; - - printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - local_ptr = rcu_dereference(test_rcu_pointer); - debug_yield_read(); - if (local_ptr) - assert(local_ptr->a == 8); - if (unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - nr_reads++; - /* QS each 1024 reads */ - if (unlikely((nr_reads & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - if (unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - /* test extra thread registration */ - rcu_register_thread(); - rcu_unregister_thread(); - - *count = nr_reads; - printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - struct test_array *new, *old; - - printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_copy_mutex_lock(); - new = test_array_alloc(); - new->a = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (unlikely(wduration)) - loop_sleep(wduration); - synchronize_rcu(); - /* can be done after unlock */ - if (old) - old->a = 0; - test_array_free(old); - rcu_copy_mutex_unlock(); - nr_writes++; - if (unlikely(!test_duration_write())) - break; - if (unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - *count = nr_writes; - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s)", argv[0]); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)"); -#endif - printf(" [-d delay] (writer period (us))"); - printf(" [-c duration] (reader C.S. duration (in loops))"); - printf(" [-e duration] (writer C.S. duration (in loops))"); - printf(" [-v] (verbose output)"); - printf(" [-a cpu#] [-a cpu#]... (affinity)"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", - "main", pthread_self(), (unsigned long)gettid()); - - test_array = calloc(1, sizeof(*test_array) * ARRAY_SIZE); - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - test_array_free(test_rcu_pointer); - free(test_array); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_qsbr_gc.c b/tests/test_qsbr_gc.c deleted file mode 100644 index 0c1c124..0000000 --- a/tests/test_qsbr_gc.c +++ /dev/null @@ -1,490 +0,0 @@ -/* - * test_urcu_gc.c - * - * Userspace RCU library - test program (with baatch reclamation) - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifdef __linux__ -#include -#endif - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#if defined(_syscall0) -_syscall0(pid_t, gettid) -#elif defined(__NR_gettid) -static inline pid_t gettid(void) -{ - return syscall(__NR_gettid); -} -#else -#warning "use pid as tid" -static inline pid_t gettid(void) -{ - return getpid(); -} -#endif - -#define _LGPL_SOURCE -#include - -struct test_array { - int a; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static struct test_array *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; -static unsigned int reclaim_batch = 1; - -struct reclaim_queue { - void **queue; /* Beginning of queue */ - void **head; /* Insert position */ -}; - -static struct reclaim_queue *pending_reclaims; - - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long l) -{ - while(l-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - -static void set_affinity(void) -{ - cpu_set_t mask; - int cpu; - int ret; - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static unsigned long long __thread nr_writes; -static unsigned long long __thread nr_reads; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; - - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_array *local_ptr; - - printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - _rcu_read_lock(); - local_ptr = _rcu_dereference(test_rcu_pointer); - debug_yield_read(); - if (local_ptr) - assert(local_ptr->a == 8); - if (unlikely(rduration)) - loop_sleep(rduration); - _rcu_read_unlock(); - nr_reads++; - /* QS each 1024 reads */ - if (unlikely((nr_reads & ((1 << 10) - 1)) == 0)) - _rcu_quiescent_state(); - if (unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = nr_reads; - printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - return ((void*)1); - -} - -static void rcu_gc_clear_queue(unsigned long wtidx) -{ - void **p; - - /* Wait for Q.S and empty queue */ - synchronize_rcu(); - - for (p = pending_reclaims[wtidx].queue; - p < pending_reclaims[wtidx].head; p++) { - /* poison */ - if (*p) - ((struct test_array *)*p)->a = 0; - free(*p); - } - pending_reclaims[wtidx].head = pending_reclaims[wtidx].queue; -} - -/* Using per-thread queue */ -static void rcu_gc_reclaim(unsigned long wtidx, void *old) -{ - /* Queue pointer */ - *pending_reclaims[wtidx].head = old; - pending_reclaims[wtidx].head++; - - if (likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue - < reclaim_batch)) - return; - - rcu_gc_clear_queue(wtidx); -} - -void *thr_writer(void *data) -{ - unsigned long wtidx = (unsigned long)data; -#ifdef TEST_LOCAL_GC - struct test_array *old = NULL; -#else - struct test_array *new, *old; -#endif - - printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { -#ifndef TEST_LOCAL_GC - new = malloc(sizeof(*new)); - new->a = 8; - old = _rcu_xchg_pointer(&test_rcu_pointer, new); -#endif - if (unlikely(wduration)) - loop_sleep(wduration); - rcu_gc_reclaim(wtidx, old); - nr_writes++; - if (unlikely(!test_duration_write())) - break; - if (unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - tot_nr_writes[wtidx] = nr_writes; - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s)", argv[0]); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)"); -#endif - printf(" [-b batch] (batch reclaim)"); - printf(" [-d delay] (writer period (us))"); - printf(" [-c duration] (reader C.S. duration (in loops))"); - printf(" [-e duration] (writer C.S. duration (in loops))"); - printf(" [-v] (verbose output)"); - printf(" [-a cpu#] [-a cpu#]... (affinity)"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'b': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - reclaim_batch = atol(argv[++i]); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", - "main", pthread_self(), (unsigned long)gettid()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - tot_nr_writes = malloc(sizeof(*tot_nr_writes) * nr_writers); - pending_reclaims = malloc(sizeof(*pending_reclaims) * nr_writers); - if (reclaim_batch * sizeof(*pending_reclaims[i].queue) - < CAA_CACHE_LINE_SIZE) - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].queue = calloc(1, CAA_CACHE_LINE_SIZE); - else - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].queue = calloc(reclaim_batch, - sizeof(*pending_reclaims[i].queue)); - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].head = pending_reclaims[i].queue; - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += tot_nr_writes[i]; - rcu_gc_clear_queue(i); - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " - "batch %u\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes, reclaim_batch); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(tot_nr_writes); - for (i = 0; i < nr_writers; i++) - free(pending_reclaims[i].queue); - free(pending_reclaims); - - return 0; -} diff --git a/tests/test_qsbr_timing.c b/tests/test_qsbr_timing.c deleted file mode 100644 index 7925548..0000000 --- a/tests/test_qsbr_timing.c +++ /dev/null @@ -1,243 +0,0 @@ -/* - * test_qsbr_timing.c - * - * Userspace QSBR - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifdef __linux__ -#include -#endif - -#if defined(_syscall0) -_syscall0(pid_t, gettid) -#elif defined(__NR_gettid) -static inline pid_t gettid(void) -{ - return syscall(__NR_gettid); -} -#else -#warning "use pid as tid" -static inline pid_t gettid(void) -{ - return getpid(); -} -#endif - -#define _LGPL_SOURCE -#include - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -struct test_array { - int a; -}; - -static struct test_array *test_rcu_pointer; - -#define OUTER_READ_LOOP 2000U -#define INNER_READ_LOOP 100000U -#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) - -#define OUTER_WRITE_LOOP 10U -#define INNER_WRITE_LOOP 200U -#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) - -static int num_read; -static int num_write; - -#define NR_READ num_read -#define NR_WRITE num_write - -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; - -void *thr_reader(void *arg) -{ - int i, j; - struct test_array *local_ptr; - cycles_t time1, time2; - - printf("thread_begin %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - sleep(2); - - rcu_register_thread(); - - time1 = caa_get_cycles(); - for (i = 0; i < OUTER_READ_LOOP; i++) { - for (j = 0; j < INNER_READ_LOOP; j++) { - _rcu_read_lock(); - local_ptr = _rcu_dereference(test_rcu_pointer); - if (local_ptr) { - assert(local_ptr->a == 8); - } - _rcu_read_unlock(); - } - _rcu_quiescent_state(); - } - time2 = caa_get_cycles(); - - rcu_unregister_thread(); - - reader_time[(unsigned long)arg] = time2 - time1; - - sleep(2); - printf("thread_end %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - return ((void*)1); - -} - -void *thr_writer(void *arg) -{ - int i, j; - struct test_array *new, *old; - cycles_t time1, time2; - - printf("thread_begin %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - sleep(2); - - for (i = 0; i < OUTER_WRITE_LOOP; i++) { - for (j = 0; j < INNER_WRITE_LOOP; j++) { - time1 = caa_get_cycles(); - new = malloc(sizeof(struct test_array)); - rcu_copy_mutex_lock(); - old = test_rcu_pointer; - if (old) { - assert(old->a == 8); - } - new->a = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - rcu_copy_mutex_unlock(); - synchronize_rcu(); - /* can be done after unlock */ - if (old) { - old->a = 0; - } - free(old); - time2 = caa_get_cycles(); - writer_time[(unsigned long)arg] += time2 - time1; - usleep(1); - } - } - - printf("thread_end %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - return ((void*)2); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - int i; - cycles_t tot_rtime = 0; - cycles_t tot_wtime = 0; - - if (argc < 2) { - printf("Usage : %s nr_readers nr_writers\n", argv[0]); - exit(-1); - } - num_read = atoi(argv[1]); - num_write = atoi(argv[2]); - - reader_time = malloc(sizeof(*reader_time) * num_read); - writer_time = malloc(sizeof(*writer_time) * num_write); - tid_reader = malloc(sizeof(*tid_reader) * num_read); - tid_writer = malloc(sizeof(*tid_writer) * num_write); - - printf("thread %-6s, thread id : %lx, tid %lu\n", - "main", pthread_self(), (unsigned long)gettid()); - - for (i = 0; i < NR_READ; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - sleep(10); - - for (i = 0; i < NR_READ; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_rtime += reader_time[i]; - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_wtime += writer_time[i]; - } - free(test_rcu_pointer); - printf("Time per read : %g cycles\n", - (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); - printf("Time per write : %g cycles\n", - (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); - - free(reader_time); - free(writer_time); - free(tid_reader); - free(tid_writer); - - return 0; -} diff --git a/tests/test_rwlock.c b/tests/test_rwlock.c index 9c8f97e..34d8c07 100644 --- a/tests/test_rwlock.c +++ b/tests/test_rwlock.c @@ -201,11 +201,11 @@ void *thr_reader(void *_count) for (;;) { pthread_rwlock_rdlock(&lock); assert(test_array.a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); pthread_rwlock_unlock(&lock); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -234,13 +234,13 @@ void *thr_writer(void *_count) pthread_rwlock_wrlock(&lock); test_array.a = 0; test_array.a = 8; - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); pthread_rwlock_unlock(&lock); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu.c b/tests/test_urcu.c index 884d77c..870f133 100644 --- a/tests/test_urcu.c +++ b/tests/test_urcu.c @@ -239,11 +239,11 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); rcu_read_unlock(); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -280,7 +280,7 @@ void *thr_writer(void *_count) new = test_array_alloc(); new->a = 8; old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); synchronize_rcu(); if (old) @@ -288,9 +288,9 @@ void *thr_writer(void *_count) test_array_free(old); rcu_copy_mutex_unlock(); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_assign.c b/tests/test_urcu_assign.c index 0d9ef85..42d70c2 100644 --- a/tests/test_urcu_assign.c +++ b/tests/test_urcu_assign.c @@ -239,11 +239,11 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); rcu_read_unlock(); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -277,7 +277,7 @@ void *thr_writer(void *_count) new->a = 8; old = test_rcu_pointer; rcu_assign_pointer(test_rcu_pointer, new); - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); synchronize_rcu(); if (old) @@ -285,9 +285,9 @@ void *thr_writer(void *_count) test_array_free(old); rcu_copy_mutex_unlock(); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_bp.c b/tests/test_urcu_bp.c index ba80ae6..857913f 100644 --- a/tests/test_urcu_bp.c +++ b/tests/test_urcu_bp.c @@ -239,11 +239,11 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); rcu_read_unlock(); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -276,7 +276,7 @@ void *thr_writer(void *_count) new = test_array_alloc(); new->a = 8; old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); synchronize_rcu(); if (old) @@ -284,9 +284,9 @@ void *thr_writer(void *_count) test_array_free(old); rcu_copy_mutex_unlock(); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_defer.c b/tests/test_urcu_defer.c index 7d71f10..1575e9c 100644 --- a/tests/test_urcu_defer.c +++ b/tests/test_urcu_defer.c @@ -210,11 +210,11 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); rcu_read_unlock(); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -261,7 +261,7 @@ void *thr_writer(void *data) new = malloc(sizeof(*new)); new->a = 8; old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); defer_rcu(free, old); defer_rcu(test_cb1, old); @@ -272,9 +272,9 @@ void *thr_writer(void *data) defer_rcu(test_cb2, (void *)-4L); defer_rcu(test_cb2, (void *)-2L); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_gc.c b/tests/test_urcu_gc.c index 4eaa61b..21c5d56 100644 --- a/tests/test_urcu_gc.c +++ b/tests/test_urcu_gc.c @@ -218,11 +218,11 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); rcu_read_unlock(); nr_reads++; - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -259,7 +259,7 @@ static void rcu_gc_reclaim(unsigned long wtidx, void *old) *pending_reclaims[wtidx].head = old; pending_reclaims[wtidx].head++; - if (likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue + if (caa_likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue < reclaim_batch)) return; @@ -291,13 +291,13 @@ void *thr_writer(void *data) new->a = 8; old = rcu_xchg_pointer(&test_rcu_pointer, new); #endif - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); rcu_gc_reclaim(wtidx, old); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_lfq.c b/tests/test_urcu_lfq.c index 5292ebd..11e7eb3 100644 --- a/tests/test_urcu_lfq.c +++ b/tests/test_urcu_lfq.c @@ -190,11 +190,11 @@ void *thr_enqueuer(void *_count) rcu_read_unlock(); nr_successful_enqueues++; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); fail: nr_enqueues++; - if (unlikely(!test_duration_enqueue())) + if (caa_unlikely(!test_duration_enqueue())) break; } @@ -255,9 +255,9 @@ void *thr_dequeuer(void *_count) } nr_dequeues++; - if (unlikely(!test_duration_dequeue())) + if (caa_unlikely(!test_duration_dequeue())) break; - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); } diff --git a/tests/test_urcu_lfs.c b/tests/test_urcu_lfs.c index c85fa44..883fd0c 100644 --- a/tests/test_urcu_lfs.c +++ b/tests/test_urcu_lfs.c @@ -189,11 +189,11 @@ void *thr_enqueuer(void *_count) cds_lfs_push_rcu(&s, &node->list); nr_successful_enqueues++; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); fail: nr_enqueues++; - if (unlikely(!test_duration_enqueue())) + if (caa_unlikely(!test_duration_enqueue())) break; } @@ -252,9 +252,9 @@ void *thr_dequeuer(void *_count) nr_successful_dequeues++; } nr_dequeues++; - if (unlikely(!test_duration_dequeue())) + if (caa_unlikely(!test_duration_dequeue())) break; - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); } diff --git a/tests/test_urcu_qsbr.c b/tests/test_urcu_qsbr.c index 1c5a696..b986fd8 100644 --- a/tests/test_urcu_qsbr.c +++ b/tests/test_urcu_qsbr.c @@ -238,14 +238,14 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); rcu_read_unlock(); nr_reads++; /* QS each 1024 reads */ - if (unlikely((nr_reads & ((1 << 10) - 1)) == 0)) + if (caa_unlikely((nr_reads & ((1 << 10) - 1)) == 0)) rcu_quiescent_state(); - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -282,7 +282,7 @@ void *thr_writer(void *_count) new = test_array_alloc(); new->a = 8; old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); synchronize_rcu(); /* can be done after unlock */ @@ -291,9 +291,9 @@ void *thr_writer(void *_count) test_array_free(old); rcu_copy_mutex_unlock(); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_qsbr_gc.c b/tests/test_urcu_qsbr_gc.c index 0c1c124..9deb0aa 100644 --- a/tests/test_urcu_qsbr_gc.c +++ b/tests/test_urcu_qsbr_gc.c @@ -214,14 +214,14 @@ void *thr_reader(void *_count) debug_yield_read(); if (local_ptr) assert(local_ptr->a == 8); - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); _rcu_read_unlock(); nr_reads++; /* QS each 1024 reads */ - if (unlikely((nr_reads & ((1 << 10) - 1)) == 0)) + if (caa_unlikely((nr_reads & ((1 << 10) - 1)) == 0)) _rcu_quiescent_state(); - if (unlikely(!test_duration_read())) + if (caa_unlikely(!test_duration_read())) break; } @@ -258,7 +258,7 @@ static void rcu_gc_reclaim(unsigned long wtidx, void *old) *pending_reclaims[wtidx].head = old; pending_reclaims[wtidx].head++; - if (likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue + if (caa_likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue < reclaim_batch)) return; @@ -290,13 +290,13 @@ void *thr_writer(void *data) new->a = 8; old = _rcu_xchg_pointer(&test_rcu_pointer, new); #endif - if (unlikely(wduration)) + if (caa_unlikely(wduration)) loop_sleep(wduration); rcu_gc_reclaim(wtidx, old); nr_writes++; - if (unlikely(!test_duration_write())) + if (caa_unlikely(!test_duration_write())) break; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); } diff --git a/tests/test_urcu_wfq.c b/tests/test_urcu_wfq.c index e042f5e..83ec635 100644 --- a/tests/test_urcu_wfq.c +++ b/tests/test_urcu_wfq.c @@ -180,11 +180,11 @@ void *thr_enqueuer(void *_count) cds_wfq_enqueue(&q, node); nr_successful_enqueues++; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); fail: nr_enqueues++; - if (unlikely(!test_duration_enqueue())) + if (caa_unlikely(!test_duration_enqueue())) break; } @@ -221,9 +221,9 @@ void *thr_dequeuer(void *_count) } nr_dequeues++; - if (unlikely(!test_duration_dequeue())) + if (caa_unlikely(!test_duration_dequeue())) break; - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); } diff --git a/tests/test_urcu_wfs.c b/tests/test_urcu_wfs.c index 6d419ca..7746a1d 100644 --- a/tests/test_urcu_wfs.c +++ b/tests/test_urcu_wfs.c @@ -180,11 +180,11 @@ void *thr_enqueuer(void *_count) cds_wfs_push(&s, node); nr_successful_enqueues++; - if (unlikely(wdelay)) + if (caa_unlikely(wdelay)) loop_sleep(wdelay); fail: nr_enqueues++; - if (unlikely(!test_duration_enqueue())) + if (caa_unlikely(!test_duration_enqueue())) break; } @@ -221,9 +221,9 @@ void *thr_dequeuer(void *_count) } nr_dequeues++; - if (unlikely(!test_duration_dequeue())) + if (caa_unlikely(!test_duration_dequeue())) break; - if (unlikely(rduration)) + if (caa_unlikely(rduration)) loop_sleep(rduration); } diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h index 182e9b1..36e3cf4 100644 --- a/urcu-call-rcu-impl.h +++ b/urcu-call-rcu-impl.h @@ -204,7 +204,7 @@ static void call_rcu_wake_up(struct call_rcu_data *crdp) { /* Write to call_rcu list before reading/writing futex */ cmm_smp_mb(); - if (unlikely(uatomic_read(&crdp->futex) == -1)) { + if (caa_unlikely(uatomic_read(&crdp->futex) == -1)) { uatomic_set(&crdp->futex, 0); futex_async(&crdp->futex, FUTEX_WAKE, 1, NULL, NULL, 0); diff --git a/urcu-defer-impl.h b/urcu-defer-impl.h index 34d99c9..4d1ca5e 100644 --- a/urcu-defer-impl.h +++ b/urcu-defer-impl.h @@ -161,7 +161,7 @@ static void mutex_lock_defer(pthread_mutex_t *mutex) */ static void wake_up_defer(void) { - if (unlikely(uatomic_read(&defer_thread_futex) == -1)) { + if (caa_unlikely(uatomic_read(&defer_thread_futex) == -1)) { uatomic_set(&defer_thread_futex, 0); futex_noasync(&defer_thread_futex, FUTEX_WAKE, 1, NULL, NULL, 0); @@ -225,11 +225,11 @@ static void rcu_defer_barrier_queue(struct defer_queue *queue, for (i = queue->tail; i != head;) { cmm_smp_rmb(); /* read head before q[]. */ p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); - if (unlikely(DQ_IS_FCT_BIT(p))) { + if (caa_unlikely(DQ_IS_FCT_BIT(p))) { DQ_CLEAR_FCT_BIT(p); queue->last_fct_out = p; p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); - } else if (unlikely(p == DQ_FCT_MARK)) { + } else if (caa_unlikely(p == DQ_FCT_MARK)) { p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); queue->last_fct_out = p; p = CMM_LOAD_SHARED(queue->q[i++ & DEFER_QUEUE_MASK]); @@ -247,7 +247,7 @@ static void _rcu_defer_barrier_thread(void) head = defer_queue.head; num_items = head - defer_queue.tail; - if (unlikely(!num_items)) + if (caa_unlikely(!num_items)) return; synchronize_rcu(); rcu_defer_barrier_queue(&defer_queue, head); @@ -286,7 +286,7 @@ void rcu_defer_barrier(void) index->last_head = CMM_LOAD_SHARED(index->head); num_items += index->last_head - index->tail; } - if (likely(!num_items)) { + if (caa_likely(!num_items)) { /* * We skip the grace period because there are no queued * callbacks to execute. @@ -318,7 +318,7 @@ void _defer_rcu(void (*fct)(void *p), void *p) * If queue is full, or reached threshold. Empty queue ourself. * Worse-case: must allow 2 supplementary entries for fct pointer. */ - if (unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) { + if (caa_unlikely(head - tail >= DEFER_QUEUE_SIZE - 2)) { assert(head - tail <= DEFER_QUEUE_SIZE); rcu_defer_barrier_thread(); assert(head - CMM_LOAD_SHARED(defer_queue.tail) == 0); @@ -340,11 +340,11 @@ void _defer_rcu(void (*fct)(void *p), void *p) * Decode: see the comments before 'struct defer_queue' * or the code in rcu_defer_barrier_queue(). */ - if (unlikely(defer_queue.last_fct_in != fct + if (caa_unlikely(defer_queue.last_fct_in != fct || DQ_IS_FCT_BIT(p) || p == DQ_FCT_MARK)) { defer_queue.last_fct_in = fct; - if (unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) { + if (caa_unlikely(DQ_IS_FCT_BIT(fct) || fct == DQ_FCT_MARK)) { _CMM_STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], DQ_FCT_MARK); _CMM_STORE_SHARED(defer_queue.q[head++ & DEFER_QUEUE_MASK], diff --git a/urcu.c b/urcu.c index 77f6888..ba013d9 100644 --- a/urcu.c +++ b/urcu.c @@ -144,7 +144,7 @@ static void mutex_unlock(pthread_mutex_t *mutex) #ifdef RCU_MEMBARRIER static void smp_mb_master(int group) { - if (likely(has_sys_membarrier)) + if (caa_likely(has_sys_membarrier)) membarrier(MEMBARRIER_EXPEDITED); else cmm_smp_mb(); diff --git a/urcu/compiler.h b/urcu/compiler.h index 6db803e..f977b95 100644 --- a/urcu/compiler.h +++ b/urcu/compiler.h @@ -20,8 +20,8 @@ #include /* for offsetof */ -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) +#define caa_likely(x) __builtin_expect(!!(x), 1) +#define caa_unlikely(x) __builtin_expect(!!(x), 0) #define cmm_barrier() asm volatile("" : : : "memory") diff --git a/urcu/static/urcu-bp.h b/urcu/static/urcu-bp.h index 832ba0f..8d22163 100644 --- a/urcu/static/urcu-bp.h +++ b/urcu/static/urcu-bp.h @@ -166,7 +166,7 @@ static inline void _rcu_read_lock(void) long tmp; /* Check if registered */ - if (unlikely(!rcu_reader)) + if (caa_unlikely(!rcu_reader)) rcu_bp_register(); cmm_barrier(); /* Ensure the compiler does not reorder us with mutex */ @@ -175,7 +175,7 @@ static inline void _rcu_read_lock(void) * rcu_gp_ctr is * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) */ - if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { + if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { _CMM_STORE_SHARED(rcu_reader->ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); /* * Set active readers count for outermost nesting level before diff --git a/urcu/static/urcu-qsbr.h b/urcu/static/urcu-qsbr.h index 489abb0..68bfc31 100644 --- a/urcu/static/urcu-qsbr.h +++ b/urcu/static/urcu-qsbr.h @@ -137,7 +137,7 @@ extern int32_t gp_futex; */ static inline void wake_up_gp(void) { - if (unlikely(_CMM_LOAD_SHARED(rcu_reader.waiting))) { + if (caa_unlikely(_CMM_LOAD_SHARED(rcu_reader.waiting))) { _CMM_STORE_SHARED(rcu_reader.waiting, 0); cmm_smp_mb(); if (uatomic_read(&gp_futex) != -1) diff --git a/urcu/static/urcu.h b/urcu/static/urcu.h index b993375..7ae0185 100644 --- a/urcu/static/urcu.h +++ b/urcu/static/urcu.h @@ -176,7 +176,7 @@ extern int has_sys_membarrier; static inline void smp_mb_slave(int group) { - if (likely(has_sys_membarrier)) + if (caa_likely(has_sys_membarrier)) cmm_barrier(); else cmm_smp_mb(); @@ -231,7 +231,7 @@ extern int32_t gp_futex; */ static inline void wake_up_gp(void) { - if (unlikely(uatomic_read(&gp_futex) == -1)) { + if (caa_unlikely(uatomic_read(&gp_futex) == -1)) { uatomic_set(&gp_futex, 0); futex_async(&gp_futex, FUTEX_WAKE, 1, NULL, NULL, 0); @@ -261,7 +261,7 @@ static inline void _rcu_read_lock(void) * rcu_gp_ctr is * RCU_GP_COUNT | (~RCU_GP_CTR_PHASE or RCU_GP_CTR_PHASE) */ - if (likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { + if (caa_likely(!(tmp & RCU_GP_CTR_NEST_MASK))) { _CMM_STORE_SHARED(rcu_reader.ctr, _CMM_LOAD_SHARED(rcu_gp_ctr)); /* * Set active readers count for outermost nesting level before @@ -282,7 +282,7 @@ static inline void _rcu_read_unlock(void) * Finish using rcu before decrementing the pointer. * See smp_mb_master(). */ - if (likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { + if (caa_likely((tmp & RCU_GP_CTR_NEST_MASK) == RCU_GP_COUNT)) { smp_mb_slave(RCU_MB_GROUP); _CMM_STORE_SHARED(rcu_reader.ctr, rcu_reader.ctr - RCU_GP_COUNT); /* write rcu_reader.ctr before read futex */ diff --git a/urcu/uatomic/x86.h b/urcu/uatomic/x86.h index e064b9e..9b67f16 100644 --- a/urcu/uatomic/x86.h +++ b/urcu/uatomic/x86.h @@ -505,9 +505,9 @@ extern int __rcu_cas_avail; extern int __rcu_cas_init(void); #define UATOMIC_COMPAT(insn) \ - ((likely(__rcu_cas_avail > 0)) \ + ((caa_likely(__rcu_cas_avail > 0)) \ ? (_uatomic_##insn) \ - : ((unlikely(__rcu_cas_avail < 0) \ + : ((caa_unlikely(__rcu_cas_avail < 0) \ ? ((__rcu_cas_init() > 0) \ ? (_uatomic_##insn) \ : (compat_uatomic_##insn)) \