From 453629a9317adef5b96c3d55e4dcd98db680997a Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Mon, 12 Jul 2010 12:16:30 -0400 Subject: [PATCH] Add lock-free RCU queue and stack Signed-off-by: Mathieu Desnoyers --- tests/Makefile.am | 6 +- tests/api_gcc.h | 11 -- tests/api_ppc.h | 11 -- tests/api_x86.h | 11 -- tests/test_urcu_lfq.c | 438 ++++++++++++++++++++++++++++++++++++++++++ tests/test_urcu_lfs.c | 423 ++++++++++++++++++++++++++++++++++++++++ urcu/compiler.h | 6 + urcu/rculfqueue.h | 136 +++++++++++++ urcu/rculfstack.h | 88 +++++++++ urcu/urcu_ref.h | 47 +++++ 10 files changed, 1143 insertions(+), 34 deletions(-) create mode 100644 tests/test_urcu_lfq.c create mode 100644 tests/test_urcu_lfs.c create mode 100644 urcu/rculfqueue.h create mode 100644 urcu/rculfstack.h create mode 100644 urcu/urcu_ref.h diff --git a/tests/Makefile.am b/tests/Makefile.am index 0ff6b3e..9fca6b1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -12,7 +12,8 @@ noinst_PROGRAMS = test_urcu test_urcu_dynamic_link test_urcu_timing \ test_urcu_mb_gc test_qsbr_gc test_qsbr_lgc test_urcu_signal_lgc \ test_urcu_mb_lgc test_qsbr_dynamic_link test_urcu_defer \ test_uatomic test_urcu_assign test_urcu_assign_dynamic_link \ - test_urcu_bp test_urcu_bp_dynamic_link test_cycles_per_loop + test_urcu_bp test_urcu_bp_dynamic_link test_cycles_per_loop \ + test_urcu_lfq test_urcu_lfs noinst_HEADERS = rcutorture.h if COMPAT_ARCH @@ -151,6 +152,9 @@ test_urcu_bp_SOURCES = test_urcu_bp.c $(URCU_BP) test_urcu_bp_dynamic_link_SOURCES = test_urcu_bp.c $(URCU_BP) test_urcu_bp_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_lfq_SOURCES = test_urcu_lfq.c $(URCU_DEFER) +test_urcu_lfs_SOURCES = test_urcu_lfs.c $(URCU_DEFER) + urcutorture.c: api.h check-am: diff --git a/tests/api_gcc.h b/tests/api_gcc.h index 17487d5..be437cc 100644 --- a/tests/api_gcc.h +++ b/tests/api_gcc.h @@ -291,13 +291,6 @@ cmpxchg(volatile long *ptr, long oldval, long newval) #include /* #include "atomic.h" */ -/* - * Compiler magic. - */ -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - /* * Default machine parameters. */ @@ -652,10 +645,6 @@ static void smp_init(void) #define LIST_POISON1 ((void *) 0x00100100) #define LIST_POISON2 ((void *) 0x00200200) -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - #if 0 /* * Simple doubly linked list implementation. diff --git a/tests/api_ppc.h b/tests/api_ppc.h index 18821e4..6548077 100644 --- a/tests/api_ppc.h +++ b/tests/api_ppc.h @@ -656,13 +656,6 @@ static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) #include /* #include "atomic.h" */ -/* - * Compiler magic. - */ -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - /* * Default machine parameters. */ @@ -1010,10 +1003,6 @@ static void smp_init(void) #define LIST_POISON1 ((void *) 0x00100100) #define LIST_POISON2 ((void *) 0x00200200) -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - #if 0 /* diff --git a/tests/api_x86.h b/tests/api_x86.h index eba0613..cdd4667 100644 --- a/tests/api_x86.h +++ b/tests/api_x86.h @@ -352,13 +352,6 @@ __asm__ __volatile__(LOCK_PREFIX "orl %0,%1" \ #include /* #include "atomic.h" */ -/* - * Compiler magic. - */ -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - /* * Default machine parameters. */ @@ -713,10 +706,6 @@ static void smp_init(void) #define LIST_POISON1 ((void *) 0x00100100) #define LIST_POISON2 ((void *) 0x00200200) -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - #if 0 /* diff --git a/tests/test_urcu_lfq.c b/tests/test_urcu_lfq.c new file mode 100644 index 0000000..87c53b6 --- /dev/null +++ b/tests/test_urcu_lfq.c @@ -0,0 +1,438 @@ +/* + * test_urcu_lfq.c + * + * Userspace RCU library - example RCU-based lock-free queue + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#if defined(_syscall0) +_syscall0(pid_t, gettid) +#elif defined(__NR_gettid) +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} +#else +#warning "use pid as tid" +static inline pid_t gettid(void) +{ + return getpid(); +} +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include +#include +#include + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long l) +{ + while(l-- != 0) + cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +#ifndef HAVE_CPU_SET_T +typedef unsigned long cpu_set_t; +# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) +# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) +#endif + +static void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static unsigned long long __thread nr_dequeues; +static unsigned long long __thread nr_enqueues; + +static unsigned long long __thread nr_successful_dequeues; +static unsigned long long __thread nr_successful_enqueues; + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +static struct rcu_lfq_queue q; + +void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "enqueuer", pthread_self(), (unsigned long)gettid()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + smp_mb(); + + for (;;) { + struct rcu_lfq_node *node = malloc(sizeof(*node)); + if (!node) + goto fail; + rcu_lfq_node_init(node); + rcu_lfq_enqueue(&q, node); + nr_successful_enqueues++; + + if (unlikely(wdelay)) + loop_sleep(wdelay); +fail: + nr_enqueues++; + if (unlikely(!test_duration_enqueue())) + break; + } + + rcu_unregister_thread(); + + count[0] = nr_enqueues; + count[1] = nr_successful_enqueues; + printf_verbose("enqueuer thread_end, thread id : %lx, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + pthread_self(), (unsigned long)gettid(), nr_enqueues, + nr_successful_enqueues); + return ((void*)1); + +} + +static void rcu_release_node(struct urcu_ref *ref) +{ + struct rcu_lfq_node *node = container_of(ref, struct rcu_lfq_node, ref); + defer_rcu(free, node); + //synchronize_rcu(); + //free(node); +} + +void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "dequeuer", pthread_self(), (unsigned long)gettid()); + + set_affinity(); + + rcu_defer_register_thread(); + rcu_register_thread(); + + while (!test_go) + { + } + smp_mb(); + + for (;;) { + struct rcu_lfq_node *node = rcu_lfq_dequeue(&q, + rcu_release_node); + + if (node) { + urcu_ref_put(&node->ref, rcu_release_node); + nr_successful_dequeues++; + } + + nr_dequeues++; + if (unlikely(!test_duration_dequeue())) + break; + if (unlikely(rduration)) + loop_sleep(rduration); + } + + rcu_unregister_thread(); + rcu_defer_unregister_thread(); + + printf_verbose("dequeuer thread_end, thread id : %lx, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + pthread_self(), (unsigned long)gettid(), nr_dequeues, + nr_successful_dequeues); + count[0] = nr_dequeues; + count[1] = nr_successful_dequeues; + return ((void*)2); +} + +static void release_node(struct urcu_ref *ref) +{ + struct rcu_lfq_node *node = container_of(ref, struct rcu_lfq_node, ref); + free(node); +} + +void test_end(struct rcu_lfq_queue *q, unsigned long long *nr_dequeues) +{ + struct rcu_lfq_node *node; + + do { + node = rcu_lfq_dequeue(q, release_node); + if (node) { + urcu_ref_put(&node->ref, release_node); + (*nr_dequeues)++; + } + } while (node); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s)", argv[0]); + printf(" [-d delay] (enqueuer period (in loops))"); + printf(" [-c duration] (dequeuer period (in loops))"); + printf(" [-v] (verbose output)"); + printf(" [-a cpu#] [-a cpu#]... (affinity)"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", + "main", pthread_self(), (unsigned long)gettid()); + + tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); + tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); + count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); + count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); + rcu_lfq_init(&q); + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + write (1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&q, &end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/tests/test_urcu_lfs.c b/tests/test_urcu_lfs.c new file mode 100644 index 0000000..8f881e7 --- /dev/null +++ b/tests/test_urcu_lfs.c @@ -0,0 +1,423 @@ +/* + * test_urcu_lfs.c + * + * Userspace RCU library - example RCU-based lock-free stack + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#if defined(_syscall0) +_syscall0(pid_t, gettid) +#elif defined(__NR_gettid) +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} +#else +#warning "use pid as tid" +static inline pid_t gettid(void) +{ + return getpid(); +} +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include +#include +#include + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long l) +{ + while(l-- != 0) + cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +#ifndef HAVE_CPU_SET_T +typedef unsigned long cpu_set_t; +# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) +# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) +#endif + +static void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static unsigned long long __thread nr_dequeues; +static unsigned long long __thread nr_enqueues; + +static unsigned long long __thread nr_successful_dequeues; +static unsigned long long __thread nr_successful_enqueues; + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +static struct rcu_lfs_stack s; + +void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "enqueuer", pthread_self(), (unsigned long)gettid()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + smp_mb(); + + for (;;) { + struct rcu_lfs_node *node = malloc(sizeof(*node)); + if (!node) + goto fail; + rcu_lfs_node_init(node); + rcu_lfs_push(&s, node); + nr_successful_enqueues++; + + if (unlikely(wdelay)) + loop_sleep(wdelay); +fail: + nr_enqueues++; + if (unlikely(!test_duration_enqueue())) + break; + } + + rcu_unregister_thread(); + + count[0] = nr_enqueues; + count[1] = nr_successful_enqueues; + printf_verbose("enqueuer thread_end, thread id : %lx, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + pthread_self(), (unsigned long)gettid(), nr_enqueues, + nr_successful_enqueues); + return ((void*)1); + +} + +void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "dequeuer", pthread_self(), (unsigned long)gettid()); + + set_affinity(); + + rcu_defer_register_thread(); + rcu_register_thread(); + + while (!test_go) + { + } + smp_mb(); + + for (;;) { + struct rcu_lfs_node *node = rcu_lfs_pop(&s); + + if (node) { + defer_rcu(free, node); + nr_successful_dequeues++; + } + + nr_dequeues++; + if (unlikely(!test_duration_dequeue())) + break; + if (unlikely(rduration)) + loop_sleep(rduration); + } + + rcu_unregister_thread(); + rcu_defer_unregister_thread(); + + printf_verbose("dequeuer thread_end, thread id : %lx, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + pthread_self(), (unsigned long)gettid(), nr_dequeues, + nr_successful_dequeues); + count[0] = nr_dequeues; + count[1] = nr_successful_dequeues; + return ((void*)2); +} + +void test_end(struct rcu_lfs_stack *s, unsigned long long *nr_dequeues) +{ + struct rcu_lfs_node *node; + + do { + node = rcu_lfs_pop(s); + if (node) { + free(node); + (*nr_dequeues)++; + } + } while (node); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s)", argv[0]); + printf(" [-d delay] (enqueuer period (in loops))"); + printf(" [-c duration] (dequeuer period (in loops))"); + printf(" [-v] (verbose output)"); + printf(" [-a cpu#] [-a cpu#]... (affinity)"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", + "main", pthread_self(), (unsigned long)gettid()); + + tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); + tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); + count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); + count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); + rcu_lfs_init(&s); + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + write (1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&s, &end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/urcu/compiler.h b/urcu/compiler.h index fcab5ea..fb8b829 100644 --- a/urcu/compiler.h +++ b/urcu/compiler.h @@ -55,4 +55,10 @@ #define BITS_PER_LONG 32 #endif +#define container_of(ptr, type, member) \ + ({ \ + const typeof(((type *)NULL)->member) * __ptr = (ptr); \ + (type *)((char *)__ptr - offsetof(type, member)); \ + }) + #endif /* _URCU_COMPILER_H */ diff --git a/urcu/rculfqueue.h b/urcu/rculfqueue.h new file mode 100644 index 0000000..f9b14fe --- /dev/null +++ b/urcu/rculfqueue.h @@ -0,0 +1,136 @@ +/* + * rculfqueue.h + * + * Userspace RCU library - Lock-Free RCU Queue + * + * Copyright 2010 - Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include +#include + +#if (!defined(_GNU_SOURCE) && !defined(_LGPL_SOURCE)) +#error "Dynamic loader LGPL wrappers not implemented yet" +#endif + +/* + * Lock-free RCU queue using reference counting. Enqueue and dequeue operations + * hold a RCU read lock to deal with cmpxchg ABA problem. This implementation + * keeps a dummy head node to ensure we can always update the queue locklessly. + * Given that this is a queue, the dummy head node must always advance as we + * dequeue entries. Therefore, we keep a reference count on each entry we are + * dequeueing, so they can be kept as dummy head node until the next dequeue, at + * which point their reference count will be decremented. + */ + +#define URCU_LFQ_PERMANENT_REF 128 + +struct rcu_lfq_node { + struct rcu_lfq_node *next; + struct urcu_ref ref; +}; + +struct rcu_lfq_queue { + struct rcu_lfq_node *head, *tail; + struct rcu_lfq_node init; /* Dummy initialization node */ +}; + +void rcu_lfq_node_init(struct rcu_lfq_node *node) +{ + node->next = NULL; + urcu_ref_init(&node->ref); +} + +void rcu_lfq_init(struct rcu_lfq_queue *q) +{ + rcu_lfq_node_init(&q->init); + /* Make sure the initial node is never freed. */ + urcu_ref_set(&q->init.ref, URCU_LFQ_PERMANENT_REF); + q->head = q->tail = &q->init; +} + +void rcu_lfq_enqueue(struct rcu_lfq_queue *q, struct rcu_lfq_node *node) +{ + urcu_ref_get(&node->ref); + + /* + * uatomic_cmpxchg() implicit memory barrier orders earlier stores to + * node before publication. + */ + + rcu_read_lock(); + for (;;) { + struct rcu_lfq_node *tail = rcu_dereference(q->tail); + struct rcu_lfq_node *next; + + /* + * Typically expect tail to be NULL. + */ + next = uatomic_cmpxchg(&tail->next, NULL, node); + if (next == NULL) { + /* + * Tail was at the end of queue, we successfully + * appended to it. + * Now move tail (another enqueue might beat + * us to it, that's fine). + */ + uatomic_cmpxchg(&q->tail, tail, node); + rcu_read_unlock(); + return; + } else { + /* + * Failure to append to current tail. Help moving tail + * further and retry. + */ + uatomic_cmpxchg(&q->tail, tail, next); + continue; + } + } +} + +/* + * The entry returned by dequeue must be taken care of by doing a urcu_ref_put, + * which calls the release primitive when the reference count drops to zero. A + * grace period must be waited before performing the actual memory reclamation + * in the release primitive. + * The entry lfq node returned by dequeue must no be re-used before the + * reference count reaches zero. + */ +struct rcu_lfq_node * +rcu_lfq_dequeue(struct rcu_lfq_queue *q, void (*release)(struct urcu_ref *)) +{ + rcu_read_lock(); + for (;;) { + struct rcu_lfq_node *head = rcu_dereference(q->head); + struct rcu_lfq_node *next = rcu_dereference(head->next); + + if (next) { + if (uatomic_cmpxchg(&q->head, head, next) == head) { + rcu_read_unlock(); + urcu_ref_put(&head->ref, release); + return next; + } else { + /* Concurrently pushed, retry */ + continue; + } + } else { + /* Empty */ + rcu_read_unlock(); + return NULL; + } + } +} diff --git a/urcu/rculfstack.h b/urcu/rculfstack.h new file mode 100644 index 0000000..f43c9d8 --- /dev/null +++ b/urcu/rculfstack.h @@ -0,0 +1,88 @@ +/* + * rculfstack.h + * + * Userspace RCU library - Lock-Free RCU Stack + * + * Copyright 2010 - Mathieu Desnoyers + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#if (!defined(_GNU_SOURCE) && !defined(_LGPL_SOURCE)) +#error "Dynamic loader LGPL wrappers not implemented yet" +#endif + +struct rcu_lfs_node { + struct rcu_lfs_node *next; +}; + +struct rcu_lfs_stack { + struct rcu_lfs_node *head; +}; + +void rcu_lfs_node_init(struct rcu_lfs_node *node) +{ +} + +void rcu_lfs_init(struct rcu_lfs_stack *s) +{ + s->head = NULL; +} + +void rcu_lfs_push(struct rcu_lfs_stack *s, struct rcu_lfs_node *node) +{ + rcu_read_lock(); + for (;;) { + struct rcu_lfs_node *head = rcu_dereference(s->head); + + node->next = head; + /* + * uatomic_cmpxchg() implicit memory barrier orders earlier + * stores to node before publication. + */ + if (uatomic_cmpxchg(&s->head, head, node) == head) { + rcu_read_unlock(); + return; + } else { + /* Failure to prepend. Retry. */ + continue; + } + } +} + +struct rcu_lfs_node * +rcu_lfs_pop(struct rcu_lfs_stack *s) +{ + rcu_read_lock(); + for (;;) { + struct rcu_lfs_node *head = rcu_dereference(s->head); + + if (head) { + struct rcu_lfs_node *next = rcu_dereference(head->next); + + if (uatomic_cmpxchg(&s->head, head, next) == head) { + rcu_read_unlock(); + return head; + } else { + /* Concurrent modification. Retry. */ + continue; + } + } else { + /* Empty stack */ + rcu_read_unlock(); + return NULL; + } + } +} diff --git a/urcu/urcu_ref.h b/urcu/urcu_ref.h new file mode 100644 index 0000000..bce7116 --- /dev/null +++ b/urcu/urcu_ref.h @@ -0,0 +1,47 @@ +#ifndef _URCU_REF_H +#define _URCU_REF_H + +/* + * Userspace RCU - Reference counting + * + * Copyright (C) 2009 Novell Inc. + * Copyright (C) 2010 Mathieu Desnoyers + * + * Author: Jan Blunck + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License version 2.1 as + * published by the Free Software Foundation. + */ + +#include +#include + +struct urcu_ref { + long refcount; /* ATOMIC */ +}; + +static inline void urcu_ref_set(struct urcu_ref *ref, long val) +{ + uatomic_set(&ref->refcount, val); +} + +static inline void urcu_ref_init(struct urcu_ref *ref) +{ + urcu_ref_set(ref, 1); +} + +static inline void urcu_ref_get(struct urcu_ref *ref) +{ + long res = uatomic_add_return(&ref->refcount, 1); + assert(res != 0); +} + +static inline void urcu_ref_put(struct urcu_ref *ref, + void (*release)(struct urcu_ref *)) +{ + if (!uatomic_sub_return(&ref->refcount, 1)) + release(ref); +} + +#endif /* _URCU_REF_H */ -- 2.34.1