From 95d8822d8b4c4a3563bf51c7c718350eb2babd20 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 22 Feb 2013 09:05:32 -0500 Subject: [PATCH] Fix tests: finer-grained use of CPU_SET, CPU_ZERO and cpu_set_t Noticed build failure at https://buildd.debian.org/status/package.php?p=liburcu : Tail of log for liburcu on hurd-i386: test_urcu.c:110:0: warning: "CPU_SET" redefined [enabled by default] In file included from /usr/include/pthread/pthread.h:50:0, from /usr/include/pthread.h:2, from test_urcu.c:26: /usr/include/sched.h:80:0: note: this is the location of the previous definition make[3]: *** [test_urcu.o] Error 1 make[2]: *** [all-recursive] Error 1 make[1]: *** [all] Error 2 dh_auto_build: make -j1 returned exit code 2 make: *** [build-arch] Error 2 dpkg-buildpackage: error: debian/rules build-arch gave error exit status 2 make[3]: Entering directory `/build/buildd-liburcu_0.7.6-1-hurd-i386-wGBAtt/liburcu-0.7.6/tests' CC test_urcu.o make[3]: Leaving directory `/build/buildd-liburcu_0.7.6-1-hurd-i386-wGBAtt/liburcu-0.7.6/tests' make[2]: Leaving directory `/build/buildd-liburcu_0.7.6-1-hurd-i386-wGBAtt/liburcu-0.7.6' Signed-off-by: Mathieu Desnoyers --- tests/Makefile.am | 2 +- tests/cpuset.h | 45 +++ tests/test_mutex.c | 8 +- tests/test_perthreadlock.c | 8 +- tests/test_rwlock.c | 8 +- tests/test_urcu.c | 8 +- tests/test_urcu_assign.c | 8 +- tests/test_urcu_bp.c | 8 +- tests/test_urcu_defer.c | 8 +- tests/test_urcu_gc.c | 8 +- tests/test_urcu_hash.h | 8 +- tests/test_urcu_lfq.c | 8 +- tests/test_urcu_lfs.c | 8 +- tests/test_urcu_lfs_rcu.c | 454 ++++++++++++++++++++++++++++ tests/test_urcu_qsbr.c | 8 +- tests/test_urcu_qsbr_gc.c | 8 +- tests/test_urcu_wfcq.c | 586 +++++++++++++++++++++++++++++++++++++ tests/test_urcu_wfq.c | 8 +- tests/test_urcu_wfs.c | 8 +- 19 files changed, 1101 insertions(+), 106 deletions(-) create mode 100644 tests/cpuset.h create mode 100644 tests/test_urcu_lfs_rcu.c create mode 100644 tests/test_urcu_wfcq.c diff --git a/tests/Makefile.am b/tests/Makefile.am index bae181c..04f6a0c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,7 +17,7 @@ noinst_PROGRAMS = test_urcu test_urcu_dynamic_link test_urcu_timing \ test_urcu_wfq_dynlink test_urcu_wfs_dynlink \ test_urcu_lfq_dynlink test_urcu_lfs_dynlink test_urcu_hash \ test_urcu_fork -noinst_HEADERS = rcutorture.h +noinst_HEADERS = rcutorture.h cpuset.h if COMPAT_ARCH COMPAT=$(top_srcdir)/compat_arch_@ARCHTYPE@.c diff --git a/tests/cpuset.h b/tests/cpuset.h new file mode 100644 index 0000000..3f1587d --- /dev/null +++ b/tests/cpuset.h @@ -0,0 +1,45 @@ +#ifndef _URCU_TESTS_CPUSET_H +#define _URCU_TESTS_CPUSET_H + +/* + * cpuset.h + * + * Userspace RCU library - test cpuset header + * + * Copyright 2009-2013 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "../config.h" + +#if defined(HAVE_SCHED_SETAFFINITY) || defined(HAVE_CPU_SET_T) \ + || defined(HAVE_CPU_ZERO) || defined(HAVE_CPU_SET) +# include +#endif + +#ifndef HAVE_CPU_SET_T +typedef unsigned long cpu_set_t; +#endif + +#ifndef HAVE_CPU_ZERO +# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) +#endif + +#ifndef HAVE_CPU_SET +# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) +#endif + +#endif /* _URCU_TESTS_CPUSET_H */ diff --git a/tests/test_mutex.c b/tests/test_mutex.c index 99e2553..4a6912f 100644 --- a/tests/test_mutex.c +++ b/tests/test_mutex.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -106,12 +106,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_perthreadlock.c b/tests/test_perthreadlock.c index 86a34e2..655c287 100644 --- a/tests/test_perthreadlock.c +++ b/tests/test_perthreadlock.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -110,12 +110,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_rwlock.c b/tests/test_rwlock.c index a6f8658..6c137cb 100644 --- a/tests/test_rwlock.c +++ b/tests/test_rwlock.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -106,12 +106,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu.c b/tests/test_urcu.c index f904ddd..62dfa18 100644 --- a/tests/test_urcu.c +++ b/tests/test_urcu.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -104,12 +104,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_assign.c b/tests/test_urcu_assign.c index b510ec8..74c6b19 100644 --- a/tests/test_urcu_assign.c +++ b/tests/test_urcu_assign.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -104,12 +104,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_bp.c b/tests/test_urcu_bp.c index 26ed741..30c79c1 100644 --- a/tests/test_urcu_bp.c +++ b/tests/test_urcu_bp.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -104,12 +104,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_defer.c b/tests/test_urcu_defer.c index d995b37..b759e84 100644 --- a/tests/test_urcu_defer.c +++ b/tests/test_urcu_defer.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -105,12 +105,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_gc.c b/tests/test_urcu_gc.c index d2de751..498abc6 100644 --- a/tests/test_urcu_gc.c +++ b/tests/test_urcu_gc.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -113,12 +113,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_hash.h b/tests/test_urcu_hash.h index fe13c36..eaf1d79 100644 --- a/tests/test_urcu_hash.h +++ b/tests/test_urcu_hash.h @@ -33,11 +33,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -195,12 +195,6 @@ extern int use_affinity; extern pthread_mutex_t affinity_mutex; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - void set_affinity(void); /* diff --git a/tests/test_urcu_lfq.c b/tests/test_urcu_lfq.c index a2b1ef6..87188d8 100644 --- a/tests/test_urcu_lfq.c +++ b/tests/test_urcu_lfq.c @@ -34,11 +34,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -98,12 +98,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_lfs.c b/tests/test_urcu_lfs.c index 262add9..6bcf1be 100644 --- a/tests/test_urcu_lfs.c +++ b/tests/test_urcu_lfs.c @@ -34,11 +34,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -98,12 +98,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_lfs_rcu.c b/tests/test_urcu_lfs_rcu.c new file mode 100644 index 0000000..b6d830c --- /dev/null +++ b/tests/test_urcu_lfs_rcu.c @@ -0,0 +1,454 @@ +/* + * test_urcu_lfs_rcu.c + * + * Userspace RCU library - example RCU-based lock-free stack + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" + +#ifdef __linux__ +#include +#endif + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#if defined(_syscall0) +_syscall0(pid_t, gettid) +#elif defined(__NR_gettid) +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} +#else +#warning "use pid as tid" +static inline pid_t gettid(void) +{ + return getpid(); +} +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +/* Remove deprecation warnings from test build. */ +#define CDS_LFS_RCU_DEPRECATED + +#include + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +struct test { + struct cds_lfs_node_rcu list; + struct rcu_head rcu; +}; + +static struct cds_lfs_stack_rcu s; + +void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "enqueuer", (unsigned long) pthread_self(), + (unsigned long) gettid()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct test *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_lfs_node_init_rcu(&node->list); + /* No rcu read-side is needed for push */ + cds_lfs_push_rcu(&s, &node->list); + URCU_TLS(nr_successful_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + rcu_unregister_thread(); + + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + printf_verbose("enqueuer thread_end, thread id : %lx, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + pthread_self(), + (unsigned long) gettid(), + URCU_TLS(nr_enqueues), URCU_TLS(nr_successful_enqueues)); + return ((void*)1); + +} + +static +void free_node_cb(struct rcu_head *head) +{ + struct test *node = + caa_container_of(head, struct test, rcu); + free(node); +} + +void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "dequeuer", (unsigned long) pthread_self(), + (unsigned long) gettid()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_lfs_node_rcu *snode; + + rcu_read_lock(); + snode = cds_lfs_pop_rcu(&s); + rcu_read_unlock(); + if (snode) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + call_rcu(&node->rcu, free_node_cb); + URCU_TLS(nr_successful_dequeues)++; + } + URCU_TLS(nr_dequeues)++; + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + rcu_unregister_thread(); + + printf_verbose("dequeuer thread_end, thread id : %lx, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + pthread_self(), + (unsigned long) gettid(), + URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + return ((void*)2); +} + +void test_end(struct cds_lfs_stack_rcu *s, unsigned long long *nr_dequeues) +{ + struct cds_lfs_node_rcu *snode; + + do { + snode = cds_lfs_pop_rcu(s); + if (snode) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + free(node); + (*nr_dequeues)++; + } + } while (snode); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s)", argv[0]); + printf(" [-d delay] (enqueuer period (in loops))"); + printf(" [-c duration] (dequeuer period (in loops))"); + printf(" [-v] (verbose output)"); + printf(" [-a cpu#] [-a cpu#]... (affinity)"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", + "main", (unsigned long) pthread_self(), + (unsigned long) gettid()); + + tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); + tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); + count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); + count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); + cds_lfs_init_rcu(&s); + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + write (1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&s, &end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free_all_cpu_call_rcu_data(); + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/tests/test_urcu_qsbr.c b/tests/test_urcu_qsbr.c index 112e3a2..6c47b72 100644 --- a/tests/test_urcu_qsbr.c +++ b/tests/test_urcu_qsbr.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -104,12 +104,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_qsbr_gc.c b/tests/test_urcu_qsbr_gc.c index 0b01f22..76fe4b7 100644 --- a/tests/test_urcu_qsbr_gc.c +++ b/tests/test_urcu_qsbr_gc.c @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -109,12 +109,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_wfcq.c b/tests/test_urcu_wfcq.c new file mode 100644 index 0000000..124968d --- /dev/null +++ b/tests/test_urcu_wfcq.c @@ -0,0 +1,586 @@ +/* + * test_urcu_wfcq.c + * + * Userspace RCU library - example RCU-based lock-free concurrent queue + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "cpuset.h" + +#ifdef __linux__ +#include +#endif + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#if defined(_syscall0) +_syscall0(pid_t, gettid) +#elif defined(__NR_gettid) +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} +#else +#warning "use pid as tid" +static inline pid_t gettid(void) +{ + return getpid(); +} +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +enum test_sync { + TEST_SYNC_NONE = 0, + TEST_SYNC_MUTEX, +}; + +static enum test_sync test_sync; + +static int test_force_sync; + +static volatile int test_go, test_stop_enqueue, test_stop_dequeue; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +static int test_dequeue, test_splice, test_wait_empty; +static int test_enqueue_stopped; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop_dequeue; +} + +static int test_duration_enqueue(void) +{ + return !test_stop_enqueue; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); +static DEFINE_URCU_TLS(unsigned long long, nr_empty_dest_enqueues); +static DEFINE_URCU_TLS(unsigned long long, nr_splice); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +static struct cds_wfcq_head __attribute__((aligned(CAA_CACHE_LINE_SIZE))) head; +static struct cds_wfcq_tail __attribute__((aligned(CAA_CACHE_LINE_SIZE))) tail; + +static void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + bool was_nonempty; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "enqueuer", (unsigned long) pthread_self(), + (unsigned long) gettid()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_wfcq_node *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_wfcq_node_init(node); + was_nonempty = cds_wfcq_enqueue(&head, &tail, node); + URCU_TLS(nr_successful_enqueues)++; + if (!was_nonempty) + URCU_TLS(nr_empty_dest_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + uatomic_inc(&test_enqueue_stopped); + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + count[2] = URCU_TLS(nr_empty_dest_enqueues); + printf_verbose("enqueuer thread_end, thread id : %lx, tid %lu, " + "enqueues %llu successful_enqueues %llu, " + "empty_dest_enqueues %llu\n", + pthread_self(), + (unsigned long) gettid(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues), + URCU_TLS(nr_empty_dest_enqueues)); + return ((void*)1); + +} + +static void do_test_dequeue(enum test_sync sync) +{ + struct cds_wfcq_node *node; + + if (sync == TEST_SYNC_MUTEX) + node = cds_wfcq_dequeue_blocking(&head, &tail); + else + node = __cds_wfcq_dequeue_blocking(&head, &tail); + + if (node) { + free(node); + URCU_TLS(nr_successful_dequeues)++; + } + URCU_TLS(nr_dequeues)++; +} + +static void do_test_splice(enum test_sync sync) +{ + struct cds_wfcq_head tmp_head; + struct cds_wfcq_tail tmp_tail; + struct cds_wfcq_node *node, *n; + enum cds_wfcq_ret ret; + + cds_wfcq_init(&tmp_head, &tmp_tail); + + if (sync == TEST_SYNC_MUTEX) + ret = cds_wfcq_splice_blocking(&tmp_head, &tmp_tail, + &head, &tail); + else + ret = __cds_wfcq_splice_blocking(&tmp_head, &tmp_tail, + &head, &tail); + + switch (ret) { + case CDS_WFCQ_RET_WOULDBLOCK: + assert(0); /* blocking call */ + break; + case CDS_WFCQ_RET_DEST_EMPTY: + URCU_TLS(nr_splice)++; + /* ok */ + break; + case CDS_WFCQ_RET_DEST_NON_EMPTY: + assert(0); /* entirely unexpected */ + break; + case CDS_WFCQ_RET_SRC_EMPTY: + /* ok, we could even skip iteration on dest if we wanted */ + break; + } + + __cds_wfcq_for_each_blocking_safe(&tmp_head, &tmp_tail, node, n) { + free(node); + URCU_TLS(nr_successful_dequeues)++; + URCU_TLS(nr_dequeues)++; + } +} + +static void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + unsigned int counter; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "dequeuer", (unsigned long) pthread_self(), + (unsigned long) gettid()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + if (test_dequeue && test_splice) { + if (counter & 1) + do_test_dequeue(test_sync); + else + do_test_splice(test_sync); + counter++; + } else { + if (test_dequeue) + do_test_dequeue(test_sync); + else + do_test_splice(test_sync); + } + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + printf_verbose("dequeuer thread_end, thread id : %lx, tid %lu, " + "dequeues %llu, successful_dequeues %llu, " + "nr_splice %llu\n", + pthread_self(), + (unsigned long) gettid(), + URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues), + URCU_TLS(nr_splice)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + count[2] = URCU_TLS(nr_splice); + return ((void*)2); +} + +static void test_end(unsigned long long *nr_dequeues) +{ + struct cds_wfcq_node *node; + + do { + node = cds_wfcq_dequeue_blocking(&head, &tail); + if (node) { + free(node); + (*nr_dequeues)++; + } + } while (node); +} + +static void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s)", argv[0]); + printf(" [-d delay] (enqueuer period (in loops))"); + printf(" [-c duration] (dequeuer period (in loops))"); + printf(" [-v] (verbose output)"); + printf(" [-a cpu#] [-a cpu#]... (affinity)"); + printf(" [-q] (test dequeue)"); + printf(" [-s] (test splice, enabled by default)"); + printf(" [-M] (use mutex external synchronization)"); + printf(" Note: default: no external synchronization used."); + printf(" [-f] (force user-provided synchronization)"); + printf(" [-w] Wait for dequeuer to empty queue"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0, + tot_empty_dest_enqueues = 0, + tot_splice = 0; + unsigned long long end_dequeues = 0; + int i, a, retval = 0; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'q': + test_dequeue = 1; + break; + case 's': + test_splice = 1; + break; + case 'M': + test_sync = TEST_SYNC_MUTEX; + break; + case 'w': + test_wait_empty = 1; + break; + case 'f': + test_force_sync = 1; + break; + } + } + + /* activate splice test by default */ + if (!test_dequeue && !test_splice) + test_splice = 1; + + if (test_sync == TEST_SYNC_NONE && nr_dequeuers > 1 && test_dequeue) { + if (test_force_sync) { + fprintf(stderr, "[WARNING] Using dequeue concurrently " + "with other dequeue or splice without external " + "synchronization. Expect run-time failure.\n"); + } else { + printf("Enforcing mutex synchronization\n"); + test_sync = TEST_SYNC_MUTEX; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + if (test_dequeue) + printf_verbose("dequeue test activated.\n"); + else + printf_verbose("splice test activated.\n"); + if (test_sync == TEST_SYNC_MUTEX) + printf_verbose("External sync: mutex.\n"); + else + printf_verbose("External sync: none.\n"); + if (test_wait_empty) + printf_verbose("Wait for dequeuers to empty queue.\n"); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", + "main", (unsigned long) pthread_self(), + (unsigned long) gettid()); + + tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); + tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); + count_enqueuer = malloc(3 * sizeof(*count_enqueuer) * nr_enqueuers); + count_dequeuer = malloc(3 * sizeof(*count_dequeuer) * nr_dequeuers); + cds_wfcq_init(&head, &tail); + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[3 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[3 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + write (1, ".", 1); + } + + test_stop_enqueue = 1; + + if (test_wait_empty) { + while (nr_enqueuers != uatomic_read(&test_enqueue_stopped)) { + sleep(1); + } + while (!cds_wfcq_empty(&head, &tail)) { + sleep(1); + } + } + + test_stop_dequeue = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[3 * i]; + tot_successful_enqueues += count_enqueuer[3 * i + 1]; + tot_empty_dest_enqueues += count_enqueuer[3 * i + 2]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[3 * i]; + tot_successful_dequeues += count_dequeuer[3 * i + 1]; + tot_splice += count_dequeuer[3 * i + 2]; + } + + test_end(&end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "enqueues to empty dest : %llu, " + "successful dequeues %llu, " + "splice : %llu\n", + tot_successful_enqueues, + tot_empty_dest_enqueues, + tot_successful_dequeues, + tot_splice); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu enqueues to empty dest %12llu " + "successful dequeues %12llu splice %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_empty_dest_enqueues, + tot_successful_dequeues, tot_splice, end_dequeues, + tot_enqueues + tot_dequeues); + + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) { + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + retval = 1; + } + + /* + * If only using splice to dequeue, the enqueuer should see + * exactly as many empty queues than the number of non-empty + * src splice. + */ + if (test_wait_empty && test_splice && !test_dequeue + && tot_empty_dest_enqueues != tot_splice) { + printf("WARNING! Discrepancy between empty enqueue (%llu) and " + "number of non-empty splice (%llu)\n", + tot_empty_dest_enqueues, + tot_splice); + retval = 1; + } + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return retval; +} diff --git a/tests/test_urcu_wfq.c b/tests/test_urcu_wfq.c index 985f0d1..c361e02 100644 --- a/tests/test_urcu_wfq.c +++ b/tests/test_urcu_wfq.c @@ -34,11 +34,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -97,12 +97,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; diff --git a/tests/test_urcu_wfs.c b/tests/test_urcu_wfs.c index d2c4705..31ce146 100644 --- a/tests/test_urcu_wfs.c +++ b/tests/test_urcu_wfs.c @@ -34,11 +34,11 @@ #include #include #include -#include #include #include #include +#include "cpuset.h" #ifdef __linux__ #include @@ -97,12 +97,6 @@ static int use_affinity = 0; pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - static void set_affinity(void) { cpu_set_t mask; -- 2.34.1