From: Mathieu Desnoyers Date: Mon, 4 May 2015 17:35:35 +0000 (-0400) Subject: Merge branch 'master' into urcu/rcuja-range-merge X-Git-Url: http://git.liburcu.org/?a=commitdiff_plain;h=169e1020838cc5b9f3df503d160ce1bf0c939b2f;hp=5bcf8326b6e13ca93429925bc38fb81c73155c54;p=userspace-rcu.git Merge branch 'master' into urcu/rcuja-range-merge Signed-off-by: Mathieu Desnoyers --- diff --git a/.gitignore b/.gitignore index 00dbc40..7b6704b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,75 +2,77 @@ urcu/arch.h urcu/uatomic.h tests/api.h -tests/urcutorture -tests/urcutorture-yield -tests/urcu-asm.S -tests/test_rwlock_timing -tests/test_urcu -tests/test_urcu_dynamic_link -tests/test_urcu_timing -tests/test_urcu_yield -tests/test_looplen -tests/test_mutex -tests/test_perthreadlock -tests/test_perthreadlock_timing -tests/test_qsbr -tests/test_qsbr_dynamic_link -tests/test_qsbr_gc -tests/test_qsbr_lgc -tests/test_qsbr_timing -tests/test_rwlock -tests/test_uatomic -tests/test_urcu_gc -tests/test_urcu_gc_mb -tests/test_urcu_lgc -tests/test_urcu_lgc_mb -tests/test_urcu_mb -tests/test_urcu_mb_defer -tests/test_urcu_assign -tests/test_urcu_assign_dynamic_link -tests/test_urcu_bp -tests/test_urcu_bp_dynamic_link -tests/rcutorture_qsbr -tests/rcutorture_urcu -tests/rcutorture_urcu_bp -tests/rcutorture_urcu_mb -tests/rcutorture_urcu_qsbr -tests/rcutorture_urcu_signal -tests/test_cycles_per_loop -tests/test_urcu_defer -tests/test_urcu_hash -tests/test_urcu_lfq -tests/test_urcu_lfq_dynlink -tests/test_urcu_lfs -tests/test_urcu_lfs_dynlink -tests/test_urcu_mb_gc -tests/test_urcu_mb_lgc -tests/test_urcu_qsbr -tests/test_urcu_qsbr_dynamic_link -tests/test_urcu_qsbr_gc -tests/test_urcu_qsbr_lgc -tests/test_urcu_qsbr_timing -tests/test_urcu_signal -tests/test_urcu_signal_dynamic_link -tests/test_urcu_signal_gc -tests/test_urcu_signal_lgc -tests/test_urcu_signal_timing -tests/test_urcu_signal_yield -tests/test_urcu_wfq -tests/test_urcu_wfq_dynlink -tests/test_urcu_wfs -tests/test_urcu_wfs_dynlink -tests/test_urcu_fork -tests/test_urcu_ja -tests/test_urcu_ja_range -tests/test_urcu_lfs_rcu -tests/test_urcu_lfs_rcu_dynlink -tests/test_urcu_multiflavor -tests/test_urcu_multiflavor_dynlink -tests/test_urcu_wfcq -tests/test_urcu_wfcq_dynlink -tests/*.log + +tests/unit/urcu-asm.S +tests/unit/test_uatomic +tests/unit/test_urcu_multiflavor +tests/unit/test_urcu_multiflavor_dynlink + +tests/regression/rcutorture_qsbr +tests/regression/rcutorture_urcu +tests/regression/rcutorture_urcu_bp +tests/regression/rcutorture_urcu_mb +tests/regression/rcutorture_urcu_qsbr +tests/regression/rcutorture_urcu_signal +tests/regression/test_urcu_fork + +tests/benchmark/test_rwlock_timing +tests/benchmark/test_urcu +tests/benchmark/test_urcu_dynamic_link +tests/benchmark/test_urcu_timing +tests/benchmark/test_urcu_yield +tests/benchmark/test_looplen +tests/benchmark/test_mutex +tests/benchmark/test_perthreadlock +tests/benchmark/test_perthreadlock_timing +tests/benchmark/test_qsbr +tests/benchmark/test_qsbr_dynamic_link +tests/benchmark/test_qsbr_gc +tests/benchmark/test_qsbr_lgc +tests/benchmark/test_qsbr_timing +tests/benchmark/test_rwlock +tests/benchmark/test_urcu_gc +tests/benchmark/test_urcu_gc_mb +tests/benchmark/test_urcu_lgc +tests/benchmark/test_urcu_lgc_mb +tests/benchmark/test_urcu_mb +tests/benchmark/test_urcu_mb_defer +tests/benchmark/test_urcu_assign +tests/benchmark/test_urcu_assign_dynamic_link +tests/benchmark/test_urcu_bp +tests/benchmark/test_urcu_bp_dynamic_link +tests/benchmark/test_cycles_per_loop +tests/benchmark/test_urcu_defer +tests/benchmark/test_urcu_hash +tests/benchmark/test_urcu_lfq +tests/benchmark/test_urcu_lfq_dynlink +tests/benchmark/test_urcu_lfs +tests/benchmark/test_urcu_lfs_dynlink +tests/benchmark/test_urcu_mb_gc +tests/benchmark/test_urcu_mb_lgc +tests/benchmark/test_urcu_qsbr +tests/benchmark/test_urcu_qsbr_dynamic_link +tests/benchmark/test_urcu_qsbr_gc +tests/benchmark/test_urcu_qsbr_lgc +tests/benchmark/test_urcu_qsbr_timing +tests/benchmark/test_urcu_signal +tests/benchmark/test_urcu_signal_dynamic_link +tests/benchmark/test_urcu_signal_gc +tests/benchmark/test_urcu_signal_lgc +tests/benchmark/test_urcu_signal_timing +tests/benchmark/test_urcu_signal_yield +tests/benchmark/test_urcu_wfq +tests/benchmark/test_urcu_wfq_dynlink +tests/benchmark/test_urcu_wfs +tests/benchmark/test_urcu_wfs_dynlink +tests/benchmark/test_urcu_ja +tests/benchmark/test_urcu_ja_range +tests/benchmark/test_urcu_lfs_rcu +tests/benchmark/test_urcu_lfs_rcu_dynlink +tests/benchmark/test_urcu_wfcq +tests/benchmark/test_urcu_wfcq_dynlink + +tests/benchmark/*.log *.so doc/examples/urcu-flavors/qsbr diff --git a/ChangeLog b/ChangeLog index 2b79afc..817582c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,214 @@ +2013-09-06 Userspace RCU 0.8.0 + * Fix: hash table growth (for small tables) should be limited + * Fix: doc/examples cross-build + * Introduce URCU_INLINE_SMALL_FUNCTIONS + * Add missing tests/common/Makefile.am + * README: document make check/regtest/bench + * tests: split in check, regtest and bench targets + * Cleanup: doc/examples makefile + * Fix: doc/examples VPATH build + * doc/examples: Move the LIBS after the OBJECTS in the Makefile + * Document build work-around on MacOS X + * Fix tests: use of uninitialized variables + * test_urcu_hash*: initialize TLS seeds + * doc/examples: cds_lfht_for_each_entry_duplicate + * doc/examples: cds_lfht_lookup + * doc/examples: cds_lfht_destroy + * doc/examples: cds_lfht_add_replace + * doc/examples: cds_lfht_add_unique + * doc/examples: cds_lfht_add/cds_lfht_del + * doc/examples: add rculfqueue example + * doc/examples: add synchronize_rcu() + * doc/examples: add bp flavor + * doc/examples: add dist toplevel makefile + * doc/examples: add membarrier flavor + * doc/examples: document call_rcu() + * doc/examples: update qsbr example + * urcu signal: remove assertion on exit + * doc/examples: signal flavor + * doc/examples: add mb flavor + * doc/examples: update qsbr + * doc/examples: introduce urcu-flavors examples directory + * doc/examples: enhance rcu-flavor-qsbr example + * doc/examples: rename qsbr-minimal to rcu-flavor-qsbr + * doc/examples: automake stop on error + * doc/examples: hlist + * hlist/rcuhlist update + * doc/examples: fix typo in list example + * rcuhlist: make pointer stores atomic + * hlist, rcuhlist: cleanup coding style + * doc/examples: lfstack + * doc/examples: update cds_wfs_pop_all_blocking + * doc/examples: cds_wfs_pop_all_blocking + * doc/examples: cds_wfs_pop + * doc/examples: add missing Makefile + * doc/examples: cds_wfs_push + * doc/wfcqueue: cds_wfcq_splice + * doc/examples: add cds_wfcq_dequeue + * doc/examples: wfcq needs to link against urcu-common + * doc/examples: update queue comment + * doc/examples: fix make clean + * gitignore: add qsbr-minimal + * doc/examples: cds_wfcq_enqueue + * doc/examples: Move LIBS to each makefile + * doc/examples: cds_list_for_each_rcu + * doc/examples: cds_list_for_each_entry_rcu + * doc/examples: cds_list_replace_rcu + * doc/examples: cds_list_add_tail_rcu + * doc/examples: cds_list_del_rcu + * doc/examples: cds_list_add_rcu + * rculist: ensure atomic updates of next pointers + * rculist: implement cds_list_add_tail_rcu + * rculist.h and list.h style cleanup + * example makefile: add missing cd .. + * Update gitignore + * Fix: examples Makefile on FreeBSD + * hash table test: don't redefine CACHE_LINE_SIZE + * tests: use thread-id.h wrapper + * Implement thread-id.h wrapper + * tests: add missing unsigned long casts to pthread_self() + * Fix: don't build examples in static builds + * Add QSBR minimal example + * compiler.h: implement CAA_ARRAY_SIZE() + * document rcu barrier + * rcu barrier: handle OOM die urcu_die + * Implement rcu_barrier() + * rculfhash: document destroy context limitations + * Add MIPS to README + * Update README + * Update README testing info about FreeBSD + * test: fix api.h missing if brackets + * tests: fix incorrect counter + * Fix: membarrier fallback symbol conflict + * Fix: Use a filled signal mask to disable all signals + * urcu-bp: introduce struct urcu_gp + * Fix: struct urcu_gp broke multiflavor + * Cleanup test usage printout + * wfstack tests: use pop "last" state info + * wfstack: return whether pop is popping the last element + * wfcqueue tests: use dequeue empty state + * wfcqueue: return whether dequeue is dequeuing last element + * urcu: avoid false sharing for rcu_gp_ctr + * urcu: make the code of urcu-qsbr as normal urcu + * rculfhash: detect if resize/destroy are called within RCU read-side C.S. + * Documentation: rculfhash: cds_lfht_resize not within read-side C.S. + * fix: rculfhash don't change qsbr online state + * Add rcu_read_ongoing() API to each urcu flavor + * Add "sparc" host cpu to configure.ac + * futex: include syscall.h instead of sys/syscall.h + * Add tab to output in order to allow easy nesting of tables. + * Remove urcu-api-list.sh from dist tarball + * Add urcu-api-list.sh script + * list: implement cds_list_for_each_safe() + * Fix: tests/api.h use cpuset.h + * Fix hurd-i386: move cpuset tests outside of sched_setaffinity conditional + * Fix tests: finer-grained use of CPU_SET, CPU_ZERO and cpu_set_t + * Test for CPU_SET + * Fix build on architectures with HAVE_SCHED_GETCPU but without HAVE_SYSCONF + * README: document that Clang 3.0 (based on LLVM 3.0) is supported + * clang: silence "unused expression result" warning + * rculfhash: add assertions on node alignment + * Spelling cleanups within comments and documentation + * Fix configure checks for Tile + * uatomic: style fix + * doc/cds-api.txt: expand documentation + * README: document each API file + * README: reorganize + * Add compilation support for the TileGX architecture + * wfstack: add nonblocking to _LGPL_SOURCE API + * Discourage use of pthread_atfork() for call_rcu handlers + * Fix call_rcu fork handling + * test: fork handling + * rculfhash: add cds_lfht_replace to the write operations in the comments + * urcu: fix comments for cds_list_for_each_prev() + * documentation: fix rcu-api.txt duplicates + * test wfcq: remove unneeded urcu.h include + * test wfs: remove unneeded urcu.h include + * urcu: declare test_urcu_multiflavor functions + * urcu: remove the wrong comma + * wfstack: implement nonblocking pop and next + * wfcqueue: document first/next return values + * wfstack: update comments about cds_wfs_empty/first being wait-free + * wfstack API: rename cds_wfs_first_blocking to cds_wfs_first + * wfstack test: test if number of push to empty vs pop_all match + * wfstack: document first/next return values + * test wfstack: enforce external mutex if needed by default + * test wfcqueue: enforce external mutex if needed by default + * urcu-mb/signal/membarrier: batch concurrent synchronize_rcu() + * urcu-wait: move queue management code into urcu-wait.h + * urcu-wait: move wait code into separate file + * urcu-qsbr: batch concurrent synchronize_rcu() + * tests: use standard malloc/free for synchronize_rcu() + * urcu-bp: move quiescent threads to separate list + * urcu-mb/signal/membarrier: move quiescent threads to separate list + * urcu-qsbr: move offline threads to separate list + * urcu-bp: improve 2-phase wait scheme + * urcu-mb/signal/membarrier: improve 2-phase wait scheme + * urcu-qsbr: improve 2-phase wait scheme + * wfcqueue: implement mutex-free splice + * wfcqueue: document empty criterion + * urcu-call-rcu: use wait-free splice return value + * test wfcqueue: add tests for queue state return value + * wfcqueue: enqueue and splice return queue state + * Fix: wfcqueue nonblocking dequeue + * wfcqueue: Fix lock and unlock functions + * runtests: Make path of time binary configurable + * urcu-qsbr: skip Q.S. reporting if already reported + * Fix TLS detection: test with linker, add --disable-compiler-tls + * Cleanup: cast pthread_self() return value to unsigned long + * Fallback mechanism not working on platform where TLS is unsupported + * Revert "Fix: cross-build: configure.ac should use --target, not --host" + * Fix: cross-build: configure.ac should use --target, not --host + * test_urcu_wfcq: add splice and nosync tests + * test_urcu_wfs: cleanup + * test_urcu_lfs: cleanup + * Fix static linking: add missing static for _defer_rcu + * tests: report error value for make check + * Add multiflavor test program + * Fix static linking: fix symbol name namespaces + * Fix static linking: add missing static to thr_defer + * Fix static linking: add missing static + * deprecation: fix build with gcc < 4.5 + * wfstack.c: update copyright notice + * Update wfstack copyright notice + * Comment fix: update associated LGPL header name + * Update cds-api.txt following API deprecations + * Deprecate wfqueue + * Deprecate rculfstack + * wfcqueue: introduce nonblocking API + * lfstack: test pop_all and pop + * lfstack: implement empty, pop_all and iterators, document API + * lfstack: implement test + * lfstack: implement lock-free stack + * wfstack: implement pop_all and iteration tests + * wfstack: implement cds_wfs_pop_all and iterators, document API + * rculfhash test: fix trivial memleak and return node leak and errors + * rculfhash: add missing extern + * Cleanup: fix cppcheck errors + * wfcqueue: remove ancient comment + * test_urcu_lfq: remove rcu_defer_register_thread() from test_urcu_lfq + * test_urcu_lfq: test for the proper pointer + * test_urcu_lfs: remove rcu_defer_register_thread() from test_urcu_lfs + * test_urcu_lfs: test for the proper pointer + * wfcqueue: clarify locking usage + * Document APIs in README + * Test cleanup: replace "l" parameter by "loops" + * Add wfcqueue header to cds.h + * Fix: urcu-bp, urcu, urcu-qsbr should include wfcqueue + * Fix: call_rcu list corruption on teardown (documentation) + * call_rcu: remove head field alignement, explain wfcqueue motivation + * wfcqueue: update credits in patch documentation + * wfcqueue documentation: hint at for_each iterators + * Fix urcu-call-rcu-impl.h: false-sharing + * call_rcu: use wfcqueue, eliminate false-sharing + * wfcqueue test + * wfcqueue: implement concurrency-efficient queue + * Ensure that read-side functions meet 10-line LGPL criterion + * tls-compat.h: document sigaltstack(2) limitation + * urcu: add notice to URCU_TLS() for it is not strictly async-signal-safe + * Document sigaltstack(2) limitation + * Documentation: update LICENSE file + 2012-08-27 Userspace RCU 0.7.4 * rculfhash API documentation: document destroy RCU read-lock constraint * Fix: rculfhash should be offline while waiting for resize to complete diff --git a/Makefile.am b/Makefile.am index 2ac790f..df6eec1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,10 +1,13 @@ ACLOCAL_AMFLAGS=-I config -INCLUDES = -I$(top_builddir)/urcu +AM_CPPFLAGS = -I$(top_builddir)/urcu #Add the -version-info directly here since we are only building # library that use the version-info -AM_LDFLAGS=-lpthread -version-info $(URCU_LIBRARY_VERSION) +AM_LDFLAGS=-version-info $(URCU_LIBRARY_VERSION) +if !LIBC_INCLUDES_PTHREAD +AM_LDFLAGS+=-lpthread +endif AM_CFLAGS=-Wall SUBDIRS = . doc tests rcuja @@ -18,8 +21,10 @@ nobase_dist_include_HEADERS = urcu/compiler.h urcu/hlist.h urcu/list.h \ urcu/ref.h urcu/cds.h urcu/urcu_ref.h urcu/urcu-futex.h \ urcu/uatomic_arch.h urcu/rculfhash.h urcu/wfcqueue.h \ urcu/rcuja.h urcu/rcuja-range.h urcu/lfstack.h \ + urcu/syscall-compat.h \ $(top_srcdir)/urcu/map/*.h \ $(top_srcdir)/urcu/static/*.h \ + urcu/rand-compat.h \ urcu/tls-compat.h nobase_nodist_include_HEADERS = urcu/arch.h urcu/uatomic.h urcu/config.h @@ -29,8 +34,7 @@ EXTRA_DIST = $(top_srcdir)/urcu/arch/*.h $(top_srcdir)/urcu/uatomic/*.h \ gpl-2.0.txt lgpl-2.1.txt lgpl-relicensing.txt \ LICENSE compat_arch_x86.c \ urcu-call-rcu-impl.h urcu-defer-impl.h \ - rculfhash-internal.h \ - $(top_srcdir)/tests/*.sh + rculfhash-internal.h if COMPAT_ARCH COMPAT=compat_arch_@ARCHTYPE@.c @@ -84,6 +88,12 @@ pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = liburcu-cds.pc liburcu.pc liburcu-bp.pc liburcu-qsbr.pc \ liburcu-signal.pc liburcu-mb.pc -dist_doc_DATA = README ChangeLog +dist_doc_DATA = README.md ChangeLog dist_noinst_DATA = CodingStyle + +.PHONY: bench regtest +bench: + cd tests && $(MAKE) $(AM_MAKEFLAGS) bench +regtest: + cd tests && $(MAKE) $(AM_MAKEFLAGS) regtest diff --git a/README b/README deleted file mode 100644 index c7403f8..0000000 --- a/README +++ /dev/null @@ -1,299 +0,0 @@ -Userspace RCU Implementation -by Mathieu Desnoyers and Paul E. McKenney - -BUILDING --------- - - ./bootstrap (skip if using tarball) - ./configure - make - make install - ldconfig - - Hints: Forcing 32-bit build: - * CFLAGS="-m32 -g -O2" ./configure - - Forcing 64-bit build: - * CFLAGS="-m64 -g -O2" ./configure - - Forcing a 32-bit build with 386 backward compatibility: - * CFLAGS="-m32 -g -O2" ./configure --host=i386-pc-linux-gnu - - Forcing a 32-bit build for Sparcv9 (typical for Sparc v9) - * CFLAGS="-m32 -Wa,-Av9a -g -O2" ./configure - - -ARCHITECTURES SUPPORTED ------------------------ - -Currently, Linux x86 (i386, i486, i586, i686), x86 64-bit, PowerPC 32/64, -S390, S390x, ARM, MIPS, Alpha, ia64 and Sparcv9 32/64 are supported. -Tested on Linux, FreeBSD 8.2/8.3/9.0/9.1/10.0 i386/amd64, and Cygwin. -Should also work on: Android, NetBSD 5, OpenBSD, Darwin (more testing -needed before claiming support for these OS). - -Linux ARM depends on running a Linux kernel 2.6.15 or better, GCC 4.4 or -better. - -The gcc compiler versions 3.3, 3.4, 4.0, 4.1, 4.2, 4.3, 4.4 and 4.5 are -supported, with the following exceptions: - -- gcc 3.3 and 3.4 have a bug that prevents them from generating volatile - accesses to offsets in a TLS structure on 32-bit x86. These versions are - therefore not compatible with liburcu on x86 32-bit (i386, i486, i586, i686). - The problem has been reported to the gcc community: - http://www.mail-archive.com/gcc-bugs@gcc.gnu.org/msg281255.html -- gcc 3.3 cannot match the "xchg" instruction on 32-bit x86 build. - See: http://kerneltrap.org/node/7507 -- Alpha, ia64 and ARM architectures depend on gcc 4.x with atomic builtins - support. For ARM this was introduced with gcc 4.4: - http://gcc.gnu.org/gcc-4.4/changes.html - -Clang version 3.0 (based on LLVM 3.0) is supported. - -For developers using the git tree: - -This source tree is based on the autotools suite from GNU to simplify -portability. Here are some things you should have on your system in order to -compile the git repository tree : - -- GNU autotools (automake >=1.10, autoconf >=2.50, autoheader >=2.50) - (make sure your system wide "automake" points to a recent version!) -- GNU Libtool >=2.2 - (for more information, go to http://www.gnu.org/software/autoconf/) - -If you get the tree from the repository, you will need to use the "bootstrap" -script in the root of the tree. It calls all the GNU tools needed to prepare the -tree configuration. - -Test scripts provided in the tests/ directory of the source tree depend -on "bash" and the "seq" program. - - -API ---- - -See the relevant API documentation files in doc/. The APIs provided by -Userspace RCU are, by prefix: - -- rcu_ : Read-Copy Update (see doc/rcu-api.txt) -- cmm_ : Concurrent Memory Model -- caa_ : Concurrent Architecture Abstraction -- cds_ : Concurrent Data Structures (see doc/cds-api.txt) -- uatomic_: Userspace Atomic (see doc/uatomic-api.txt) - - -QUICK START GUIDE ------------------ - -Usage of all urcu libraries - - * Define _LGPL_SOURCE (only) if your code is LGPL or GPL compatible - before including the urcu.h or urcu-qsbr.h header. If your application - is distributed under another license, function calls will be generated - instead of inlines, so your application can link with the library. - * Linking with one of the libraries below is always necessary even for - LGPL and GPL applications. - -Usage of liburcu - - * #include - * Link the application with "-lurcu". - * This is the preferred version of the library, in terms of - grace-period detection speed, read-side speed and flexibility. - Dynamically detects kernel support for sys_membarrier(). Falls back - on urcu-mb scheme if support is not present, which has slower - read-side. - -Usage of liburcu-qsbr - - * #include - * Link with "-lurcu-qsbr". - * The QSBR flavor of RCU needs to have each reader thread executing - rcu_quiescent_state() periodically to progress. rcu_thread_online() - and rcu_thread_offline() can be used to mark long periods for which - the threads are not active. It provides the fastest read-side at the - expense of more intrusiveness in the application code. - -Usage of liburcu-mb - - * #include - * Compile any _LGPL_SOURCE code using this library with "-DRCU_MB". - * Link with "-lurcu-mb". - * This version of the urcu library uses memory barriers on the writer - and reader sides. This results in faster grace-period detection, but - results in slower reads. - -Usage of liburcu-signal - - * #include - * Compile any _LGPL_SOURCE code using this library with "-DRCU_SIGNAL". - * Link the application with "-lurcu-signal". - * Version of the library that requires a signal, typically SIGUSR1. Can - be overridden with -DSIGRCU by modifying Makefile.build.inc. - -Usage of liburcu-bp - - * #include - * Link with "-lurcu-bp". - * The BP library flavor stands for "bulletproof". It is specifically - designed to help tracing library to hook on applications without - requiring to modify these applications. rcu_init(), - rcu_register_thread() and rcu_unregister_thread() all become nops. - The state is dealt with by the library internally at the expense of - read-side and write-side performance. - -Initialization - - Each thread that has reader critical sections (that uses - rcu_read_lock()/rcu_read_unlock() must first register to the URCU - library. This is done by calling rcu_register_thread(). Unregistration - must be performed before exiting the thread by using - rcu_unregister_thread(). - -Reading - - Reader critical sections must be protected by locating them between - calls to rcu_read_lock() and rcu_read_unlock(). Inside that lock, - rcu_dereference() may be called to read an RCU protected pointer. - -Writing - - rcu_assign_pointer() and rcu_xchg_pointer() may be called anywhere. - After, synchronize_rcu() must be called. When it returns, the old - values are not in usage anymore. - -Usage of liburcu-defer - - * Follow instructions for either liburcu, liburcu-qsbr, - liburcu-mb, liburcu-signal, or liburcu-bp above. - The liburcu-defer functionality is pulled into each of - those library modules. - * Provides defer_rcu() primitive to enqueue delayed callbacks. Queued - callbacks are executed in batch periodically after a grace period. - Do _not_ use defer_rcu() within a read-side critical section, because - it may call synchronize_rcu() if the thread queue is full. - This can lead to deadlock or worse. - * Requires that rcu_defer_barrier() must be called in library destructor - if a library queues callbacks and is expected to be unloaded with - dlclose(). - * Its API is currently experimental. It may change in future library - releases. - -Usage of urcu-call-rcu - - * Follow instructions for either liburcu, liburcu-qsbr, - liburcu-mb, liburcu-signal, or liburcu-bp above. - The urcu-call-rcu functionality is provided for each of - these library modules. - * Provides the call_rcu() primitive to enqueue delayed callbacks - in a manner similar to defer_rcu(), but without ever delaying - for a grace period. On the other hand, call_rcu()'s best-case - overhead is not quite as good as that of defer_rcu(). - * Provides call_rcu() to allow asynchronous handling of RCU - grace periods. A number of additional functions are provided - to manage the helper threads used by call_rcu(), but reasonable - defaults are used if these additional functions are not invoked. - See rcu-api.txt in userspace-rcu documentation for more details. - -Being careful with signals - - The liburcu library uses signals internally. The signal handler is - registered with the SA_RESTART flag. However, these signals may cause - some non-restartable system calls to fail with errno = EINTR. Care - should be taken to restart system calls manually if they fail with this - error. A list of non-restartable system calls may be found in - signal(7). The liburcu-mb and liburcu-qsbr versions of the Userspace RCU - library do not require any signal. - - Read-side critical sections are allowed in a signal handler, - except those setup with sigaltstack(2), with liburcu and - liburcu-mb. Be careful, however, to disable these signals - between thread creation and calls to rcu_register_thread(), because a - signal handler nesting on an unregistered thread would not be - allowed to call rcu_read_lock(). - - Read-side critical sections are _not_ allowed in a signal handler with - liburcu-qsbr, unless signals are disabled explicitly around each - rcu_quiescent_state() calls, when threads are put offline and around - calls to synchronize_rcu(). Even then, we do not recommend it. - -Interaction with mutexes - - One must be careful to do not cause deadlocks due to interaction of - synchronize_rcu() and RCU read-side with mutexes. If synchronize_rcu() - is called with a mutex held, this mutex (or any mutex which has this - mutex in its dependency chain) should not be acquired from within a RCU - read-side critical section. - - This is especially important to understand in the context of the - QSBR flavor: a registered reader thread being "online" by - default should be considered as within a RCU read-side critical - section unless explicitly put "offline". Therefore, if - synchronize_rcu() is called with a mutex held, this mutex, as - well as any mutex which has this mutex in its dependency chain - should only be taken when the RCU reader thread is "offline" - (this can be performed by calling rcu_thread_offline()). - -Interaction with fork() - - Special care must be taken for applications performing fork() without - any following exec(). This is caused by the fact that Linux only clones - the thread calling fork(), and thus never replicates any of the other - parent thread into the child process. Most liburcu implementations - require that all registrations (as reader, defer_rcu and call_rcu - threads) should be released before a fork() is performed, except for the - rather common scenario where fork() is immediately followed by exec() in - the child process. The only implementation not subject to that rule is - liburcu-bp, which is designed to handle fork() by calling - rcu_bp_before_fork, rcu_bp_after_fork_parent and - rcu_bp_after_fork_child. - - Applications that use call_rcu() and that fork() without - doing an immediate exec() must take special action. The parent - must invoke call_rcu_before_fork() before the fork() and - call_rcu_after_fork_parent() after the fork(). The child - process must invoke call_rcu_after_fork_child(). - Even though these three APIs are suitable for passing to - pthread_atfork(), use of pthread_atfork() is *STRONGLY - DISCOURAGED* for programs calling the glibc memory allocator - (malloc(), calloc(), free(), ...) within call_rcu callbacks. - This is due to limitations in the way glibc memory allocator - handles calls to the memory allocator from concurrent threads - while the pthread_atfork() handlers are executing. - Combining e.g.: - * call to free() from callbacks executed within call_rcu worker - threads, - * executing call_rcu atfork handlers within the glibc pthread - atfork mechanism, - will sometimes trigger interesting process hangs. This usually - hangs on a memory allocator lock within glibc. - -Thread Local Storage (TLS) - - Userspace RCU can fall back on pthread_getspecific() to emulate - TLS variables on systems where it is not available. This behavior - can be forced by specifying --disable-compiler-tls as configure - argument. - -Usage of DEBUG_RCU - - DEBUG_RCU is used to add internal debugging self-checks to the - RCU library. This define adds a performance penalty when enabled. - Can be enabled by uncommenting the corresponding line in - Makefile.build.inc. - -Usage of DEBUG_YIELD - - DEBUG_YIELD is used to add random delays in the code for testing - purposes. - -SMP support - - By default the library is configured to use synchronization primitives - adequate for SMP systems. On uniprocessor systems, support for SMP - systems can be disabled with: - - ./configure --disable-smp-support - - theoretically yielding slightly better performance. diff --git a/README.md b/README.md new file mode 100644 index 0000000..f6b290f --- /dev/null +++ b/README.md @@ -0,0 +1,399 @@ +Userspace RCU Implementation +============================ + +by Mathieu Desnoyers and Paul E. McKenney + + +Building +-------- + + ./bootstrap # skip if using tarball + ./configure + make + make install + ldconfig + +Hints: + + - Forcing 32-bit build: + + CFLAGS="-m32 -g -O2" ./configure + + - Forcing 64-bit build: + + CFLAGS="-m64 -g -O2" ./configure + + - Forcing a 32-bit build with 386 backward compatibility: + + CFLAGS="-m32 -g -O2" ./configure --host=i386-pc-linux-gnu + + - Forcing a 32-bit build for Sparcv9 (typical for Sparc v9) + + CFLAGS="-m32 -Wa,-Av9a -g -O2" ./configure + + +Architectures supported +----------------------- + +Currently, the following architectures are supported: + + - Linux x86 (i386, i486, i586, i686) + - x86 64-bit + - PowerPC 32/64 + - S390, S390x + - ARM 32/64 + - MIPS + - Alpha + - ia64 + - Sparcv9 32/64 + - Tilera + - hppa/PA-RISC + +Tested on Linux, FreeBSD 8.2/8.3/9.0/9.1/10.0 i386/amd64, and Cygwin. +Should also work on: + + - Android + - NetBSD 5 + - OpenBSD + - Darwin + +(more testing needed before claiming support for these OS). + +Linux ARM depends on running a Linux kernel 2.6.15 or better, GCC 4.4 or +better. + +The GCC compiler versions 3.3, 3.4, 4.0, 4.1, 4.2, 4.3, 4.4 and 4.5 are +supported, with the following exceptions: + + - GCC 3.3 and 3.4 have a bug that prevents them from generating volatile + accesses to offsets in a TLS structure on 32-bit x86. These versions are + therefore not compatible with `liburcu` on x86 32-bit + (i386, i486, i586, i686). + The problem has been reported to the GCC community: + http://www.mail-archive.com/gcc-bugs@gcc.gnu.org/msg281255.html + - GCC 3.3 cannot match the "xchg" instruction on 32-bit x86 build. + See http://kerneltrap.org/node/7507 + - Alpha, ia64 and ARM architectures depend on GCC 4.x with atomic builtins + support. For ARM this was introduced with GCC 4.4: + http://gcc.gnu.org/gcc-4.4/changes.html. + +Clang version 3.0 (based on LLVM 3.0) is supported. + +Building on MacOS X (Darwin) requires a work-around for processor +detection: + + - 32-bit: + + ./configure --build=i686-apple-darwin11 + + - 64-bit: + + ./configure --build=x86_64-apple-darwin11 + +For developers using the Git tree: + +This source tree is based on the autotools suite from GNU to simplify +portability. Here are some things you should have on your system in order to +compile the git repository tree : + + - GNU autotools (automake >=1.10, autoconf >=2.50, autoheader >=2.50) + (make sure your system wide `automake` points to a recent version!) + - GNU Libtool >=2.2 + (for more information, go to http://www.gnu.org/software/autoconf/) + +If you get the tree from the repository, you will need to use the `bootstrap` +script in the root of the tree. It calls all the GNU tools needed to prepare +the tree configuration. + +Test scripts provided in the `tests/` directory of the source tree depend +on `bash` and the `seq` program. + + +API +--- + +See the relevant API documentation files in `doc/`. The APIs provided by +Userspace RCU are, by prefix: + + - `rcu_`: Read-Copy Update (see [`doc/rcu-api.md`](doc/rcu-api.md)) + - `cmm_`: Concurrent Memory Model + - `caa_`: Concurrent Architecture Abstraction + - `cds_`: Concurrent Data Structures + (see [`doc/cds-api.md`](doc/cds-api.md)) + - `uatomic_`: Userspace Atomic + (see [`doc/uatomic-api.md`](doc/uatomic-api.md)) + + +Quick start guide +----------------- + +### Usage of all urcu libraries: + + - Define `_LGPL_SOURCE` (only) if your code is LGPL or GPL compatible + before including the `urcu.h` or `urcu-qsbr.h` header. If your application + is distributed under another license, function calls will be generated + instead of inlines, so your application can link with the library. + - Linking with one of the libraries below is always necessary even for + LGPL and GPL applications. + - Define `URCU_INLINE_SMALL_FUNCTIONS` before including Userspace RCU + headers if you want Userspace RCU to inline small functions (10 + lines or less) into the application. It can be used by applications + distributed under any kind of license, and does *not* make the + application a derived work of Userspace RCU. + +Those small inlined functions are guaranteed to match the library +content as long as the library major version is unchanged. +Therefore, the application *must* be compiled with headers matching +the library major version number. Applications using +`URCU_INLINE_SMALL_FUNCTIONS` may be unable to use debugging +features of Userspace RCU without being recompiled. + + +### Usage of `liburcu` + + 1. `#include ` + 2. Link the application with `-lurcu` + +This is the preferred version of the library, in terms of +grace-period detection speed, read-side speed and flexibility. +Dynamically detects kernel support for `sys_membarrier()`. Falls back +on `urcu-mb` scheme if support is not present, which has slower +read-side. + + +### Usage of `liburcu-qsbr` + + 1. `#include ` + 2. Link with `-lurcu-qsbr` + +The QSBR flavor of RCU needs to have each reader thread executing +`rcu_quiescent_state()` periodically to progress. `rcu_thread_online()` +and `rcu_thread_offline()` can be used to mark long periods for which +the threads are not active. It provides the fastest read-side at the +expense of more intrusiveness in the application code. + + +### Usage of `liburcu-mb` + + 1. `#include ` + 2. Compile any `_LGPL_SOURCE` code using this library with `-DRCU_MB` + 3. Link with `-lurcu-mb` + +This version of the urcu library uses memory barriers on the writer +and reader sides. This results in faster grace-period detection, but +results in slower reads. + + +### Usage of `liburcu-signal` + + 1. `#include ` + 2. Compile any `_LGPL_SOURCE` code using this library with `-DRCU_SIGNAL` + 3. Link the application with `-lurcu-signal` + +Version of the library that requires a signal, typically `SIGUSR1`. Can +be overridden with `-DSIGRCU` by modifying `Makefile.build.inc`. + + +### Usage of `liburcu-bp` + + 1. `#include ` + 2. Link with `-lurcu-bp` + +The BP library flavor stands for "bulletproof". It is specifically +designed to help tracing library to hook on applications without +requiring to modify these applications. `rcu_init()`, +`rcu_register_thread()` and `rcu_unregister_thread()` all become nops. +The state is dealt with by the library internally at the expense of +read-side and write-side performance. + + +### Initialization + +Each thread that has reader critical sections (that uses +`rcu_read_lock()`/`rcu_read_unlock()` must first register to the URCU +library. This is done by calling `rcu_register_thread()`. Unregistration +must be performed before exiting the thread by using +`rcu_unregister_thread()`. + + +### Reading + +Reader critical sections must be protected by locating them between +calls to `rcu_read_lock()` and `rcu_read_unlock()`. Inside that lock, +`rcu_dereference()` may be called to read an RCU protected pointer. + + +### Writing + +`rcu_assign_pointer()` and `rcu_xchg_pointer()` may be called anywhere. +After, `synchronize_rcu()` must be called. When it returns, the old +values are not in usage anymore. + + +### Usage of `liburcu-defer` + + - Follow instructions for either `liburcu`, `liburcu-qsbr`, + `liburcu-mb`, `liburcu-signal`, or `liburcu-bp` above. + The `liburcu-defer` functionality is pulled into each of + those library modules. + - Provides `defer_rcu()` primitive to enqueue delayed callbacks. Queued + callbacks are executed in batch periodically after a grace period. + Do _not_ use `defer_rcu()` within a read-side critical section, because + it may call `synchronize_rcu()` if the thread queue is full. + This can lead to deadlock or worse. + - Requires that `rcu_defer_barrier()` must be called in library destructor + if a library queues callbacks and is expected to be unloaded with + `dlclose()`. + +Its API is currently experimental. It may change in future library releases. + + +### Usage of `urcu-call-rcu` + + - Follow instructions for either `liburcu`, `liburcu-qsbr`, + `liburcu-mb`, `liburcu-signal`, or `liburcu-bp` above. + The `urcu-call-rcu` functionality is pulled into each of + those library modules. + - Provides the `call_rcu()` primitive to enqueue delayed callbacks + in a manner similar to `defer_rcu()`, but without ever delaying + for a grace period. On the other hand, `call_rcu()`'s best-case + overhead is not quite as good as that of `defer_rcu()`. + - Provides `call_rcu()` to allow asynchronous handling of RCU + grace periods. A number of additional functions are provided + to manage the helper threads used by `call_rcu()`, but reasonable + defaults are used if these additional functions are not invoked. + See [`doc/rcu-api.md`](doc/rcu-api.md) in userspace-rcu documentation + for more details. + + +### Being careful with signals + +The `liburcu` library uses signals internally. The signal handler is +registered with the `SA_RESTART` flag. However, these signals may cause +some non-restartable system calls to fail with `errno = EINTR`. Care +should be taken to restart system calls manually if they fail with this +error. A list of non-restartable system calls may be found in +`signal(7)`. The `liburcu-mb` and `liburcu-qsbr` versions of the Userspace RCU +library do not require any signal. + +Read-side critical sections are allowed in a signal handler, +except those setup with `sigaltstack(2)`, with `liburcu` and +`liburcu-mb`. Be careful, however, to disable these signals +between thread creation and calls to `rcu_register_thread()`, because a +signal handler nesting on an unregistered thread would not be +allowed to call `rcu_read_lock()`. + +Read-side critical sections are _not_ allowed in a signal handler with +`liburcu-qsbr`, unless signals are disabled explicitly around each +`rcu_quiescent_state()` calls, when threads are put offline and around +calls to `synchronize_rcu()`. Even then, we do not recommend it. + + +### Interaction with mutexes + +One must be careful to do not cause deadlocks due to interaction of +`synchronize_rcu()` and RCU read-side with mutexes. If `synchronize_rcu()` +is called with a mutex held, this mutex (or any mutex which has this +mutex in its dependency chain) should not be acquired from within a RCU +read-side critical section. + +This is especially important to understand in the context of the +QSBR flavor: a registered reader thread being "online" by +default should be considered as within a RCU read-side critical +section unless explicitly put "offline". Therefore, if +`synchronize_rcu()` is called with a mutex held, this mutex, as +well as any mutex which has this mutex in its dependency chain +should only be taken when the RCU reader thread is "offline" +(this can be performed by calling `rcu_thread_offline()`). + + +### Interaction with `fork()` + +Special care must be taken for applications performing `fork()` without +any following `exec()`. This is caused by the fact that Linux only clones +the thread calling `fork()`, and thus never replicates any of the other +parent thread into the child process. Most `liburcu` implementations +require that all registrations (as reader, `defer_rcu` and `call_rcu` +threads) should be released before a `fork()` is performed, except for the +rather common scenario where `fork()` is immediately followed by `exec()` in +the child process. The only implementation not subject to that rule is +`liburcu-bp`, which is designed to handle `fork()` by calling +`rcu_bp_before_fork`, `rcu_bp_after_fork_parent` and +`rcu_bp_after_fork_child`. + +Applications that use `call_rcu()` and that `fork()` without +doing an immediate `exec()` must take special action. The parent +must invoke `call_rcu_before_fork()` before the `fork()` and +`call_rcu_after_fork_parent()` after the `fork()`. The child +process must invoke `call_rcu_after_fork_child()`. +Even though these three APIs are suitable for passing to +`pthread_atfork()`, use of `pthread_atfork()` is **STRONGLY +DISCOURAGED** for programs calling the glibc memory allocator +(`malloc()`, `calloc()`, `free()`, ...) within `call_rcu` callbacks. +This is due to limitations in the way glibc memory allocator +handles calls to the memory allocator from concurrent threads +while the `pthread_atfork()` handlers are executing. + +Combining e.g.: + + - call to `free()` from callbacks executed within `call_rcu` worker + threads, + - executing `call_rcu` atfork handlers within the glibc pthread + atfork mechanism, + +will sometimes trigger interesting process hangs. This usually +hangs on a memory allocator lock within glibc. + + +### Thread Local Storage (TLS) + +Userspace RCU can fall back on `pthread_getspecific()` to emulate +TLS variables on systems where it is not available. This behavior +can be forced by specifying `--disable-compiler-tls` as configure +argument. + + +### Usage of `DEBUG_RCU` + +`DEBUG_RCU` is used to add internal debugging self-checks to the +RCU library. This define adds a performance penalty when enabled. +Can be enabled by uncommenting the corresponding line in +`Makefile.build.inc`. + + +### Usage of `DEBUG_YIELD` + +`DEBUG_YIELD` is used to add random delays in the code for testing +purposes. + + +### SMP support + +By default the library is configured to use synchronization primitives +adequate for SMP systems. On uniprocessor systems, support for SMP +systems can be disabled with: + + ./configure --disable-smp-support + +theoretically yielding slightly better performance. + + +Make targets +------------ + +In addition to the usual `make check` target, Userspace RCU features +`make regtest` and `make bench` targets: + + - `make check`: short tests, meant to be run when rebuilding or + porting Userspace RCU. + - `make regtest`: long (many hours) test, meant to be run when + modifying Userspace RCU or porting it to a new architecture or + operating system. + - `make bench`: long (many hours) benchmarks. + + +Contacts +-------- + +You can contact the maintainers on the following mailing list: +`lttng-dev@lists.lttng.org`. diff --git a/compat_arch_x86.c b/compat_arch_x86.c index 7d3b83a..3e73f9c 100644 --- a/compat_arch_x86.c +++ b/compat_arch_x86.c @@ -26,6 +26,14 @@ #include #include +/* + * Using attribute "weak" for __rcu_cas_avail and + * __urcu_x86_compat_mutex. Those are globally visible by the entire + * program, even though many shared objects may have their own version. + * The first version that gets loaded will be used by the entire + * program (executable and all shared objects). + */ + /* * It does not really matter if the constructor is called before using * the library, as long as the caller checks if __rcu_cas_avail < 0 and calls @@ -38,9 +46,11 @@ int __attribute__((constructor)) __rcu_cas_init(void); * 1: available * 0: unavailable */ +__attribute__((weak)) int __rcu_cas_avail = -1; -static pthread_mutex_t compat_mutex = PTHREAD_MUTEX_INITIALIZER; +__attribute__((weak)) +pthread_mutex_t __urcu_x86_compat_mutex = PTHREAD_MUTEX_INITIALIZER; /* * get_eflags/set_eflags/compare_and_swap_is_available imported from glibc @@ -84,7 +94,7 @@ static void mutex_lock_signal_save(pthread_mutex_t *mutex, sigset_t *oldmask) assert(!ret); ret = pthread_sigmask(SIG_BLOCK, &newmask, oldmask); assert(!ret); - ret = pthread_mutex_lock(&compat_mutex); + ret = pthread_mutex_lock(&__urcu_x86_compat_mutex); assert(!ret); } @@ -92,7 +102,7 @@ static void mutex_lock_signal_restore(pthread_mutex_t *mutex, sigset_t *oldmask) { int ret; - ret = pthread_mutex_unlock(&compat_mutex); + ret = pthread_mutex_unlock(&__urcu_x86_compat_mutex); assert(!ret); ret = pthread_sigmask(SIG_SETMASK, oldmask, NULL); assert(!ret); @@ -103,7 +113,7 @@ unsigned long _compat_uatomic_set(void *addr, unsigned long _new, int len) sigset_t mask; unsigned long result; - mutex_lock_signal_save(&compat_mutex, &mask); + mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask); switch (len) { case 1: *(unsigned char *)addr = (unsigned char)_new; @@ -125,7 +135,7 @@ unsigned long _compat_uatomic_set(void *addr, unsigned long _new, int len) result = 0; __asm__ __volatile__("ud2"); } - mutex_lock_signal_restore(&compat_mutex, &mask); + mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask); return result; } @@ -134,7 +144,7 @@ unsigned long _compat_uatomic_xchg(void *addr, unsigned long _new, int len) sigset_t mask; unsigned long retval; - mutex_lock_signal_save(&compat_mutex, &mask); + mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask); switch (len) { case 1: retval = *(unsigned char *)addr; @@ -156,7 +166,7 @@ unsigned long _compat_uatomic_xchg(void *addr, unsigned long _new, int len) retval = 0; /* silence gcc warnings */ __asm__ __volatile__("ud2"); } - mutex_lock_signal_restore(&compat_mutex, &mask); + mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask); return retval; } @@ -166,7 +176,7 @@ unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old, unsigned long retval; sigset_t mask; - mutex_lock_signal_save(&compat_mutex, &mask); + mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask); switch (len) { case 1: { @@ -200,7 +210,7 @@ unsigned long _compat_uatomic_cmpxchg(void *addr, unsigned long old, retval = 0; /* silence gcc warnings */ __asm__ __volatile__("ud2"); } - mutex_lock_signal_restore(&compat_mutex, &mask); + mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask); return retval; } @@ -208,7 +218,7 @@ void _compat_uatomic_or(void *addr, unsigned long v, int len) { sigset_t mask; - mutex_lock_signal_save(&compat_mutex, &mask); + mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask); switch (len) { case 1: *(unsigned char *)addr |= (unsigned char)v; @@ -226,14 +236,14 @@ void _compat_uatomic_or(void *addr, unsigned long v, int len) */ __asm__ __volatile__("ud2"); } - mutex_lock_signal_restore(&compat_mutex, &mask); + mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask); } void _compat_uatomic_and(void *addr, unsigned long v, int len) { sigset_t mask; - mutex_lock_signal_save(&compat_mutex, &mask); + mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask); switch (len) { case 1: *(unsigned char *)addr &= (unsigned char)v; @@ -251,7 +261,7 @@ void _compat_uatomic_and(void *addr, unsigned long v, int len) */ __asm__ __volatile__("ud2"); } - mutex_lock_signal_restore(&compat_mutex, &mask); + mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask); } unsigned long _compat_uatomic_add_return(void *addr, unsigned long v, int len) @@ -259,7 +269,7 @@ unsigned long _compat_uatomic_add_return(void *addr, unsigned long v, int len) sigset_t mask; unsigned long result; - mutex_lock_signal_save(&compat_mutex, &mask); + mutex_lock_signal_save(&__urcu_x86_compat_mutex, &mask); switch (len) { case 1: *(unsigned char *)addr += (unsigned char)v; @@ -281,7 +291,7 @@ unsigned long _compat_uatomic_add_return(void *addr, unsigned long v, int len) result = 0; /* silence gcc warnings */ __asm__ __volatile__("ud2"); } - mutex_lock_signal_restore(&compat_mutex, &mask); + mutex_lock_signal_restore(&__urcu_x86_compat_mutex, &mask); return result; } diff --git a/compat_futex.c b/compat_futex.c index bb928e6..9a08624 100644 --- a/compat_futex.c +++ b/compat_futex.c @@ -31,8 +31,18 @@ #include #include -static pthread_mutex_t compat_futex_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t compat_futex_cond = PTHREAD_COND_INITIALIZER; +/* + * Using attribute "weak" for __urcu_compat_futex_lock and + * __urcu_compat_futex_cond. Those are globally visible by the entire + * program, even though many shared objects may have their own version. + * The first version that gets loaded will be used by the entire program + * (executable and all shared objects). + */ + +__attribute__((weak)) +pthread_mutex_t __urcu_compat_futex_lock = PTHREAD_MUTEX_INITIALIZER; +__attribute__((weak)) +pthread_cond_t __urcu_compat_futex_cond = PTHREAD_COND_INITIALIZER; /* * _NOT SIGNAL-SAFE_. pthread_cond is not signal-safe anyway. Though. @@ -58,22 +68,31 @@ int compat_futex_noasync(int32_t *uaddr, int op, int32_t val, */ cmm_smp_mb(); - ret = pthread_mutex_lock(&compat_futex_lock); + ret = pthread_mutex_lock(&__urcu_compat_futex_lock); assert(!ret); switch (op) { case FUTEX_WAIT: - if (*uaddr != val) - goto end; - pthread_cond_wait(&compat_futex_cond, &compat_futex_lock); + /* + * Wait until *uaddr is changed to something else than "val". + * Comparing *uaddr content against val figures out which + * thread has been awakened. + */ + while (*uaddr == val) + pthread_cond_wait(&__urcu_compat_futex_cond, + &__urcu_compat_futex_lock); break; case FUTEX_WAKE: - pthread_cond_broadcast(&compat_futex_cond); + /* + * Each wake is sending a broadcast, thus attempting wakeup of + * all awaiting threads, independently of their respective + * uaddr. + */ + pthread_cond_broadcast(&__urcu_compat_futex_cond); break; default: gret = -EINVAL; } -end: - ret = pthread_mutex_unlock(&compat_futex_lock); + ret = pthread_mutex_unlock(&__urcu_compat_futex_lock); assert(!ret); return gret; } diff --git a/configure.ac b/configure.ac index 017cdf1..c3912bf 100644 --- a/configure.ac +++ b/configure.ac @@ -2,17 +2,18 @@ # Process this file with autoconf to produce a configure script. -AC_INIT([userspace-rcu],[0.7.4],[mathieu dot desnoyers at efficios dot com]) +AC_INIT([userspace-rcu],[0.8.0],[mathieu dot desnoyers at efficios dot com]) # Following the numbering scheme proposed by libtool for the library version # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -AC_SUBST([URCU_LIBRARY_VERSION], [2:0:0]) +AC_SUBST([URCU_LIBRARY_VERSION], [3:0:0]) AC_CONFIG_AUX_DIR([config]) AC_CONFIG_MACRO_DIR([config]) AC_CANONICAL_TARGET AC_CANONICAL_HOST AM_INIT_AUTOMAKE([foreign dist-bzip2 no-dist-gzip]) +AM_MAINTAINER_MODE([enable]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) m4_include([config/ax_tls.m4]) @@ -52,11 +53,20 @@ LT_INIT AC_C_INLINE AC_TYPE_PID_T AC_TYPE_SIZE_T +AC_TYPE_INT32_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T +AC_TYPE_UINT8_T # Checks for library functions. AC_FUNC_MALLOC AC_FUNC_MMAP -AC_CHECK_FUNCS([bzero gettimeofday munmap sched_getcpu strtoul sysconf]) +AC_CHECK_FUNCS( + [bzero gettimeofday munmap sched_getcpu strtoul sysconf gettid memeset strerror] +) + +# Check for headers +AC_CHECK_HEADERS([limits.h stddef.h sys/time.h]) # Find arch type AS_CASE([$host_cpu], @@ -69,6 +79,7 @@ AS_CASE([$host_cpu], [powerpc], [ARCHTYPE="ppc"], [ppc64], [ARCHTYPE="ppc"], [powerpc64], [ARCHTYPE="ppc"], + [powerpc64le], [ARCHTYPE="gcc"], [ppc], [ARCHTYPE="ppc"], [s390], [ARCHTYPE="s390"], [s390x], [ARCHTYPE="s390"], @@ -77,11 +88,18 @@ AS_CASE([$host_cpu], [alpha*], [ARCHTYPE="alpha"], [ia64], [ARCHTYPE="gcc"], [arm*], [ARCHTYPE="arm"], + [aarch64], [ARCHTYPE="gcc"], [mips*], [ARCHTYPE="mips"], [tile*], [ARCHTYPE="gcc"], + [hppa*], [ARCHTYPE="hppa"], [ARCHTYPE="unknown"] ) +AS_CASE([$host],[*-*-linux-androideabi], + [AM_CONDITIONAL(TARGET_IS_ANDROID, true)], + [AM_CONDITIONAL(TARGET_IS_ANDROID, false)] +) + AC_SUBST(ARCHTYPE) AC_SUBST(SUBARCHTYPE) @@ -135,7 +153,10 @@ AS_IF([test "x$ARCHTYPE" = "xx86"],[ #For now, using lock; addl compatibility mode even for i686, because the #Pentium III is seen as a i686, but lacks mfence instruction. #Only using fence for x86_64. - AS_IF([test "x$host_cpu" != "xi386" -a "x$host_cpu" != "xi486" -a "x$host_cpu" != "xi586" -a "x$host_cpu" != "xi686"],[ + # + #k1om is the name for the Intel MIC family (Xeon Phi). It is an x86_64 + #variant but lacks fence instructions. + AS_IF([test "x$host_cpu" != "xi386" -a "x$host_cpu" != "xi486" -a "x$host_cpu" != "xi586" -a "x$host_cpu" != "xi686" -a "x$host_vendor" != "xk1om"],[ AC_MSG_RESULT([yes]) AC_DEFINE([CONFIG_RCU_HAVE_FENCE], [1]) ],[ @@ -160,6 +181,15 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ compat_futex_test=1 ]) +# Check for pthread +AC_CHECK_LIB([pthread], [pthread_create], + [AM_CONDITIONAL(LIBC_INCLUDES_PTHREAD, false)], + [AC_CHECK_LIB([c], [pthread_create], + [AM_CONDITIONAL(LIBC_INCLUDES_PTHREAD, true)], + [AC_MSG_ERROR([Cannot find libpthread. Use [LDFLAGS]=-Ldir to specify its location.])] + )] +) + AM_CONDITIONAL([COMPAT_FUTEX], [test "x$compat_futex_test" = "x1"]) AM_CONDITIONAL([COMPAT_ARCH], [test "x$SUBARCHTYPE" = "xx86compat"]) AM_CONDITIONAL([NO_SHARED], [test "x$enable_shared" = "xno"]) @@ -294,6 +324,10 @@ AC_CONFIG_FILES([ doc/examples/Makefile tests/Makefile rcuja/Makefile + tests/common/Makefile + tests/unit/Makefile + tests/benchmark/Makefile + tests/regression/Makefile liburcu.pc liburcu-bp.pc liburcu-cds.pc diff --git a/doc/Makefile.am b/doc/Makefile.am index d98f16b..ce1774c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,3 +1,3 @@ SUBDIRS = examples -dist_doc_DATA = rcu-api.txt cds-api.txt uatomic-api.txt +dist_doc_DATA = rcu-api.md cds-api.md uatomic-api.md diff --git a/doc/cds-api.md b/doc/cds-api.md new file mode 100644 index 0000000..49a3c7c --- /dev/null +++ b/doc/cds-api.md @@ -0,0 +1,84 @@ +Userspace RCU Concurrent Data Structures (CDS) API +================================================== + +by Mathieu Desnoyers and Paul E. McKenney + +This document describes briefly the data structures contained with the +userspace RCU library. + + +Data structure files +-------------------- + +### `urcu/list.h` + +Doubly-linked list, which requires mutual exclusion on +updates and reads. + + +### `urcu/rculist.h` + +Doubly-linked list, which requires mutual exclusion on +updates, allows RCU read traversals. + + +### `urcu/hlist.h` + +Doubly-linked list, with single pointer list head. Requires +mutual exclusion on updates and reads. Useful for implementing hash tables. +Downside over `list.h`: lookup of tail in O(n). + + +### `urcu/rcuhlist.h` + +Doubly-linked list, with single pointer list head. +Requires mutual exclusion on updates, allows RCU read traversals. Useful +for implementing hash tables. Downside over rculist.h: lookup of tail in O(n). + + +### `urcu/wfstack.h` + +Stack with wait-free push and wait-free pop_all. Both +blocking and non-blocking pop and traversal operations are provided. This +stack does _not_ specifically rely on RCU. Various synchronization techniques +can be used to deal with pop ABA. Those are detailed in the API. + + +### `urcu/wfcqueue.h` + +Concurrent queue with wait-free enqueue. Both blocking and +non-blocking dequeue, splice (move all elements from one queue +to another), and traversal operations are provided. + +This queue does _not_ specifically rely on RCU. Mutual exclusion +is used to protect dequeue, splice (from source queue) and +traversal (see API for details). + + - Note: deprecates `urcu/wfqueue.h`. + + +### `urcu/lfstack.h` + +Stack with lock-free push, lock-free pop, wait-free pop_all, +wait-free traversal. Various synchronization techniques can be +used to deal with pop ABA. Those are detailed in the API. +This stack does _not_ specifically rely on RCU. + + - Note: deprecates `urcu/rculfstack.h`. + + +### `urcu/rculfqueue.h` + +RCU queue with lock-free enqueue, lock-free dequeue. +This queue relies on RCU for existence guarantees. + + +### `urcu/rculfhash.h` + +Lock-Free Resizable RCU Hash Table. RCU used to provide +existance guarantees. Provides scalable updates, and scalable +RCU read-side lookups and traversals. Unique and duplicate keys +are supported. Provides "uniquify add" and "replace add" +operations, along with associated read-side traversal uniqueness +guarantees. Automatic hash table resize based on number of +elements is supported. See the API for more details. diff --git a/doc/cds-api.txt b/doc/cds-api.txt deleted file mode 100644 index 8896e20..0000000 --- a/doc/cds-api.txt +++ /dev/null @@ -1,70 +0,0 @@ -Userspace RCU Concurrent Data Structures (CDS) API -by Mathieu Desnoyers and Paul E. McKenney - - -This document describes briefly the data structures contained with the -userspace RCU library. - -urcu/list.h: - - Doubly-linked list, which requires mutual exclusion on updates - and reads. - -urcu/rculist.h: - - Doubly-linked list, which requires mutual exclusion on updates, - allows RCU read traversals. - -urcu/hlist.h: - - Doubly-linked list, with single pointer list head. Requires - mutual exclusion on updates and reads. Useful for implementing - hash tables. Downside over list.h: lookup of tail in O(n). - -urcu/rcuhlist.h: - - Doubly-linked list, with single pointer list head. Requires - mutual exclusion on updates, allows RCU read traversals. Useful - for implementing hash tables. Downside over rculist.h: lookup of - tail in O(n). - -urcu/wfstack.h: - - Stack with wait-free push and wait-free pop_all. Both blocking - and non-blocking pop and traversal operations are provided. - This stack does _not_ specifically rely on RCU. - Various synchronization techniques can be used to deal with - pop ABA. Those are detailed in the API. - -urcu/wfcqueue.h: - - Concurrent queue with wait-free enqueue. Both blocking and - non-blocking dequeue, splice (move all elements from one queue - to another), and traversal operations are provided. - This queue does _not_ specifically rely on RCU. Mutual exclusion - is used to protect dequeue, splice (from source queue) and - traversal (see API for details). - (note: deprecates urcu/wfqueue.h) - -urcu/lfstack.h: - - Stack with lock-free push, lock-free pop, wait-free pop_all, - wait-free traversal. Various synchronization techniques can be - used to deal with pop ABA. Those are detailed in the API. - This stack does _not_ specifically rely on RCU. - (note: deprecates urcu/rculfstack.h) - -urcu/rculfqueue.h: - - RCU queue with lock-free enqueue, lock-free dequeue. - This queue relies on RCU for existence guarantees. - -urcu/rculfhash.h: - - Lock-Free Resizable RCU Hash Table. RCU used to provide - existance guarantees. Provides scalable updates, and scalable - RCU read-side lookups and traversals. Unique and duplicate keys - are supported. Provides "uniquify add" and "replace add" - operations, along with associated read-side traversal uniqueness - guarantees. Automatic hash table resize based on number of - elements is supported. See the API for more details. diff --git a/doc/examples/Makefile.am b/doc/examples/Makefile.am index 16477b5..3ae6842 100644 --- a/doc/examples/Makefile.am +++ b/doc/examples/Makefile.am @@ -115,9 +115,42 @@ if NO_SHARED # Don't build examples if shared libraries support was explicitly # disabled. else + +SUBDIRS_PROXY = hlist list urcu-flavors wfcqueue rculfqueue \ + wfstack lfstack rculfhash + +# Copies are for VPATH build support. all-local: - $(MAKE) -f dist-files/Makefile AM_CPPFLAGS="-I../../../urcu/ -I../../../" AM_LDFLAGS='-L../../../.libs/ -Wl,-rpath="$(PWD)/../../.libs/"' $(AM_MAKEFLAGS) all + @if [ x"$(srcdir)" != x"$(builddir)" ]; then \ + cp -pfR $(srcdir)/dist-files $(builddir); \ + cp -pf $(srcdir)/Makefile.examples.template $(builddir); \ + for subdir in $(SUBDIRS_PROXY); do \ + cp -pfR $(srcdir)/$$subdir $(builddir); \ + done; \ + fi; \ + if [ x"$(shell echo "$(top_srcdir)" | grep "^/" | wc -l)" = x"1" ]; then \ + echo "Examples: absolute top_srcdir path $(top_srcdir)"; \ + rel_src_subdir=""; \ + else \ + echo "Examples: relative top_srcdir path $(top_srcdir)"; \ + rel_src_subdir="../"; \ + fi; \ + if [ x"$(shell echo "$(top_builddir)" | grep "^/" | wc -l)" = x"1" ]; then \ + echo "Examples: absolute top_builddir path $(top_builddir)"; \ + rel_build_subdir=""; \ + else \ + echo "Examples: relative top_builddir path $(top_builddir)"; \ + rel_build_subdir="../"; \ + fi; \ + $(MAKE) -f dist-files/Makefile AM_CC="$(CC)" AM_CPPFLAGS="$(CPPFLAGS) -I$$rel_src_subdir/$(top_srcdir)/ -I$$rel_build_subdir$(top_builddir)/" AM_CFLAGS='$(CFLAGS)' AM_LDFLAGS='$(LDFLAGS) -L../../../.libs/ -Wl,-rpath "$(PWD)/../../.libs/"' $(AM_MAKEFLAGS) all; clean-local: - $(MAKE) -f dist-files/Makefile $(AM_MAKEFLAGS) clean; + @$(MAKE) -f dist-files/Makefile $(AM_MAKEFLAGS) clean; \ + if [ x"$(srcdir)" != x"$(builddir)" ]; then \ + for subdir in $(SUBDIRS_PROXY); do \ + rm -rf $(builddir)/$$subdir; \ + done; \ + rm -f $(builddir)/Makefile.examples.template; \ + rm -rf $(builddir)/dist-files; \ + fi; endif diff --git a/doc/examples/Makefile.examples.template b/doc/examples/Makefile.examples.template index badecff..6dd2bac 100644 --- a/doc/examples/Makefile.examples.template +++ b/doc/examples/Makefile.examples.template @@ -11,14 +11,16 @@ # # This makefile is purposefully kept simple to support GNU and BSD make. -CC = gcc +ifdef AM_CC +CC = $(AM_CC) +endif CFLAGS = -g -O2 -Wall all: $(BINARY) $(BINARY): $(OBJECTS) $(CC) $(CFLAGS) $(LDFLAGS) $(AM_CFLAGS) $(AM_LDFLAGS) \ - $(LIBS) -o $@ $(OBJECTS) + -o $@ $(OBJECTS) $(LIBS) $(OBJECTS): $(SOURCES) $(DEPS) $(CC) $(CPPFLAGS) $(CFLAGS) $(AM_CPPFLAGS) $(AM_CFLAGS) \ diff --git a/doc/rcu-api.md b/doc/rcu-api.md new file mode 100644 index 0000000..ea316d1 --- /dev/null +++ b/doc/rcu-api.md @@ -0,0 +1,240 @@ +Userspace RCU API +================= + +by Mathieu Desnoyers and Paul E. McKenney + + +API +--- + +```c +void rcu_init(void); +``` + +This must be called before any of the following functions +are invoked. + + +```c +void rcu_read_lock(void); +``` + +Begin an RCU read-side critical section. These critical +sections may be nested. + + +```c +void rcu_read_unlock(void); +``` + +End an RCU read-side critical section. + + +```c +void rcu_register_thread(void); +``` + +Each thread must invoke this function before its first call to +`rcu_read_lock()`. Threads that never call `rcu_read_lock()` need +not invoke this function. In addition, `rcu-bp` ("bullet proof" +RCU) does not require any thread to invoke `rcu_register_thread()`. + + +```c +void rcu_unregister_thread(void); +``` + +Each thread that invokes `rcu_register_thread()` must invoke +`rcu_unregister_thread()` before `invoking pthread_exit()` +or before returning from its top-level function. + + +```c +void synchronize_rcu(void); +``` + +Wait until every pre-existing RCU read-side critical section +has completed. Note that this primitive will not necessarily +wait for RCU read-side critical sections that have not yet +started: this is not a reader-writer lock. The duration +actually waited is called an RCU grace period. + + +```c +void call_rcu(struct rcu_head *head, + void (*func)(struct rcu_head *head)); +``` + +Registers the callback indicated by "head". This means +that `func` will be invoked after the end of a future +RCU grace period. The `rcu_head` structure referenced +by `head` will normally be a field in a larger RCU-protected +structure. A typical implementation of `func` is as +follows: + +```c +void func(struct rcu_head *head) +{ + struct foo *p = container_of(head, struct foo, rcu); + + free(p); +} +``` + +This RCU callback function can be registered as follows +given a pointer `p` to the enclosing structure: + +```c +call_rcu(&p->rcu, func); +``` + +`call_rcu` should be called from registered RCU read-side threads. +For the QSBR flavor, the caller should be online. + + +```c +void rcu_barrier(void); +``` + +Wait for all `call_rcu()` work initiated prior to `rcu_barrier()` by +_any_ thread on the system to have completed before `rcu_barrier()` +returns. `rcu_barrier()` should never be called from a `call_rcu()` +thread. This function can be used, for instance, to ensure that +all memory reclaim involving a shared object has completed +before allowing `dlclose()` of this shared object to complete. + + +```c +struct call_rcu_data *create_call_rcu_data(unsigned long flags, + int cpu_affinity); +``` + +Returns a handle that can be passed to the following +primitives. The `flags` argument can be zero, or can be +`URCU_CALL_RCU_RT` if the worker threads associated with the +new helper thread are to get real-time response. The argument +`cpu_affinity` specifies a CPU on which the `call_rcu` thread should +be affined to. It is ignored if negative. + + +```c +void call_rcu_data_free(struct call_rcu_data *crdp); +``` + +Terminates a `call_rcu()` helper thread and frees its associated +data. The caller must have ensured that this thread is no longer +in use, for example, by passing `NULL` to `set_thread_call_rcu_data()` +and `set_cpu_call_rcu_data()` as required. + + +```c +struct call_rcu_data *get_default_call_rcu_data(void); +``` + +Returns the handle for the default `call_rcu()` helper thread. +Creates it if necessary. + + +```c +struct call_rcu_data *get_cpu_call_rcu_data(int cpu); +``` + +Returns the handle for the current CPU's `call_rcu()` helper +thread, or `NULL` if the current CPU has no helper thread +currently assigned. The call to this function and use of the +returned `call_rcu_data` should be protected by RCU read-side +lock. + + +```c +struct call_rcu_data *get_thread_call_rcu_data(void); +``` + +Returns the handle for the current thread's hard-assigned +`call_rcu()` helper thread, or `NULL` if the current thread is +instead using a per-CPU or the default helper thread. + + +```c +struct call_rcu_data *get_call_rcu_data(void); +``` + +Returns the handle for the current thread's `call_rcu()` helper +thread, which is either, in increasing order of preference: +per-thread hard-assigned helper thread, per-CPU helper thread, +or default helper thread. `get_call_rcu_data` should be called +from registered RCU read-side threads. For the QSBR flavor, the +caller should be online. + + +```c +pthread_t get_call_rcu_thread(struct call_rcu_data *crdp); +``` + +Returns the helper thread's pthread identifier linked to a call +rcu helper thread data. + + +```c +void set_thread_call_rcu_data(struct call_rcu_data *crdp); +``` + +Sets the current thread's hard-assigned `call_rcu()` helper to the +handle specified by `crdp`. Note that `crdp` can be `NULL` to +disassociate this thread from its helper. Once a thread is +disassociated from its helper, further `call_rcu()` invocations +use the current CPU's helper if there is one and the default +helper otherwise. + + +```c +int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp); +``` + +Sets the specified CPU's `call_rcu()` helper to the handle +specified by `crdp`. Again, `crdp` can be `NULL` to disassociate +this CPU from its helper thread. Once a CPU has been +disassociated from its helper, further `call_rcu()` invocations +that would otherwise have used this CPU's helper will instead +use the default helper. + +The caller must wait for a grace-period to pass between return from +`set_cpu_call_rcu_data()` and call to `call_rcu_data_free()` passing the +previous call rcu data as argument. + + +```c +int create_all_cpu_call_rcu_data(unsigned long flags); +``` + +Creates a separate `call_rcu()` helper thread for each CPU. +After this primitive is invoked, the global default `call_rcu()` +helper thread will not be called. + +The `set_thread_call_rcu_data()`, `set_cpu_call_rcu_data()`, and +`create_all_cpu_call_rcu_data()` functions may be combined to set up +pretty much any desired association between worker and `call_rcu()` +helper threads. If a given executable calls only `call_rcu()`, +then that executable will have only the single global default +`call_rcu()` helper thread. This will suffice in most cases. + + +```c +void free_all_cpu_call_rcu_data(void); +``` + +Clean up all the per-CPU `call_rcu` threads. Should be paired with +`create_all_cpu_call_rcu_data()` to perform teardown. Note that +this function invokes `synchronize_rcu()` internally, so the +caller should be careful not to hold mutexes (or mutexes within a +dependency chain) that are also taken within a RCU read-side +critical section, or in a section where QSBR threads are online. + + +```c +void call_rcu_after_fork_child(void); +``` + +Should be used as `pthread_atfork()` handler for programs using +`call_rcu` and performing `fork()` or `clone()` without a following +`exec()`. diff --git a/doc/rcu-api.txt b/doc/rcu-api.txt deleted file mode 100644 index b647316..0000000 --- a/doc/rcu-api.txt +++ /dev/null @@ -1,171 +0,0 @@ -Userspace RCU API -by Mathieu Desnoyers and Paul E. McKenney - - -void rcu_init(void); - - This must be called before any of the following functions - are invoked. - -void rcu_read_lock(void); - - Begin an RCU read-side critical section. These critical - sections may be nested. - -void rcu_read_unlock(void); - - End an RCU read-side critical section. - -void rcu_register_thread(void); - - Each thread must invoke this function before its first call to - rcu_read_lock(). Threads that never call rcu_read_lock() need - not invoke this function. In addition, rcu-bp ("bullet proof" - RCU) does not require any thread to invoke rcu_register_thread(). - -void rcu_unregister_thread(void); - - Each thread that invokes rcu_register_thread() must invoke - rcu_unregister_thread() before invoking pthread_exit() - or before returning from its top-level function. - -void synchronize_rcu(void); - - Wait until every pre-existing RCU read-side critical section - has completed. Note that this primitive will not necessarily - wait for RCU read-side critical sections that have not yet - started: this is not a reader-writer lock. The duration - actually waited is called an RCU grace period. - -void call_rcu(struct rcu_head *head, - void (*func)(struct rcu_head *head)); - - Registers the callback indicated by "head". This means - that "func" will be invoked after the end of a future - RCU grace period. The rcu_head structure referenced - by "head" will normally be a field in a larger RCU-protected - structure. A typical implementation of "func" is as - follows: - - void func(struct rcu_head *head) - { - struct foo *p = container_of(head, struct foo, rcu); - - free(p); - } - - This RCU callback function can be registered as follows - given a pointer "p" to the enclosing structure: - - call_rcu(&p->rcu, func); - - call_rcu should be called from registered RCU read-side threads. - For the QSBR flavor, the caller should be online. - -void rcu_barrier(void); - - Wait for all call_rcu() work initiated prior to rcu_barrier() by - _any_ thread on the system to have completed before rcu_barrier() - returns. rcu_barrier() should never be called from a call_rcu() - thread. This function can be used, for instance, to ensure that - all memory reclaim involving a shared object has completed - before allowing dlclose() of this shared object to complete. - -struct call_rcu_data *create_call_rcu_data(unsigned long flags, - int cpu_affinity); - - Returns a handle that can be passed to the following - primitives. The "flags" argument can be zero, or can be - URCU_CALL_RCU_RT if the worker threads associated with the - new helper thread are to get real-time response. The argument - "cpu_affinity" specifies a cpu on which the call_rcu thread should - be affined to. It is ignored if negative. - -void call_rcu_data_free(struct call_rcu_data *crdp); - - Terminates a call_rcu() helper thread and frees its associated - data. The caller must have ensured that this thread is no longer - in use, for example, by passing NULL to set_thread_call_rcu_data() - and set_cpu_call_rcu_data() as required. - -struct call_rcu_data *get_default_call_rcu_data(void); - - Returns the handle for the default call_rcu() helper thread. - Creates it if necessary. - -struct call_rcu_data *get_cpu_call_rcu_data(int cpu); - - Returns the handle for the current cpu's call_rcu() helper - thread, or NULL if the current CPU has no helper thread - currently assigned. The call to this function and use of the - returned call_rcu_data should be protected by RCU read-side - lock. - -struct call_rcu_data *get_thread_call_rcu_data(void); - - Returns the handle for the current thread's hard-assigned - call_rcu() helper thread, or NULL if the current thread is - instead using a per-CPU or the default helper thread. - -struct call_rcu_data *get_call_rcu_data(void); - - Returns the handle for the current thread's call_rcu() helper - thread, which is either, in increasing order of preference: - per-thread hard-assigned helper thread, per-cpu helper thread, - or default helper thread. get_call_rcu_data should be called - from registered RCU read-side threads. For the QSBR flavor, the - caller should be online. - -pthread_t get_call_rcu_thread(struct call_rcu_data *crdp); - - Returns the helper thread's pthread identifier linked to a call - rcu helper thread data. - -void set_thread_call_rcu_data(struct call_rcu_data *crdp); - - Sets the current thread's hard-assigned call_rcu() helper to the - handle specified by "crdp". Note that "crdp" can be NULL to - disassociate this thread from its helper. Once a thread is - disassociated from its helper, further call_rcu() invocations - use the current CPU's helper if there is one and the default - helper otherwise. - -int set_cpu_call_rcu_data(int cpu, struct call_rcu_data *crdp); - - Sets the specified CPU's call_rcu() helper to the handle - specified by "crdp". Again, "crdp" can be NULL to disassociate - this CPU from its helper thread. Once a CPU has been - disassociated from its helper, further call_rcu() invocations - that would otherwise have used this CPU's helper will instead - use the default helper. The caller must wait for a grace-period - to pass between return from set_cpu_call_rcu_data() and call to - call_rcu_data_free() passing the previous call rcu data as - argument. - -int create_all_cpu_call_rcu_data(unsigned long flags) - - Creates a separate call_rcu() helper thread for each CPU. - After this primitive is invoked, the global default call_rcu() - helper thread will not be called. - - The set_thread_call_rcu_data(), set_cpu_call_rcu_data(), and - create_all_cpu_call_rcu_data() functions may be combined to set up - pretty much any desired association between worker and call_rcu() - helper threads. If a given executable calls only call_rcu(), - then that executable will have only the single global default - call_rcu() helper thread. This will suffice in most cases. - -void free_all_cpu_call_rcu_data(void); - - Clean up all the per-CPU call_rcu threads. Should be paired with - create_all_cpu_call_rcu_data() to perform teardown. Note that - this function invokes synchronize_rcu() internally, so the - caller should be careful not to hold mutexes (or mutexes within a - dependency chain) that are also taken within a RCU read-side - critical section, or in a section where QSBR threads are online. - -void call_rcu_after_fork_child(void); - - Should be used as pthread_atfork() handler for programs using - call_rcu and performing fork() or clone() without a following - exec(). diff --git a/doc/uatomic-api.md b/doc/uatomic-api.md new file mode 100644 index 0000000..2dd63c6 --- /dev/null +++ b/doc/uatomic-api.md @@ -0,0 +1,124 @@ +Userspace RCU Atomic Operations API +=================================== + +by Mathieu Desnoyers and Paul E. McKenney + +This document describes the `` API. Those are the atomic +operations provided by the Userspace RCU library. The general rule +regarding memory barriers is that only `uatomic_xchg()`, +`uatomic_cmpxchg()`, `uatomic_add_return()`, and `uatomic_sub_return()` imply +full memory barriers before and after the atomic operation. Other +primitives don't guarantee any memory barrier. + +Only atomic operations performed on integers (`int` and `long`, signed +and unsigned) are supported on all architectures. Some architectures +also support 1-byte and 2-byte atomic operations. Those respectively +have `UATOMIC_HAS_ATOMIC_BYTE` and `UATOMIC_HAS_ATOMIC_SHORT` defined when +`uatomic.h` is included. An architecture trying to perform an atomic write +to a type size not supported by the architecture will trigger an illegal +instruction. + +In the description below, `type` is a type that can be atomically +written to by the architecture. It needs to be at most word-sized, and +its alignment needs to greater or equal to its size. + + +API +--- + +```c +void uatomic_set(type *addr, type v) +``` + +Atomically write `v` into `addr`. By "atomically", we mean that no +concurrent operation that reads from addr will see partial +effects of `uatomic_set()`. + + +```c +type uatomic_read(type *addr) +``` + +Atomically read `v` from `addr`. By "atomically", we mean that +`uatomic_read()` cannot see a partial effect of any concurrent +uatomic update. + + +```c +type uatomic_cmpxchg(type *addr, type old, type new) +``` + +An atomic read-modify-write operation that performs this +sequence of operations atomically: check if `addr` contains `old`. +If true, then replace the content of `addr` by `new`. Return the +value previously contained by `addr`. This function imply a full +memory barrier before and after the atomic operation. + + +```c +type uatomic_xchg(type *addr, type new) +``` + +An atomic read-modify-write operation that performs this sequence +of operations atomically: replace the content of `addr` by `new`, +and return the value previously contained by `addr`. This +function imply a full memory barrier before and after the atomic +operation. + + +```c +type uatomic_add_return(type *addr, type v) +type uatomic_sub_return(type *addr, type v) +``` + +An atomic read-modify-write operation that performs this +sequence of operations atomically: increment/decrement the +content of `addr` by `v`, and return the resulting value. This +function imply a full memory barrier before and after the atomic +operation. + + +```c +void uatomic_and(type *addr, type mask) +void uatomic_or(type *addr, type mask) +``` + +Atomically write the result of bitwise "and"/"or" between the +content of `addr` and `mask` into `addr`. + +These operations do not necessarily imply memory barriers. +If memory barriers are needed, they may be provided by explicitly using +`cmm_smp_mb__before_uatomic_and()`, `cmm_smp_mb__after_uatomic_and()`, +`cmm_smp_mb__before_uatomic_or()`, and `cmm_smp_mb__after_uatomic_or()`. +These explicit barriers are no-ops on architectures in which the underlying +atomic instructions implicitly supply the needed memory barriers. + + +```c +void uatomic_add(type *addr, type v) +void uatomic_sub(type *addr, type v) +``` + +Atomically increment/decrement the content of `addr` by `v`. +These operations do not necessarily imply memory barriers. +If memory barriers are needed, they may be provided by +explicitly using `cmm_smp_mb__before_uatomic_add()`, +`cmm_smp_mb__after_uatomic_add()`, `cmm_smp_mb__before_uatomic_sub()`, and +`cmm_smp_mb__after_uatomic_sub()`. These explicit barriers are +no-ops on architectures in which the underlying atomic +instructions implicitly supply the needed memory barriers. + + +```c +void uatomic_inc(type *addr) +void uatomic_dec(type *addr) +``` + +Atomically increment/decrement the content of `addr` by 1. +These operations do not necessarily imply memory barriers. +If memory barriers are needed, they may be provided by +explicitly using `cmm_smp_mb__before_uatomic_inc()`, +`cmm_smp_mb__after_uatomic_inc()`, `cmm_smp_mb__before_uatomic_dec()`, +and `cmm_smp_mb__after_uatomic_dec()`. These explicit barriers are +no-ops on architectures in which the underlying atomic +instructions implicitly supply the needed memory barriers. diff --git a/doc/uatomic-api.txt b/doc/uatomic-api.txt deleted file mode 100644 index edb7d4f..0000000 --- a/doc/uatomic-api.txt +++ /dev/null @@ -1,102 +0,0 @@ -Userspace RCU Atomic Operations API -by Mathieu Desnoyers and Paul E. McKenney - - -This document describes the API. Those are the atomic -operations provided by the Userspace RCU library. The general rule -regarding memory barriers is that only uatomic_xchg(), -uatomic_cmpxchg(), uatomic_add_return(), and uatomic_sub_return() imply -full memory barriers before and after the atomic operation. Other -primitives don't guarantee any memory barrier. - -Only atomic operations performed on integers ("int" and "long", signed -and unsigned) are supported on all architectures. Some architectures -also support 1-byte and 2-byte atomic operations. Those respectively -have UATOMIC_HAS_ATOMIC_BYTE and UATOMIC_HAS_ATOMIC_SHORT defined when -uatomic.h is included. An architecture trying to perform an atomic write -to a type size not supported by the architecture will trigger an illegal -instruction. - -In the description below, "type" is a type that can be atomically -written to by the architecture. It needs to be at most word-sized, and -its alignment needs to greater or equal to its size. - -void uatomic_set(type *addr, type v) - - Atomically write @v into @addr. By "atomically", we mean that no - concurrent operation that reads from addr will see partial - effects of uatomic_set(). - -type uatomic_read(type *addr) - - Atomically read @v from @addr. By "atomically", we mean that - uatomic_read() cannot see a partial effect of any concurrent - uatomic update. - -type uatomic_cmpxchg(type *addr, type old, type new) - - An atomic read-modify-write operation that performs this - sequence of operations atomically: check if @addr contains @old. - If true, then replace the content of @addr by @new. Return the - value previously contained by @addr. This function imply a full - memory barrier before and after the atomic operation. - -type uatomic_xchg(type *addr, type new) - - An atomic read-modify-write operation that performs this sequence - of operations atomically: replace the content of @addr by @new, - and return the value previously contained by @addr. This - function imply a full memory barrier before and after the atomic - operation. - -type uatomic_add_return(type *addr, type v) -type uatomic_sub_return(type *addr, type v) - - An atomic read-modify-write operation that performs this - sequence of operations atomically: increment/decrement the - content of @addr by @v, and return the resulting value. This - function imply a full memory barrier before and after the atomic - operation. - -void uatomic_and(type *addr, type mask) -void uatomic_or(type *addr, type mask) - - Atomically write the result of bitwise "and"/"or" between the - content of @addr and @mask into @addr. - These operations do not necessarily imply memory barriers. - If memory barriers are needed, they may be provided by - explicitly using - cmm_smp_mb__before_uatomic_and(), - cmm_smp_mb__after_uatomic_and(), - cmm_smp_mb__before_uatomic_or(), and - cmm_smp_mb__after_uatomic_or(). These explicit barriers are - no-ops on architectures in which the underlying atomic - instructions implicitly supply the needed memory barriers. - -void uatomic_add(type *addr, type v) -void uatomic_sub(type *addr, type v) - - Atomically increment/decrement the content of @addr by @v. - These operations do not necessarily imply memory barriers. - If memory barriers are needed, they may be provided by - explicitly using - cmm_smp_mb__before_uatomic_add(), - cmm_smp_mb__after_uatomic_add(), - cmm_smp_mb__before_uatomic_sub(), and - cmm_smp_mb__after_uatomic_sub(). These explicit barriers are - no-ops on architectures in which the underlying atomic - instructions implicitly supply the needed memory barriers. - -void uatomic_inc(type *addr) -void uatomic_dec(type *addr) - - Atomically increment/decrement the content of @addr by 1. - These operations do not necessarily imply memory barriers. - If memory barriers are needed, they may be provided by - explicitly using - cmm_smp_mb__before_uatomic_inc(), - cmm_smp_mb__after_uatomic_inc(), - cmm_smp_mb__before_uatomic_dec(), and - cmm_smp_mb__after_uatomic_dec(). These explicit barriers are - no-ops on architectures in which the underlying atomic - instructions implicitly supply the needed memory barriers. diff --git a/lfstack.c b/lfstack.c index db2c2cf..3dac178 100644 --- a/lfstack.c +++ b/lfstack.c @@ -40,12 +40,17 @@ void cds_lfs_init(struct cds_lfs_stack *s) _cds_lfs_init(s); } -bool cds_lfs_empty(struct cds_lfs_stack *s) +void __cds_lfs_init(struct __cds_lfs_stack *s) +{ + ___cds_lfs_init(s); +} + +bool cds_lfs_empty(cds_lfs_stack_ptr_t s) { return _cds_lfs_empty(s); } -bool cds_lfs_push(struct cds_lfs_stack *s, struct cds_lfs_node *node) +bool cds_lfs_push(cds_lfs_stack_ptr_t s, struct cds_lfs_node *node) { return _cds_lfs_push(s, node); } @@ -70,12 +75,12 @@ void cds_lfs_pop_unlock(struct cds_lfs_stack *s) _cds_lfs_pop_unlock(s); } -struct cds_lfs_node *__cds_lfs_pop(struct cds_lfs_stack *s) +struct cds_lfs_node *__cds_lfs_pop(cds_lfs_stack_ptr_t s) { return ___cds_lfs_pop(s); } -struct cds_lfs_head *__cds_lfs_pop_all(struct cds_lfs_stack *s) +struct cds_lfs_head *__cds_lfs_pop_all(cds_lfs_stack_ptr_t s) { return ___cds_lfs_pop_all(s); } diff --git a/rculfhash.c b/rculfhash.c index 7b8e2fe..4c6c970 100644 --- a/rculfhash.c +++ b/rculfhash.c @@ -426,7 +426,7 @@ unsigned int fls_u32(uint32_t x) { int r; - asm("bsrl %1,%0\n\t" + __asm__ ("bsrl %1,%0\n\t" "jnz 1f\n\t" "movl $-1,%0\n\t" "1:\n\t" @@ -442,7 +442,7 @@ unsigned int fls_u64(uint64_t x) { long r; - asm("bsrq %1,%0\n\t" + __asm__ ("bsrq %1,%0\n\t" "jnz 1f\n\t" "movq $-1,%0\n\t" "1:\n\t" @@ -563,6 +563,7 @@ void cds_lfht_resize_lazy_count(struct cds_lfht *ht, unsigned long size, static long nr_cpus_mask = -1; static long split_count_mask = -1; +static int split_count_order = -1; #if defined(HAVE_SYSCONF) static void ht_init_nr_cpus_mask(void) @@ -597,6 +598,8 @@ void alloc_split_items_count(struct cds_lfht *ht) split_count_mask = DEFAULT_SPLIT_COUNT_MASK; else split_count_mask = nr_cpus_mask; + split_count_order = + cds_lfht_get_count_order_ulong(split_count_mask + 1); } assert(split_count_mask >= 0); @@ -713,14 +716,39 @@ void check_resize(struct cds_lfht *ht, unsigned long size, uint32_t chain_len) * Use bucket-local length for small table expand and for * environments lacking per-cpu data support. */ - if (count >= (1UL << COUNT_COMMIT_ORDER)) + if (count >= (1UL << (COUNT_COMMIT_ORDER + split_count_order))) return; if (chain_len > 100) dbg_printf("WARNING: large chain length: %u.\n", chain_len); - if (chain_len >= CHAIN_LEN_RESIZE_THRESHOLD) - cds_lfht_resize_lazy_grow(ht, size, - cds_lfht_get_count_order_u32(chain_len - (CHAIN_LEN_TARGET - 1))); + if (chain_len >= CHAIN_LEN_RESIZE_THRESHOLD) { + int growth; + + /* + * Ideal growth calculated based on chain length. + */ + growth = cds_lfht_get_count_order_u32(chain_len + - (CHAIN_LEN_TARGET - 1)); + if ((ht->flags & CDS_LFHT_ACCOUNTING) + && (size << growth) + >= (1UL << (COUNT_COMMIT_ORDER + + split_count_order))) { + /* + * If ideal growth expands the hash table size + * beyond the "small hash table" sizes, use the + * maximum small hash table size to attempt + * expanding the hash table. This only applies + * when node accounting is available, otherwise + * the chain length is used to expand the hash + * table in every case. + */ + growth = COUNT_COMMIT_ORDER + split_count_order + - cds_lfht_get_count_order_ulong(size); + if (growth <= 0) + return; + } + cds_lfht_resize_lazy_grow(ht, size, growth); + } } static @@ -1143,11 +1171,15 @@ void partition_resize_helper(struct cds_lfht *ht, unsigned long i, void (*fct)(struct cds_lfht *ht, unsigned long i, unsigned long start, unsigned long len)) { - unsigned long partition_len; + unsigned long partition_len, start = 0; struct partition_resize_work *work; int thread, ret; unsigned long nr_threads; + assert(nr_cpus_mask != -1); + if (nr_cpus_mask < 0 || len < 2 * MIN_PARTITION_PER_THREAD) + goto fallback; + /* * Note: nr_cpus_mask + 1 is always power of 2. * We spawn just the number of threads we need to satisfy the minimum @@ -1161,7 +1193,10 @@ void partition_resize_helper(struct cds_lfht *ht, unsigned long i, } partition_len = len >> cds_lfht_get_count_order_ulong(nr_threads); work = calloc(nr_threads, sizeof(*work)); - assert(work); + if (!work) { + dbg_printf("error allocating for resize, single-threading\n"); + goto fallback; + } for (thread = 0; thread < nr_threads; thread++) { work[thread].ht = ht; work[thread].i = i; @@ -1170,6 +1205,17 @@ void partition_resize_helper(struct cds_lfht *ht, unsigned long i, work[thread].fct = fct; ret = pthread_create(&(work[thread].thread_id), ht->resize_attr, partition_resize_thread, &work[thread]); + if (ret == EAGAIN) { + /* + * Out of resources: wait and join the threads + * we've created, then handle leftovers. + */ + dbg_printf("error spawning for resize, single-threading\n"); + start = work[thread].start; + len -= start; + nr_threads = thread; + break; + } assert(!ret); } for (thread = 0; thread < nr_threads; thread++) { @@ -1177,6 +1223,18 @@ void partition_resize_helper(struct cds_lfht *ht, unsigned long i, assert(!ret); } free(work); + + /* + * A pthread_create failure above will either lead in us having + * no threads to join or starting at a non-zero offset, + * fallback to single thread processing of leftovers. + */ + if (start == 0 && nr_threads > 0) + return; +fallback: + ht->flavor->thread_online(); + fct(ht, i, start, len); + ht->flavor->thread_offline(); } /* @@ -1214,13 +1272,6 @@ static void init_table_populate(struct cds_lfht *ht, unsigned long i, unsigned long len) { - assert(nr_cpus_mask != -1); - if (nr_cpus_mask < 0 || len < 2 * MIN_PARTITION_PER_THREAD) { - ht->flavor->thread_online(); - init_table_populate_partition(ht, i, 0, len); - ht->flavor->thread_offline(); - return; - } partition_resize_helper(ht, i, len, init_table_populate_partition); } @@ -1313,14 +1364,6 @@ void remove_table_partition(struct cds_lfht *ht, unsigned long i, static void remove_table(struct cds_lfht *ht, unsigned long i, unsigned long len) { - - assert(nr_cpus_mask != -1); - if (nr_cpus_mask < 0 || len < 2 * MIN_PARTITION_PER_THREAD) { - ht->flavor->thread_online(); - remove_table_partition(ht, i, 0, len); - ht->flavor->thread_offline(); - return; - } partition_resize_helper(ht, i, len, remove_table_partition); } diff --git a/tests/Makefile.am b/tests/Makefile.am index f4c84d2..d8fc5ef 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,246 +1,8 @@ -AM_LDFLAGS=-lpthread -AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -g +SUBDIRS = common unit benchmark regression -noinst_PROGRAMS = test_urcu test_urcu_dynamic_link test_urcu_timing \ - test_urcu_signal test_urcu_signal_dynamic_link test_urcu_signal_timing \ - test_rwlock_timing test_rwlock test_perthreadlock_timing \ - test_perthreadlock test_urcu_yield test_urcu_signal_yield test_urcu_mb \ - test_urcu_qsbr_timing test_urcu_qsbr rcutorture_urcu rcutorture_urcu_signal \ - rcutorture_urcu_mb rcutorture_urcu_bp rcutorture_urcu_qsbr \ - test_mutex test_looplen test_urcu_gc test_urcu_signal_gc \ - test_urcu_lgc \ - test_urcu_mb_gc test_urcu_qsbr_gc test_urcu_qsbr_lgc test_urcu_signal_lgc \ - test_urcu_mb_lgc test_urcu_qsbr_dynamic_link test_urcu_defer \ - test_uatomic test_urcu_assign test_urcu_assign_dynamic_link \ - test_urcu_bp test_urcu_bp_dynamic_link test_cycles_per_loop \ - test_urcu_lfq test_urcu_wfq test_urcu_lfs test_urcu_wfs \ - test_urcu_lfs_rcu \ - test_urcu_wfcq \ - test_urcu_wfq_dynlink test_urcu_wfs_dynlink \ - test_urcu_wfcq_dynlink \ - test_urcu_lfq_dynlink test_urcu_lfs_dynlink test_urcu_hash \ - test_urcu_lfs_rcu_dynlink \ - test_urcu_multiflavor test_urcu_multiflavor_dynlink \ - test_urcu_fork \ - test_urcu_ja test_urcu_ja_range -noinst_HEADERS = rcutorture.h test_urcu_multiflavor.h cpuset.h thread-id.h +.PHONY: bench regtest -if COMPAT_ARCH -COMPAT=$(top_srcdir)/compat_arch_@ARCHTYPE@.c -else -COMPAT= -endif - -if COMPAT_FUTEX -COMPAT+=$(top_srcdir)/compat_futex.c -endif - -URCU=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfcqueue.c $(COMPAT) -URCU_QSBR=$(top_srcdir)/urcu-qsbr.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfcqueue.c $(COMPAT) -# URCU_MB uses urcu.c but -DRCU_MB must be defined -URCU_MB=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfcqueue.c $(COMPAT) -# URCU_SIGNAL uses urcu.c but -DRCU_SIGNAL must be defined -URCU_SIGNAL=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfcqueue.c $(COMPAT) -URCU_BP=$(top_srcdir)/urcu-bp.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfcqueue.c $(COMPAT) -URCU_DEFER=$(top_srcdir)/urcu.c $(top_srcdir)/urcu-pointer.c $(top_srcdir)/wfcqueue.c $(COMPAT) - -URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la -URCU_LIB=$(top_builddir)/liburcu.la -URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la -URCU_MB_LIB=$(top_builddir)/liburcu-mb.la -URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la -URCU_BP_LIB=$(top_builddir)/liburcu-bp.la -URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la - -EXTRA_DIST = $(top_srcdir)/tests/api.h runall.sh runhash.sh runja.sh - -test_urcu_SOURCES = test_urcu.c $(URCU) - -test_urcu_dynamic_link_SOURCES = test_urcu.c $(URCU) -test_urcu_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) - -test_urcu_timing_SOURCES = test_urcu_timing.c $(URCU) - -test_urcu_yield_SOURCES = test_urcu.c $(URCU) -test_urcu_yield_CFLAGS = -DDEBUG_YIELD $(AM_CFLAGS) - - -test_urcu_qsbr_SOURCES = test_urcu_qsbr.c $(URCU_QSBR) - -test_urcu_qsbr_timing_SOURCES = test_urcu_qsbr_timing.c $(URCU_QSBR) - - -test_urcu_mb_SOURCES = test_urcu.c $(URCU_MB) -test_urcu_mb_CFLAGS = -DRCU_MB $(AM_CFLAGS) - - -test_urcu_signal_SOURCES = test_urcu.c $(URCU_SIGNAL) -test_urcu_signal_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) - -test_urcu_signal_dynamic_link_SOURCES = test_urcu.c $(URCU_SIGNAL) -test_urcu_signal_dynamic_link_CFLAGS = -DRCU_SIGNAL -DDYNAMIC_LINK_TEST \ - $(AM_CFLAGS) - -test_urcu_signal_timing_SOURCES = test_urcu_timing.c $(URCU_SIGNAL) -test_urcu_signal_timing_CFLAGS= -DRCU_SIGNAL $(AM_CFLAGS) - -test_urcu_signal_yield_SOURCES = test_urcu.c $(URCU_SIGNAL) -test_urcu_signal_yield_CFLAGS = -DRCU_SIGNAL -DDEBUG_YIELD $(AM_CFLAGS) - -test_urcu_fork_SOURCES = test_urcu_fork.c $(URCU) - -test_rwlock_timing_SOURCES = test_rwlock_timing.c $(URCU_SIGNAL) - -test_rwlock_SOURCES = test_rwlock.c $(URCU_SIGNAL) - -test_perthreadlock_timing_SOURCES = test_perthreadlock_timing.c $(URCU_SIGNAL) - -test_perthreadlock_SOURCES = test_perthreadlock.c $(URCU_SIGNAL) - - -rcutorture_urcu_SOURCES = urcutorture.c -rcutorture_urcu_CFLAGS = -DRCU_MEMBARRIER $(AM_CFLAGS) -rcutorture_urcu_LDADD = $(URCU) - -rcutorture_urcu_mb_SOURCES = urcutorture.c -rcutorture_urcu_mb_CFLAGS = -DRCU_MB $(AM_CFLAGS) -rcutorture_urcu_mb_LDADD = $(URCU_MB_LIB) - -rcutorture_urcu_qsbr_SOURCES = urcutorture.c -rcutorture_urcu_qsbr_CFLAGS = -DTORTURE_QSBR -DRCU_QSBR $(AM_CFLAGS) -rcutorture_urcu_qsbr_LDADD = $(URCU_QSBR_LIB) - -rcutorture_urcu_signal_SOURCES = urcutorture.c -rcutorture_urcu_signal_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) -rcutorture_urcu_signal_LDADD = $(URCU_SIGNAL_LIB) - -rcutorture_urcu_bp_SOURCES = urcutorture.c -rcutorture_urcu_bp_CFLAGS = -DRCU_BP $(AM_CFLAGS) -rcutorture_urcu_bp_LDADD = $(URCU_BP_LIB) - -test_mutex_SOURCES = test_mutex.c $(URCU) - -test_looplen_SOURCES = test_looplen.c - -test_urcu_gc_SOURCES = test_urcu_gc.c $(URCU) - -test_urcu_signal_gc_SOURCES = test_urcu_gc.c $(URCU_SIGNAL) -test_urcu_signal_gc_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) - -test_urcu_mb_gc_SOURCES = test_urcu_gc.c $(URCU_MB) -test_urcu_mb_gc_CFLAGS = -DRCU_MB $(AM_CFLAGS) - -test_urcu_qsbr_gc_SOURCES = test_urcu_qsbr_gc.c $(URCU_QSBR) - -test_urcu_qsbr_lgc_SOURCES = test_urcu_qsbr_gc.c $(URCU_QSBR) -test_urcu_qsbr_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) - -test_urcu_lgc_SOURCES = test_urcu_gc.c $(URCU) -test_urcu_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) - -test_urcu_signal_lgc_SOURCES = test_urcu_gc.c $(URCU_SIGNAL) -test_urcu_signal_lgc_CFLAGS = -DRCU_SIGNAL -DTEST_LOCAL_GC $(AM_CFLAGS) - -test_urcu_mb_lgc_SOURCES = test_urcu_gc.c $(URCU_MB) -test_urcu_mb_lgc_CFLAGS = -DTEST_LOCAL_GC -DRCU_MB $(AM_CFLAGS) - -test_urcu_qsbr_dynamic_link_SOURCES = test_urcu_qsbr.c $(URCU_QSBR) -test_urcu_qsbr_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) - -test_urcu_defer_SOURCES = test_urcu_defer.c $(URCU_DEFER) - -test_uatomic_SOURCES = test_uatomic.c $(COMPAT) - -test_cycles_per_loop_SOURCES = test_cycles_per_loop.c - -test_urcu_assign_SOURCES = test_urcu_assign.c $(URCU) - -test_urcu_assign_dynamic_link_SOURCES = test_urcu_assign.c $(URCU) -test_urcu_assign_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) - -test_urcu_bp_SOURCES = test_urcu_bp.c $(URCU_BP) - -test_urcu_bp_dynamic_link_SOURCES = test_urcu_bp.c $(URCU_BP) -test_urcu_bp_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) - -test_urcu_lfq_SOURCES = test_urcu_lfq.c $(URCU) -test_urcu_lfq_LDADD = $(URCU_CDS_LIB) - -test_urcu_lfq_dynlink_SOURCES = test_urcu_lfq.c $(URCU) -test_urcu_lfq_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_lfq_dynlink_LDADD = $(URCU_CDS_LIB) - -test_urcu_wfq_SOURCES = test_urcu_wfq.c $(COMPAT) -test_urcu_wfq_LDADD = $(URCU_COMMON_LIB) - -test_urcu_wfq_dynlink_SOURCES = test_urcu_wfq.c -test_urcu_wfq_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_wfq_dynlink_LDADD = $(URCU_COMMON_LIB) - -test_urcu_wfcq_SOURCES = test_urcu_wfcq.c $(COMPAT) -test_urcu_wfcq_LDADD = $(URCU_COMMON_LIB) - -test_urcu_wfcq_dynlink_SOURCES = test_urcu_wfcq.c -test_urcu_wfcq_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_wfcq_dynlink_LDADD = $(URCU_COMMON_LIB) - -test_urcu_lfs_SOURCES = test_urcu_lfs.c $(URCU) -test_urcu_lfs_LDADD = $(URCU_CDS_LIB) - -test_urcu_lfs_rcu_SOURCES = test_urcu_lfs_rcu.c $(URCU) -test_urcu_lfs_rcu_LDADD = $(URCU_CDS_LIB) - -test_urcu_lfs_dynlink_SOURCES = test_urcu_lfs.c $(URCU) -test_urcu_lfs_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_lfs_dynlink_LDADD = $(URCU_CDS_LIB) - -test_urcu_lfs_rcu_dynlink_SOURCES = test_urcu_lfs_rcu.c $(URCU) -test_urcu_lfs_rcu_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_lfs_rcu_dynlink_LDADD = $(URCU_CDS_LIB) - -test_urcu_wfs_SOURCES = test_urcu_wfs.c $(COMPAT) -test_urcu_wfs_LDADD = $(URCU_COMMON_LIB) - -test_urcu_wfs_dynlink_SOURCES = test_urcu_wfs.c -test_urcu_wfs_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_wfs_dynlink_LDADD = $(URCU_COMMON_LIB) - -test_urcu_hash_SOURCES = test_urcu_hash.c test_urcu_hash.h \ - test_urcu_hash_rw.c test_urcu_hash_unique.c $(COMPAT) -test_urcu_hash_CFLAGS = -DRCU_QSBR $(AM_CFLAGS) -test_urcu_hash_LDADD = $(URCU_QSBR_LIB) $(URCU_CDS_LIB) - -test_urcu_multiflavor_SOURCES = test_urcu_multiflavor.c \ - test_urcu_multiflavor-memb.c \ - test_urcu_multiflavor-mb.c \ - test_urcu_multiflavor-signal.c \ - test_urcu_multiflavor-qsbr.c \ - test_urcu_multiflavor-bp.c -test_urcu_multiflavor_LDADD = $(URCU_LIB) $(URCU_MB_LIB) \ - $(URCU_SIGNAL_LIB) $(URCU_QSBR_LIB) $(URCU_BP_LIB) - -test_urcu_multiflavor_dynlink_SOURCES = test_urcu_multiflavor.c \ - test_urcu_multiflavor-memb.c \ - test_urcu_multiflavor-mb.c \ - test_urcu_multiflavor-signal.c \ - test_urcu_multiflavor-qsbr.c \ - test_urcu_multiflavor-bp.c -test_urcu_multiflavor_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) -test_urcu_multiflavor_dynlink_LDADD = $(URCU_LIB) $(URCU_MB_LIB) \ - $(URCU_SIGNAL_LIB) $(URCU_QSBR_LIB) $(URCU_BP_LIB) - -test_urcu_ja_SOURCES = test_urcu_ja.c test_urcu_ja.h \ - $(COMPAT) -test_urcu_ja_CFLAGS = -DRCU_QSBR $(AM_CFLAGS) -test_urcu_ja_LDADD = $(URCU_QSBR_LIB) $(URCU_CDS_LIB) - -test_urcu_ja_range_SOURCES = test_urcu_ja_range.c test_urcu_ja_range.h \ - $(COMPAT) -test_urcu_ja_range_CFLAGS = -DRCU_QSBR $(AM_CFLAGS) -test_urcu_ja_range_LDADD = $(URCU_QSBR_LIB) $(URCU_CDS_LIB) - -urcutorture.c: api.h - -check-am: - ./test_uatomic - ./runall.sh +bench: + cd benchmark && $(MAKE) $(AM_MAKEFLAGS) bench +regtest: + cd regression && $(MAKE) $(AM_MAKEFLAGS) regtest diff --git a/tests/api.h b/tests/api.h deleted file mode 100644 index dced0d8..0000000 --- a/tests/api.h +++ /dev/null @@ -1,311 +0,0 @@ -#ifndef _INCLUDE_API_H -#define _INCLUDE_API_H - -#define _GNU_SOURCE -#include "../config.h" - -/* - * common.h: Common Linux kernel-isms. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; but version 2 of the License only due - * to code included from the Linux kernel. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Copyright (c) 2006 Paul E. McKenney, IBM. - * - * Much code taken from the Linux kernel. For such code, the option - * to redistribute under later versions of GPL might not be available. - */ - -#include -#include -#include "cpuset.h" - -/* - * Machine parameters. - */ - -#define ____cacheline_internodealigned_in_smp \ - __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) - -/* - * api_pthreads.h: API mapping to pthreads environment. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. However, please note that much - * of the code in this file derives from the Linux kernel, and that such - * code may not be available except under GPLv2. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Copyright (c) 2006 Paul E. McKenney, IBM. - */ - -#include -#include -#include -#include -#include -#include -#include -/* #include "atomic.h" */ - -/* - * Exclusive locking primitives. - */ - -typedef pthread_mutex_t spinlock_t; - -#define DEFINE_SPINLOCK(lock) spinlock_t lock = PTHREAD_MUTEX_INITIALIZER; -#define __SPIN_LOCK_UNLOCKED(lockp) PTHREAD_MUTEX_INITIALIZER - -static void spin_lock_init(spinlock_t *sp) -{ - if (pthread_mutex_init(sp, NULL) != 0) { - perror("spin_lock_init:pthread_mutex_init"); - exit(-1); - } -} - -static void spin_lock(spinlock_t *sp) -{ - if (pthread_mutex_lock(sp) != 0) { - perror("spin_lock:pthread_mutex_lock"); - exit(-1); - } -} - -static void spin_unlock(spinlock_t *sp) -{ - if (pthread_mutex_unlock(sp) != 0) { - perror("spin_unlock:pthread_mutex_unlock"); - exit(-1); - } -} - -#define spin_lock_irqsave(l, f) do { f = 1; spin_lock(l); } while (0) -#define spin_unlock_irqrestore(l, f) do { f = 0; spin_unlock(l); } while (0) - -/* - * Thread creation/destruction primitives. - */ - -typedef pthread_t thread_id_t; - -#define NR_THREADS 128 - -#define __THREAD_ID_MAP_EMPTY ((thread_id_t) 0) -#define __THREAD_ID_MAP_WAITING ((thread_id_t) 1) -thread_id_t __thread_id_map[NR_THREADS]; -spinlock_t __thread_id_map_mutex; - -#define for_each_thread(t) \ - for (t = 0; t < NR_THREADS; t++) - -#define for_each_running_thread(t) \ - for (t = 0; t < NR_THREADS; t++) \ - if ((__thread_id_map[t] != __THREAD_ID_MAP_EMPTY) && \ - (__thread_id_map[t] != __THREAD_ID_MAP_WAITING)) - -#define for_each_tid(t, tid) \ - for (t = 0; t < NR_THREADS; t++) \ - if ((((tid) = __thread_id_map[t]) != __THREAD_ID_MAP_EMPTY) && \ - ((tid) != __THREAD_ID_MAP_WAITING)) - -pthread_key_t thread_id_key; - -static int __smp_thread_id(void) -{ - int i; - thread_id_t tid = pthread_self(); - - for (i = 0; i < NR_THREADS; i++) { - if (__thread_id_map[i] == tid) { - long v = i + 1; /* must be non-NULL. */ - - if (pthread_setspecific(thread_id_key, (void *)v) != 0) { - perror("pthread_setspecific"); - exit(-1); - } - return i; - } - } - spin_lock(&__thread_id_map_mutex); - for (i = 0; i < NR_THREADS; i++) { - if (__thread_id_map[i] == tid) { - spin_unlock(&__thread_id_map_mutex); - return i; - } - } - spin_unlock(&__thread_id_map_mutex); - fprintf(stderr, "smp_thread_id: Rogue thread, id: %lu(%#lx)\n", - (unsigned long) tid, (unsigned long) tid); - exit(-1); -} - -static int smp_thread_id(void) -{ - void *id; - - id = pthread_getspecific(thread_id_key); - if (id == NULL) - return __smp_thread_id(); - return (long)(id - 1); -} - -static thread_id_t create_thread(void *(*func)(void *), void *arg) -{ - thread_id_t tid; - int i; - - spin_lock(&__thread_id_map_mutex); - for (i = 0; i < NR_THREADS; i++) { - if (__thread_id_map[i] == __THREAD_ID_MAP_EMPTY) - break; - } - if (i >= NR_THREADS) { - spin_unlock(&__thread_id_map_mutex); - fprintf(stderr, "Thread limit of %d exceeded!\n", NR_THREADS); - exit(-1); - } - __thread_id_map[i] = __THREAD_ID_MAP_WAITING; - spin_unlock(&__thread_id_map_mutex); - if (pthread_create(&tid, NULL, func, arg) != 0) { - perror("create_thread:pthread_create"); - exit(-1); - } - __thread_id_map[i] = tid; - return tid; -} - -static void *wait_thread(thread_id_t tid) -{ - int i; - void *vp; - - for (i = 0; i < NR_THREADS; i++) { - if (__thread_id_map[i] == tid) - break; - } - if (i >= NR_THREADS){ - fprintf(stderr, "wait_thread: bad tid = %lu(%#lx)\n", - (unsigned long)tid, (unsigned long)tid); - exit(-1); - } - if (pthread_join(tid, &vp) != 0) { - perror("wait_thread:pthread_join"); - exit(-1); - } - __thread_id_map[i] = __THREAD_ID_MAP_EMPTY; - return vp; -} - -static void wait_all_threads(void) -{ - int i; - thread_id_t tid; - - for (i = 1; i < NR_THREADS; i++) { - tid = __thread_id_map[i]; - if (tid != __THREAD_ID_MAP_EMPTY && - tid != __THREAD_ID_MAP_WAITING) - (void)wait_thread(tid); - } -} - -static void run_on(int cpu) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * timekeeping -- very crude -- should use MONOTONIC... - */ - -long long get_microseconds(void) -{ - struct timeval tv; - - if (gettimeofday(&tv, NULL) != 0) - abort(); - return ((long long)tv.tv_sec) * 1000000LL + (long long)tv.tv_usec; -} - -/* - * Per-thread variables. - */ - -#define DEFINE_PER_THREAD(type, name) \ - struct { \ - __typeof__(type) v \ - __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))); \ - } __per_thread_##name[NR_THREADS]; -#define DECLARE_PER_THREAD(type, name) extern DEFINE_PER_THREAD(type, name) - -#define per_thread(name, thread) __per_thread_##name[thread].v -#define __get_thread_var(name) per_thread(name, smp_thread_id()) - -#define init_per_thread(name, v) \ - do { \ - int __i_p_t_i; \ - for (__i_p_t_i = 0; __i_p_t_i < NR_THREADS; __i_p_t_i++) \ - per_thread(name, __i_p_t_i) = v; \ - } while (0) - -DEFINE_PER_THREAD(int, smp_processor_id); - -/* - * Bug checks. - */ - -#define BUG_ON(c) do { if (!(c)) abort(); } while (0) - -/* - * Initialization -- Must be called before calling any primitives. - */ - -static void smp_init(void) -{ - int i; - - spin_lock_init(&__thread_id_map_mutex); - __thread_id_map[0] = pthread_self(); - for (i = 1; i < NR_THREADS; i++) - __thread_id_map[i] = __THREAD_ID_MAP_EMPTY; - init_per_thread(smp_processor_id, 0); - if (pthread_key_create(&thread_id_key, NULL) != 0) { - perror("pthread_key_create"); - exit(-1); - } -} - -#endif diff --git a/tests/benchmark/Makefile.am b/tests/benchmark/Makefile.am new file mode 100644 index 0000000..85d454d --- /dev/null +++ b/tests/benchmark/Makefile.am @@ -0,0 +1,200 @@ +if !LIBC_INCLUDES_PTHREAD +AM_LDFLAGS=-lpthread +endif +AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/common -g + +noinst_PROGRAMS = test_urcu test_urcu_dynamic_link test_urcu_timing \ + test_urcu_signal test_urcu_signal_dynamic_link test_urcu_signal_timing \ + test_rwlock_timing test_rwlock test_perthreadlock_timing \ + test_perthreadlock test_urcu_yield test_urcu_signal_yield test_urcu_mb \ + test_urcu_qsbr_timing test_urcu_qsbr \ + test_mutex test_looplen test_urcu_gc test_urcu_signal_gc \ + test_urcu_lgc \ + test_urcu_mb_gc test_urcu_qsbr_gc test_urcu_qsbr_lgc test_urcu_signal_lgc \ + test_urcu_mb_lgc test_urcu_qsbr_dynamic_link test_urcu_defer \ + test_urcu_assign test_urcu_assign_dynamic_link \ + test_urcu_bp test_urcu_bp_dynamic_link test_cycles_per_loop \ + test_urcu_lfq test_urcu_wfq test_urcu_lfs test_urcu_wfs \ + test_urcu_lfs_rcu \ + test_urcu_wfcq \ + test_urcu_wfq_dynlink test_urcu_wfs_dynlink \ + test_urcu_wfcq_dynlink \ + test_urcu_lfq_dynlink test_urcu_lfs_dynlink test_urcu_hash \ + test_urcu_lfs_rcu_dynlink + +URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la +URCU_LIB=$(top_builddir)/liburcu.la +URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la +URCU_MB_LIB=$(top_builddir)/liburcu-mb.la +URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la +URCU_BP_LIB=$(top_builddir)/liburcu-bp.la +URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la + +DEBUG_YIELD_LIB=$(builddir)/../common/libdebug-yield.la + +EXTRA_DIST = *.sh + +test_urcu_SOURCES = test_urcu.c +test_urcu_LDADD = $(URCU_LIB) + +test_urcu_dynamic_link_SOURCES = test_urcu.c +test_urcu_dynamic_link_LDADD = $(URCU_LIB) +test_urcu_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) + +test_urcu_timing_SOURCES = test_urcu_timing.c +test_urcu_timing_LDADD = $(URCU_LIB) + +test_urcu_yield_SOURCES = test_urcu.c +test_urcu_yield_LDADD = $(URCU_LIB) $(DEBUG_YIELD_LIB) +test_urcu_yield_CFLAGS = -DDEBUG_YIELD $(AM_CFLAGS) + + +test_urcu_qsbr_SOURCES = test_urcu_qsbr.c +test_urcu_qsbr_LDADD = $(URCU_QSBR_LIB) + +test_urcu_qsbr_timing_SOURCES = test_urcu_qsbr_timing.c +test_urcu_qsbr_timing_LDADD = $(URCU_QSBR_LIB) + + +test_urcu_mb_SOURCES = test_urcu.c +test_urcu_mb_LDADD = $(URCU_MB_LIB) +test_urcu_mb_CFLAGS = -DRCU_MB $(AM_CFLAGS) + + +test_urcu_signal_SOURCES = test_urcu.c +test_urcu_signal_LDADD = $(URCU_SIGNAL_LIB) +test_urcu_signal_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) + +test_urcu_signal_dynamic_link_SOURCES = test_urcu.c +test_urcu_signal_dynamic_link_LDADD = $(URCU_SIGNAL_LIB) +test_urcu_signal_dynamic_link_CFLAGS = -DRCU_SIGNAL -DDYNAMIC_LINK_TEST \ + $(AM_CFLAGS) + +test_urcu_signal_timing_SOURCES = test_urcu_timing.c +test_urcu_signal_timing_LDADD = $(URCU_SIGNAL_LIB) +test_urcu_signal_timing_CFLAGS= -DRCU_SIGNAL $(AM_CFLAGS) + +test_urcu_signal_yield_SOURCES = test_urcu.c +test_urcu_signal_yield_LDADD = $(URCU_SIGNAL_LIB) $(DEBUG_YIELD_LIB) +test_urcu_signal_yield_CFLAGS = -DRCU_SIGNAL -DDEBUG_YIELD $(AM_CFLAGS) + +test_rwlock_timing_SOURCES = test_rwlock_timing.c +test_rwlock_timing_LDADD = $(URCU_SIGNAL_LIB) + +test_rwlock_SOURCES = test_rwlock.c +test_rwlock_LDADD = $(URCU_SIGNAL_LIB) + +test_perthreadlock_timing_SOURCES = test_perthreadlock_timing.c +test_perthreadlock_timing_LDADD = $(URCU_SIGNAL_LIB) + +test_perthreadlock_SOURCES = test_perthreadlock.c +test_perthreadlock_LDADD = $(URCU_SIGNAL_LIB) + +test_mutex_SOURCES = test_mutex.c + +test_looplen_SOURCES = test_looplen.c + +test_urcu_gc_SOURCES = test_urcu_gc.c +test_urcu_gc_LDADD = $(URCU_LIB) + +test_urcu_signal_gc_SOURCES = test_urcu_gc.c +test_urcu_signal_gc_LDADD = $(URCU_SIGNAL_LIB) +test_urcu_signal_gc_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) + +test_urcu_mb_gc_SOURCES = test_urcu_gc.c +test_urcu_mb_gc_LDADD = $(URCU_MB_LIB) +test_urcu_mb_gc_CFLAGS = -DRCU_MB $(AM_CFLAGS) + +test_urcu_qsbr_gc_SOURCES = test_urcu_qsbr_gc.c +test_urcu_qsbr_gc_LDADD = $(URCU_QSBR_LIB) + +test_urcu_qsbr_lgc_SOURCES = test_urcu_qsbr_gc.c +test_urcu_qsbr_lgc_LDADD = $(URCU_QSBR_LIB) +test_urcu_qsbr_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) + +test_urcu_lgc_SOURCES = test_urcu_gc.c +test_urcu_lgc_LDADD = $(URCU_LIB) +test_urcu_lgc_CFLAGS = -DTEST_LOCAL_GC $(AM_CFLAGS) + +test_urcu_signal_lgc_SOURCES = test_urcu_gc.c +test_urcu_signal_lgc_LDADD = $(URCU_SIGNAL_LIB) +test_urcu_signal_lgc_CFLAGS = -DRCU_SIGNAL -DTEST_LOCAL_GC $(AM_CFLAGS) + +test_urcu_mb_lgc_SOURCES = test_urcu_gc.c +test_urcu_mb_lgc_LDADD = $(URCU_MB_LIB) +test_urcu_mb_lgc_CFLAGS = -DTEST_LOCAL_GC -DRCU_MB $(AM_CFLAGS) + +test_urcu_qsbr_dynamic_link_SOURCES = test_urcu_qsbr.c +test_urcu_qsbr_dynamic_link_LDADD = $(URCU_QSBR_LIB) +test_urcu_qsbr_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) + +test_urcu_defer_SOURCES = test_urcu_defer.c +test_urcu_defer_LDADD = $(URCU_LIB) + +test_cycles_per_loop_SOURCES = test_cycles_per_loop.c + +test_urcu_assign_SOURCES = test_urcu_assign.c +test_urcu_assign_LDADD = $(URCU_LIB) + +test_urcu_assign_dynamic_link_SOURCES = test_urcu_assign.c +test_urcu_assign_dynamic_link_LDADD = $(URCU_LIB) +test_urcu_assign_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) + +test_urcu_bp_SOURCES = test_urcu_bp.c +test_urcu_bp_LDADD = $(URCU_BP_LIB) + +test_urcu_bp_dynamic_link_SOURCES = test_urcu_bp.c +test_urcu_bp_dynamic_link_LDADD = $(URCU_BP_LIB) +test_urcu_bp_dynamic_link_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) + +test_urcu_lfq_SOURCES = test_urcu_lfq.c +test_urcu_lfq_LDADD = $(URCU_LIB) $(URCU_CDS_LIB) + +test_urcu_lfq_dynlink_SOURCES = test_urcu_lfq.c +test_urcu_lfq_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_lfq_dynlink_LDADD = $(URCU_LIB) $(URCU_CDS_LIB) + +test_urcu_wfq_SOURCES = test_urcu_wfq.c +test_urcu_wfq_LDADD = $(URCU_COMMON_LIB) + +test_urcu_wfq_dynlink_SOURCES = test_urcu_wfq.c +test_urcu_wfq_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_wfq_dynlink_LDADD = $(URCU_COMMON_LIB) + +test_urcu_wfcq_SOURCES = test_urcu_wfcq.c +test_urcu_wfcq_LDADD = $(URCU_COMMON_LIB) + +test_urcu_wfcq_dynlink_SOURCES = test_urcu_wfcq.c +test_urcu_wfcq_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_wfcq_dynlink_LDADD = $(URCU_COMMON_LIB) + +test_urcu_lfs_SOURCES = test_urcu_lfs.c +test_urcu_lfs_LDADD = $(URCU_LIB) $(URCU_CDS_LIB) + +test_urcu_lfs_rcu_SOURCES = test_urcu_lfs_rcu.c +test_urcu_lfs_rcu_LDADD = $(URCU_LIB) $(URCU_CDS_LIB) + +test_urcu_lfs_dynlink_SOURCES = test_urcu_lfs.c +test_urcu_lfs_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_lfs_dynlink_LDADD = $(URCU_LIB) $(URCU_CDS_LIB) + +test_urcu_lfs_rcu_dynlink_SOURCES = test_urcu_lfs_rcu.c +test_urcu_lfs_rcu_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_lfs_rcu_dynlink_LDADD = $(URCU_LIB) $(URCU_CDS_LIB) + +test_urcu_wfs_SOURCES = test_urcu_wfs.c +test_urcu_wfs_LDADD = $(URCU_COMMON_LIB) + +test_urcu_wfs_dynlink_SOURCES = test_urcu_wfs.c +test_urcu_wfs_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_wfs_dynlink_LDADD = $(URCU_COMMON_LIB) + +test_urcu_hash_SOURCES = test_urcu_hash.c test_urcu_hash.h \ + test_urcu_hash_rw.c test_urcu_hash_unique.c +test_urcu_hash_CFLAGS = -DRCU_QSBR $(AM_CFLAGS) +test_urcu_hash_LDADD = $(URCU_QSBR_LIB) $(URCU_COMMON_LIB) $(URCU_CDS_LIB) + +.PHONY: bench + +bench: + ./runall.sh diff --git a/tests/benchmark/common.sh b/tests/benchmark/common.sh new file mode 100644 index 0000000..9eb03f8 --- /dev/null +++ b/tests/benchmark/common.sh @@ -0,0 +1,12 @@ +# +# This file is meant to be sourced from other tests scripts. +# + +if [ -x "$URCU_TEST_TIME_BIN" ]; then + test_time_bin="$URCU_TEST_TIME_BIN" +elif [ -x "/usr/bin/time" ]; then + test_time_bin="/usr/bin/time" +else + test_time_bin="" +fi + diff --git a/tests/benchmark/runall.sh b/tests/benchmark/runall.sh new file mode 100755 index 0000000..92e57bc --- /dev/null +++ b/tests/benchmark/runall.sh @@ -0,0 +1,107 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +NUM_CPUS=8 + +#extra options, e.g. for setting affinity on even CPUs : +#EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) + +#ppc64 striding, use with NUM_CPUS=8 + +#stride 1 +#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) +#stride 2 +#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) +#stride 4 +#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) +#stride 8 +#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) + +#Vary update fraction +#x: vary update fraction from 0 to 0.0001 + #fix number of readers and reader C.S. length, vary delay between updates +#y: ops/s + +rm -f runall.log +rm -fr runall.detail.log + + +echo Executing batch RCU test + +DURATION=10 +BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 + 131072 262144" +NR_WRITERS=$((${NUM_CPUS} / 2)) + +rm -f batch-rcu.log + +NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) +for BATCH_SIZE in ${BATCH_ARRAY}; do + echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log + (./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log) || exit 1 +done + +#setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON TEST ABOVE ** +EXTRA_OPTS="${EXTRA_OPTS} -b 32768" + +echo Executing update fraction test + +DURATION=10 +WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 + 65536 131072 262144 524288 1048576 2097152 4194304 8388608 + 16777216 33554432 67108864 134217728" +NR_WRITERS=$((${NUM_CPUS} / 2)) + +rm -f update-fraction.log + +NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) +for WDELAY in ${WDELAY_ARRAY}; do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log + (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log) || exit 1 +done + +#Test scalability : +# x: vary number of readers from 0 to num cpus +# y: ops/s +# 0 writer. + +echo Executing scalability test + +NR_WRITERS=0 +DURATION=10 + +rm -f scalability.log + +for NR_READERS in $(seq 1 ${NUM_CPUS}); do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log + (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log) || exit 1 +done + + +# x: Vary reader C.S. length from 0 to 100 us +# y: ops/s +# 8 readers +# 0 writers + +echo Executing reader C.S. length test + +NR_READERS=${NUM_CPUS} +NR_WRITERS=0 +DURATION=10 +#in loops. +READERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" + +rm -f readercslen.log + +for READERCSLEN in ${READERCSLEN_ARRAY}; do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log" >> runall.log + (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log) || exit 1 +done + +echo Executing Hash table test +./runhash.sh || exit 1 + +echo Executing Judy array test +./runja.sh || exit 1 diff --git a/tests/benchmark/runhash.sh b/tests/benchmark/runhash.sh new file mode 100755 index 0000000..0ba20ef --- /dev/null +++ b/tests/benchmark/runhash.sh @@ -0,0 +1,136 @@ +#!/bin/sh + +# TODO: missing tests: +# - send kill signals during tests to change the behavior between +# add/remove/random +# - validate that "nr_leaked" is always 0 in SUMMARY for all tests + +# 30 seconds per test +TIME_UNITS=30 + +TESTPROG=./test_urcu_hash + +#thread multiplier +THREAD_MUL=1 + +EXTRA_PARAMS=-v + +# ** test update coherency with single-value table + +# rw test, single key, replace and del randomly, 4 threads, auto resize. +# key range: init, lookup, and update: 0 to 0 +${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 + +# rw test, single key, add unique and del randomly, 4 threads, auto resize. +# key range: init, lookup, and update: 0 to 0 +${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -u -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 + +# rw test, single key, replace and del randomly, 2 lookup threads, 2 update threads, auto resize. +# key range: init, lookup, and update: 0 to 0 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 + +# rw test, single key, add and del randomly, 2 lookup threads, 2 update threads, auto resize. +# key range: init, lookup, and update: 0 to 0 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 + + +# ** test updates vs lookups with default table + +# rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. +# max 1048576 buckets +# key range: init, lookup, and update: 0 to 999999 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. +# max 1048576 buckets +# key range: init, lookup, and update: 0 to 999999 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add_unique and del randomly, auto resize. +# max 1048576 buckets +# key range: init, lookup, and update: 0 to 999999 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -u ${EXTRA_PARAMS} || exit 1 + + +# test memory management backends + +# rw test, 2 lookup, 2 update threads, add only, auto resize. +# max buckets: 1048576 +# key range: init, lookup, and update: 0 to 99999999 +# mm backend: "order" +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ + -M 100000000 -N 100000000 -O 100000000 -B order ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add only, auto resize. +# max buckets: 1048576 +# key range: init, lookup, and update: 0 to 99999999 +# mm backend: "chunk" +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ + -M 100000000 -N 100000000 -O 100000000 -B chunk ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add only, auto resize. +# max buckets: 1048576 +# key range: init, lookup, and update: 0 to 99999999 +# mm backend: "mmap" +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ + -M 100000000 -N 100000000 -O 100000000 -B mmap ${EXTRA_PARAMS} || exit 1 + + +# ** key range tests + +# rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. +# max 1048576 buckets +# key range: init, and update: 0 to 999999 +# key range: lookup: 1000000 to 1999999 +# NOTE: reader threads in this test should never have a successful +# lookup. TODO +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -R 1000000 ${EXTRA_PARAMS} || exit 1 + +# ** small key range + +# rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. +# max 1048576 buckets +# key range: init, update, and lookups: 0 to 9 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add_unique and del randomly, auto resize. +# max 1048576 buckets +# key range: init, update, and lookups: 0 to 9 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 -u ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. +# max 1048576 buckets +# key range: init, update, and lookups: 0 to 9 +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 -s ${EXTRA_PARAMS} || exit 1 + +# ** lookup for known keys + +# rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. +# max 1048576 buckets +# lookup range is entirely populated. +# key range: init, and lookups: 0 to 9 +# key range: updates: 10 to 19 +# NOTE: reader threads in this test should always have successful +# lookups. TODO +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 -R 0 -T 0 -S 10 -k 10 -s ${EXTRA_PARAMS} || exit 1 + +# ** Uniqueness test + +# rw test, 2 lookup, 2 update threads, add_unique, add_replace and del randomly, auto resize. +# max 1048576 buckets +# asserts that no duplicates are observed by reader threads +# standard length hash chains +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -U ${EXTRA_PARAMS} || exit 1 + +# rw test, 2 lookup, 2 update threads, add_unique, add_replace and del randomly, auto resize. +# max 1048576 buckets +# asserts that no duplicates are observed by reader threads +# create long hash chains: using modulo 4 on keys as hash +${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -U -C 4 ${EXTRA_PARAMS} || exit 1 diff --git a/tests/benchmark/runpaul-phase1.sh b/tests/benchmark/runpaul-phase1.sh new file mode 100755 index 0000000..d2c8649 --- /dev/null +++ b/tests/benchmark/runpaul-phase1.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +NUM_CPUS=64 + +#extra options, e.g. for setting affinity on even CPUs : +EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) + +#ppc64 striding, use with NUM_CPUS=8 + +#stride 1 +#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) +#stride 2 +#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) +#stride 4 +#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) +#stride 8 +#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) + +#Vary update fraction +#x: vary update fraction from 0 to 0.0001 + #fix number of readers and reader C.S. length, vary delay between updates +#y: ops/s + +rm -f runall.log +rm -fr runall.detail.log + + +echo Executing batch RCU test + +DURATION=10 +BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 + 131072 262144" +NR_WRITERS=$((${NUM_CPUS} / 2)) + +rm -f batch-rcu.log + +NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) +for BATCH_SIZE in ${BATCH_ARRAY}; do + echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log + ./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log +done diff --git a/tests/benchmark/runpaul-phase2.sh b/tests/benchmark/runpaul-phase2.sh new file mode 100755 index 0000000..bba9e3e --- /dev/null +++ b/tests/benchmark/runpaul-phase2.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +NUM_CPUS=64 + +#extra options, e.g. for setting affinity on even CPUs : +EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) + +#ppc64 striding, use with NUM_CPUS=8 + +#stride 1 +#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) +#stride 2 +#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) +#stride 4 +#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) +#stride 8 +#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) + +#Vary update fraction +#x: vary update fraction from 0 to 0.0001 + #fix number of readers and reader C.S. length, vary delay between updates +#y: ops/s + +rm -f runall.log +rm -fr runall.detail.log + +#setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON PHASE 1 RESULT ** +EXTRA_OPTS="${EXTRA_OPTS} -b 32768" + +echo Executing update fraction test + +DURATION=10 +WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 + 65536 131072 262144 524288 1048576 2097152 4194304 8388608 + 16777216 33554432 67108864 134217728" +NR_WRITERS=$((${NUM_CPUS} / 2)) + +rm -f update-fraction.log + +NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) +for WDELAY in ${WDELAY_ARRAY}; do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log + ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log +done diff --git a/tests/benchmark/runpaul-phase3.sh b/tests/benchmark/runpaul-phase3.sh new file mode 100755 index 0000000..7c5f055 --- /dev/null +++ b/tests/benchmark/runpaul-phase3.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +NUM_CPUS=64 + +#extra options, e.g. for setting affinity on even CPUs : +EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) + +#ppc64 striding, use with NUM_CPUS=8 + +#stride 1 +#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) +#stride 2 +#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) +#stride 4 +#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) +#stride 8 +#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) + +#Vary update fraction +#x: vary update fraction from 0 to 0.0001 + #fix number of readers and reader C.S. length, vary delay between updates +#y: ops/s + +rm -f runall.log +rm -fr runall.detail.log + +#setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON PHASE 1 RESULT ** +EXTRA_OPTS="${EXTRA_OPTS} -b 32768" + +#Test scalability : +# x: vary number of readers from 0 to num cpus +# y: ops/s +# 0 writer. + +echo Executing scalability test + +NR_WRITERS=0 +DURATION=10 + +rm -f scalability.log + +for NR_READERS in $(seq 1 ${NUM_CPUS}); do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log + ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log +done + + diff --git a/tests/benchmark/runpaul-phase4.sh b/tests/benchmark/runpaul-phase4.sh new file mode 100755 index 0000000..ede402c --- /dev/null +++ b/tests/benchmark/runpaul-phase4.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +export NUM_CPUS=8 + +#extra options, e.g. for setting affinity on even CPUs : +#EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) + +#ppc64 striding, use with NUM_CPUS=8 + +rm -f *.log + +#stride 1 +export EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) +sh subphase4.sh $* +mkdir ppc64-8cores-stride1 +mv *.log ppc64-8cores-stride1/ + + +#stride 2 +export EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) +sh subphase4.sh $* +mkdir ppc64-8cores-stride2 +mv *.log ppc64-8cores-stride2/ + + +#stride 4 +export EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) +sh subphase4.sh $* +mkdir ppc64-8cores-stride4 +mv *.log ppc64-8cores-stride4/ + + +#stride 8 +export EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) +sh subphase4.sh $* +mkdir ppc64-8cores-stride8 +mv *.log ppc64-8cores-stride8/ diff --git a/tests/benchmark/runpaul-phase5.sh b/tests/benchmark/runpaul-phase5.sh new file mode 100644 index 0000000..bb4bfe7 --- /dev/null +++ b/tests/benchmark/runpaul-phase5.sh @@ -0,0 +1,11 @@ +# test run after write-size update + +sh runpaul-phase1.sh +mkdir runpaul-phase1 +mv *.log runpaul-phase1/ + +sh runpaul-phase2.sh +mkdir runpaul-phase2 +mv *.log runpaul-phase2/ + +sh runpaul-phase4.sh diff --git a/tests/benchmark/runpaul-phase6.sh b/tests/benchmark/runpaul-phase6.sh new file mode 100644 index 0000000..5f65072 --- /dev/null +++ b/tests/benchmark/runpaul-phase6.sh @@ -0,0 +1,7 @@ +sh runpaul-phase1.sh +mkdir runpaul-phase1 +mv *.log runpaul-phase1/ + +sh runpaul-phase2.sh +mkdir runpaul-phase2 +mv *.log runpaul-phase2/ diff --git a/tests/benchmark/runpaul-phase7.sh b/tests/benchmark/runpaul-phase7.sh new file mode 100755 index 0000000..4c301da --- /dev/null +++ b/tests/benchmark/runpaul-phase7.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +export NUM_CPUS=64 +#export NUM_CPUS=8 + +#extra options, e.g. for setting affinity on even CPUs : +EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) +#EXTRA_OPTS=$(for a in $(seq 0 1 7); do echo -n "-a ${a} "; done) + +rm -f *.log + +# x: Vary writer C.S. length from 0 to 100 us +# y: reads/s +# 4 readers +# 4 writers + +echo Executing writer C.S. length test + +NR_READERS=$((${NUM_CPUS} / 2)) +NR_WRITERS=$((${NUM_CPUS} / 2)) +DURATION=10 +WDELAY=10 +#in loops. +WRITERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" + +rm -f writercslen.log + +for WRITERCSLEN in ${WRITERCSLEN_ARRAY}; do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -d ${WDELAY} -e ${WRITERCSLEN} | tee -a writercslen.log" >> runall.log + ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -d ${WDELAY} -e ${WRITERCSLEN} | tee -a writercslen.log +done + + + +mkdir ppc64-writercslen +mv *.log ppc64-writercslen/ +#mkdir xeon-writercslen +#mv *.log xeon-writercslen/ diff --git a/tests/benchmark/runtests-batch.sh b/tests/benchmark/runtests-batch.sh new file mode 100755 index 0000000..3d295fb --- /dev/null +++ b/tests/benchmark/runtests-batch.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +. ./common.sh + +log_file="runall.detail.log" + +# Check if time bin is non-empty +if [ -n "$test_time_bin" ]; then + time_command="$test_time_bin" +else + time_command="" +fi + +#for a in test_urcu_gc test_urcu_gc_mb test_urcu_qsbr_gc; do +for a in test_urcu_gc; do + echo "./${a} $*" | tee -a "$log_file" + $time_command ./${a} $* 2>> $log_file +done + diff --git a/tests/benchmark/runtests.sh b/tests/benchmark/runtests.sh new file mode 100755 index 0000000..38e798c --- /dev/null +++ b/tests/benchmark/runtests.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +. ./common.sh + +log_file="runall.detail.log" + +# Check if time bin is non-empty +if [ -n "$test_time_bin" ]; then + time_command="$test_time_bin" +else + time_command="" +fi + +for a in test_urcu_gc test_urcu_signal_gc test_urcu_mb_gc test_urcu_qsbr_gc \ + test_urcu_lgc test_urcu_signal_lgc test_urcu_mb_lgc test_urcu_qsbr_lgc \ + test_urcu test_urcu_signal test_urcu_mb test_urcu_qsbr \ + test_rwlock test_perthreadlock test_mutex; do + echo "./${a} $*" | tee -a "$log_file" + $time_command ./${a} $* 2>> $log_file +done + diff --git a/tests/benchmark/subphase4.sh b/tests/benchmark/subphase4.sh new file mode 100755 index 0000000..d0d7587 --- /dev/null +++ b/tests/benchmark/subphase4.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +#run all tests + +#set to number of active CPUS +#NUM_CPUS=8 + +#extra options, e.g. for setting affinity on even CPUs : +#EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) + +#ppc64 striding, use with NUM_CPUS=8 + +#stride 1 +#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) +#stride 2 +#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) +#stride 4 +#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) +#stride 8 +#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) + +#Vary update fraction +#x: vary update fraction from 0 to 0.0001 + #fix number of readers and reader C.S. length, vary delay between updates +#y: ops/s + +rm -f runall.log +rm -fr runall.detail.log + + +echo Executing batch RCU test + +DURATION=10 +BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 + 131072 262144" +NR_WRITERS=$((${NUM_CPUS} / 2)) + +rm -f batch-rcu.log + +NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) +for BATCH_SIZE in ${BATCH_ARRAY}; do + echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log + ./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log +done + +#setting gc each 4096. ** UPDATE FOR YOUR ARCHITECTURE BASED ON TEST ABOVE ** +EXTRA_OPTS="${EXTRA_OPTS} -b 32768" + +echo Executing update fraction test + +DURATION=10 +WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 + 65536 131072 262144 524288 1048576 2097152 4194304 8388608 + 16777216 33554432 67108864 134217728" +NR_WRITERS=$((${NUM_CPUS} / 2)) + +rm -f update-fraction.log + +NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) +for WDELAY in ${WDELAY_ARRAY}; do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log + ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log +done + +#Test scalability : +# x: vary number of readers from 0 to num cpus +# y: ops/s +# 0 writer. + +echo Executing scalability test + +NR_WRITERS=0 +DURATION=10 + +rm -f scalability.log + +for NR_READERS in $(seq 1 ${NUM_CPUS}); do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log + ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log +done + + +# x: Vary reader C.S. length from 0 to 100 us +# y: ops/s +# 8 readers +# 0 writers + +echo Executing reader C.S. length test + +NR_READERS=${NUM_CPUS} +NR_WRITERS=0 +DURATION=10 +#in loops. +READERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" + +rm -f readercslen.log + +for READERCSLEN in ${READERCSLEN_ARRAY}; do + echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log" >> runall.log + ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log +done diff --git a/tests/benchmark/test_cycles_per_loop.c b/tests/benchmark/test_cycles_per_loop.c new file mode 100644 index 0000000..6ff100b --- /dev/null +++ b/tests/benchmark/test_cycles_per_loop.c @@ -0,0 +1,44 @@ +/* + * test_cycles_per_loop.c + * + * Userspace RCU library - test cycles per loop + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include + +#define NR_LOOPS 1000000UL + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +int main() +{ + cycles_t time1, time2; + + time1 = caa_get_cycles(); + loop_sleep(NR_LOOPS); + time2 = caa_get_cycles(); + printf("CPU clock cycles per loop: %g\n", (time2 - time1) / + (double)NR_LOOPS); + return 0; +} diff --git a/tests/benchmark/test_looplen.c b/tests/benchmark/test_looplen.c new file mode 100644 index 0000000..16674e7 --- /dev/null +++ b/tests/benchmark/test_looplen.c @@ -0,0 +1,74 @@ +/* + * test_looplen.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#else +#define debug_yield_read() +#endif +#include + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +#define LOOPS 1048576 +#define TESTS 10 + +int main(int argc, char **argv) +{ + unsigned long i; + cycles_t time1, time2; + cycles_t time_tot = 0; + double cpl; + + for (i = 0; i < TESTS; i++) { + time1 = caa_get_cycles(); + loop_sleep(LOOPS); + time2 = caa_get_cycles(); + time_tot += time2 - time1; + } + cpl = ((double)time_tot) / (double)TESTS / (double)LOOPS; + + printf("CALIBRATION : %g cycles per loop\n", cpl); + printf("time_tot = %llu, LOOPS = %d, TESTS = %d\n", + time_tot, LOOPS, TESTS); + + return 0; +} diff --git a/tests/benchmark/test_mutex.c b/tests/benchmark/test_mutex.c new file mode 100644 index 0000000..574d0e8 --- /dev/null +++ b/tests/benchmark/test_mutex.c @@ -0,0 +1,386 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +struct test_array { + int a; +}; + +static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static volatile struct test_array test_array = { 8 }; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_reads; + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *data) +{ + unsigned long tidx = (unsigned long)data; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + + for (;;) { + pthread_mutex_lock(&lock); + assert(test_array.a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + pthread_mutex_unlock(&lock); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + tot_nr_reads[tidx] = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *data) +{ + unsigned long wtidx = (unsigned long)data; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + pthread_mutex_lock(&lock); + test_array.a = 0; + test_array.a = 8; + if (caa_unlikely(wduration)) + loop_sleep(wduration); + pthread_mutex_unlock(&lock); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + tot_nr_writes[wtidx] = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + cmm_smp_mb(); + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + count_writer = calloc(nr_writers, sizeof(*count_writer)); + tot_nr_reads = calloc(nr_readers, sizeof(*tot_nr_reads)); + tot_nr_writes = calloc(nr_writers, sizeof(*tot_nr_writes)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + (void *)(long)i); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += tot_nr_reads[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += tot_nr_writes[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + free(tot_nr_reads); + free(tot_nr_writes); + return 0; +} diff --git a/tests/benchmark/test_perthreadlock.c b/tests/benchmark/test_perthreadlock.c new file mode 100644 index 0000000..cc5f47d --- /dev/null +++ b/tests/benchmark/test_perthreadlock.c @@ -0,0 +1,397 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +struct test_array { + int a; +}; + +struct per_thread_lock { + pthread_mutex_t lock; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); /* cache-line aligned */ + +static struct per_thread_lock *per_thread_lock; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static volatile struct test_array test_array = { 8 }; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_reads; + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *data) +{ + unsigned long tidx = (unsigned long)data; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + + for (;;) { + pthread_mutex_lock(&per_thread_lock[tidx].lock); + assert(test_array.a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + pthread_mutex_unlock(&per_thread_lock[tidx].lock); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + tot_nr_reads[tidx] = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *data) +{ + unsigned long wtidx = (unsigned long)data; + long tidx; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + for (tidx = 0; tidx < nr_readers; tidx++) { + pthread_mutex_lock(&per_thread_lock[tidx].lock); + } + test_array.a = 0; + test_array.a = 8; + if (caa_unlikely(wduration)) + loop_sleep(wduration); + for (tidx = (long)nr_readers - 1; tidx >= 0; tidx--) { + pthread_mutex_unlock(&per_thread_lock[tidx].lock); + } + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + tot_nr_writes[wtidx] = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + cmm_smp_mb(); + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + tot_nr_reads = calloc(nr_readers, sizeof(*tot_nr_reads)); + tot_nr_writes = calloc(nr_writers, sizeof(*tot_nr_writes)); + per_thread_lock = calloc(nr_readers, sizeof(*per_thread_lock)); + for (i = 0; i < nr_readers; i++) { + err = pthread_mutex_init(&per_thread_lock[i].lock, NULL); + if (err != 0) + exit(1); + } + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + (void *)(long)i); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += tot_nr_reads[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += tot_nr_writes[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + + free(tid_reader); + free(tid_writer); + free(tot_nr_reads); + free(tot_nr_writes); + free(per_thread_lock); + return 0; +} diff --git a/tests/benchmark/test_perthreadlock_timing.c b/tests/benchmark/test_perthreadlock_timing.c new file mode 100644 index 0000000..97bdd04 --- /dev/null +++ b/tests/benchmark/test_perthreadlock_timing.c @@ -0,0 +1,198 @@ +/* + * test_perthreadloc_timing.c + * + * Per thread locks - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "thread-id.h" + +#include + +struct test_array { + int a; +}; + +static struct test_array test_array = { 8 }; + +struct per_thread_lock { + pthread_mutex_t lock; +} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); /* cache-line aligned */ + +static struct per_thread_lock *per_thread_lock; + +#define OUTER_READ_LOOP 200U +#define INNER_READ_LOOP 100000U +#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) + +#define OUTER_WRITE_LOOP 10U +#define INNER_WRITE_LOOP 200U +#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) + +static int num_read; +static int num_write; + +#define NR_READ num_read +#define NR_WRITE num_write + +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; + +void *thr_reader(void *arg) +{ + int i, j; + cycles_t time1, time2; + long tidx = (long)arg; + + printf("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + sleep(2); + + time1 = caa_get_cycles(); + for (i = 0; i < OUTER_READ_LOOP; i++) { + for (j = 0; j < INNER_READ_LOOP; j++) { + pthread_mutex_lock(&per_thread_lock[tidx].lock); + assert(test_array.a == 8); + pthread_mutex_unlock(&per_thread_lock[tidx].lock); + } + } + time2 = caa_get_cycles(); + + reader_time[tidx] = time2 - time1; + + sleep(2); + printf("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *arg) +{ + int i, j; + long tidx; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + sleep(2); + + for (i = 0; i < OUTER_WRITE_LOOP; i++) { + for (j = 0; j < INNER_WRITE_LOOP; j++) { + time1 = caa_get_cycles(); + for (tidx = 0; tidx < NR_READ; tidx++) { + pthread_mutex_lock(&per_thread_lock[tidx].lock); + } + test_array.a = 8; + for (tidx = NR_READ - 1; tidx >= 0; tidx--) { + pthread_mutex_unlock(&per_thread_lock[tidx].lock); + } + time2 = caa_get_cycles(); + writer_time[(unsigned long)arg] += time2 - time1; + usleep(1); + } + } + + printf("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + return ((void*)2); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + int i; + cycles_t tot_rtime = 0; + cycles_t tot_wtime = 0; + + if (argc < 2) { + printf("Usage : %s nr_readers nr_writers\n", argv[0]); + exit(-1); + } + num_read = atoi(argv[1]); + num_write = atoi(argv[2]); + + reader_time = calloc(num_read, sizeof(*reader_time)); + writer_time = calloc(num_write, sizeof(*writer_time)); + tid_reader = calloc(num_read, sizeof(*tid_reader)); + tid_writer = calloc(num_write, sizeof(*tid_writer)); + + printf("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + per_thread_lock = malloc(sizeof(struct per_thread_lock) * NR_READ); + + for (i = 0; i < NR_READ; i++) { + pthread_mutex_init(&per_thread_lock[i].lock, NULL); + } + for (i = 0; i < NR_READ; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + (void *)(long)i); + if (err != 0) + exit(1); + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + sleep(10); + + for (i = 0; i < NR_READ; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_rtime += reader_time[i]; + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_wtime += writer_time[i]; + } + printf("Time per read : %g cycles\n", + (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); + printf("Time per write : %g cycles\n", + (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); + free(per_thread_lock); + + free(reader_time); + free(writer_time); + free(tid_reader); + free(tid_writer); + + return 0; +} diff --git a/tests/benchmark/test_rwlock.c b/tests/benchmark/test_rwlock.c new file mode 100644 index 0000000..6f52e5f --- /dev/null +++ b/tests/benchmark/test_rwlock.c @@ -0,0 +1,378 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +struct test_array { + int a; +}; + +pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static volatile struct test_array test_array = { 8 }; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + + for (;;) { + pthread_rwlock_rdlock(&lock); + assert(test_array.a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + pthread_rwlock_unlock(&lock); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + pthread_rwlock_wrlock(&lock); + test_array.a = 0; + test_array.a = 8; + if (caa_unlikely(wduration)) + loop_sleep(wduration); + pthread_rwlock_unlock(&lock); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + *count = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + cmm_smp_mb(); + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + count_writer = calloc(nr_writers, sizeof(*count_writer)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + return 0; +} diff --git a/tests/benchmark/test_rwlock_timing.c b/tests/benchmark/test_rwlock_timing.c new file mode 100644 index 0000000..d916071 --- /dev/null +++ b/tests/benchmark/test_rwlock_timing.c @@ -0,0 +1,183 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "thread-id.h" + +#include + +struct test_array { + int a; +}; + +pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER; + +static struct test_array test_array = { 8 }; + +#define OUTER_READ_LOOP 200U +#define INNER_READ_LOOP 100000U +#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) + +#define OUTER_WRITE_LOOP 10U +#define INNER_WRITE_LOOP 200U +#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) + +static int num_read; +static int num_write; + +#define NR_READ num_read +#define NR_WRITE num_write + +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; + +void *thr_reader(void *arg) +{ + int i, j; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + sleep(2); + + time1 = caa_get_cycles(); + for (i = 0; i < OUTER_READ_LOOP; i++) { + for (j = 0; j < INNER_READ_LOOP; j++) { + pthread_rwlock_rdlock(&lock); + assert(test_array.a == 8); + pthread_rwlock_unlock(&lock); + } + } + time2 = caa_get_cycles(); + + reader_time[(unsigned long)arg] = time2 - time1; + + sleep(2); + printf("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *arg) +{ + int i, j; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + sleep(2); + + for (i = 0; i < OUTER_WRITE_LOOP; i++) { + for (j = 0; j < INNER_WRITE_LOOP; j++) { + time1 = caa_get_cycles(); + pthread_rwlock_wrlock(&lock); + test_array.a = 8; + pthread_rwlock_unlock(&lock); + time2 = caa_get_cycles(); + writer_time[(unsigned long)arg] += time2 - time1; + usleep(1); + } + } + + printf("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + return ((void*)2); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + int i; + cycles_t tot_rtime = 0; + cycles_t tot_wtime = 0; + + if (argc < 2) { + printf("Usage : %s nr_readers nr_writers\n", argv[0]); + exit(-1); + } + num_read = atoi(argv[1]); + num_write = atoi(argv[2]); + + reader_time = calloc(num_read, sizeof(*reader_time)); + writer_time = calloc(num_write, sizeof(*writer_time)); + tid_reader = calloc(num_read, sizeof(*tid_reader)); + tid_writer = calloc(num_write, sizeof(*tid_writer)); + + printf("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + for (i = 0; i < NR_READ; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + (void *)(long)i); + if (err != 0) + exit(1); + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + sleep(10); + + for (i = 0; i < NR_READ; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_rtime += reader_time[i]; + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_wtime += writer_time[i]; + } + printf("Time per read : %g cycles\n", + (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); + printf("Time per write : %g cycles\n", + (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); + + free(reader_time); + free(writer_time); + free(tid_reader); + free(tid_writer); + + return 0; +} diff --git a/tests/benchmark/test_urcu.c b/tests/benchmark/test_urcu.c new file mode 100644 index 0000000..28dc75a --- /dev/null +++ b/tests/benchmark/test_urcu.c @@ -0,0 +1,399 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include <../common/debug-yield.h> + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static int *test_rcu_pointer; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + int *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + assert(!rcu_read_ongoing()); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + assert(rcu_read_ongoing()); + local_ptr = rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(*local_ptr == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + /* test extra thread registration */ + rcu_register_thread(); + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *_count) +{ + unsigned long long *count = _count; + int *new, *old; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + new = malloc(sizeof(int)); + assert(new); + *new = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); + if (caa_unlikely(wduration)) + loop_sleep(wduration); + synchronize_rcu(); + if (old) + *old = 0; + free(old); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + *count = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + count_writer = calloc(nr_writers, sizeof(*count_writer)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + free(test_rcu_pointer); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + return 0; +} diff --git a/tests/benchmark/test_urcu_assign.c b/tests/benchmark/test_urcu_assign.c new file mode 100644 index 0000000..37b5f7a --- /dev/null +++ b/tests/benchmark/test_urcu_assign.c @@ -0,0 +1,434 @@ +/* + * test_urcu_assign.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +struct test_array { + int a; +}; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static struct test_array *test_rcu_pointer; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +/* + * malloc/free are reusing memory areas too quickly, which does not let us + * test races appropriately. Use a large circular array for allocations. + * ARRAY_SIZE is larger than nr_writers, and we keep the mutex across + * both alloc and free, which insures we never run over our tail. + */ +#define ARRAY_SIZE (1048576 * nr_writers) +#define ARRAY_POISON 0xDEADBEEF +static int array_index; +static struct test_array *test_array; + +static struct test_array *test_array_alloc(void) +{ + struct test_array *ret; + int index; + + index = array_index % ARRAY_SIZE; + assert(test_array[index].a == ARRAY_POISON || + test_array[index].a == 0); + ret = &test_array[index]; + array_index++; + if (array_index == ARRAY_SIZE) + array_index = 0; + return ret; +} + +static void test_array_free(struct test_array *ptr) +{ + if (!ptr) + return; + ptr->a = ARRAY_POISON; +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct test_array *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + local_ptr = rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(local_ptr->a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *_count) +{ + unsigned long long *count = _count; + struct test_array *new, *old; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_copy_mutex_lock(); + new = test_array_alloc(); + new->a = 8; + old = test_rcu_pointer; + rcu_assign_pointer(test_rcu_pointer, new); + if (caa_unlikely(wduration)) + loop_sleep(wduration); + synchronize_rcu(); + if (old) + old->a = 0; + test_array_free(old); + rcu_copy_mutex_unlock(); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + *count = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + test_array = calloc(1, sizeof(*test_array) * ARRAY_SIZE); + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + count_writer = calloc(nr_writers, sizeof(*count_writer)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + test_array_free(test_rcu_pointer); + free(test_array); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + return 0; +} diff --git a/tests/benchmark/test_urcu_bp.c b/tests/benchmark/test_urcu_bp.c new file mode 100644 index 0000000..9c336e5 --- /dev/null +++ b/tests/benchmark/test_urcu_bp.c @@ -0,0 +1,394 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static int *test_rcu_pointer; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + int *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + assert(!rcu_read_ongoing()); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + assert(rcu_read_ongoing()); + local_ptr = rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(*local_ptr == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *_count) +{ + unsigned long long *count = _count; + int *new, *old; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + new = malloc(sizeof(int)); + *new = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); + if (caa_unlikely(wduration)) + loop_sleep(wduration); + synchronize_rcu(); + if (old) + *old = 0; + free(old); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + *count = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + count_writer = calloc(nr_writers, sizeof(*count_writer)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + free(test_rcu_pointer); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + return 0; +} diff --git a/tests/benchmark/test_urcu_defer.c b/tests/benchmark/test_urcu_defer.c new file mode 100644 index 0000000..bc11941 --- /dev/null +++ b/tests/benchmark/test_urcu_defer.c @@ -0,0 +1,421 @@ +/* + * test_urcu_defer.c + * + * Userspace RCU library - test program (with automatic reclamation) + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include +#include + +struct test_array { + int a; +}; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static struct test_array *test_rcu_pointer; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct test_array *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + local_ptr = rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(local_ptr->a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +static void test_cb2(void *data) +{ +} + +static void test_cb1(void *data) +{ +} + +void *thr_writer(void *data) +{ + unsigned long wtidx = (unsigned long)data; + struct test_array *new, *old = NULL; + int ret; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + ret = rcu_defer_register_thread(); + if (ret) { + printf("Error in rcu_defer_register_thread\n"); + exit(-1); + } + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + new = malloc(sizeof(*new)); + new->a = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); + if (caa_unlikely(wduration)) + loop_sleep(wduration); + defer_rcu(free, old); + defer_rcu(test_cb1, old); + defer_rcu(test_cb1, (void *)-2L); + defer_rcu(test_cb1, (void *)-2L); + defer_rcu(test_cb1, old); + defer_rcu(test_cb2, (void *)-2L); + defer_rcu(test_cb2, (void *)-4L); + defer_rcu(test_cb2, (void *)-2L); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + rcu_defer_unregister_thread(); + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + tot_nr_writes[wtidx] = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + tot_nr_writes = calloc(nr_writers, sizeof(*tot_nr_writes)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += tot_nr_writes[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(tot_nr_writes); + + return 0; +} diff --git a/tests/benchmark/test_urcu_gc.c b/tests/benchmark/test_urcu_gc.c new file mode 100644 index 0000000..b9cda58 --- /dev/null +++ b/tests/benchmark/test_urcu_gc.c @@ -0,0 +1,465 @@ +/* + * test_urcu_gc.c + * + * Userspace RCU library - test program (with batch reclamation) + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +struct test_array { + int a; +}; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static struct test_array *test_rcu_pointer; + +static unsigned int reclaim_batch = 1; + +struct reclaim_queue { + void **queue; /* Beginning of queue */ + void **head; /* Insert position */ +}; + +static struct reclaim_queue *pending_reclaims; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct test_array *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + local_ptr = rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(local_ptr->a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +static void rcu_gc_clear_queue(unsigned long wtidx) +{ + void **p; + + /* Wait for Q.S and empty queue */ + synchronize_rcu(); + + for (p = pending_reclaims[wtidx].queue; + p < pending_reclaims[wtidx].head; p++) { + /* poison */ + if (*p) + ((struct test_array *)*p)->a = 0; + free(*p); + } + pending_reclaims[wtidx].head = pending_reclaims[wtidx].queue; +} + +/* Using per-thread queue */ +static void rcu_gc_reclaim(unsigned long wtidx, void *old) +{ + /* Queue pointer */ + *pending_reclaims[wtidx].head = old; + pending_reclaims[wtidx].head++; + + if (caa_likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue + < reclaim_batch)) + return; + + rcu_gc_clear_queue(wtidx); +} + +void *thr_writer(void *data) +{ + unsigned long wtidx = (unsigned long)data; +#ifdef TEST_LOCAL_GC + struct test_array *old = NULL; +#else + struct test_array *new, *old; +#endif + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { +#ifndef TEST_LOCAL_GC + new = malloc(sizeof(*new)); + new->a = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); +#endif + if (caa_unlikely(wduration)) + loop_sleep(wduration); + rcu_gc_reclaim(wtidx, old); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + tot_nr_writes[wtidx] = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'b': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + reclaim_batch = atol(argv[++i]); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + tot_nr_writes = calloc(nr_writers, sizeof(*tot_nr_writes)); + pending_reclaims = calloc(nr_writers, sizeof(*pending_reclaims)); + if (reclaim_batch * sizeof(*pending_reclaims[i].queue) + < CAA_CACHE_LINE_SIZE) + for (i = 0; i < nr_writers; i++) + pending_reclaims[i].queue = calloc(1, CAA_CACHE_LINE_SIZE); + else + for (i = 0; i < nr_writers; i++) + pending_reclaims[i].queue = calloc(reclaim_batch, + sizeof(*pending_reclaims[i].queue)); + for (i = 0; i < nr_writers; i++) + pending_reclaims[i].head = pending_reclaims[i].queue; + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += tot_nr_writes[i]; + rcu_gc_clear_queue(i); + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " + "batch %u\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes, reclaim_batch); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(tot_nr_writes); + for (i = 0; i < nr_writers; i++) + free(pending_reclaims[i].queue); + free(pending_reclaims); + + return 0; +} diff --git a/tests/benchmark/test_urcu_hash.c b/tests/benchmark/test_urcu_hash.c new file mode 100644 index 0000000..d610d05 --- /dev/null +++ b/tests/benchmark/test_urcu_hash.c @@ -0,0 +1,770 @@ +/* + * test_urcu_hash.c + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "test_urcu_hash.h" + +enum test_hash { + TEST_HASH_RW, + TEST_HASH_UNIQUE, +}; + +struct test_hash_cb { + void (*sigusr1)(int signo); + void (*sigusr2)(int signo); + void *(*thr_reader)(void *_count); + void *(*thr_writer)(void *_count); + int (*populate_hash)(void); +}; + +static +struct test_hash_cb test_hash_cb[] = { + [TEST_HASH_RW] = { + test_hash_rw_sigusr1_handler, + test_hash_rw_sigusr2_handler, + test_hash_rw_thr_reader, + test_hash_rw_thr_writer, + test_hash_rw_populate_hash, + }, + [TEST_HASH_UNIQUE] = { + test_hash_unique_sigusr1_handler, + test_hash_unique_sigusr2_handler, + test_hash_unique_thr_reader, + test_hash_unique_thr_writer, + test_hash_unique_populate_hash, + }, + +}; + +static enum test_hash test_choice = TEST_HASH_RW; + +void (*get_sigusr1_cb(void))(int) +{ + return test_hash_cb[test_choice].sigusr1; +} + +void (*get_sigusr2_cb(void))(int) +{ + return test_hash_cb[test_choice].sigusr2; +} + +void *(*get_thr_reader_cb(void))(void *) +{ + return test_hash_cb[test_choice].thr_reader; +} + +void *(*get_thr_writer_cb(void))(void *) +{ + return test_hash_cb[test_choice].thr_writer; +} + +int (*get_populate_hash_cb(void))(void) +{ + return test_hash_cb[test_choice].populate_hash; +} + +DEFINE_URCU_TLS(unsigned int, rand_lookup); +DEFINE_URCU_TLS(unsigned long, nr_add); +DEFINE_URCU_TLS(unsigned long, nr_addexist); +DEFINE_URCU_TLS(unsigned long, nr_del); +DEFINE_URCU_TLS(unsigned long, nr_delnoent); +DEFINE_URCU_TLS(unsigned long, lookup_fail); +DEFINE_URCU_TLS(unsigned long, lookup_ok); + +struct cds_lfht *test_ht; + +volatile int test_go, test_stop; + +unsigned long wdelay; + +unsigned long duration; + +/* read-side C.S. duration, in loops */ +unsigned long rduration; + +unsigned long init_hash_size = DEFAULT_HASH_SIZE; +unsigned long min_hash_alloc_size = DEFAULT_MIN_ALLOC_SIZE; +unsigned long max_hash_buckets_size = (1UL << 20); +unsigned long init_populate; +int opt_auto_resize; +int add_only, add_unique, add_replace; +const struct cds_lfht_mm_type *memory_backend; + +unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; +unsigned long init_pool_size = DEFAULT_RAND_POOL, + lookup_pool_size = DEFAULT_RAND_POOL, + write_pool_size = DEFAULT_RAND_POOL; +int validate_lookup; +unsigned long nr_hash_chains; /* 0: normal table, other: number of hash chains */ + +int count_pipe[2]; + +int verbose_mode; + +unsigned int cpu_affinities[NR_CPUS]; +unsigned int next_aff = 0; +int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +DEFINE_URCU_TLS(unsigned long long, nr_writes); +DEFINE_URCU_TLS(unsigned long long, nr_reads); + +unsigned int nr_readers; +unsigned int nr_writers; + +static pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +unsigned long test_compare(const void *key1, size_t key1_len, + const void *key2, size_t key2_len) +{ + if (caa_unlikely(key1_len != key2_len)) + return -1; + assert(key1_len == sizeof(unsigned long)); + if (key1 == key2) + return 0; + else + return 1; +} + +void *thr_count(void *arg) +{ + printf_verbose("thread_begin %s, tid %lu\n", + "counter", urcu_get_thread_id()); + + rcu_register_thread(); + + for (;;) { + unsigned long count; + long approx_before, approx_after; + ssize_t len; + char buf[1]; + + rcu_thread_offline(); + len = read(count_pipe[0], buf, 1); + rcu_thread_online(); + if (caa_unlikely(!test_duration_read())) + break; + if (len != 1) + continue; + /* Accounting */ + printf("Counting nodes... "); + fflush(stdout); + rcu_read_lock(); + cds_lfht_count_nodes(test_ht, &approx_before, &count, + &approx_after); + rcu_read_unlock(); + printf("done.\n"); + printf("Approximation before node accounting: %ld nodes.\n", + approx_before); + printf("Accounting of nodes in the hash table: " + "%lu nodes.\n", + count); + printf("Approximation after node accounting: %ld nodes.\n", + approx_after); + } + rcu_unregister_thread(); + return NULL; +} + +void free_node_cb(struct rcu_head *head) +{ + struct lfht_test_node *node = + caa_container_of(head, struct lfht_test_node, head); + free(node); +} + +static +void test_delete_all_nodes(struct cds_lfht *ht) +{ + struct cds_lfht_iter iter; + struct lfht_test_node *node; + unsigned long count = 0; + + cds_lfht_for_each_entry(ht, &iter, node, node) { + int ret; + + ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); + assert(!ret); + call_rcu(&node->head, free_node_cb); + count++; + } + printf("deleted %lu nodes.\n", count); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf(" [-h size] (initial number of buckets)\n"); + printf(" [-m size] (minimum number of allocated buckets)\n"); + printf(" [-n size] (maximum number of buckets)\n"); +printf(" [not -u nor -s] Add entries (supports redundant keys).\n"); + printf(" [-u] Uniquify add (no redundant keys).\n"); + printf(" [-s] Replace (swap) entries.\n"); + printf(" [-i] Add only (no removal).\n"); + printf(" [-k nr_nodes] Number of nodes to insert initially.\n"); + printf(" [-A] Automatically resize hash table.\n"); + printf(" [-B order|chunk|mmap] Specify the memory backend.\n"); + printf(" [-R offset] Lookup pool offset.\n"); + printf(" [-S offset] Write pool offset.\n"); + printf(" [-T offset] Init pool offset.\n"); + printf(" [-M size] Lookup pool size.\n"); + printf(" [-N size] Write pool size.\n"); + printf(" [-O size] Init pool size.\n"); + printf(" [-V] Validate lookups of init values.\n"); + printf(" (use with filled init pool, same lookup range,\n"); + printf(" with different write range)\n"); + printf(" [-U] Uniqueness test.\n"); + printf(" [-C] Number of hash chains.\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + pthread_t *tid_reader, *tid_writer; + pthread_t tid_count; + void *tret; + unsigned long long *count_reader; + struct wr_count *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0, + tot_add = 0, tot_add_exist = 0, tot_remove = 0; + unsigned long count; + long approx_before, approx_after; + int i, a, ret, err, mainret = 0; + struct sigaction act; + unsigned int remain; + unsigned int nr_readers_created = 0, nr_writers_created = 0; + long long nr_leaked; + + if (argc < 4) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'h': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + init_hash_size = atol(argv[++i]); + break; + case 'm': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + min_hash_alloc_size = atol(argv[++i]); + break; + case 'n': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + max_hash_buckets_size = atol(argv[++i]); + break; + case 'u': + if (add_replace) { + printf("Please specify at most one of -s or -u.\n"); + exit(-1); + } + add_unique = 1; + break; + case 's': + if (add_unique) { + printf("Please specify at most one of -s or -u.\n"); + exit(-1); + } + add_replace = 1; + break; + case 'i': + add_only = 1; + break; + case 'k': + init_populate = atol(argv[++i]); + break; + case 'A': + opt_auto_resize = 1; + break; + case 'B': + if (argc < i + 2) { + show_usage(argc, argv); + mainret = 1; + goto end; + } + i++; + if (!strcmp("order", argv[i])) + memory_backend = &cds_lfht_mm_order; + else if (!strcmp("chunk", argv[i])) + memory_backend = &cds_lfht_mm_chunk; + else if (!strcmp("mmap", argv[i])) + memory_backend = &cds_lfht_mm_mmap; + else { + printf("Please specify memory backend with order|chunk|mmap.\n"); + mainret = 1; + goto end; + } + break; + case 'R': + lookup_pool_offset = atol(argv[++i]); + break; + case 'S': + write_pool_offset = atol(argv[++i]); + break; + case 'T': + init_pool_offset = atol(argv[++i]); + break; + case 'M': + lookup_pool_size = atol(argv[++i]); + break; + case 'N': + write_pool_size = atol(argv[++i]); + break; + case 'O': + init_pool_size = atol(argv[++i]); + break; + case 'V': + validate_lookup = 1; + break; + case 'U': + test_choice = TEST_HASH_UNIQUE; + break; + case 'C': + nr_hash_chains = atol(argv[++i]); + break; + } + } + + /* Check if hash size is power of 2 */ + if (init_hash_size && init_hash_size & (init_hash_size - 1)) { + printf("Error: Initial number of buckets (%lu) is not a power of 2.\n", + init_hash_size); + mainret = 1; + goto end; + } + + if (min_hash_alloc_size && min_hash_alloc_size & (min_hash_alloc_size - 1)) { + printf("Error: Minimum number of allocated buckets (%lu) is not a power of 2.\n", + min_hash_alloc_size); + mainret = 1; + goto end; + } + + if (max_hash_buckets_size && max_hash_buckets_size & (max_hash_buckets_size - 1)) { + printf("Error: Maximum number of buckets (%lu) is not a power of 2.\n", + max_hash_buckets_size); + mainret = 1; + goto end; + } + + memset(&act, 0, sizeof(act)); + ret = sigemptyset(&act.sa_mask); + if (ret == -1) { + perror("sigemptyset"); + mainret = 1; + goto end; + } + act.sa_handler = get_sigusr1_cb(); + act.sa_flags = SA_RESTART; + ret = sigaction(SIGUSR1, &act, NULL); + if (ret == -1) { + perror("sigaction"); + mainret = 1; + goto end; + } + + act.sa_handler = get_sigusr2_cb(); + act.sa_flags = SA_RESTART; + ret = sigaction(SIGUSR2, &act, NULL); + if (ret == -1) { + perror("sigaction"); + mainret = 1; + goto end; + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("Mode:%s%s.\n", + add_only ? " add only" : " add/remove", + add_unique ? " uniquify" : ( add_replace ? " replace" : " insert")); + printf_verbose("Initial number of buckets: %lu buckets.\n", init_hash_size); + printf_verbose("Minimum number of allocated buckets: %lu buckets.\n", min_hash_alloc_size); + printf_verbose("Maximum number of buckets: %lu buckets.\n", max_hash_buckets_size); + printf_verbose("Init pool size offset %lu size %lu.\n", + init_pool_offset, init_pool_size); + printf_verbose("Lookup pool size offset %lu size %lu.\n", + lookup_pool_offset, lookup_pool_size); + printf_verbose("Update pool size offset %lu size %lu.\n", + write_pool_offset, write_pool_size); + printf_verbose("Number of hash chains: %lu.\n", + nr_hash_chains); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + if (!tid_reader) { + mainret = 1; + goto end; + } + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + if (!tid_writer) { + mainret = 1; + goto end_free_tid_reader; + } + count_reader = calloc(nr_readers, sizeof(*count_reader)); + if (!count_reader) { + mainret = 1; + goto end_free_tid_writer; + } + count_writer = calloc(nr_writers, sizeof(*count_writer)); + if (!count_writer) { + mainret = 1; + goto end_free_count_reader; + } + + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + if (memory_backend) { + test_ht = _cds_lfht_new(init_hash_size, min_hash_alloc_size, + max_hash_buckets_size, + (opt_auto_resize ? CDS_LFHT_AUTO_RESIZE : 0) | + CDS_LFHT_ACCOUNTING, memory_backend, + &rcu_flavor, NULL); + } else { + test_ht = cds_lfht_new(init_hash_size, min_hash_alloc_size, + max_hash_buckets_size, + (opt_auto_resize ? CDS_LFHT_AUTO_RESIZE : 0) | + CDS_LFHT_ACCOUNTING, NULL); + } + if (!test_ht) { + printf("Error allocating hash table.\n"); + mainret = 1; + goto end_free_call_rcu_data; + } + + /* + * Hash Population needs to be seen as a RCU reader + * thread from the point of view of resize. + */ + rcu_register_thread(); + ret = (get_populate_hash_cb())(); + assert(!ret); + + rcu_thread_offline(); + + next_aff = 0; + + ret = pipe(count_pipe); + if (ret == -1) { + perror("pipe"); + mainret = 1; + goto end_online; + } + + /* spawn counter thread */ + err = pthread_create(&tid_count, NULL, thr_count, + NULL); + if (err != 0) { + errno = err; + mainret = 1; + perror("pthread_create"); + goto end_close_pipe; + } + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], + NULL, get_thr_reader_cb(), + &count_reader[i]); + if (err != 0) { + errno = err; + mainret = 1; + perror("pthread_create"); + goto end_pthread_join; + } + nr_readers_created++; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], + NULL, get_thr_writer_cb(), + &count_writer[i]); + if (err != 0) { + errno = err; + mainret = 1; + perror("pthread_create"); + goto end_pthread_join; + } + nr_writers_created++; + } + + cmm_smp_mb(); + + test_go = 1; + + remain = duration; + do { + remain = sleep(remain); + } while (remain > 0); + + test_stop = 1; + +end_pthread_join: + for (i = 0; i < nr_readers_created; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) { + errno = err; + mainret = 1; + perror("pthread_join"); + } + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers_created; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) { + errno = err; + mainret = 1; + perror("pthread_join"); + } + tot_writes += count_writer[i].update_ops; + tot_add += count_writer[i].add; + tot_add_exist += count_writer[i].add_exist; + tot_remove += count_writer[i].remove; + } + + /* teardown counter thread */ + act.sa_handler = SIG_IGN; + act.sa_flags = SA_RESTART; + ret = sigaction(SIGUSR2, &act, NULL); + if (ret == -1) { + mainret = 1; + perror("sigaction"); + } + { + char msg[1] = { 0x42 }; + ssize_t ret; + + do { + ret = write(count_pipe[1], msg, 1); /* wakeup thread */ + } while (ret == -1L && errno == EINTR); + } + err = pthread_join(tid_count, &tret); + if (err != 0) { + errno = err; + mainret = 1; + perror("pthread_join"); + } + +end_close_pipe: + for (i = 0; i < 2; i++) { + err = close(count_pipe[i]); + if (err) { + mainret = 1; + perror("close pipe"); + } + } + fflush(stdout); +end_online: + rcu_thread_online(); + rcu_read_lock(); + printf("Counting nodes... "); + cds_lfht_count_nodes(test_ht, &approx_before, &count, &approx_after); + printf("done.\n"); + test_delete_all_nodes(test_ht); + rcu_read_unlock(); + rcu_thread_offline(); + if (count) { + printf("Approximation before node accounting: %ld nodes.\n", + approx_before); + printf("Nodes deleted from hash table before destroy: " + "%lu nodes.\n", + count); + printf("Approximation after node accounting: %ld nodes.\n", + approx_after); + } + + ret = cds_lfht_destroy(test_ht, NULL); + if (ret) { + printf_verbose("final delete aborted\n"); + mainret = 1; + } else { + printf_verbose("final delete success\n"); + } + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + nr_leaked = (long long) tot_add + init_populate - tot_remove - count; + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " + "nr_add %12llu nr_add_fail %12llu nr_remove %12llu nr_leaked %12lld\n", + argv[0], duration, nr_readers, rduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes, tot_add, tot_add_exist, tot_remove, + nr_leaked); + if (nr_leaked != 0) { + mainret = 1; + printf("WARNING: %lld nodes were leaked!\n", nr_leaked); + } + + rcu_unregister_thread(); +end_free_call_rcu_data: + free_all_cpu_call_rcu_data(); + free(count_writer); +end_free_count_reader: + free(count_reader); +end_free_tid_writer: + free(tid_writer); +end_free_tid_reader: + free(tid_reader); +end: + if (!mainret) + exit(EXIT_SUCCESS); + else + exit(EXIT_FAILURE); +} diff --git a/tests/benchmark/test_urcu_hash.h b/tests/benchmark/test_urcu_hash.h new file mode 100644 index 0000000..a722485 --- /dev/null +++ b/tests/benchmark/test_urcu_hash.h @@ -0,0 +1,392 @@ +#ifndef _TEST_URCU_HASH_H +#define _TEST_URCU_HASH_H + +/* + * test_urcu_hash.h + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +#define DEFAULT_HASH_SIZE 32 +#define DEFAULT_MIN_ALLOC_SIZE 1 +#define DEFAULT_RAND_POOL 1000000 + +/* + * Note: the hash seed should be a random value for hash tables + * targeting production environments to provide protection against + * denial of service attacks. We keep it a static value within this test + * program to compare identical benchmark runs. + */ +#define TEST_HASH_SEED 0x42UL + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifdef POISON_FREE +#define poison_free(ptr) \ + do { \ + memset(ptr, 0x42, sizeof(*(ptr))); \ + free(ptr); \ + } while (0) +#else +#define poison_free(ptr) free(ptr) +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#else +#define debug_yield_read() +#endif +#include +#include +#include + +struct wr_count { + unsigned long update_ops; + unsigned long add; + unsigned long add_exist; + unsigned long remove; +}; + +extern DECLARE_URCU_TLS(unsigned int, rand_lookup); +extern DECLARE_URCU_TLS(unsigned long, nr_add); +extern DECLARE_URCU_TLS(unsigned long, nr_addexist); +extern DECLARE_URCU_TLS(unsigned long, nr_del); +extern DECLARE_URCU_TLS(unsigned long, nr_delnoent); +extern DECLARE_URCU_TLS(unsigned long, lookup_fail); +extern DECLARE_URCU_TLS(unsigned long, lookup_ok); + +extern struct cds_lfht *test_ht; + +struct test_data { + int a; + int b; +}; + +struct lfht_test_node { + struct cds_lfht_node node; + void *key; + unsigned int key_len; + /* cache-cold for iteration */ + struct rcu_head head; +}; + +static inline struct lfht_test_node * +to_test_node(struct cds_lfht_node *node) +{ + return caa_container_of(node, struct lfht_test_node, node); +} + +static inline +void lfht_test_node_init(struct lfht_test_node *node, void *key, + size_t key_len) +{ + cds_lfht_node_init(&node->node); + node->key = key; + node->key_len = key_len; +} + +static inline struct lfht_test_node * +cds_lfht_iter_get_test_node(struct cds_lfht_iter *iter) +{ + return to_test_node(cds_lfht_iter_get_node(iter)); +} + +extern volatile int test_go, test_stop; + +extern unsigned long wdelay; + +extern unsigned long duration; + +/* read-side C.S. duration, in loops */ +extern unsigned long rduration; + +extern unsigned long init_hash_size; +extern unsigned long min_hash_alloc_size; +extern unsigned long max_hash_buckets_size; +extern unsigned long init_populate; +extern int opt_auto_resize; +extern int add_only, add_unique, add_replace; +extern const struct cds_lfht_mm_type *memory_backend; + +extern unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; +extern unsigned long init_pool_size, + lookup_pool_size, + write_pool_size; +extern int validate_lookup; + +extern unsigned long nr_hash_chains; + +extern int count_pipe[2]; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +extern int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +extern unsigned int cpu_affinities[NR_CPUS]; +extern unsigned int next_aff; +extern int use_affinity; + +extern pthread_mutex_t affinity_mutex; + +void set_affinity(void); + +/* + * returns 0 if test should end. + */ +static inline int test_duration_write(void) +{ + return !test_stop; +} + +static inline int test_duration_read(void) +{ + return !test_stop; +} + +extern DECLARE_URCU_TLS(unsigned long long, nr_writes); +extern DECLARE_URCU_TLS(unsigned long long, nr_reads); + +extern unsigned int nr_readers; +extern unsigned int nr_writers; + +void rcu_copy_mutex_lock(void); +void rcu_copy_mutex_unlock(void); + +/* + * Hash function + * Source: http://burtleburtle.net/bob/c/lookup3.c + * Originally Public Domain + */ + +#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k)))) + +#define mix(a, b, c) \ +do { \ + a -= c; a ^= rot(c, 4); c += b; \ + b -= a; b ^= rot(a, 6); a += c; \ + c -= b; c ^= rot(b, 8); b += a; \ + a -= c; a ^= rot(c, 16); c += b; \ + b -= a; b ^= rot(a, 19); a += c; \ + c -= b; c ^= rot(b, 4); b += a; \ +} while (0) + +#define final(a, b, c) \ +{ \ + c ^= b; c -= rot(b, 14); \ + a ^= c; a -= rot(c, 11); \ + b ^= a; b -= rot(a, 25); \ + c ^= b; c -= rot(b, 16); \ + a ^= c; a -= rot(c, 4);\ + b ^= a; b -= rot(a, 14); \ + c ^= b; c -= rot(b, 24); \ +} + +static inline __attribute__((unused)) +uint32_t hash_u32( + const uint32_t *k, /* the key, an array of uint32_t values */ + size_t length, /* the length of the key, in uint32_ts */ + uint32_t initval) /* the previous hash, or an arbitrary value */ +{ + uint32_t a, b, c; + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + (((uint32_t) length) << 2) + initval; + + /*----------------------------------------- handle most of the key */ + while (length > 3) { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a, b, c); + length -= 3; + k += 3; + } + + /*----------------------------------- handle the last 3 uint32_t's */ + switch (length) { /* all the case statements fall through */ + case 3: c += k[2]; + case 2: b += k[1]; + case 1: a += k[0]; + final(a, b, c); + case 0: /* case 0: nothing left to add */ + break; + } + /*---------------------------------------------- report the result */ + return c; +} + +static inline +void hashword2( + const uint32_t *k, /* the key, an array of uint32_t values */ + size_t length, /* the length of the key, in uint32_ts */ + uint32_t *pc, /* IN: seed OUT: primary hash value */ + uint32_t *pb) /* IN: more seed OUT: secondary hash value */ +{ + uint32_t a, b, c; + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + ((uint32_t) (length << 2)) + *pc; + c += *pb; + + /*----------------------------------------- handle most of the key */ + while (length > 3) { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a, b, c); + length -= 3; + k += 3; + } + + /*----------------------------------- handle the last 3 uint32_t's */ + switch (length) { /* all the case statements fall through */ + case 3: c += k[2]; + case 2: b += k[1]; + case 1: a += k[0]; + final(a, b, c); + case 0: /* case 0: nothing left to add */ + break; + } + /*---------------------------------------------- report the result */ + *pc = c; + *pb = b; +} + +#if (CAA_BITS_PER_LONG == 32) +static inline +unsigned long test_hash_mix(const void *_key, size_t length, unsigned long seed) +{ + unsigned int key = (unsigned int) _key; + + assert(length == sizeof(unsigned int)); + return hash_u32(&key, 1, seed); +} +#else +static inline +unsigned long test_hash_mix(const void *_key, size_t length, unsigned long seed) +{ + union { + uint64_t v64; + uint32_t v32[2]; + } v; + union { + uint64_t v64; + uint32_t v32[2]; + } key; + + assert(length == sizeof(unsigned long)); + v.v64 = (uint64_t) seed; + key.v64 = (uint64_t) _key; + hashword2(key.v32, 2, &v.v32[0], &v.v32[1]); + return v.v64; +} +#endif + +/* + * Hash function with nr_hash_chains != 0 for testing purpose only! + * Creates very long hash chains, deteriorating the hash table into a + * few linked lists, depending on the nr_hash_chains value. The purpose + * of this test is to check how the hash table behaves with hash chains + * containing different values, which is a rare case in a normal hash + * table. + */ +static inline +unsigned long test_hash(const void *_key, size_t length, + unsigned long seed) +{ + if (nr_hash_chains == 0) { + return test_hash_mix(_key, length, seed); + } else { + unsigned long v; + + assert(length == sizeof(unsigned long)); + v = (unsigned long) _key; + return v % nr_hash_chains; + } +} + +unsigned long test_compare(const void *key1, size_t key1_len, + const void *key2, size_t key2_len); + +static inline +int test_match(struct cds_lfht_node *node, const void *key) +{ + struct lfht_test_node *test_node = to_test_node(node); + + return !test_compare(test_node->key, test_node->key_len, + key, sizeof(unsigned long)); +} + +static inline +void cds_lfht_test_lookup(struct cds_lfht *ht, void *key, size_t key_len, + struct cds_lfht_iter *iter) +{ + assert(key_len == sizeof(unsigned long)); + + cds_lfht_lookup(ht, test_hash(key, key_len, TEST_HASH_SEED), + test_match, key, iter); +} + +void free_node_cb(struct rcu_head *head); + +/* rw test */ +void test_hash_rw_sigusr1_handler(int signo); +void test_hash_rw_sigusr2_handler(int signo); +void *test_hash_rw_thr_reader(void *_count); +void *test_hash_rw_thr_writer(void *_count); +int test_hash_rw_populate_hash(void); + +/* unique test */ +void test_hash_unique_sigusr1_handler(int signo); +void test_hash_unique_sigusr2_handler(int signo); +void *test_hash_unique_thr_reader(void *_count); +void *test_hash_unique_thr_writer(void *_count); +int test_hash_unique_populate_hash(void); + +#endif /* _TEST_URCU_HASH_H */ diff --git a/tests/benchmark/test_urcu_hash_rw.c b/tests/benchmark/test_urcu_hash_rw.c new file mode 100644 index 0000000..8802b9c --- /dev/null +++ b/tests/benchmark/test_urcu_hash_rw.c @@ -0,0 +1,284 @@ +/* + * test_urcu_hash_rw.c + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "test_urcu_hash.h" + +enum urcu_hash_addremove { + AR_RANDOM = 0, + AR_ADD = 1, + AR_REMOVE = -1, +}; /* 1: add, -1 remove, 0: random */ + +static enum urcu_hash_addremove addremove; /* 1: add, -1 remove, 0: random */ + +void test_hash_rw_sigusr1_handler(int signo) +{ + switch (addremove) { + case AR_ADD: + printf("Add/Remove: random.\n"); + addremove = AR_RANDOM; + break; + case AR_RANDOM: + printf("Add/Remove: remove only.\n"); + addremove = AR_REMOVE; + break; + case AR_REMOVE: + printf("Add/Remove: add only.\n"); + addremove = AR_ADD; + break; + } +} + +void test_hash_rw_sigusr2_handler(int signo) +{ + char msg[1] = { 0x42 }; + ssize_t ret; + + do { + ret = write(count_pipe[1], msg, 1); /* wakeup thread */ + } while (ret == -1L && errno == EINTR); +} + +void *test_hash_rw_thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct lfht_test_node *node; + struct cds_lfht_iter iter; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + cds_lfht_test_lookup(test_ht, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % lookup_pool_size) + lookup_pool_offset), + sizeof(void *), &iter); + node = cds_lfht_iter_get_test_node(&iter); + if (node == NULL) { + if (validate_lookup) { + printf("[ERROR] Lookup cannot find initial node.\n"); + exit(-1); + } + URCU_TLS(lookup_fail)++; + } else { + URCU_TLS(lookup_ok)++; + } + rcu_debug_yield_read(); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + printf_verbose("read tid : %lx, lookupfail %lu, lookupok %lu\n", + urcu_get_thread_id(), + URCU_TLS(lookup_fail), + URCU_TLS(lookup_ok)); + return ((void*)1); + +} + +void *test_hash_rw_thr_writer(void *_count) +{ + struct lfht_test_node *node; + struct cds_lfht_node *ret_node; + struct cds_lfht_iter iter; + struct wr_count *count = _count; + int ret; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + if ((addremove == AR_ADD || add_only) + || (addremove == AR_RANDOM && rand_r(&URCU_TLS(rand_lookup)) & 1)) { + node = malloc(sizeof(struct lfht_test_node)); + lfht_test_node_init(node, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), + sizeof(void *)); + rcu_read_lock(); + if (add_unique) { + ret_node = cds_lfht_add_unique(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); + } else { + if (add_replace) + ret_node = cds_lfht_add_replace(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); + else + cds_lfht_add(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + &node->node); + } + rcu_read_unlock(); + if (add_unique && ret_node != &node->node) { + free(node); + URCU_TLS(nr_addexist)++; + } else { + if (add_replace && ret_node) { + call_rcu(&to_test_node(ret_node)->head, + free_node_cb); + URCU_TLS(nr_addexist)++; + } else { + URCU_TLS(nr_add)++; + } + } + } else { + /* May delete */ + rcu_read_lock(); + cds_lfht_test_lookup(test_ht, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), + sizeof(void *), &iter); + ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); + rcu_read_unlock(); + if (ret == 0) { + node = cds_lfht_iter_get_test_node(&iter); + call_rcu(&node->head, free_node_cb); + URCU_TLS(nr_del)++; + } else + URCU_TLS(nr_delnoent)++; + } +#if 0 + //if (URCU_TLS(nr_writes) % 100000 == 0) { + if (URCU_TLS(nr_writes) % 1000 == 0) { + rcu_read_lock(); + if (rand_r(&URCU_TLS(rand_lookup)) & 1) { + ht_resize(test_ht, 1); + } else { + ht_resize(test_ht, -1); + } + rcu_read_unlock(); + } +#endif //0 + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, " + "nr_delnoent %lu\n", urcu_get_thread_id(), + URCU_TLS(nr_add), + URCU_TLS(nr_addexist), + URCU_TLS(nr_del), + URCU_TLS(nr_delnoent)); + count->update_ops = URCU_TLS(nr_writes); + count->add = URCU_TLS(nr_add); + count->add_exist = URCU_TLS(nr_addexist); + count->remove = URCU_TLS(nr_del); + return ((void*)2); +} + +int test_hash_rw_populate_hash(void) +{ + struct lfht_test_node *node; + struct cds_lfht_node *ret_node; + + if (!init_populate) + return 0; + + printf("Starting rw test\n"); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + if ((add_unique || add_replace) && init_populate * 10 > init_pool_size) { + printf("WARNING: required to populate %lu nodes (-k), but random " +"pool is quite small (%lu values) and we are in add_unique (-u) or add_replace (-s) mode. Try with a " +"larger random pool (-p option). This may take a while...\n", init_populate, init_pool_size); + } + + while (URCU_TLS(nr_add) < init_populate) { + node = malloc(sizeof(struct lfht_test_node)); + lfht_test_node_init(node, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % init_pool_size) + init_pool_offset), + sizeof(void *)); + rcu_read_lock(); + if (add_unique) { + ret_node = cds_lfht_add_unique(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); + } else { + if (add_replace) + ret_node = cds_lfht_add_replace(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); + else + cds_lfht_add(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + &node->node); + } + rcu_read_unlock(); + if (add_unique && ret_node != &node->node) { + free(node); + URCU_TLS(nr_addexist)++; + } else { + if (add_replace && ret_node) { + call_rcu(&to_test_node(ret_node)->head, free_node_cb); + URCU_TLS(nr_addexist)++; + } else { + URCU_TLS(nr_add)++; + } + } + URCU_TLS(nr_writes)++; + } + return 0; +} diff --git a/tests/benchmark/test_urcu_hash_unique.c b/tests/benchmark/test_urcu_hash_unique.c new file mode 100644 index 0000000..adbbde0 --- /dev/null +++ b/tests/benchmark/test_urcu_hash_unique.c @@ -0,0 +1,279 @@ +/* + * test_urcu_hash_unique.c + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "test_urcu_hash.h" + +enum urcu_hash_addremove { + AR_RANDOM = 0, + AR_ADD = 1, + AR_REMOVE = -1, +}; /* 1: add, -1 remove, 0: random */ + +static enum urcu_hash_addremove addremove; /* 1: add, -1 remove, 0: random */ + +void test_hash_unique_sigusr1_handler(int signo) +{ + switch (addremove) { + case AR_ADD: + printf("Add/Remove: random.\n"); + addremove = AR_RANDOM; + break; + case AR_RANDOM: + printf("Add/Remove: remove only.\n"); + addremove = AR_REMOVE; + break; + case AR_REMOVE: + printf("Add/Remove: add only.\n"); + addremove = AR_ADD; + break; + } +} + +void test_hash_unique_sigusr2_handler(int signo) +{ + char msg[1] = { 0x42 }; + ssize_t ret; + + do { + ret = write(count_pipe[1], msg, 1); /* wakeup thread */ + } while (ret == -1L && errno == EINTR); +} + +void *test_hash_unique_thr_reader(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct lfht_test_node *node; + struct cds_lfht_iter iter; + /* + * iterate on whole table, ensuring that no duplicate is + * found. + */ + rcu_read_lock(); + cds_lfht_for_each_entry(test_ht, &iter, node, node) { + struct cds_lfht_iter dup_iter; + + dup_iter = iter; + cds_lfht_next_duplicate(test_ht, test_match, + node->key, &dup_iter); + if (dup_iter.node != NULL) { + printf("[ERROR] Duplicate key %p found\n", node->key); + } + } + rcu_read_unlock(); + + rcu_debug_yield_read(); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + printf_verbose("read tid : %lu, lookupfail %lu, lookupok %lu\n", + urcu_get_thread_id(), URCU_TLS(lookup_fail), + URCU_TLS(lookup_ok)); + return ((void*)1); + +} + +void *test_hash_unique_thr_writer(void *_count) +{ + struct lfht_test_node *node; + struct cds_lfht_node *ret_node; + struct cds_lfht_iter iter; + struct wr_count *count = _count; + int ret; + int loc_add_unique; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + /* + * add unique/add replace with new node key from range. + */ + if (1 || (addremove == AR_ADD || add_only) + || (addremove == AR_RANDOM && rand_r(&URCU_TLS(rand_lookup)) & 1)) { + node = malloc(sizeof(struct lfht_test_node)); + lfht_test_node_init(node, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), + sizeof(void *)); + rcu_read_lock(); + loc_add_unique = rand_r(&URCU_TLS(rand_lookup)) & 1; + if (loc_add_unique) { + ret_node = cds_lfht_add_unique(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); + } else { + ret_node = cds_lfht_add_replace(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); +#if 0 //generate an error on purpose + cds_lfht_add(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + &node->node); + ret_node = NULL; +#endif //0 + } + rcu_read_unlock(); + if (loc_add_unique) { + if (ret_node != &node->node) { + free(node); + URCU_TLS(nr_addexist)++; + } else { + URCU_TLS(nr_add)++; + } + } else { + if (ret_node) { + call_rcu(&to_test_node(ret_node)->head, + free_node_cb); + URCU_TLS(nr_addexist)++; + } else { + URCU_TLS(nr_add)++; + } + } + } else { + /* May delete */ + rcu_read_lock(); + cds_lfht_test_lookup(test_ht, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), + sizeof(void *), &iter); + ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); + rcu_read_unlock(); + if (ret == 0) { + node = cds_lfht_iter_get_test_node(&iter); + call_rcu(&node->head, free_node_cb); + URCU_TLS(nr_del)++; + } else + URCU_TLS(nr_delnoent)++; + } +#if 0 + //if (URCU_TLS(nr_writes) % 100000 == 0) { + if (URCU_TLS(nr_writes) % 1000 == 0) { + rcu_read_lock(); + if (rand_r(&URCU_TLS(rand_lookup)) & 1) { + ht_resize(test_ht, 1); + } else { + ht_resize(test_ht, -1); + } + rcu_read_unlock(); + } +#endif //0 + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, " + "nr_delnoent %lu\n", urcu_get_thread_id(), + URCU_TLS(nr_add), + URCU_TLS(nr_addexist), + URCU_TLS(nr_del), + URCU_TLS(nr_delnoent)); + count->update_ops = URCU_TLS(nr_writes); + count->add = URCU_TLS(nr_add); + count->add_exist = URCU_TLS(nr_addexist); + count->remove = URCU_TLS(nr_del); + return ((void*)2); +} + +int test_hash_unique_populate_hash(void) +{ + struct lfht_test_node *node; + struct cds_lfht_node *ret_node; + + printf("Starting uniqueness test.\n"); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + if (!init_populate) + return 0; + + if (init_populate * 10 > init_pool_size) { + printf("WARNING: required to populate %lu nodes (-k), but random " +"pool is quite small (%lu values) and we are in add_unique (-u) or add_replace (-s) mode. Try with a " +"larger random pool (-p option). This may take a while...\n", init_populate, init_pool_size); + } + + while (URCU_TLS(nr_add) < init_populate) { + node = malloc(sizeof(struct lfht_test_node)); + lfht_test_node_init(node, + (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % init_pool_size) + init_pool_offset), + sizeof(void *)); + rcu_read_lock(); + ret_node = cds_lfht_add_replace(test_ht, + test_hash(node->key, node->key_len, TEST_HASH_SEED), + test_match, node->key, &node->node); + rcu_read_unlock(); + if (ret_node) { + call_rcu(&to_test_node(ret_node)->head, free_node_cb); + URCU_TLS(nr_addexist)++; + } else { + URCU_TLS(nr_add)++; + } + URCU_TLS(nr_writes)++; + } + return 0; +} diff --git a/tests/benchmark/test_urcu_lfq.c b/tests/benchmark/test_urcu_lfq.c new file mode 100644 index 0000000..a505ccd --- /dev/null +++ b/tests/benchmark/test_urcu_lfq.c @@ -0,0 +1,436 @@ +/* + * test_urcu_lfq.c + * + * Userspace RCU library - example RCU-based lock-free queue + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include +#include + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +struct test { + struct cds_lfq_node_rcu list; + struct rcu_head rcu; +}; + +static struct cds_lfq_queue_rcu q; + +void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "enqueuer", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct test *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_lfq_node_init_rcu(&node->list); + rcu_read_lock(); + cds_lfq_enqueue_rcu(&q, &node->list); + rcu_read_unlock(); + URCU_TLS(nr_successful_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + rcu_unregister_thread(); + + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + printf_verbose("enqueuer thread_end, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues)); + return ((void*)1); + +} + +static +void free_node_cb(struct rcu_head *head) +{ + struct test *node = + caa_container_of(head, struct test, rcu); + free(node); +} + +void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "dequeuer", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_lfq_node_rcu *qnode; + + rcu_read_lock(); + qnode = cds_lfq_dequeue_rcu(&q); + rcu_read_unlock(); + + if (qnode) { + struct test *node; + + node = caa_container_of(qnode, struct test, list); + call_rcu(&node->rcu, free_node_cb); + URCU_TLS(nr_successful_dequeues)++; + } + + URCU_TLS(nr_dequeues)++; + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + rcu_unregister_thread(); + printf_verbose("dequeuer thread_end, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_dequeues), + URCU_TLS(nr_successful_dequeues)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + return ((void*)2); +} + +void test_end(struct cds_lfq_queue_rcu *q, unsigned long long *nr_dequeues) +{ + struct cds_lfq_node_rcu *snode; + + do { + snode = cds_lfq_dequeue_rcu(q); + if (snode) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + free(node); /* no more concurrent access */ + (*nr_dequeues)++; + } + } while (snode); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (enqueuer period (in loops))\n"); + printf(" [-c duration] (dequeuer period (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer)); + tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer)); + count_enqueuer = calloc(nr_enqueuers, 2 * sizeof(*count_enqueuer)); + count_dequeuer = calloc(nr_dequeuers, 2 * sizeof(*count_dequeuer)); + cds_lfq_init_rcu(&q, call_rcu); + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + (void) write(1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&q, &end_dequeues); + err = cds_lfq_destroy_rcu(&q); + assert(!err); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free_all_cpu_call_rcu_data(); + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/tests/benchmark/test_urcu_lfs.c b/tests/benchmark/test_urcu_lfs.c new file mode 100644 index 0000000..723e4a4 --- /dev/null +++ b/tests/benchmark/test_urcu_lfs.c @@ -0,0 +1,518 @@ +/* + * test_urcu_lfs.c + * + * Userspace RCU library - example lock-free stack + * + * Copyright 2010-2012 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include +#include + +/* + * External synchronization used. + */ +enum test_sync { + TEST_SYNC_NONE = 0, + TEST_SYNC_RCU, +}; + +static enum test_sync test_sync; + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +static int test_pop, test_pop_all; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +struct test { + struct cds_lfs_node list; + struct rcu_head rcu; +}; + +static struct cds_lfs_stack s; + +static void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "enqueuer", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct test *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_lfs_node_init(&node->list); + cds_lfs_push(&s, &node->list); + URCU_TLS(nr_successful_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + rcu_unregister_thread(); + + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + printf_verbose("enqueuer thread_end, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues)); + return ((void*)1); + +} + +static +void free_node_cb(struct rcu_head *head) +{ + struct test *node = + caa_container_of(head, struct test, rcu); + free(node); +} + +static +void do_test_pop(enum test_sync sync) +{ + struct cds_lfs_node *snode; + + if (sync == TEST_SYNC_RCU) + rcu_read_lock(); + snode = __cds_lfs_pop(&s); + if (sync == TEST_SYNC_RCU) + rcu_read_unlock(); + if (snode) { + struct test *node; + + node = caa_container_of(snode, + struct test, list); + if (sync == TEST_SYNC_RCU) + call_rcu(&node->rcu, free_node_cb); + else + free(node); + URCU_TLS(nr_successful_dequeues)++; + } + URCU_TLS(nr_dequeues)++; +} + +static +void do_test_pop_all(enum test_sync sync) +{ + struct cds_lfs_node *snode; + struct cds_lfs_head *head; + struct cds_lfs_node *n; + + head = __cds_lfs_pop_all(&s); + cds_lfs_for_each_safe(head, snode, n) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + if (sync == TEST_SYNC_RCU) + call_rcu(&node->rcu, free_node_cb); + else + free(node); + URCU_TLS(nr_successful_dequeues)++; + URCU_TLS(nr_dequeues)++; + } + +} + +static void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + unsigned int counter = 0; + + printf_verbose("thread_begin %s, tid %lu\n", + "dequeuer", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + assert(test_pop || test_pop_all); + + for (;;) { + if (test_pop && test_pop_all) { + /* both pop and pop all */ + if (counter & 1) + do_test_pop(test_sync); + else + do_test_pop_all(test_sync); + counter++; + } else { + if (test_pop) + do_test_pop(test_sync); + else + do_test_pop_all(test_sync); + } + + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + rcu_unregister_thread(); + + printf_verbose("dequeuer thread_end, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_dequeues), + URCU_TLS(nr_successful_dequeues)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + return ((void*)2); +} + +static void test_end(struct cds_lfs_stack *s, unsigned long long *nr_dequeues) +{ + struct cds_lfs_node *snode; + + do { + snode = __cds_lfs_pop(s); + if (snode) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + free(node); + (*nr_dequeues)++; + } + } while (snode); +} + +static void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (enqueuer period (in loops))\n"); + printf(" [-c duration] (dequeuer period (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf(" [-p] (test pop)\n"); + printf(" [-P] (test pop_all, enabled by default)\n"); + printf(" [-R] (use RCU external synchronization)\n"); + printf(" Note: default: no external synchronization used.\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'p': + test_pop = 1; + break; + case 'P': + test_pop_all = 1; + break; + case 'R': + test_sync = TEST_SYNC_RCU; + break; + } + } + + /* activate pop_all test by default */ + if (!test_pop && !test_pop_all) + test_pop_all = 1; + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + if (test_pop) + printf_verbose("pop test activated.\n"); + if (test_pop_all) + printf_verbose("pop_all test activated.\n"); + if (test_sync == TEST_SYNC_RCU) + printf_verbose("External sync: RCU.\n"); + else + printf_verbose("External sync: none.\n"); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer)); + tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer)); + count_enqueuer = calloc(nr_enqueuers, 2 * sizeof(*count_enqueuer)); + count_dequeuer = calloc(nr_dequeuers, 2 * sizeof(*count_dequeuer)); + cds_lfs_init(&s); + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + (void) write(1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&s, &end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free_all_cpu_call_rcu_data(); + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/tests/benchmark/test_urcu_lfs_rcu.c b/tests/benchmark/test_urcu_lfs_rcu.c new file mode 100644 index 0000000..1ec7be4 --- /dev/null +++ b/tests/benchmark/test_urcu_lfs_rcu.c @@ -0,0 +1,435 @@ +/* + * test_urcu_lfs_rcu.c + * + * Userspace RCU library - example RCU-based lock-free stack + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +/* Remove deprecation warnings from test build. */ +#define CDS_LFS_RCU_DEPRECATED + +#include + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +struct test { + struct cds_lfs_node_rcu list; + struct rcu_head rcu; +}; + +static struct cds_lfs_stack_rcu s; + +void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "enqueuer", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct test *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_lfs_node_init_rcu(&node->list); + /* No rcu read-side is needed for push */ + cds_lfs_push_rcu(&s, &node->list); + URCU_TLS(nr_successful_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + rcu_unregister_thread(); + + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + printf_verbose("enqueuer thread_end, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues)); + return ((void*)1); + +} + +static +void free_node_cb(struct rcu_head *head) +{ + struct test *node = + caa_container_of(head, struct test, rcu); + free(node); +} + +void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "dequeuer", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_lfs_node_rcu *snode; + + rcu_read_lock(); + snode = cds_lfs_pop_rcu(&s); + rcu_read_unlock(); + if (snode) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + call_rcu(&node->rcu, free_node_cb); + URCU_TLS(nr_successful_dequeues)++; + } + URCU_TLS(nr_dequeues)++; + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + rcu_unregister_thread(); + + printf_verbose("dequeuer thread_end, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_dequeues), + URCU_TLS(nr_successful_dequeues)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + return ((void*)2); +} + +void test_end(struct cds_lfs_stack_rcu *s, unsigned long long *nr_dequeues) +{ + struct cds_lfs_node_rcu *snode; + + do { + snode = cds_lfs_pop_rcu(s); + if (snode) { + struct test *node; + + node = caa_container_of(snode, struct test, list); + free(node); + (*nr_dequeues)++; + } + } while (snode); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (enqueuer period (in loops))\n"); + printf(" [-c duration] (dequeuer period (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer)); + tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer)); + count_enqueuer = calloc(nr_enqueuers, 2 * sizeof(*count_enqueuer)); + count_dequeuer = calloc(nr_dequeuers, 2 * sizeof(*count_dequeuer)); + cds_lfs_init_rcu(&s); + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + (void) write(1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&s, &end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free_all_cpu_call_rcu_data(); + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/tests/benchmark/test_urcu_qsbr.c b/tests/benchmark/test_urcu_qsbr.c new file mode 100644 index 0000000..c6a6098 --- /dev/null +++ b/tests/benchmark/test_urcu_qsbr.c @@ -0,0 +1,405 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include "urcu-qsbr.h" + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static int *test_rcu_pointer; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + int *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + assert(rcu_read_ongoing()); + rcu_thread_offline(); + assert(!rcu_read_ongoing()); + rcu_thread_online(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + assert(rcu_read_ongoing()); + local_ptr = rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(*local_ptr == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + /* QS each 1024 reads */ + if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + /* test extra thread registration */ + rcu_register_thread(); + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *_count) +{ + unsigned long long *count = _count; + int *new, *old; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + new = malloc(sizeof(int)); + assert(new); + *new = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); + if (caa_unlikely(wduration)) + loop_sleep(wduration); + synchronize_rcu(); + if (old) + *old = 0; + free(old); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + *count = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + count_writer = calloc(nr_writers, sizeof(*count_writer)); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i]; + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + free(test_rcu_pointer); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + return 0; +} diff --git a/tests/benchmark/test_urcu_qsbr_gc.c b/tests/benchmark/test_urcu_qsbr_gc.c new file mode 100644 index 0000000..198dbb5 --- /dev/null +++ b/tests/benchmark/test_urcu_qsbr_gc.c @@ -0,0 +1,467 @@ +/* + * test_urcu_gc.c + * + * Userspace RCU library - test program (with baatch reclamation) + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" +#include "../common/debug-yield.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#define _LGPL_SOURCE +#include + +struct test_array { + int a; +}; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static struct test_array *test_rcu_pointer; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; +static unsigned int reclaim_batch = 1; + +struct reclaim_queue { + void **queue; /* Beginning of queue */ + void **head; /* Insert position */ +}; + +static struct reclaim_queue *pending_reclaims; + + +/* write-side C.S. duration, in loops */ +static unsigned long wduration; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_writes); +static DEFINE_URCU_TLS(unsigned long long, nr_reads); + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; +static +unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; + + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct test_array *local_ptr; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + _rcu_read_lock(); + local_ptr = _rcu_dereference(test_rcu_pointer); + rcu_debug_yield_read(); + if (local_ptr) + assert(local_ptr->a == 8); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + _rcu_read_unlock(); + URCU_TLS(nr_reads)++; + /* QS each 1024 reads */ + if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) + _rcu_quiescent_state(); + if (caa_unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +static void rcu_gc_clear_queue(unsigned long wtidx) +{ + void **p; + + /* Wait for Q.S and empty queue */ + synchronize_rcu(); + + for (p = pending_reclaims[wtidx].queue; + p < pending_reclaims[wtidx].head; p++) { + /* poison */ + if (*p) + ((struct test_array *)*p)->a = 0; + free(*p); + } + pending_reclaims[wtidx].head = pending_reclaims[wtidx].queue; +} + +/* Using per-thread queue */ +static void rcu_gc_reclaim(unsigned long wtidx, void *old) +{ + /* Queue pointer */ + *pending_reclaims[wtidx].head = old; + pending_reclaims[wtidx].head++; + + if (caa_likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue + < reclaim_batch)) + return; + + rcu_gc_clear_queue(wtidx); +} + +void *thr_writer(void *data) +{ + unsigned long wtidx = (unsigned long)data; +#ifdef TEST_LOCAL_GC + struct test_array *old = NULL; +#else + struct test_array *new, *old; +#endif + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { +#ifndef TEST_LOCAL_GC + new = malloc(sizeof(*new)); + new->a = 8; + old = _rcu_xchg_pointer(&test_rcu_pointer, new); +#endif + if (caa_unlikely(wduration)) + loop_sleep(wduration); + rcu_gc_reclaim(wtidx, old); + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + } + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + tot_nr_writes[wtidx] = URCU_TLS(nr_writes); + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-r] [-w] (yield reader and/or writer)\n"); + printf(" [-b batch] (batch reclaim)\n"); + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-e duration] (writer C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'r': + rcu_debug_yield_enable(RCU_YIELD_READ); + break; + case 'w': + rcu_debug_yield_enable(RCU_YIELD_WRITE); + break; + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'b': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + reclaim_batch = atol(argv[++i]); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'e': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wduration = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_reader = calloc(nr_readers, sizeof(*tid_reader)); + tid_writer = calloc(nr_writers, sizeof(*tid_writer)); + count_reader = calloc(nr_readers, sizeof(*count_reader)); + tot_nr_writes = calloc(nr_writers, sizeof(*tot_nr_writes)); + pending_reclaims = calloc(nr_writers, sizeof(*pending_reclaims)); + if (reclaim_batch * sizeof(*pending_reclaims[i].queue) + < CAA_CACHE_LINE_SIZE) + for (i = 0; i < nr_writers; i++) + pending_reclaims[i].queue = calloc(1, CAA_CACHE_LINE_SIZE); + else + for (i = 0; i < nr_writers; i++) + pending_reclaims[i].queue = calloc(reclaim_batch, + sizeof(*pending_reclaims[i].queue)); + for (i = 0; i < nr_writers; i++) + pending_reclaims[i].head = pending_reclaims[i].queue; + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += tot_nr_writes[i]; + rcu_gc_clear_queue(i); + } + + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " + "batch %u\n", + argv[0], duration, nr_readers, rduration, wduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes, reclaim_batch); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(tot_nr_writes); + for (i = 0; i < nr_writers; i++) + free(pending_reclaims[i].queue); + free(pending_reclaims); + + return 0; +} diff --git a/tests/benchmark/test_urcu_qsbr_timing.c b/tests/benchmark/test_urcu_qsbr_timing.c new file mode 100644 index 0000000..9b5195d --- /dev/null +++ b/tests/benchmark/test_urcu_qsbr_timing.c @@ -0,0 +1,225 @@ +/* + * test_qsbr_timing.c + * + * Userspace QSBR - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "thread-id.h" + +#define _LGPL_SOURCE +#include + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +struct test_array { + int a; +}; + +static struct test_array *test_rcu_pointer; + +#define OUTER_READ_LOOP 2000U +#define INNER_READ_LOOP 100000U +#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) + +#define OUTER_WRITE_LOOP 10U +#define INNER_WRITE_LOOP 200U +#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) + +static int num_read; +static int num_write; + +#define NR_READ num_read +#define NR_WRITE num_write + +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; + +void *thr_reader(void *arg) +{ + int i, j; + struct test_array *local_ptr; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + sleep(2); + + rcu_register_thread(); + + time1 = caa_get_cycles(); + for (i = 0; i < OUTER_READ_LOOP; i++) { + for (j = 0; j < INNER_READ_LOOP; j++) { + _rcu_read_lock(); + local_ptr = _rcu_dereference(test_rcu_pointer); + if (local_ptr) { + assert(local_ptr->a == 8); + } + _rcu_read_unlock(); + } + _rcu_quiescent_state(); + } + time2 = caa_get_cycles(); + + rcu_unregister_thread(); + + reader_time[(unsigned long)arg] = time2 - time1; + + sleep(2); + printf("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *arg) +{ + int i, j; + struct test_array *new, *old; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + sleep(2); + + for (i = 0; i < OUTER_WRITE_LOOP; i++) { + for (j = 0; j < INNER_WRITE_LOOP; j++) { + time1 = caa_get_cycles(); + new = malloc(sizeof(struct test_array)); + rcu_copy_mutex_lock(); + old = test_rcu_pointer; + if (old) { + assert(old->a == 8); + } + new->a = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); + rcu_copy_mutex_unlock(); + synchronize_rcu(); + /* can be done after unlock */ + if (old) { + old->a = 0; + } + free(old); + time2 = caa_get_cycles(); + writer_time[(unsigned long)arg] += time2 - time1; + usleep(1); + } + } + + printf("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + return ((void*)2); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + int i; + cycles_t tot_rtime = 0; + cycles_t tot_wtime = 0; + + if (argc < 2) { + printf("Usage : %s nr_readers nr_writers\n", argv[0]); + exit(-1); + } + num_read = atoi(argv[1]); + num_write = atoi(argv[2]); + + reader_time = calloc(num_read, sizeof(*reader_time)); + writer_time = calloc(num_write, sizeof(*writer_time)); + tid_reader = calloc(num_read, sizeof(*tid_reader)); + tid_writer = calloc(num_write, sizeof(*tid_writer)); + + printf("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + for (i = 0; i < NR_READ; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + (void *)(long)i); + if (err != 0) + exit(1); + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + sleep(10); + + for (i = 0; i < NR_READ; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_rtime += reader_time[i]; + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_wtime += writer_time[i]; + } + free(test_rcu_pointer); + printf("Time per read : %g cycles\n", + (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); + printf("Time per write : %g cycles\n", + (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); + + free(reader_time); + free(writer_time); + free(tid_reader); + free(tid_writer); + + return 0; +} diff --git a/tests/benchmark/test_urcu_timing.c b/tests/benchmark/test_urcu_timing.c new file mode 100644 index 0000000..e31b676 --- /dev/null +++ b/tests/benchmark/test_urcu_timing.c @@ -0,0 +1,224 @@ +/* + * test_urcu.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "thread-id.h" + +#define _LGPL_SOURCE +#include + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +struct test_array { + int a; +}; + +static struct test_array *test_rcu_pointer; + +#define OUTER_READ_LOOP 2000U +#define INNER_READ_LOOP 100000U +#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) + +#define OUTER_WRITE_LOOP 10U +#define INNER_WRITE_LOOP 200U +#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) + +static int num_read; +static int num_write; + +#define NR_READ num_read +#define NR_WRITE num_write + +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; +static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; + +void *thr_reader(void *arg) +{ + int i, j; + struct test_array *local_ptr; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + sleep(2); + + rcu_register_thread(); + + time1 = caa_get_cycles(); + for (i = 0; i < OUTER_READ_LOOP; i++) { + for (j = 0; j < INNER_READ_LOOP; j++) { + rcu_read_lock(); + local_ptr = rcu_dereference(test_rcu_pointer); + if (local_ptr) { + assert(local_ptr->a == 8); + } + rcu_read_unlock(); + } + } + time2 = caa_get_cycles(); + + rcu_unregister_thread(); + + reader_time[(unsigned long)arg] = time2 - time1; + + sleep(2); + printf("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + return ((void*)1); + +} + +void *thr_writer(void *arg) +{ + int i, j; + struct test_array *new, *old; + cycles_t time1, time2; + + printf("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + sleep(2); + + for (i = 0; i < OUTER_WRITE_LOOP; i++) { + for (j = 0; j < INNER_WRITE_LOOP; j++) { + time1 = caa_get_cycles(); + new = malloc(sizeof(struct test_array)); + rcu_copy_mutex_lock(); + old = test_rcu_pointer; + if (old) { + assert(old->a == 8); + } + new->a = 8; + old = rcu_xchg_pointer(&test_rcu_pointer, new); + rcu_copy_mutex_unlock(); + synchronize_rcu(); + /* can be done after unlock */ + if (old) { + old->a = 0; + } + free(old); + time2 = caa_get_cycles(); + writer_time[(unsigned long)arg] += time2 - time1; + usleep(1); + } + } + + printf("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + return ((void*)2); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + int i; + cycles_t tot_rtime = 0; + cycles_t tot_wtime = 0; + + if (argc < 2) { + printf("Usage : %s nr_readers nr_writers\n", argv[0]); + exit(-1); + } + num_read = atoi(argv[1]); + num_write = atoi(argv[2]); + + reader_time = calloc(num_read, sizeof(*reader_time)); + writer_time = calloc(num_write, sizeof(*writer_time)); + tid_reader = calloc(num_read, sizeof(*tid_reader)); + tid_writer = calloc(num_write, sizeof(*tid_writer)); + + printf("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + for (i = 0; i < NR_READ; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + (void *)(long)i); + if (err != 0) + exit(1); + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + (void *)(long)i); + if (err != 0) + exit(1); + } + + sleep(10); + + for (i = 0; i < NR_READ; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_rtime += reader_time[i]; + } + for (i = 0; i < NR_WRITE; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_wtime += writer_time[i]; + } + free(test_rcu_pointer); + printf("Time per read : %g cycles\n", + (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); + printf("Time per write : %g cycles\n", + (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); + + free(reader_time); + free(writer_time); + free(tid_reader); + free(tid_writer); + + return 0; +} diff --git a/tests/benchmark/test_urcu_wfcq.c b/tests/benchmark/test_urcu_wfcq.c new file mode 100644 index 0000000..bdd5b09 --- /dev/null +++ b/tests/benchmark/test_urcu_wfcq.c @@ -0,0 +1,581 @@ +/* + * test_urcu_wfcq.c + * + * Userspace RCU library - example RCU-based lock-free concurrent queue + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +enum test_sync { + TEST_SYNC_NONE = 0, + TEST_SYNC_MUTEX, +}; + +static enum test_sync test_sync; + +static int test_force_sync; + +static volatile int test_go, test_stop_enqueue, test_stop_dequeue; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +static int test_dequeue, test_splice, test_wait_empty; +static int test_enqueue_stopped; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop_dequeue; +} + +static int test_duration_enqueue(void) +{ + return !test_stop_enqueue; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); +static DEFINE_URCU_TLS(unsigned long long, nr_empty_dest_enqueues); +static DEFINE_URCU_TLS(unsigned long long, nr_splice); +static DEFINE_URCU_TLS(unsigned long long, nr_dequeue_last); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +static struct cds_wfcq_head __attribute__((aligned(CAA_CACHE_LINE_SIZE))) head; +static struct cds_wfcq_tail __attribute__((aligned(CAA_CACHE_LINE_SIZE))) tail; + +static void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + bool was_nonempty; + + printf_verbose("thread_begin %s, tid %lu\n", + "enqueuer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_wfcq_node *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_wfcq_node_init(node); + was_nonempty = cds_wfcq_enqueue(&head, &tail, node); + URCU_TLS(nr_successful_enqueues)++; + if (!was_nonempty) + URCU_TLS(nr_empty_dest_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + uatomic_inc(&test_enqueue_stopped); + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + count[2] = URCU_TLS(nr_empty_dest_enqueues); + printf_verbose("enqueuer thread_end, tid %lu, " + "enqueues %llu successful_enqueues %llu, " + "empty_dest_enqueues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues), + URCU_TLS(nr_empty_dest_enqueues)); + return ((void*)1); + +} + +static void do_test_dequeue(enum test_sync sync) +{ + struct cds_wfcq_node *node; + int state; + + if (sync == TEST_SYNC_MUTEX) + node = cds_wfcq_dequeue_with_state_blocking(&head, &tail, + &state); + else + node = __cds_wfcq_dequeue_with_state_blocking(&head, &tail, + &state); + + if (state & CDS_WFCQ_STATE_LAST) + URCU_TLS(nr_dequeue_last)++; + + if (node) { + free(node); + URCU_TLS(nr_successful_dequeues)++; + } + URCU_TLS(nr_dequeues)++; +} + +static void do_test_splice(enum test_sync sync) +{ + struct cds_wfcq_head tmp_head; + struct cds_wfcq_tail tmp_tail; + struct cds_wfcq_node *node, *n; + enum cds_wfcq_ret ret; + + cds_wfcq_init(&tmp_head, &tmp_tail); + + if (sync == TEST_SYNC_MUTEX) + ret = cds_wfcq_splice_blocking(&tmp_head, &tmp_tail, + &head, &tail); + else + ret = __cds_wfcq_splice_blocking(&tmp_head, &tmp_tail, + &head, &tail); + + switch (ret) { + case CDS_WFCQ_RET_WOULDBLOCK: + assert(0); /* blocking call */ + break; + case CDS_WFCQ_RET_DEST_EMPTY: + URCU_TLS(nr_splice)++; + URCU_TLS(nr_dequeue_last)++; + /* ok */ + break; + case CDS_WFCQ_RET_DEST_NON_EMPTY: + assert(0); /* entirely unexpected */ + break; + case CDS_WFCQ_RET_SRC_EMPTY: + /* ok, we could even skip iteration on dest if we wanted */ + break; + } + + __cds_wfcq_for_each_blocking_safe(&tmp_head, &tmp_tail, node, n) { + free(node); + URCU_TLS(nr_successful_dequeues)++; + URCU_TLS(nr_dequeues)++; + } +} + +static void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + unsigned int counter = 0; + + printf_verbose("thread_begin %s, tid %lu\n", + "dequeuer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + if (test_dequeue && test_splice) { + if (counter & 1) + do_test_dequeue(test_sync); + else + do_test_splice(test_sync); + counter++; + } else { + if (test_dequeue) + do_test_dequeue(test_sync); + else + do_test_splice(test_sync); + } + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + printf_verbose("dequeuer thread_end, tid %lu, " + "dequeues %llu, successful_dequeues %llu, " + "nr_splice %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues), + URCU_TLS(nr_splice)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + count[2] = URCU_TLS(nr_splice); + count[3] = URCU_TLS(nr_dequeue_last); + return ((void*)2); +} + +static void test_end(unsigned long long *nr_dequeues, + unsigned long long *nr_dequeue_last) +{ + struct cds_wfcq_node *node; + int state; + + do { + node = cds_wfcq_dequeue_with_state_blocking(&head, &tail, + &state); + if (node) { + if (state & CDS_WFCQ_STATE_LAST) + (*nr_dequeue_last)++; + free(node); + (*nr_dequeues)++; + } + } while (node); +} + +static void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (enqueuer period (in loops))\n"); + printf(" [-c duration] (dequeuer period (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf(" [-q] (test dequeue)\n"); + printf(" [-s] (test splice, enabled by default)\n"); + printf(" [-M] (use mutex external synchronization)\n"); + printf(" Note: default: no external synchronization used.\n"); + printf(" [-f] (force user-provided synchronization)\n"); + printf(" [-w] Wait for dequeuer to empty queue\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0, + tot_empty_dest_enqueues = 0, + tot_splice = 0, tot_dequeue_last = 0; + unsigned long long end_dequeues = 0; + int i, a, retval = 0; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'q': + test_dequeue = 1; + break; + case 's': + test_splice = 1; + break; + case 'M': + test_sync = TEST_SYNC_MUTEX; + break; + case 'w': + test_wait_empty = 1; + break; + case 'f': + test_force_sync = 1; + break; + } + } + + /* activate splice test by default */ + if (!test_dequeue && !test_splice) + test_splice = 1; + + if (test_sync == TEST_SYNC_NONE && nr_dequeuers > 1 && test_dequeue) { + if (test_force_sync) { + fprintf(stderr, "[WARNING] Using dequeue concurrently " + "with other dequeue or splice without external " + "synchronization. Expect run-time failure.\n"); + } else { + printf("Enforcing mutex synchronization\n"); + test_sync = TEST_SYNC_MUTEX; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + if (test_dequeue) + printf_verbose("dequeue test activated.\n"); + else + printf_verbose("splice test activated.\n"); + if (test_sync == TEST_SYNC_MUTEX) + printf_verbose("External sync: mutex.\n"); + else + printf_verbose("External sync: none.\n"); + if (test_wait_empty) + printf_verbose("Wait for dequeuers to empty queue.\n"); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer)); + tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer)); + count_enqueuer = calloc(nr_enqueuers, 3 * sizeof(*count_enqueuer)); + count_dequeuer = calloc(nr_dequeuers, 4 * sizeof(*count_dequeuer)); + cds_wfcq_init(&head, &tail); + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[3 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[4 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + (void) write(1, ".", 1); + } + + test_stop_enqueue = 1; + + if (test_wait_empty) { + while (nr_enqueuers != uatomic_read(&test_enqueue_stopped)) { + sleep(1); + } + while (!cds_wfcq_empty(&head, &tail)) { + sleep(1); + } + } + + test_stop_dequeue = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[3 * i]; + tot_successful_enqueues += count_enqueuer[3 * i + 1]; + tot_empty_dest_enqueues += count_enqueuer[3 * i + 2]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[4 * i]; + tot_successful_dequeues += count_dequeuer[4 * i + 1]; + tot_splice += count_dequeuer[4 * i + 2]; + tot_dequeue_last += count_dequeuer[4 * i + 3]; + } + + test_end(&end_dequeues, &tot_dequeue_last); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "enqueues to empty dest : %llu, " + "successful dequeues %llu, " + "splice : %llu, dequeue_last : %llu\n", + tot_successful_enqueues, + tot_empty_dest_enqueues, + tot_successful_dequeues, + tot_splice, tot_dequeue_last); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu enqueues to empty dest %12llu " + "successful dequeues %12llu splice %12llu " + "dequeue_last %llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_empty_dest_enqueues, + tot_successful_dequeues, tot_splice, tot_dequeue_last, + end_dequeues, + tot_enqueues + tot_dequeues); + + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) { + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + retval = 1; + } + + /* + * If only using splice to dequeue, the enqueuer should see + * exactly as many empty queues than the number of non-empty + * src splice. + */ + if (tot_empty_dest_enqueues != tot_dequeue_last) { + printf("WARNING! Discrepancy between empty enqueue (%llu) and " + "number of dequeue of last element (%llu)\n", + tot_empty_dest_enqueues, + tot_dequeue_last); + retval = 1; + } + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return retval; +} diff --git a/tests/benchmark/test_urcu_wfq.c b/tests/benchmark/test_urcu_wfq.c new file mode 100644 index 0000000..deabdbf --- /dev/null +++ b/tests/benchmark/test_urcu_wfq.c @@ -0,0 +1,401 @@ +/* + * test_urcu_wfq.c + * + * Userspace RCU library - example RCU-based lock-free queue + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif + +/* Remove deprecation warnings from test build. */ +#define CDS_WFQ_DEPRECATED + +#include +#include + +static volatile int test_go, test_stop; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop; +} + +static int test_duration_enqueue(void) +{ + return !test_stop; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +static struct cds_wfq_queue q; + +void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "enqueuer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_wfq_node *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_wfq_node_init(node); + cds_wfq_enqueue(&q, node); + URCU_TLS(nr_successful_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + printf_verbose("enqueuer thread_end, tid %lu, " + "enqueues %llu successful_enqueues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues)); + return ((void*)1); + +} + +void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + + printf_verbose("thread_begin %s, tid %lu\n", + "dequeuer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_wfq_node *node = cds_wfq_dequeue_blocking(&q); + + if (node) { + free(node); + URCU_TLS(nr_successful_dequeues)++; + } + + URCU_TLS(nr_dequeues)++; + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + printf_verbose("dequeuer thread_end, tid %lu, " + "dequeues %llu, successful_dequeues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_dequeues), + URCU_TLS(nr_successful_dequeues)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + return ((void*)2); +} + +void test_end(struct cds_wfq_queue *q, unsigned long long *nr_dequeues) +{ + struct cds_wfq_node *node; + + do { + node = cds_wfq_dequeue_blocking(q); + if (node) { + free(node); + (*nr_dequeues)++; + } + } while (node); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (enqueuer period (in loops))\n"); + printf(" [-c duration] (dequeuer period (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0; + unsigned long long end_dequeues = 0; + int i, a; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer)); + tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer)); + count_enqueuer = calloc(nr_enqueuers, 2 * sizeof(*count_enqueuer)); + count_dequeuer = calloc(nr_dequeuers, 2 * sizeof(*count_dequeuer)); + cds_wfq_init(&q); + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[2 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[2 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + (void) write(1, ".", 1); + } + + test_stop = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[2 * i]; + tot_successful_enqueues += count_enqueuer[2 * i + 1]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[2 * i]; + tot_successful_dequeues += count_dequeuer[2 * i + 1]; + } + + test_end(&q, &end_dequeues); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "successful dequeues %llu\n", + tot_successful_enqueues, tot_successful_dequeues); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu successful dequeues %12llu " + "end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_successful_dequeues, end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return 0; +} diff --git a/tests/benchmark/test_urcu_wfs.c b/tests/benchmark/test_urcu_wfs.c new file mode 100644 index 0000000..f643671 --- /dev/null +++ b/tests/benchmark/test_urcu_wfs.c @@ -0,0 +1,565 @@ +/* + * test_urcu_wfs.c + * + * Userspace RCU library - example RCU-based lock-free stack + * + * Copyright February 2010 - Mathieu Desnoyers + * Copyright February 2010 - Paolo Bonzini + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "cpuset.h" +#include "thread-id.h" + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif +#include + +/* + * External synchronization used. + */ +enum test_sync { + TEST_SYNC_NONE = 0, + TEST_SYNC_MUTEX, +}; + +static enum test_sync test_sync; + +static int test_force_sync; + +static volatile int test_go, test_stop_enqueue, test_stop_dequeue; + +static unsigned long rduration; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long wdelay; + +static inline void loop_sleep(unsigned long loops) +{ + while (loops-- != 0) + caa_cpu_relax(); +} + +static int verbose_mode; + +static int test_pop, test_pop_all, test_wait_empty; +static int test_enqueue_stopped; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + int cpu, ret; +#endif /* HAVE_SCHED_SETAFFINITY */ + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * returns 0 if test should end. + */ +static int test_duration_dequeue(void) +{ + return !test_stop_dequeue; +} + +static int test_duration_enqueue(void) +{ + return !test_stop_enqueue; +} + +static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); + +static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); +static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); +static DEFINE_URCU_TLS(unsigned long long, nr_empty_dest_enqueues); +static DEFINE_URCU_TLS(unsigned long long, nr_pop_all); +static DEFINE_URCU_TLS(unsigned long long, nr_pop_last); + +static unsigned int nr_enqueuers; +static unsigned int nr_dequeuers; + +static struct cds_wfs_stack s; + +static void *thr_enqueuer(void *_count) +{ + unsigned long long *count = _count; + bool was_nonempty; + + printf_verbose("thread_begin %s, tid %lu\n", + "enqueuer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + struct cds_wfs_node *node = malloc(sizeof(*node)); + if (!node) + goto fail; + cds_wfs_node_init(node); + was_nonempty = cds_wfs_push(&s, node); + URCU_TLS(nr_successful_enqueues)++; + if (!was_nonempty) + URCU_TLS(nr_empty_dest_enqueues)++; + + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); +fail: + URCU_TLS(nr_enqueues)++; + if (caa_unlikely(!test_duration_enqueue())) + break; + } + + uatomic_inc(&test_enqueue_stopped); + count[0] = URCU_TLS(nr_enqueues); + count[1] = URCU_TLS(nr_successful_enqueues); + count[2] = URCU_TLS(nr_empty_dest_enqueues); + printf_verbose("enqueuer thread_end, tid %lu, " + "enqueues %llu successful_enqueues %llu, " + "empty_dest_enqueues %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_enqueues), + URCU_TLS(nr_successful_enqueues), + URCU_TLS(nr_empty_dest_enqueues)); + return ((void*)1); + +} + +static void do_test_pop(enum test_sync sync) +{ + struct cds_wfs_node *node; + int state; + + if (sync == TEST_SYNC_MUTEX) + cds_wfs_pop_lock(&s); + node = __cds_wfs_pop_with_state_blocking(&s, &state); + if (sync == TEST_SYNC_MUTEX) + cds_wfs_pop_unlock(&s); + + if (node) { + if (state & CDS_WFS_STATE_LAST) + URCU_TLS(nr_pop_last)++; + free(node); + URCU_TLS(nr_successful_dequeues)++; + } + URCU_TLS(nr_dequeues)++; +} + +static void do_test_pop_all(enum test_sync sync) +{ + struct cds_wfs_head *head; + struct cds_wfs_node *node, *n; + + if (sync == TEST_SYNC_MUTEX) + cds_wfs_pop_lock(&s); + head = __cds_wfs_pop_all(&s); + if (sync == TEST_SYNC_MUTEX) + cds_wfs_pop_unlock(&s); + + /* Check if empty */ + if (cds_wfs_first(head) == NULL) + return; + + URCU_TLS(nr_pop_all)++; + URCU_TLS(nr_pop_last)++; + + cds_wfs_for_each_blocking_safe(head, node, n) { + free(node); + URCU_TLS(nr_successful_dequeues)++; + URCU_TLS(nr_dequeues)++; + } +} + +static void *thr_dequeuer(void *_count) +{ + unsigned long long *count = _count; + unsigned int counter = 0; + + printf_verbose("thread_begin %s, tid %lu\n", + "dequeuer", urcu_get_thread_id()); + + set_affinity(); + + while (!test_go) + { + } + cmm_smp_mb(); + + assert(test_pop || test_pop_all); + + for (;;) { + if (test_pop && test_pop_all) { + if (counter & 1) + do_test_pop(test_sync); + else + do_test_pop_all(test_sync); + counter++; + } else { + if (test_pop) + do_test_pop(test_sync); + else + do_test_pop_all(test_sync); + } + + if (caa_unlikely(!test_duration_dequeue())) + break; + if (caa_unlikely(rduration)) + loop_sleep(rduration); + } + + printf_verbose("dequeuer thread_end, tid %lu, " + "dequeues %llu, successful_dequeues %llu " + "pop_all %llu pop_last %llu\n", + urcu_get_thread_id(), + URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues), + URCU_TLS(nr_pop_all), + URCU_TLS(nr_pop_last)); + count[0] = URCU_TLS(nr_dequeues); + count[1] = URCU_TLS(nr_successful_dequeues); + count[2] = URCU_TLS(nr_pop_all); + count[3] = URCU_TLS(nr_pop_last); + return ((void*)2); +} + +static void test_end(struct cds_wfs_stack *s, unsigned long long *nr_dequeues, + unsigned long long *nr_pop_last) +{ + struct cds_wfs_node *node; + int state; + + do { + node = cds_wfs_pop_with_state_blocking(s, &state); + if (node) { + if (state & CDS_WFS_STATE_LAST) + (*nr_pop_last)++; + free(node); + (*nr_dequeues)++; + } + } while (node); +} + +static void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", + argv[0]); + printf("OPTIONS:\n"); + printf(" [-d delay] (enqueuer period (in loops))\n"); + printf(" [-c duration] (dequeuer period (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf(" [-p] (test pop)\n"); + printf(" [-P] (test pop_all, enabled by default)\n"); + printf(" [-M] (use mutex external synchronization)\n"); + printf(" Note: default: no external synchronization used.\n"); + printf(" [-f] (force user-provided synchronization)\n"); + printf(" [-w] Wait for dequeuer to empty stack\n"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_enqueuer, *tid_dequeuer; + void *tret; + unsigned long long *count_enqueuer, *count_dequeuer; + unsigned long long tot_enqueues = 0, tot_dequeues = 0; + unsigned long long tot_successful_enqueues = 0, + tot_successful_dequeues = 0, + tot_empty_dest_enqueues = 0, + tot_pop_all = 0, tot_pop_last = 0; + unsigned long long end_dequeues = 0; + int i, a, retval = 0; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_dequeuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_enqueuers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'p': + test_pop = 1; + break; + case 'P': + test_pop_all = 1; + break; + case 'M': + test_sync = TEST_SYNC_MUTEX; + break; + case 'w': + test_wait_empty = 1; + break; + case 'f': + test_force_sync = 1; + break; + } + } + + /* activate pop_all test by default */ + if (!test_pop && !test_pop_all) + test_pop_all = 1; + + if (test_sync == TEST_SYNC_NONE && nr_dequeuers > 1 && test_pop) { + if (test_force_sync) { + fprintf(stderr, "[WARNING] Using pop concurrently " + "with other pop or pop_all without external " + "synchronization. Expect run-time failure.\n"); + } else { + printf("Enforcing mutex synchronization\n"); + test_sync = TEST_SYNC_MUTEX; + } + } + + printf_verbose("running test for %lu seconds, %u enqueuers, " + "%u dequeuers.\n", + duration, nr_enqueuers, nr_dequeuers); + if (test_pop) + printf_verbose("pop test activated.\n"); + if (test_pop_all) + printf_verbose("pop_all test activated.\n"); + if (test_sync == TEST_SYNC_MUTEX) + printf_verbose("External sync: mutex.\n"); + else + printf_verbose("External sync: none.\n"); + if (test_wait_empty) + printf_verbose("Wait for dequeuers to empty stack.\n"); + printf_verbose("Writer delay : %lu loops.\n", rduration); + printf_verbose("Reader duration : %lu loops.\n", wdelay); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer)); + tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer)); + count_enqueuer = calloc(nr_enqueuers, 3 * sizeof(*count_enqueuer)); + count_dequeuer = calloc(nr_dequeuers, 4 * sizeof(*count_dequeuer)); + cds_wfs_init(&s); + + next_aff = 0; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, + &count_enqueuer[3 * i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, + &count_dequeuer[4 * i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + for (i = 0; i < duration; i++) { + sleep(1); + if (verbose_mode) + (void) write(1, ".", 1); + } + + test_stop_enqueue = 1; + + if (test_wait_empty) { + while (nr_enqueuers != uatomic_read(&test_enqueue_stopped)) { + sleep(1); + } + while (!cds_wfs_empty(&s)) { + sleep(1); + } + } + + test_stop_dequeue = 1; + + for (i = 0; i < nr_enqueuers; i++) { + err = pthread_join(tid_enqueuer[i], &tret); + if (err != 0) + exit(1); + tot_enqueues += count_enqueuer[3 * i]; + tot_successful_enqueues += count_enqueuer[3 * i + 1]; + tot_empty_dest_enqueues += count_enqueuer[3 * i + 2]; + } + for (i = 0; i < nr_dequeuers; i++) { + err = pthread_join(tid_dequeuer[i], &tret); + if (err != 0) + exit(1); + tot_dequeues += count_dequeuer[4 * i]; + tot_successful_dequeues += count_dequeuer[4 * i + 1]; + tot_pop_all += count_dequeuer[4 * i + 2]; + tot_pop_last += count_dequeuer[4 * i + 3]; + } + + test_end(&s, &end_dequeues, &tot_pop_last); + + printf_verbose("total number of enqueues : %llu, dequeues %llu\n", + tot_enqueues, tot_dequeues); + printf_verbose("total number of successful enqueues : %llu, " + "enqueues to empty dest : %llu, " + "successful dequeues %llu, " + "pop_all : %llu, pop_last : %llu\n", + tot_successful_enqueues, + tot_empty_dest_enqueues, + tot_successful_dequeues, + tot_pop_all, tot_pop_last); + printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " + "nr_dequeuers %3u " + "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " + "successful enqueues %12llu enqueues to empty dest %12llu " + "successful dequeues %12llu pop_all %12llu " + "pop_last %llu end_dequeues %llu nr_ops %12llu\n", + argv[0], duration, nr_enqueuers, wdelay, + nr_dequeuers, rduration, tot_enqueues, tot_dequeues, + tot_successful_enqueues, + tot_empty_dest_enqueues, + tot_successful_dequeues, tot_pop_all, tot_pop_last, + end_dequeues, + tot_enqueues + tot_dequeues); + if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) { + printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " + "succ. dequeues + end dequeues %llu.\n", + tot_successful_enqueues, + tot_successful_dequeues + end_dequeues); + retval = 1; + } + /* + * The enqueuer should see exactly as many empty queues than the + * number of non-empty stacks dequeued. + */ + if (tot_empty_dest_enqueues != tot_pop_last) { + printf("WARNING! Discrepancy between empty enqueue (%llu) and " + "number of pop last (%llu)\n", + tot_empty_dest_enqueues, + tot_pop_last); + retval = 1; + } + free(count_enqueuer); + free(count_dequeuer); + free(tid_enqueuer); + free(tid_dequeuer); + return retval; +} diff --git a/tests/common.sh b/tests/common.sh deleted file mode 100644 index 9eb03f8..0000000 --- a/tests/common.sh +++ /dev/null @@ -1,12 +0,0 @@ -# -# This file is meant to be sourced from other tests scripts. -# - -if [ -x "$URCU_TEST_TIME_BIN" ]; then - test_time_bin="$URCU_TEST_TIME_BIN" -elif [ -x "/usr/bin/time" ]; then - test_time_bin="/usr/bin/time" -else - test_time_bin="" -fi - diff --git a/tests/common/Makefile.am b/tests/common/Makefile.am new file mode 100644 index 0000000..b55e527 --- /dev/null +++ b/tests/common/Makefile.am @@ -0,0 +1,9 @@ +AM_CPPFLAGS = -I$(top_srcdir) -I$(top_builddir) + +noinst_HEADERS = cpuset.h thread-id.h + +noinst_LTLIBRARIES = libdebug-yield.la + +libdebug_yield_la_SOURCES = debug-yield.c debug-yield.h + +EXTRA_DIST = api.h diff --git a/tests/common/api.h b/tests/common/api.h new file mode 100644 index 0000000..38d92a7 --- /dev/null +++ b/tests/common/api.h @@ -0,0 +1,311 @@ +#ifndef _INCLUDE_API_H +#define _INCLUDE_API_H + +#define _GNU_SOURCE +#include "config.h" + +/* + * common.h: Common Linux kernel-isms. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; but version 2 of the License only due + * to code included from the Linux kernel. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright (c) 2006 Paul E. McKenney, IBM. + * + * Much code taken from the Linux kernel. For such code, the option + * to redistribute under later versions of GPL might not be available. + */ + +#include +#include +#include "cpuset.h" + +/* + * Machine parameters. + */ + +#define ____cacheline_internodealigned_in_smp \ + __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) + +/* + * api_pthreads.h: API mapping to pthreads environment. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. However, please note that much + * of the code in this file derives from the Linux kernel, and that such + * code may not be available except under GPLv2. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright (c) 2006 Paul E. McKenney, IBM. + */ + +#include +#include +#include +#include +#include +#include +#include +/* #include "atomic.h" */ + +/* + * Exclusive locking primitives. + */ + +typedef pthread_mutex_t spinlock_t; + +#define DEFINE_SPINLOCK(lock) spinlock_t lock = PTHREAD_MUTEX_INITIALIZER; +#define __SPIN_LOCK_UNLOCKED(lockp) PTHREAD_MUTEX_INITIALIZER + +static void spin_lock_init(spinlock_t *sp) +{ + if (pthread_mutex_init(sp, NULL) != 0) { + perror("spin_lock_init:pthread_mutex_init"); + exit(-1); + } +} + +static void spin_lock(spinlock_t *sp) +{ + if (pthread_mutex_lock(sp) != 0) { + perror("spin_lock:pthread_mutex_lock"); + exit(-1); + } +} + +static void spin_unlock(spinlock_t *sp) +{ + if (pthread_mutex_unlock(sp) != 0) { + perror("spin_unlock:pthread_mutex_unlock"); + exit(-1); + } +} + +#define spin_lock_irqsave(l, f) do { f = 1; spin_lock(l); } while (0) +#define spin_unlock_irqrestore(l, f) do { f = 0; spin_unlock(l); } while (0) + +/* + * Thread creation/destruction primitives. + */ + +typedef pthread_t thread_id_t; + +#define NR_THREADS 128 + +#define __THREAD_ID_MAP_EMPTY ((thread_id_t) 0) +#define __THREAD_ID_MAP_WAITING ((thread_id_t) 1) +thread_id_t __thread_id_map[NR_THREADS]; +spinlock_t __thread_id_map_mutex; + +#define for_each_thread(t) \ + for (t = 0; t < NR_THREADS; t++) + +#define for_each_running_thread(t) \ + for (t = 0; t < NR_THREADS; t++) \ + if ((__thread_id_map[t] != __THREAD_ID_MAP_EMPTY) && \ + (__thread_id_map[t] != __THREAD_ID_MAP_WAITING)) + +#define for_each_tid(t, tid) \ + for (t = 0; t < NR_THREADS; t++) \ + if ((((tid) = __thread_id_map[t]) != __THREAD_ID_MAP_EMPTY) && \ + ((tid) != __THREAD_ID_MAP_WAITING)) + +pthread_key_t thread_id_key; + +static int __smp_thread_id(void) +{ + int i; + thread_id_t tid = pthread_self(); + + for (i = 0; i < NR_THREADS; i++) { + if (__thread_id_map[i] == tid) { + long v = i + 1; /* must be non-NULL. */ + + if (pthread_setspecific(thread_id_key, (void *)v) != 0) { + perror("pthread_setspecific"); + exit(-1); + } + return i; + } + } + spin_lock(&__thread_id_map_mutex); + for (i = 0; i < NR_THREADS; i++) { + if (__thread_id_map[i] == tid) { + spin_unlock(&__thread_id_map_mutex); + return i; + } + } + spin_unlock(&__thread_id_map_mutex); + fprintf(stderr, "smp_thread_id: Rogue thread, id: %lu(%#lx)\n", + (unsigned long) tid, (unsigned long) tid); + exit(-1); +} + +static int smp_thread_id(void) +{ + void *id; + + id = pthread_getspecific(thread_id_key); + if (id == NULL) + return __smp_thread_id(); + return (long)(id - 1); +} + +static thread_id_t create_thread(void *(*func)(void *), void *arg) +{ + thread_id_t tid; + int i; + + spin_lock(&__thread_id_map_mutex); + for (i = 0; i < NR_THREADS; i++) { + if (__thread_id_map[i] == __THREAD_ID_MAP_EMPTY) + break; + } + if (i >= NR_THREADS) { + spin_unlock(&__thread_id_map_mutex); + fprintf(stderr, "Thread limit of %d exceeded!\n", NR_THREADS); + exit(-1); + } + __thread_id_map[i] = __THREAD_ID_MAP_WAITING; + spin_unlock(&__thread_id_map_mutex); + if (pthread_create(&tid, NULL, func, arg) != 0) { + perror("create_thread:pthread_create"); + exit(-1); + } + __thread_id_map[i] = tid; + return tid; +} + +static void *wait_thread(thread_id_t tid) +{ + int i; + void *vp; + + for (i = 0; i < NR_THREADS; i++) { + if (__thread_id_map[i] == tid) + break; + } + if (i >= NR_THREADS){ + fprintf(stderr, "wait_thread: bad tid = %lu(%#lx)\n", + (unsigned long)tid, (unsigned long)tid); + exit(-1); + } + if (pthread_join(tid, &vp) != 0) { + perror("wait_thread:pthread_join"); + exit(-1); + } + __thread_id_map[i] = __THREAD_ID_MAP_EMPTY; + return vp; +} + +static void wait_all_threads(void) +{ + int i; + thread_id_t tid; + + for (i = 1; i < NR_THREADS; i++) { + tid = __thread_id_map[i]; + if (tid != __THREAD_ID_MAP_EMPTY && + tid != __THREAD_ID_MAP_WAITING) + (void)wait_thread(tid); + } +} + +static void run_on(int cpu) +{ +#if HAVE_SCHED_SETAFFINITY + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +/* + * timekeeping -- very crude -- should use MONOTONIC... + */ + +long long get_microseconds(void) +{ + struct timeval tv; + + if (gettimeofday(&tv, NULL) != 0) + abort(); + return ((long long)tv.tv_sec) * 1000000LL + (long long)tv.tv_usec; +} + +/* + * Per-thread variables. + */ + +#define DEFINE_PER_THREAD(type, name) \ + struct { \ + __typeof__(type) v \ + __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))); \ + } __per_thread_##name[NR_THREADS] +#define DECLARE_PER_THREAD(type, name) extern DEFINE_PER_THREAD(type, name) + +#define per_thread(name, thread) __per_thread_##name[thread].v +#define __get_thread_var(name) per_thread(name, smp_thread_id()) + +#define init_per_thread(name, v) \ + do { \ + int __i_p_t_i; \ + for (__i_p_t_i = 0; __i_p_t_i < NR_THREADS; __i_p_t_i++) \ + per_thread(name, __i_p_t_i) = v; \ + } while (0) + +DEFINE_PER_THREAD(int, smp_processor_id); + +/* + * Bug checks. + */ + +#define BUG_ON(c) do { if (!(c)) abort(); } while (0) + +/* + * Initialization -- Must be called before calling any primitives. + */ + +static void smp_init(void) +{ + int i; + + spin_lock_init(&__thread_id_map_mutex); + __thread_id_map[0] = pthread_self(); + for (i = 1; i < NR_THREADS; i++) + __thread_id_map[i] = __THREAD_ID_MAP_EMPTY; + init_per_thread(smp_processor_id, 0); + if (pthread_key_create(&thread_id_key, NULL) != 0) { + perror("pthread_key_create"); + exit(-1); + } +} + +#endif diff --git a/tests/common/cpuset.h b/tests/common/cpuset.h new file mode 100644 index 0000000..4d1f211 --- /dev/null +++ b/tests/common/cpuset.h @@ -0,0 +1,45 @@ +#ifndef _URCU_TESTS_CPUSET_H +#define _URCU_TESTS_CPUSET_H + +/* + * cpuset.h + * + * Userspace RCU library - test cpuset header + * + * Copyright 2009-2013 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "config.h" + +#if defined(HAVE_SCHED_SETAFFINITY) || defined(HAVE_CPU_SET_T) \ + || defined(HAVE_CPU_ZERO) || defined(HAVE_CPU_SET) +# include +#endif + +#ifndef HAVE_CPU_SET_T +typedef unsigned long cpu_set_t; +#endif + +#ifndef HAVE_CPU_ZERO +# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) +#endif + +#ifndef HAVE_CPU_SET +# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) +#endif + +#endif /* _URCU_TESTS_CPUSET_H */ diff --git a/tests/common/debug-yield.c b/tests/common/debug-yield.c new file mode 100644 index 0000000..8f06a9d --- /dev/null +++ b/tests/common/debug-yield.c @@ -0,0 +1,31 @@ +/* + * debug-yield.c + * + * Userspace RCU library tests - Debugging code + * + * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. + */ + +#include +#include "debug-yield.h" + +unsigned int rcu_yield_active; + +DEFINE_URCU_TLS(unsigned int, rcu_rand_yield); diff --git a/tests/common/debug-yield.h b/tests/common/debug-yield.h new file mode 100644 index 0000000..c60e4e0 --- /dev/null +++ b/tests/common/debug-yield.h @@ -0,0 +1,101 @@ +#ifndef URCU_TESTS_DEBUG_YIELD_H +#define URCU_TESTS_DEBUG_YIELD_H + +/* + * debug-yield.h + * + * Userspace RCU library tests - Debugging header + * + * Copyright (c) 2009 Mathieu Desnoyers + * Copyright (c) 2009 Paul E. McKenney, IBM Corporation. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * IBM's contributions to this file may be relicensed under LGPLv2 or later. + */ + +#include +#include +#include +#include + +#define RCU_YIELD_READ (1 << 0) +#define RCU_YIELD_WRITE (1 << 1) + +/* + * Updates with RCU_SIGNAL are much slower. Account this in the delay. + */ +#ifdef RCU_SIGNAL +/* maximum sleep delay, in us */ +#define MAX_SLEEP 30000 +#else +#define MAX_SLEEP 50 +#endif + +extern unsigned int rcu_yield_active; +extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield); + +#ifdef DEBUG_YIELD +static inline void rcu_debug_yield_read(void) +{ + if (rcu_yield_active & RCU_YIELD_READ) + if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) + usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); +} + +static inline void rcu_debug_yield_write(void) +{ + if (rcu_yield_active & RCU_YIELD_WRITE) + if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) + usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); +} + +static inline void rcu_debug_yield_enable(unsigned int flags) +{ + rcu_yield_active |= flags; +} + +static inline void rcu_debug_yield_disable(unsigned int flags) +{ + rcu_yield_active &= ~flags; +} + +static inline void rcu_debug_yield_init(void) +{ + URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self(); +} +#else /* DEBUG_YIELD */ +static inline void rcu_debug_yield_read(void) +{ +} + +static inline void rcu_debug_yield_write(void) +{ +} + +static inline void rcu_debug_yield_enable(unsigned int flags) +{ +} + +static inline void rcu_debug_yield_disable(unsigned int flags) +{ +} + +static inline void rcu_debug_yield_init(void) +{ +} +#endif + +#endif /* URCU_TESTS_DEBUG_YIELD_H */ diff --git a/tests/common/thread-id.h b/tests/common/thread-id.h new file mode 100644 index 0000000..04db5bb --- /dev/null +++ b/tests/common/thread-id.h @@ -0,0 +1,62 @@ +#ifndef _TEST_THREAD_ID_H +#define _TEST_THREAD_ID_H + +/* + * thread-id.h + * + * Userspace RCU library - thread ID + * + * Copyright 2013 - Mathieu Desnoyers + * + * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED + * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. + * + * Permission is hereby granted to use or copy this program + * for any purpose, provided the above notices are retained on all copies. + * Permission to modify the code and to distribute modified code is granted, + * provided the above notices are retained, and a notice that the code was + * modified is included with the above copyright notice. + */ +#include + +#ifdef __linux__ +# include + +# if defined(HAVE_GETTID) +/* + * Do not redefine gettid() as it is already included + * in bionic through . Some other libc + * may also already contain an implementation of gettid. + */ +# elif defined(_syscall0) +_syscall0(pid_t, gettid) +# elif defined(__NR_gettid) +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} +# endif + +static inline +unsigned long urcu_get_thread_id(void) +{ + return (unsigned long) gettid(); +} +#elif defined(__FreeBSD__) +# include + +static inline +unsigned long urcu_get_thread_id(void) +{ + return (unsigned long) pthread_getthreadid_np(); +} +#else +# warning "use pid as thread ID" +static inline +unsigned long urcu_get_thread_id(void) +{ + return (unsigned long) getpid(); +} +#endif + +#endif /* _TEST_THREAD_ID_H */ diff --git a/tests/cpuset.h b/tests/cpuset.h deleted file mode 100644 index 3f1587d..0000000 --- a/tests/cpuset.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef _URCU_TESTS_CPUSET_H -#define _URCU_TESTS_CPUSET_H - -/* - * cpuset.h - * - * Userspace RCU library - test cpuset header - * - * Copyright 2009-2013 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include "../config.h" - -#if defined(HAVE_SCHED_SETAFFINITY) || defined(HAVE_CPU_SET_T) \ - || defined(HAVE_CPU_ZERO) || defined(HAVE_CPU_SET) -# include -#endif - -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -#endif - -#ifndef HAVE_CPU_ZERO -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -#endif - -#ifndef HAVE_CPU_SET -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - -#endif /* _URCU_TESTS_CPUSET_H */ diff --git a/tests/rcutorture.h b/tests/rcutorture.h deleted file mode 100644 index f8548d8..0000000 --- a/tests/rcutorture.h +++ /dev/null @@ -1,503 +0,0 @@ -/* - * rcutorture.h: simple user-level performance/stress test of RCU. - * - * Usage: - * ./rcu rperf [ ] - * Run a read-side performance test with the specified - * number of readers spaced by . - * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered - * CPUs from 0 to 30. - * ./rcu uperf [ ] - * Run an update-side performance test with the specified - * number of updaters and specified CPU spacing. - * ./rcu perf [ ] - * Run a combined read/update performance test with the specified - * number of readers and one updater and specified CPU spacing. - * The readers run on the low-numbered CPUs and the updater - * of the highest-numbered CPU. - * - * The above tests produce output as follows: - * - * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1 - * ns/read: 43.4707 ns/update: 6848.1 - * - * The first line lists the total number of RCU reads and updates executed - * during the test, the number of reader threads, the number of updater - * threads, and the duration of the test in seconds. The second line - * lists the average duration of each type of operation in nanoseconds, - * or "nan" if the corresponding type of operation was not performed. - * - * ./rcu stress - * Run a stress test with the specified number of readers and - * one updater. None of the threads are affinitied to any - * particular CPU. - * - * This test produces output as follows: - * - * n_reads: 114633217 n_updates: 3903415 n_mberror: 0 - * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0 - * - * The first line lists the number of RCU read and update operations - * executed, followed by the number of memory-ordering violations - * (which will be zero in a correct RCU implementation). The second - * line lists the number of readers observing progressively more stale - * data. A correct RCU implementation will have all but the first two - * numbers non-zero. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Copyright (c) 2008 Paul E. McKenney, IBM Corporation. - */ - -/* - * Test variables. - */ - -#include - -DEFINE_PER_THREAD(long long, n_reads_pt); -DEFINE_PER_THREAD(long long, n_updates_pt); - -long long n_reads = 0LL; -long n_updates = 0L; -int nthreadsrunning; -char argsbuf[64]; - -#define GOFLAG_INIT 0 -#define GOFLAG_RUN 1 -#define GOFLAG_STOP 2 - -volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) - = GOFLAG_INIT; - -#define RCU_READ_RUN 1000 - -//MD -#define RCU_READ_NESTABLE - -#ifdef RCU_READ_NESTABLE -#define rcu_read_lock_nest() rcu_read_lock() -#define rcu_read_unlock_nest() rcu_read_unlock() -#else /* #ifdef RCU_READ_NESTABLE */ -#define rcu_read_lock_nest() -#define rcu_read_unlock_nest() -#endif /* #else #ifdef RCU_READ_NESTABLE */ - -#ifdef TORTURE_QSBR -#define mark_rcu_quiescent_state rcu_quiescent_state -#define put_thread_offline rcu_thread_offline -#define put_thread_online rcu_thread_online -#endif - -#ifndef mark_rcu_quiescent_state -#define mark_rcu_quiescent_state() do ; while (0) -#endif /* #ifdef mark_rcu_quiescent_state */ - -#ifndef put_thread_offline -#define put_thread_offline() do ; while (0) -#define put_thread_online() do ; while (0) -#define put_thread_online_delay() do ; while (0) -#else /* #ifndef put_thread_offline */ -#define put_thread_online_delay() synchronize_rcu() -#endif /* #else #ifndef put_thread_offline */ - -/* - * Performance test. - */ - -void *rcu_read_perf_test(void *arg) -{ - struct call_rcu_data *crdp; - int i; - int me = (long)arg; - long long n_reads_local = 0; - - rcu_register_thread(); - run_on(me); - uatomic_inc(&nthreadsrunning); - put_thread_offline(); - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - put_thread_online(); - while (goflag == GOFLAG_RUN) { - for (i = 0; i < RCU_READ_RUN; i++) { - rcu_read_lock(); - /* rcu_read_lock_nest(); */ - /* rcu_read_unlock_nest(); */ - rcu_read_unlock(); - } - n_reads_local += RCU_READ_RUN; - mark_rcu_quiescent_state(); - } - __get_thread_var(n_reads_pt) += n_reads_local; - put_thread_offline(); - crdp = get_thread_call_rcu_data(); - set_thread_call_rcu_data(NULL); - call_rcu_data_free(crdp); - rcu_unregister_thread(); - - return (NULL); -} - -void *rcu_update_perf_test(void *arg) -{ - long long n_updates_local = 0; - - if ((random() & 0xf00) == 0) { - struct call_rcu_data *crdp; - - crdp = create_call_rcu_data(0, -1); - if (crdp != NULL) { - fprintf(stderr, - "Using per-thread call_rcu() worker.\n"); - set_thread_call_rcu_data(crdp); - } - } - uatomic_inc(&nthreadsrunning); - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - while (goflag == GOFLAG_RUN) { - synchronize_rcu(); - n_updates_local++; - } - __get_thread_var(n_updates_pt) += n_updates_local; - return NULL; -} - -void perftestinit(void) -{ - init_per_thread(n_reads_pt, 0LL); - init_per_thread(n_updates_pt, 0LL); - uatomic_set(&nthreadsrunning, 0); -} - -void perftestrun(int nthreads, int nreaders, int nupdaters) -{ - int t; - int duration = 1; - - cmm_smp_mb(); - while (uatomic_read(&nthreadsrunning) < nthreads) - poll(NULL, 0, 1); - goflag = GOFLAG_RUN; - cmm_smp_mb(); - sleep(duration); - cmm_smp_mb(); - goflag = GOFLAG_STOP; - cmm_smp_mb(); - wait_all_threads(); - for_each_thread(t) { - n_reads += per_thread(n_reads_pt, t); - n_updates += per_thread(n_updates_pt, t); - } - printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n", - n_reads, n_updates, nreaders, nupdaters, duration); - printf("ns/read: %g ns/update: %g\n", - ((duration * 1000*1000*1000.*(double)nreaders) / - (double)n_reads), - ((duration * 1000*1000*1000.*(double)nupdaters) / - (double)n_updates)); - if (get_cpu_call_rcu_data(0)) { - fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); - free_all_cpu_call_rcu_data(); - } - exit(0); -} - -void perftest(int nreaders, int cpustride) -{ - int i; - long arg; - - perftestinit(); - for (i = 0; i < nreaders; i++) { - arg = (long)(i * cpustride); - create_thread(rcu_read_perf_test, (void *)arg); - } - arg = (long)(i * cpustride); - create_thread(rcu_update_perf_test, (void *)arg); - perftestrun(i + 1, nreaders, 1); -} - -void rperftest(int nreaders, int cpustride) -{ - int i; - long arg; - - perftestinit(); - init_per_thread(n_reads_pt, 0LL); - for (i = 0; i < nreaders; i++) { - arg = (long)(i * cpustride); - create_thread(rcu_read_perf_test, (void *)arg); - } - perftestrun(i, nreaders, 0); -} - -void uperftest(int nupdaters, int cpustride) -{ - int i; - long arg; - - perftestinit(); - init_per_thread(n_reads_pt, 0LL); - for (i = 0; i < nupdaters; i++) { - arg = (long)(i * cpustride); - create_thread(rcu_update_perf_test, (void *)arg); - } - perftestrun(i, 0, nupdaters); -} - -/* - * Stress test. - */ - -#define RCU_STRESS_PIPE_LEN 10 - -struct rcu_stress { - int pipe_count; - int mbtest; -}; - -struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } }; -struct rcu_stress *rcu_stress_current; -int rcu_stress_idx = 0; - -int n_mberror = 0; -DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count); - -int garbage = 0; - -void *rcu_read_stress_test(void *arg) -{ - int i; - int itercnt = 0; - struct rcu_stress *p; - int pc; - - rcu_register_thread(); - put_thread_offline(); - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - put_thread_online(); - while (goflag == GOFLAG_RUN) { - rcu_read_lock(); - p = rcu_dereference(rcu_stress_current); - if (p->mbtest == 0) - n_mberror++; - rcu_read_lock_nest(); - for (i = 0; i < 100; i++) - garbage++; - rcu_read_unlock_nest(); - pc = p->pipe_count; - rcu_read_unlock(); - if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0)) - pc = RCU_STRESS_PIPE_LEN; - __get_thread_var(rcu_stress_count)[pc]++; - __get_thread_var(n_reads_pt)++; - mark_rcu_quiescent_state(); - if ((++itercnt % 0x1000) == 0) { - put_thread_offline(); - put_thread_online_delay(); - put_thread_online(); - } - } - put_thread_offline(); - rcu_unregister_thread(); - - return (NULL); -} - -static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER; - -void rcu_update_stress_test_rcu(struct rcu_head *head) -{ - if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_lock"); - exit(-1); - } - if (pthread_cond_signal(&call_rcu_test_cond) != 0) { - perror("pthread_cond_signal"); - exit(-1); - } - if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_unlock"); - exit(-1); - } -} - -void *rcu_update_stress_test(void *arg) -{ - int i; - struct rcu_stress *p; - struct rcu_head rh; - - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - while (goflag == GOFLAG_RUN) { - i = rcu_stress_idx + 1; - if (i >= RCU_STRESS_PIPE_LEN) - i = 0; - p = &rcu_stress_array[i]; - p->mbtest = 0; - cmm_smp_mb(); - p->pipe_count = 0; - p->mbtest = 1; - rcu_assign_pointer(rcu_stress_current, p); - rcu_stress_idx = i; - for (i = 0; i < RCU_STRESS_PIPE_LEN; i++) - if (i != rcu_stress_idx) - rcu_stress_array[i].pipe_count++; - if (n_updates & 0x1) - synchronize_rcu(); - else { - if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_lock"); - exit(-1); - } - call_rcu(&rh, rcu_update_stress_test_rcu); - if (pthread_cond_wait(&call_rcu_test_cond, - &call_rcu_test_mutex) != 0) { - perror("pthread_cond_wait"); - exit(-1); - } - if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_unlock"); - exit(-1); - } - } - n_updates++; - } - return NULL; -} - -void *rcu_fake_update_stress_test(void *arg) -{ - if ((random() & 0xf00) == 0) { - struct call_rcu_data *crdp; - - crdp = create_call_rcu_data(0, -1); - if (crdp != NULL) { - fprintf(stderr, - "Using per-thread call_rcu() worker.\n"); - set_thread_call_rcu_data(crdp); - } - } - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - while (goflag == GOFLAG_RUN) { - synchronize_rcu(); - poll(NULL, 0, 1); - } - return NULL; -} - -void stresstest(int nreaders) -{ - int i; - int t; - long long *p; - long long sum; - - init_per_thread(n_reads_pt, 0LL); - for_each_thread(t) { - p = &per_thread(rcu_stress_count,t)[0]; - for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) - p[i] = 0LL; - } - rcu_stress_current = &rcu_stress_array[0]; - rcu_stress_current->pipe_count = 0; - rcu_stress_current->mbtest = 1; - for (i = 0; i < nreaders; i++) - create_thread(rcu_read_stress_test, NULL); - create_thread(rcu_update_stress_test, NULL); - for (i = 0; i < 5; i++) - create_thread(rcu_fake_update_stress_test, NULL); - cmm_smp_mb(); - goflag = GOFLAG_RUN; - cmm_smp_mb(); - sleep(10); - cmm_smp_mb(); - goflag = GOFLAG_STOP; - cmm_smp_mb(); - wait_all_threads(); - for_each_thread(t) - n_reads += per_thread(n_reads_pt, t); - printf("n_reads: %lld n_updates: %ld n_mberror: %d\n", - n_reads, n_updates, n_mberror); - printf("rcu_stress_count:"); - for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) { - sum = 0LL; - for_each_thread(t) { - sum += per_thread(rcu_stress_count, t)[i]; - } - printf(" %lld", sum); - } - printf("\n"); - if (get_cpu_call_rcu_data(0)) { - fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); - free_all_cpu_call_rcu_data(); - } - exit(0); -} - -/* - * Mainprogram. - */ - -void usage(int argc, char *argv[]) -{ - fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]); - exit(-1); -} - -int main(int argc, char *argv[]) -{ - int nreaders = 1; - int cpustride = 1; - - smp_init(); - //rcu_init(); - srandom(time(NULL)); - if (random() & 0x100) { - fprintf(stderr, "Allocating per-CPU call_rcu threads.\n"); - if (create_all_cpu_call_rcu_data(0)) - perror("create_all_cpu_call_rcu_data"); - } - -#ifdef DEBUG_YIELD - yield_active |= YIELD_READ; - yield_active |= YIELD_WRITE; -#endif - - if (argc > 1) { - nreaders = strtoul(argv[1], NULL, 0); - if (argc == 2) - perftest(nreaders, cpustride); - if (argc > 3) - cpustride = strtoul(argv[3], NULL, 0); - if (strcmp(argv[2], "perf") == 0) - perftest(nreaders, cpustride); - else if (strcmp(argv[2], "rperf") == 0) - rperftest(nreaders, cpustride); - else if (strcmp(argv[2], "uperf") == 0) - uperftest(nreaders, cpustride); - else if (strcmp(argv[2], "stress") == 0) - stresstest(nreaders); - usage(argc, argv); - } - perftest(nreaders, cpustride); - return 0; -} diff --git a/tests/regression/Makefile.am b/tests/regression/Makefile.am new file mode 100644 index 0000000..1b99abd --- /dev/null +++ b/tests/regression/Makefile.am @@ -0,0 +1,71 @@ +if !LIBC_INCLUDES_PTHREAD +AM_LDFLAGS=-lpthread +endif +AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/common -g + +noinst_PROGRAMS = test_urcu_fork \ + rcutorture_urcu \ + rcutorture_urcu_signal \ + rcutorture_urcu_mb \ + rcutorture_urcu_bp \ + rcutorture_urcu_qsbr \ + test_urcu_ja \ + test_urcu_ja_range + +noinst_HEADERS = rcutorture.h + +URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la +URCU_LIB=$(top_builddir)/liburcu.la +URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la +URCU_MB_LIB=$(top_builddir)/liburcu-mb.la +URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la +URCU_BP_LIB=$(top_builddir)/liburcu-bp.la +URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la + +test_urcu_fork_SOURCES = test_urcu_fork.c +test_urcu_fork_LDADD = $(URCU_LIB) + +rcutorture_urcu_SOURCES = urcutorture.c +rcutorture_urcu_CFLAGS = -DRCU_MEMBARRIER $(AM_CFLAGS) +rcutorture_urcu_LDADD = $(URCU_LIB) + +rcutorture_urcu_mb_SOURCES = urcutorture.c +rcutorture_urcu_mb_CFLAGS = -DRCU_MB $(AM_CFLAGS) +rcutorture_urcu_mb_LDADD = $(URCU_MB_LIB) + +rcutorture_urcu_qsbr_SOURCES = urcutorture.c +rcutorture_urcu_qsbr_CFLAGS = -DTORTURE_QSBR -DRCU_QSBR $(AM_CFLAGS) +rcutorture_urcu_qsbr_LDADD = $(URCU_QSBR_LIB) + +rcutorture_urcu_signal_SOURCES = urcutorture.c +rcutorture_urcu_signal_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) +rcutorture_urcu_signal_LDADD = $(URCU_SIGNAL_LIB) + +rcutorture_urcu_bp_SOURCES = urcutorture.c +rcutorture_urcu_bp_CFLAGS = -DRCU_BP $(AM_CFLAGS) +rcutorture_urcu_bp_LDADD = $(URCU_BP_LIB) + +test_urcu_ja_SOURCES = test_urcu_ja.c test_urcu_ja.h \ + $(COMPAT) +test_urcu_ja_CFLAGS = -DRCU_QSBR $(AM_CFLAGS) +test_urcu_ja_LDADD = $(URCU_QSBR_LIB) $(URCU_CDS_LIB) + +test_urcu_ja_range_SOURCES = test_urcu_ja_range.c test_urcu_ja_range.h \ + $(COMPAT) +test_urcu_ja_range_CFLAGS = -DRCU_QSBR $(AM_CFLAGS) +test_urcu_ja_range_LDADD = $(URCU_QSBR_LIB) $(URCU_CDS_LIB) + +urcutorture.c: ../common/api.h + +.PHONY: regtest + +# For now, run the benchmarks too as regression tests. +# TODO: split benchmarks from regression tests +regtest: + ./test_urcu_fork + ./rcutorture_urcu + ./rcutorture_urcu_signal + ./rcutorture_urcu_mb + ./rcutorture_urcu_bp + ./rcutorture_urcu_qsbr + cd ../benchmark && ./runall.sh && cd .. diff --git a/tests/regression/rcutorture.h b/tests/regression/rcutorture.h new file mode 100644 index 0000000..f8548d8 --- /dev/null +++ b/tests/regression/rcutorture.h @@ -0,0 +1,503 @@ +/* + * rcutorture.h: simple user-level performance/stress test of RCU. + * + * Usage: + * ./rcu rperf [ ] + * Run a read-side performance test with the specified + * number of readers spaced by . + * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered + * CPUs from 0 to 30. + * ./rcu uperf [ ] + * Run an update-side performance test with the specified + * number of updaters and specified CPU spacing. + * ./rcu perf [ ] + * Run a combined read/update performance test with the specified + * number of readers and one updater and specified CPU spacing. + * The readers run on the low-numbered CPUs and the updater + * of the highest-numbered CPU. + * + * The above tests produce output as follows: + * + * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1 + * ns/read: 43.4707 ns/update: 6848.1 + * + * The first line lists the total number of RCU reads and updates executed + * during the test, the number of reader threads, the number of updater + * threads, and the duration of the test in seconds. The second line + * lists the average duration of each type of operation in nanoseconds, + * or "nan" if the corresponding type of operation was not performed. + * + * ./rcu stress + * Run a stress test with the specified number of readers and + * one updater. None of the threads are affinitied to any + * particular CPU. + * + * This test produces output as follows: + * + * n_reads: 114633217 n_updates: 3903415 n_mberror: 0 + * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0 + * + * The first line lists the number of RCU read and update operations + * executed, followed by the number of memory-ordering violations + * (which will be zero in a correct RCU implementation). The second + * line lists the number of readers observing progressively more stale + * data. A correct RCU implementation will have all but the first two + * numbers non-zero. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Copyright (c) 2008 Paul E. McKenney, IBM Corporation. + */ + +/* + * Test variables. + */ + +#include + +DEFINE_PER_THREAD(long long, n_reads_pt); +DEFINE_PER_THREAD(long long, n_updates_pt); + +long long n_reads = 0LL; +long n_updates = 0L; +int nthreadsrunning; +char argsbuf[64]; + +#define GOFLAG_INIT 0 +#define GOFLAG_RUN 1 +#define GOFLAG_STOP 2 + +volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) + = GOFLAG_INIT; + +#define RCU_READ_RUN 1000 + +//MD +#define RCU_READ_NESTABLE + +#ifdef RCU_READ_NESTABLE +#define rcu_read_lock_nest() rcu_read_lock() +#define rcu_read_unlock_nest() rcu_read_unlock() +#else /* #ifdef RCU_READ_NESTABLE */ +#define rcu_read_lock_nest() +#define rcu_read_unlock_nest() +#endif /* #else #ifdef RCU_READ_NESTABLE */ + +#ifdef TORTURE_QSBR +#define mark_rcu_quiescent_state rcu_quiescent_state +#define put_thread_offline rcu_thread_offline +#define put_thread_online rcu_thread_online +#endif + +#ifndef mark_rcu_quiescent_state +#define mark_rcu_quiescent_state() do ; while (0) +#endif /* #ifdef mark_rcu_quiescent_state */ + +#ifndef put_thread_offline +#define put_thread_offline() do ; while (0) +#define put_thread_online() do ; while (0) +#define put_thread_online_delay() do ; while (0) +#else /* #ifndef put_thread_offline */ +#define put_thread_online_delay() synchronize_rcu() +#endif /* #else #ifndef put_thread_offline */ + +/* + * Performance test. + */ + +void *rcu_read_perf_test(void *arg) +{ + struct call_rcu_data *crdp; + int i; + int me = (long)arg; + long long n_reads_local = 0; + + rcu_register_thread(); + run_on(me); + uatomic_inc(&nthreadsrunning); + put_thread_offline(); + while (goflag == GOFLAG_INIT) + poll(NULL, 0, 1); + put_thread_online(); + while (goflag == GOFLAG_RUN) { + for (i = 0; i < RCU_READ_RUN; i++) { + rcu_read_lock(); + /* rcu_read_lock_nest(); */ + /* rcu_read_unlock_nest(); */ + rcu_read_unlock(); + } + n_reads_local += RCU_READ_RUN; + mark_rcu_quiescent_state(); + } + __get_thread_var(n_reads_pt) += n_reads_local; + put_thread_offline(); + crdp = get_thread_call_rcu_data(); + set_thread_call_rcu_data(NULL); + call_rcu_data_free(crdp); + rcu_unregister_thread(); + + return (NULL); +} + +void *rcu_update_perf_test(void *arg) +{ + long long n_updates_local = 0; + + if ((random() & 0xf00) == 0) { + struct call_rcu_data *crdp; + + crdp = create_call_rcu_data(0, -1); + if (crdp != NULL) { + fprintf(stderr, + "Using per-thread call_rcu() worker.\n"); + set_thread_call_rcu_data(crdp); + } + } + uatomic_inc(&nthreadsrunning); + while (goflag == GOFLAG_INIT) + poll(NULL, 0, 1); + while (goflag == GOFLAG_RUN) { + synchronize_rcu(); + n_updates_local++; + } + __get_thread_var(n_updates_pt) += n_updates_local; + return NULL; +} + +void perftestinit(void) +{ + init_per_thread(n_reads_pt, 0LL); + init_per_thread(n_updates_pt, 0LL); + uatomic_set(&nthreadsrunning, 0); +} + +void perftestrun(int nthreads, int nreaders, int nupdaters) +{ + int t; + int duration = 1; + + cmm_smp_mb(); + while (uatomic_read(&nthreadsrunning) < nthreads) + poll(NULL, 0, 1); + goflag = GOFLAG_RUN; + cmm_smp_mb(); + sleep(duration); + cmm_smp_mb(); + goflag = GOFLAG_STOP; + cmm_smp_mb(); + wait_all_threads(); + for_each_thread(t) { + n_reads += per_thread(n_reads_pt, t); + n_updates += per_thread(n_updates_pt, t); + } + printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n", + n_reads, n_updates, nreaders, nupdaters, duration); + printf("ns/read: %g ns/update: %g\n", + ((duration * 1000*1000*1000.*(double)nreaders) / + (double)n_reads), + ((duration * 1000*1000*1000.*(double)nupdaters) / + (double)n_updates)); + if (get_cpu_call_rcu_data(0)) { + fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); + free_all_cpu_call_rcu_data(); + } + exit(0); +} + +void perftest(int nreaders, int cpustride) +{ + int i; + long arg; + + perftestinit(); + for (i = 0; i < nreaders; i++) { + arg = (long)(i * cpustride); + create_thread(rcu_read_perf_test, (void *)arg); + } + arg = (long)(i * cpustride); + create_thread(rcu_update_perf_test, (void *)arg); + perftestrun(i + 1, nreaders, 1); +} + +void rperftest(int nreaders, int cpustride) +{ + int i; + long arg; + + perftestinit(); + init_per_thread(n_reads_pt, 0LL); + for (i = 0; i < nreaders; i++) { + arg = (long)(i * cpustride); + create_thread(rcu_read_perf_test, (void *)arg); + } + perftestrun(i, nreaders, 0); +} + +void uperftest(int nupdaters, int cpustride) +{ + int i; + long arg; + + perftestinit(); + init_per_thread(n_reads_pt, 0LL); + for (i = 0; i < nupdaters; i++) { + arg = (long)(i * cpustride); + create_thread(rcu_update_perf_test, (void *)arg); + } + perftestrun(i, 0, nupdaters); +} + +/* + * Stress test. + */ + +#define RCU_STRESS_PIPE_LEN 10 + +struct rcu_stress { + int pipe_count; + int mbtest; +}; + +struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } }; +struct rcu_stress *rcu_stress_current; +int rcu_stress_idx = 0; + +int n_mberror = 0; +DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count); + +int garbage = 0; + +void *rcu_read_stress_test(void *arg) +{ + int i; + int itercnt = 0; + struct rcu_stress *p; + int pc; + + rcu_register_thread(); + put_thread_offline(); + while (goflag == GOFLAG_INIT) + poll(NULL, 0, 1); + put_thread_online(); + while (goflag == GOFLAG_RUN) { + rcu_read_lock(); + p = rcu_dereference(rcu_stress_current); + if (p->mbtest == 0) + n_mberror++; + rcu_read_lock_nest(); + for (i = 0; i < 100; i++) + garbage++; + rcu_read_unlock_nest(); + pc = p->pipe_count; + rcu_read_unlock(); + if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0)) + pc = RCU_STRESS_PIPE_LEN; + __get_thread_var(rcu_stress_count)[pc]++; + __get_thread_var(n_reads_pt)++; + mark_rcu_quiescent_state(); + if ((++itercnt % 0x1000) == 0) { + put_thread_offline(); + put_thread_online_delay(); + put_thread_online(); + } + } + put_thread_offline(); + rcu_unregister_thread(); + + return (NULL); +} + +static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER; + +void rcu_update_stress_test_rcu(struct rcu_head *head) +{ + if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { + perror("pthread_mutex_lock"); + exit(-1); + } + if (pthread_cond_signal(&call_rcu_test_cond) != 0) { + perror("pthread_cond_signal"); + exit(-1); + } + if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { + perror("pthread_mutex_unlock"); + exit(-1); + } +} + +void *rcu_update_stress_test(void *arg) +{ + int i; + struct rcu_stress *p; + struct rcu_head rh; + + while (goflag == GOFLAG_INIT) + poll(NULL, 0, 1); + while (goflag == GOFLAG_RUN) { + i = rcu_stress_idx + 1; + if (i >= RCU_STRESS_PIPE_LEN) + i = 0; + p = &rcu_stress_array[i]; + p->mbtest = 0; + cmm_smp_mb(); + p->pipe_count = 0; + p->mbtest = 1; + rcu_assign_pointer(rcu_stress_current, p); + rcu_stress_idx = i; + for (i = 0; i < RCU_STRESS_PIPE_LEN; i++) + if (i != rcu_stress_idx) + rcu_stress_array[i].pipe_count++; + if (n_updates & 0x1) + synchronize_rcu(); + else { + if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { + perror("pthread_mutex_lock"); + exit(-1); + } + call_rcu(&rh, rcu_update_stress_test_rcu); + if (pthread_cond_wait(&call_rcu_test_cond, + &call_rcu_test_mutex) != 0) { + perror("pthread_cond_wait"); + exit(-1); + } + if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { + perror("pthread_mutex_unlock"); + exit(-1); + } + } + n_updates++; + } + return NULL; +} + +void *rcu_fake_update_stress_test(void *arg) +{ + if ((random() & 0xf00) == 0) { + struct call_rcu_data *crdp; + + crdp = create_call_rcu_data(0, -1); + if (crdp != NULL) { + fprintf(stderr, + "Using per-thread call_rcu() worker.\n"); + set_thread_call_rcu_data(crdp); + } + } + while (goflag == GOFLAG_INIT) + poll(NULL, 0, 1); + while (goflag == GOFLAG_RUN) { + synchronize_rcu(); + poll(NULL, 0, 1); + } + return NULL; +} + +void stresstest(int nreaders) +{ + int i; + int t; + long long *p; + long long sum; + + init_per_thread(n_reads_pt, 0LL); + for_each_thread(t) { + p = &per_thread(rcu_stress_count,t)[0]; + for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) + p[i] = 0LL; + } + rcu_stress_current = &rcu_stress_array[0]; + rcu_stress_current->pipe_count = 0; + rcu_stress_current->mbtest = 1; + for (i = 0; i < nreaders; i++) + create_thread(rcu_read_stress_test, NULL); + create_thread(rcu_update_stress_test, NULL); + for (i = 0; i < 5; i++) + create_thread(rcu_fake_update_stress_test, NULL); + cmm_smp_mb(); + goflag = GOFLAG_RUN; + cmm_smp_mb(); + sleep(10); + cmm_smp_mb(); + goflag = GOFLAG_STOP; + cmm_smp_mb(); + wait_all_threads(); + for_each_thread(t) + n_reads += per_thread(n_reads_pt, t); + printf("n_reads: %lld n_updates: %ld n_mberror: %d\n", + n_reads, n_updates, n_mberror); + printf("rcu_stress_count:"); + for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) { + sum = 0LL; + for_each_thread(t) { + sum += per_thread(rcu_stress_count, t)[i]; + } + printf(" %lld", sum); + } + printf("\n"); + if (get_cpu_call_rcu_data(0)) { + fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); + free_all_cpu_call_rcu_data(); + } + exit(0); +} + +/* + * Mainprogram. + */ + +void usage(int argc, char *argv[]) +{ + fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]); + exit(-1); +} + +int main(int argc, char *argv[]) +{ + int nreaders = 1; + int cpustride = 1; + + smp_init(); + //rcu_init(); + srandom(time(NULL)); + if (random() & 0x100) { + fprintf(stderr, "Allocating per-CPU call_rcu threads.\n"); + if (create_all_cpu_call_rcu_data(0)) + perror("create_all_cpu_call_rcu_data"); + } + +#ifdef DEBUG_YIELD + yield_active |= YIELD_READ; + yield_active |= YIELD_WRITE; +#endif + + if (argc > 1) { + nreaders = strtoul(argv[1], NULL, 0); + if (argc == 2) + perftest(nreaders, cpustride); + if (argc > 3) + cpustride = strtoul(argv[3], NULL, 0); + if (strcmp(argv[2], "perf") == 0) + perftest(nreaders, cpustride); + else if (strcmp(argv[2], "rperf") == 0) + rperftest(nreaders, cpustride); + else if (strcmp(argv[2], "uperf") == 0) + uperftest(nreaders, cpustride); + else if (strcmp(argv[2], "stress") == 0) + stresstest(nreaders); + usage(argc, argv); + } + perftest(nreaders, cpustride); + return 0; +} diff --git a/tests/regression/test_urcu_fork.c b/tests/regression/test_urcu_fork.c new file mode 100644 index 0000000..06786b0 --- /dev/null +++ b/tests/regression/test_urcu_fork.c @@ -0,0 +1,185 @@ +/* + * test_urcu_fork.c + * + * Userspace RCU library - test program (fork) + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#else +#define rcu_debug_yield_read() +#endif +#include + +/* We generate children 3 levels deep */ +#define FORK_DEPTH 3 +/* Each generation spawns 10 children */ +#define NR_FORK 10 + +static int fork_generation; + +struct test_node { + int somedata; + struct rcu_head head; +}; + +static void cb(struct rcu_head *head) +{ + struct test_node *node; + + fprintf(stderr, "rcu callback invoked in pid: %d\n", + (int) getpid()); + node = caa_container_of(head, struct test_node, head); + free(node); +} + +static void test_rcu(void) +{ + struct test_node *node; + + rcu_register_thread(); + + synchronize_rcu(); + + rcu_read_lock(); + rcu_read_unlock(); + + node = malloc(sizeof(*node)); + assert(node); + + call_rcu(&node->head, cb); + + synchronize_rcu(); + + rcu_unregister_thread(); +} + +/* + * Return 0 if child, > 0 if parent, < 0 on error. + */ +static int do_fork(const char *execname) +{ + pid_t pid; + + fprintf(stderr, "%s parent pid: %d, before fork\n", + execname, (int) getpid()); + + call_rcu_before_fork(); + pid = fork(); + if (pid == 0) { + /* child */ + fork_generation++; + + call_rcu_after_fork_child(); + fprintf(stderr, "%s child pid: %d, after fork\n", + execname, (int) getpid()); + test_rcu(); + fprintf(stderr, "%s child pid: %d, after rcu test\n", + execname, (int) getpid()); + if (fork_generation >= FORK_DEPTH) + exit(EXIT_SUCCESS); + return 0; + } else if (pid > 0) { + int status; + + /* parent */ + call_rcu_after_fork_parent(); + fprintf(stderr, "%s parent pid: %d, after fork\n", + execname, (int) getpid()); + test_rcu(); + fprintf(stderr, "%s parent pid: %d, after rcu test\n", + execname, (int) getpid()); + for (;;) { + pid = wait(&status); + if (pid < 0) { + perror("wait"); + return -1; + } + if (WIFEXITED(status)) { + fprintf(stderr, "child %u exited normally with status %u\n", + pid, WEXITSTATUS(status)); + if (WEXITSTATUS(status)) + return -1; + break; + } else if (WIFSIGNALED(status)) { + fprintf(stderr, "child %u was terminated by signal %u\n", + pid, WTERMSIG(status)); + return -1; + } else { + continue; + } + } + return 1; + } else { + perror("fork"); + return -1; + } +} + +int main(int argc, char **argv) +{ + unsigned int i; + +#if 0 + /* pthread_atfork does not work with malloc/free in callbacks */ + ret = pthread_atfork(call_rcu_before_fork, + call_rcu_after_fork_parent, + call_rcu_after_fork_child); + if (ret) { + errno = ret; + perror("pthread_atfork"); + exit(EXIT_FAILURE); + } +#endif + +restart: + for (i = 0; i < NR_FORK; i++) { + int ret; + + test_rcu(); + synchronize_rcu(); + ret = do_fork(argv[0]); + if (ret == 0) /* child */ + goto restart; + else if (ret < 0) + goto error; + /* else parent, continue. */ + } + exit(EXIT_SUCCESS); + +error: + exit(EXIT_FAILURE); +} diff --git a/tests/regression/test_urcu_ja.c b/tests/regression/test_urcu_ja.c new file mode 100644 index 0000000..0572527 --- /dev/null +++ b/tests/regression/test_urcu_ja.c @@ -0,0 +1,1339 @@ +/* + * test_urcu_ja.c + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "test_urcu_ja.h" +#include +#include + +DEFINE_URCU_TLS(unsigned int, rand_lookup); +DEFINE_URCU_TLS(unsigned long, nr_add); +DEFINE_URCU_TLS(unsigned long, nr_addexist); +DEFINE_URCU_TLS(unsigned long, nr_del); +DEFINE_URCU_TLS(unsigned long, nr_delnoent); +DEFINE_URCU_TLS(unsigned long, lookup_fail); +DEFINE_URCU_TLS(unsigned long, lookup_ok); + +struct cds_ja *test_ja; + +volatile int test_go, test_stop; + +unsigned long wdelay; + +unsigned long duration; + +/* read-side C.S. duration, in loops */ +unsigned long rduration; + +unsigned long init_populate; +int add_only; + +unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; +unsigned long init_pool_size = DEFAULT_RAND_POOL, + lookup_pool_size = DEFAULT_RAND_POOL, + write_pool_size = DEFAULT_RAND_POOL; +int validate_lookup; +int sanity_test; +unsigned int key_bits = 32; + +int count_pipe[2]; + +int verbose_mode; + +unsigned int cpu_affinities[NR_CPUS]; +unsigned int next_aff = 0; +int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +DEFINE_URCU_TLS(unsigned long long, nr_writes); +DEFINE_URCU_TLS(unsigned long long, nr_reads); + +unsigned int nr_readers; +unsigned int nr_writers; + +static unsigned int add_ratio = 50; +static uint64_t key_mul = 1ULL; + +static int add_unique, add_replace; + +static pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +static int leak_detection; +static unsigned long test_nodes_allocated, test_nodes_freed; + +void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +static +struct ja_test_node *node_alloc(void) +{ + struct ja_test_node *node; + + node = calloc(sizeof(*node), 1); + if (leak_detection && node) + uatomic_inc(&test_nodes_allocated); + return node; +} + +static +void free_test_node(struct ja_test_node *node) +{ + poison_free(node); + if (leak_detection) + uatomic_inc(&test_nodes_freed); +} + +static +void free_test_node_cb(struct rcu_head *head) +{ + struct ja_test_node *node = + caa_container_of(head, struct ja_test_node, head); + free_test_node(node); +} + +static +void rcu_free_test_node(struct ja_test_node *test_node) +{ + call_rcu(&test_node->head, free_test_node_cb); +} + +static +void free_node(struct cds_ja_node *node) +{ + struct ja_test_node *test_node = to_test_node(node); + + free_test_node(test_node); +} + +#if 0 +static +void test_delete_all_nodes(struct cds_lfht *ht) +{ + struct cds_lfht_iter iter; + struct lfht_test_node *node; + unsigned long count = 0; + + cds_lfht_for_each_entry(ht, &iter, node, node) { + int ret; + + ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); + assert(!ret); + call_rcu(&node->head, free_node_cb); + count++; + } + printf("deleted %lu nodes.\n", count); +} +#endif + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s)\n", argv[0]); +#ifdef DEBUG_YIELD + printf(" [-r] [-w] (yield reader and/or writer)\n"); +#endif + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf(" [-u] Add unique keys.\n"); + printf(" [-s] Replace existing keys.\n"); +printf(" [not -u nor -s] Add entries (supports redundant keys).\n"); + printf(" [-r ratio] Add ratio (in %% of add+removal).\n"); + printf(" [-k] Populate init nodes.\n"); + printf(" [-R offset] Lookup pool offset.\n"); + printf(" [-S offset] Write pool offset.\n"); + printf(" [-T offset] Init pool offset.\n"); + printf(" [-M size] Lookup pool size.\n"); + printf(" [-N size] Write pool size.\n"); + printf(" [-O size] Init pool size.\n"); + printf(" [-V] Validate lookups of init values (use with filled init pool, same lookup range, with different write range).\n"); + printf(" [-t] Do sanity test.\n"); + printf(" [-B] Key bits for multithread test (default: 32).\n"); + printf(" [-m factor] Key multiplication factor.\n"); + printf(" [-l] Memory leak detection.\n"); + printf("\n\n"); +} + +static +int test_free_all_nodes(struct cds_ja *ja) +{ + uint64_t key; + struct cds_ja_node *ja_node; + int ret = 0; + + rcu_read_lock(); + cds_ja_for_each_key_rcu(test_ja, key, ja_node) { + struct cds_ja_node *tmp_node; + + cds_ja_for_each_duplicate_safe_rcu(ja_node, tmp_node) { + ret = cds_ja_del(test_ja, key, ja_node); + if (ret) { + fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); + goto end; + } + /* Alone using Judy array, OK to free now */ + free_node(ja_node); + } + } +end: + rcu_read_unlock(); + return ret; +} + +static +int test_8bit_key(void) +{ + int ret, i; + uint64_t key; + uint64_t ka[] = { 5, 17, 100, 222 }; + uint64_t ka_test_offset = 5; + struct cds_ja_node *ja_node; + + /* Test with 8-bit key */ + test_ja = cds_ja_new(8); + if (!test_ja) { + printf("Error allocating judy array.\n"); + return -1; + } + + /* Add keys */ + printf("Test #1: add keys (8-bit).\n"); + for (key = 0; key < 200; key++) { + struct ja_test_node *node = node_alloc(); + + ja_test_node_init(node, key); + rcu_read_lock(); + ret = cds_ja_add(test_ja, key, &node->node); + rcu_read_unlock(); + if (ret) { + fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", + ret, key); + assert(0); + } + } + printf("OK\n"); + + printf("Test #2: successful key lookup (8-bit).\n"); + for (key = 0; key < 200; key++) { + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (!ja_node) { + fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); + assert(0); + } + rcu_read_unlock(); + } + printf("OK\n"); + printf("Test #3: unsuccessful key lookup (8-bit).\n"); + for (key = 200; key < 240; key++) { + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (ja_node) { + fprintf(stderr, + "Error unexpected lookup node %" PRIu64 "\n", + key); + assert(0); + } + rcu_read_unlock(); + } + printf("OK\n"); + printf("Test #4: remove keys (8-bit).\n"); + for (key = 0; key < 200; key++) { + struct ja_test_node *node; + + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (!ja_node) { + fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + ret = cds_ja_del(test_ja, key, &node->node); + if (ret) { + fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); + assert(0); + } + rcu_free_test_node(node); + ja_node = cds_ja_lookup(test_ja, key); + if (ja_node) { + fprintf(stderr, "Error lookup %" PRIu64 ": %p (after delete) failed. Node is not expected.\n", key, ja_node); + assert(0); + } + rcu_read_unlock(); + } + printf("OK\n"); + + printf("Test #5: lookup below/above equal (8-bit).\n"); + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node = node_alloc(); + + key = ka[i]; + ja_test_node_init(node, key); + rcu_read_lock(); + ret = cds_ja_add(test_ja, key, &node->node); + rcu_read_unlock(); + if (ret) { + fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", + ret, key); + assert(0); + } + } + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node; + uint64_t result_key; + + key = ka[i] + ka_test_offset; + rcu_read_lock(); + ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + rcu_read_unlock(); + } + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node; + uint64_t result_key; + + key = ka[i] - ka_test_offset; + rcu_read_lock(); + ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + rcu_read_unlock(); + } + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node; + uint64_t result_key; + + key = ka[i]; /* without offset */ + rcu_read_lock(); + ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + + ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + rcu_read_unlock(); + } + + printf("OK\n"); + + ret = test_free_all_nodes(test_ja); + if (ret) { + fprintf(stderr, "Error freeing all nodes\n"); + return -1; + } + + ret = cds_ja_destroy(test_ja); + if (ret) { + fprintf(stderr, "Error destroying judy array\n"); + return -1; + } + return 0; +} + +static +int test_16bit_key(void) +{ + int ret, i; + uint64_t key; + uint64_t ka[] = { 105, 206, 4000, 4111, 59990, 65435 }; + uint64_t ka_test_offset = 100; + struct cds_ja_node *ja_node; + + /* Test with 16-bit key */ + test_ja = cds_ja_new(16); + if (!test_ja) { + printf("Error allocating judy array.\n"); + return -1; + } + + /* Add keys */ + printf("Test #1: add keys (16-bit).\n"); + for (key = 0; key < 10000; key++) { + //for (key = 0; key < 65536; key+=256) { + struct ja_test_node *node = node_alloc(); + + ja_test_node_init(node, key); + rcu_read_lock(); + ret = cds_ja_add(test_ja, key, &node->node); + rcu_read_unlock(); + if (ret) { + fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", + ret, key); + assert(0); + } + } + printf("OK\n"); + + printf("Test #2: successful key lookup (16-bit).\n"); + for (key = 0; key < 10000; key++) { + //for (key = 0; key < 65536; key+=256) { + struct cds_ja_node *ja_node; + + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (!ja_node) { + fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); + assert(0); + } + rcu_read_unlock(); + } + printf("OK\n"); + printf("Test #3: unsuccessful key lookup (16-bit).\n"); + for (key = 11000; key <= 11002; key++) { + struct cds_ja_node *ja_node; + + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (ja_node) { + fprintf(stderr, + "Error unexpected lookup node %" PRIu64 "\n", + key); + assert(0); + } + rcu_read_unlock(); + } + printf("OK\n"); + printf("Test #4: remove keys (16-bit).\n"); + for (key = 0; key < 10000; key++) { + //for (key = 0; key < 65536; key+=256) { + struct ja_test_node *node; + + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (!ja_node) { + fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + ret = cds_ja_del(test_ja, key, &node->node); + if (ret) { + fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); + assert(0); + } + rcu_free_test_node(node); + ja_node = cds_ja_lookup(test_ja, key); + if (ja_node) { + fprintf(stderr, "Error lookup %" PRIu64 ": %p (after delete) failed. Node is not expected.\n", key, ja_node); + assert(0); + } + rcu_read_unlock(); + } + printf("OK\n"); + + printf("Test #5: lookup below/above equal (16-bit).\n"); + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node = node_alloc(); + + key = ka[i]; + ja_test_node_init(node, key); + rcu_read_lock(); + ret = cds_ja_add(test_ja, key, &node->node); + rcu_read_unlock(); + if (ret) { + fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", + ret, key); + assert(0); + } + } + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node; + uint64_t result_key; + + key = ka[i] + ka_test_offset; + rcu_read_lock(); + ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + rcu_read_unlock(); + } + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node; + uint64_t result_key; + + key = ka[i] - ka_test_offset; + rcu_read_lock(); + ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" above or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " above or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + rcu_read_unlock(); + } + + for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { + struct ja_test_node *node; + uint64_t result_key; + + key = ka[i]; /* without offset */ + rcu_read_lock(); + ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + + ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); + if (!ja_node) { + fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" above or equal to %" PRIu64 ".\n", + ka[i], key); + assert(0); + } + node = caa_container_of(ja_node, struct ja_test_node, node); + if (node->key != ka[i] || result_key != ka[i]) { + fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " above or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", + ka[i], key, node->key, result_key); + assert(0); + } + rcu_read_unlock(); + } + + printf("OK\n"); + + ret = test_free_all_nodes(test_ja); + if (ret) { + fprintf(stderr, "Error freeing all nodes\n"); + return -1; + } + + ret = cds_ja_destroy(test_ja); + if (ret) { + fprintf(stderr, "Error destroying judy array\n"); + return -1; + } + return 0; +} + +/* + * nr_dup is number of nodes per key. + */ +static +int test_sparse_key(unsigned int bits, int nr_dup) +{ + uint64_t key, max_key; + int zerocount, i, ret; + struct cds_ja_node *ja_node; + + if (bits == 64) + max_key = UINT64_MAX; + else + max_key = (1ULL << bits) - 1; + + printf("Sparse key test begins for %u-bit keys\n", bits); + /* Test with 16-bit key */ + test_ja = cds_ja_new(bits); + if (!test_ja) { + printf("Error allocating judy array.\n"); + return -1; + } + + /* Add keys */ + printf("Test #1: add keys (%u-bit).\n", bits); + for (i = 0; i < nr_dup; i++) { + zerocount = 0; + for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { + struct ja_test_node *node = node_alloc(); + + ja_test_node_init(node, key); + rcu_read_lock(); + ret = cds_ja_add(test_ja, key, &node->node); + rcu_read_unlock(); + if (ret) { + fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", + ret, key); + assert(0); + } + if (key == 0) + zerocount++; + } + } + printf("OK\n"); + + printf("Test #2: successful key lookup (%u-bit).\n", bits); + zerocount = 0; + for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { + struct ja_test_node *node; + int count = 0; + + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + if (!ja_node) { + fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); + assert(0); + } + cds_ja_for_each_duplicate_rcu(ja_node) { + count++; + } + if (count != nr_dup) { + fprintf(stderr, "Unexpected number of match for key %" PRIu64 ", expected %d, got %d.\n", key, nr_dup, count); + } + rcu_read_unlock(); + if (key == 0) + zerocount++; + } + printf("OK\n"); + if (bits > 8) { + printf("Test #3: unsuccessful key lookup (%u-bit).\n", bits); + zerocount = 0; + for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key + 42); + if (ja_node) { + fprintf(stderr, + "Error unexpected lookup node %" PRIu64 "\n", + key + 42); + assert(0); + } + rcu_read_unlock(); + if (key == 0) + zerocount++; + } + printf("OK\n"); + } + printf("Test #4: remove keys (%u-bit).\n", bits); + zerocount = 0; + for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { + int count = 0; + + rcu_read_lock(); + ja_node = cds_ja_lookup(test_ja, key); + + cds_ja_for_each_duplicate_rcu(ja_node) { + struct cds_ja_node *test_ja_node; + struct ja_test_node *node; + + count++; + node = caa_container_of(ja_node, + struct ja_test_node, node); + ret = cds_ja_del(test_ja, key, &node->node); + if (ret) { + fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); + assert(0); + } + rcu_free_test_node(node); + test_ja_node = cds_ja_lookup(test_ja, key); + if (count < nr_dup && !test_ja_node) { + fprintf(stderr, "Error: no node found after deletion of some nodes of a key\n"); + assert(0); + } + } + ja_node = cds_ja_lookup(test_ja, key); + if (ja_node) { + fprintf(stderr, "Error lookup %" PRIu64 ": %p (after delete) failed. Node is not expected.\n", key, ja_node); + assert(0); + } + rcu_read_unlock(); + if (key == 0) + zerocount++; + } + printf("OK\n"); + + ret = test_free_all_nodes(test_ja); + if (ret) { + fprintf(stderr, "Error freeing all nodes\n"); + return -1; + } + + ret = cds_ja_destroy(test_ja); + if (ret) { + fprintf(stderr, "Error destroying judy array\n"); + return -1; + } + printf("Test ends\n"); + + return 0; +} + +static +int do_sanity_test(void) +{ + int i, j, ret; + + printf("Sanity test start.\n"); + + for (i = 0; i < 3; i++) { + ret = test_8bit_key(); + if (ret) { + return ret; + } + rcu_quiescent_state(); + } + ret = test_16bit_key(); + if (ret) { + return ret; + } + rcu_quiescent_state(); + + /* key bits */ + for (i = 8; i <= 64; i *= 2) { + /* nr of nodes per key */ + for (j = 1; j < 4; j++) { + ret = test_sparse_key(i, j); + if (ret) { + return ret; + } + rcu_quiescent_state(); + } + } + printf("Sanity test end.\n"); + + return 0; +} + +enum urcu_ja_addremove { + AR_RANDOM = 0, + AR_ADD = 1, + AR_REMOVE = -1, +}; /* 1: add, -1 remove, 0: random */ + +static enum urcu_ja_addremove addremove; /* 1: add, -1 remove, 0: random */ + +static +void test_ja_rw_sigusr1_handler(int signo) +{ + switch (addremove) { + case AR_ADD: + printf("Add/Remove: random.\n"); + addremove = AR_RANDOM; + break; + case AR_RANDOM: + printf("Add/Remove: remove only.\n"); + addremove = AR_REMOVE; + break; + case AR_REMOVE: + printf("Add/Remove: add only.\n"); + addremove = AR_ADD; + break; + } +} + +static +void *test_ja_rw_thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct cds_ja_node *ja_node; + uint64_t key; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + + /* note: only looking up ulong keys */ + key = ((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % lookup_pool_size) + lookup_pool_offset; + key *= key_mul; + ja_node = cds_ja_lookup(test_ja, key); + if (!ja_node) { + if (validate_lookup) { + printf("[ERROR] Lookup cannot find initial node.\n"); + exit(-1); + } + URCU_TLS(lookup_fail)++; + } else { + URCU_TLS(lookup_ok)++; + } + rcu_debug_yield_read(); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + printf_verbose("readid : %lx, lookupfail %lu, lookupok %lu\n", + pthread_self(), URCU_TLS(lookup_fail), + URCU_TLS(lookup_ok)); + return ((void*)1); +} + +static +int is_add(void) +{ + return ((unsigned int) rand_r(&URCU_TLS(rand_lookup)) % 100) < add_ratio; +} + +static +void *test_ja_rw_thr_writer(void *_count) +{ + struct wr_count *count = _count; + uint64_t key; + int ret; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + if ((addremove == AR_ADD) + || (addremove == AR_RANDOM && is_add())) { + struct ja_test_node *node = node_alloc(); + struct cds_ja_node *ret_node; + + /* note: only inserting ulong keys */ + key = ((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset; + key *= key_mul; + ja_test_node_init(node, key); + rcu_read_lock(); + if (add_unique) { + ret_node = cds_ja_add_unique(test_ja, key, &node->node); + if (ret_node != &node->node) { + free_test_node(node); + URCU_TLS(nr_addexist)++; + } else { + URCU_TLS(nr_add)++; + } + } else if (add_replace) { + assert(0); /* not implemented yet. */ + } else { + ret = cds_ja_add(test_ja, key, &node->node); + if (ret) { + fprintf(stderr, "Error in cds_ja_add: %d\n", ret); + free_test_node(node); + } else { + URCU_TLS(nr_add)++; + } + } + rcu_read_unlock(); + } else { + struct cds_ja_node *ja_node; + struct ja_test_node *node; + + /* May delete */ + /* note: only deleting ulong keys */ + key = ((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset; + key *= key_mul; + + rcu_read_lock(); + + ja_node = cds_ja_lookup(test_ja, key); + /* Remove first entry */ + if (ja_node) { + node = caa_container_of(ja_node, + struct ja_test_node, node); + ret = cds_ja_del(test_ja, key, &node->node); + if (!ret) { + rcu_free_test_node(node); + URCU_TLS(nr_del)++; + } else { + URCU_TLS(nr_delnoent)++; + } + } else { + URCU_TLS(nr_delnoent)++; + } + rcu_read_unlock(); + } + + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + printf_verbose("info id %lx: nr_add %lu, nr_addexist %lu, nr_del %lu, " + "nr_delnoent %lu\n", pthread_self(), URCU_TLS(nr_add), + URCU_TLS(nr_addexist), URCU_TLS(nr_del), + URCU_TLS(nr_delnoent)); + count->update_ops = URCU_TLS(nr_writes); + count->add = URCU_TLS(nr_add); + count->add_exist = URCU_TLS(nr_addexist); + count->remove = URCU_TLS(nr_del); + return ((void*)2); +} + +static +int do_mt_populate_ja(void) +{ + uint64_t iter; + int ret; + + if (!init_populate) + return 0; + + printf("Starting rw test\n"); + + for (iter = init_pool_offset; iter < init_pool_offset + init_pool_size; iter++) { + struct ja_test_node *node = node_alloc(); + uint64_t key; + + /* note: only inserting ulong keys */ + key = (unsigned long) iter; + key *= key_mul; + ja_test_node_init(node, key); + rcu_read_lock(); + ret = cds_ja_add(test_ja, key, &node->node); + URCU_TLS(nr_add)++; + URCU_TLS(nr_writes)++; + rcu_read_unlock(); + /* Hash table resize only occurs in call_rcu thread */ + if (!(iter % 100)) + rcu_quiescent_state(); + if (ret) { + fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", + ret, key); + assert(0); + } + } + return 0; +} + +static +int do_mt_test(void) +{ + pthread_t *tid_reader, *tid_writer; + void *tret; + int ret, i, err; + unsigned long long *count_reader; + struct wr_count *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0, + tot_add = 0, tot_add_exist = 0, tot_remove = 0; + unsigned int remain; + + tid_reader = malloc(sizeof(*tid_reader) * nr_readers); + tid_writer = malloc(sizeof(*tid_writer) * nr_writers); + count_reader = malloc(sizeof(*count_reader) * nr_readers); + count_writer = malloc(sizeof(*count_writer) * nr_writers); + + printf("Allocating Judy Array for %u-bit keys\n", key_bits); + test_ja = cds_ja_new(key_bits); + if (!test_ja) { + printf("Error allocating judy array.\n"); + ret = -1; + goto end; + } + + do_mt_populate_ja(); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], + NULL, test_ja_rw_thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], + NULL, test_ja_rw_thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + rcu_thread_offline_qsbr(); + + remain = duration; + do { + remain = sleep(remain); + } while (remain > 0); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i].update_ops; + tot_add += count_writer[i].add; + tot_add_exist += count_writer[i].add_exist; + tot_remove += count_writer[i].remove; + } + rcu_thread_online_qsbr(); + + ret = test_free_all_nodes(test_ja); + if (ret) { + fprintf(stderr, "Error freeing all nodes\n"); + return -1; + } + + ret = cds_ja_destroy(test_ja); + if (ret) { + fprintf(stderr, "Error destroying judy array\n"); + goto end; + } + + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + ret = 0; +end: + return ret; +} + +static +int check_memory_leaks(void) +{ + unsigned long na, nf; + + na = uatomic_read(&test_nodes_allocated); + nf = uatomic_read(&test_nodes_freed); + if (na != nf) { + fprintf(stderr, "Memory leak of %ld test nodes detected. Allocated: %lu, freed: %lu\n", + na - nf, na, nf); + return -1; + } + return 0; +} + +int main(int argc, char **argv) +{ + int i, j, a, ret, err; + uint64_t key; + struct sigaction act; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { +#ifdef DEBUG_YIELD + case 'r': + yield_active |= YIELD_READ; + break; + case 'w': + yield_active |= YIELD_WRITE; + break; +#endif + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'r': + add_ratio = atoi(argv[++i]); + break; + case 'k': + init_populate = 1; + break; + case 'R': + lookup_pool_offset = atol(argv[++i]); + break; + case 'S': + write_pool_offset = atol(argv[++i]); + break; + case 'T': + init_pool_offset = atol(argv[++i]); + break; + case 'M': + lookup_pool_size = atol(argv[++i]); + break; + case 'N': + write_pool_size = atol(argv[++i]); + break; + case 'O': + init_pool_size = atol(argv[++i]); + break; + case 'V': + validate_lookup = 1; + break; + case 't': + sanity_test = 1; + break; + case 'B': + key_bits = atol(argv[++i]); + break; + case 'm': + key_mul = atoll(argv[++i]); + break; + case 'u': + add_unique = 1; + break; + case 's': + add_replace = 1; + break; + case 'l': + leak_detection = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("Add ratio: %u%%.\n", add_ratio); + printf_verbose("Mode:%s%s.\n", + " add/remove", + add_unique ? " uniquify" : ( add_replace ? " replace" : " insert")); + printf_verbose("Key multiplication factor: %" PRIu64 ".\n", key_mul); + printf_verbose("Init pool size offset %lu size %lu.\n", + init_pool_offset, init_pool_size); + printf_verbose("Lookup pool size offset %lu size %lu.\n", + lookup_pool_offset, lookup_pool_size); + printf_verbose("Update pool size offset %lu size %lu.\n", + write_pool_offset, write_pool_size); + if (validate_lookup) + printf_verbose("Validating lookups.\n"); + if (leak_detection) + printf_verbose("Memory leak dection activated.\n"); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + memset(&act, 0, sizeof(act)); + ret = sigemptyset(&act.sa_mask); + if (ret == -1) { + perror("sigemptyset"); + return -1; + } + act.sa_handler = test_ja_rw_sigusr1_handler; + act.sa_flags = SA_RESTART; + ret = sigaction(SIGUSR1, &act, NULL); + if (ret == -1) { + perror("sigaction"); + return -1; + } + + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + rcu_register_thread(); + + if (sanity_test) { + ret = do_sanity_test(); + } else { + ret = do_mt_test(); + } + + /* Wait for in-flight call_rcu free to complete for leak detection */ + rcu_barrier(); + + ret |= check_memory_leaks(); + + rcu_unregister_thread(); + free_all_cpu_call_rcu_data(); + + if (ret) { + printf("Test ended with error: %d\n", ret); + } + return ret; +} diff --git a/tests/regression/test_urcu_ja.h b/tests/regression/test_urcu_ja.h new file mode 100644 index 0000000..1dab429 --- /dev/null +++ b/tests/regression/test_urcu_ja.h @@ -0,0 +1,176 @@ +#ifndef _TEST_URCU_JA_H +#define _TEST_URCU_JA_H + +/* + * test_urcu_ja.h + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "thread-id.h" + +#define DEFAULT_RAND_POOL 1000000 + +/* Make this big enough to include the POWER5+ L3 cacheline size of 256B */ +#define CACHE_LINE_SIZE 4096 + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifdef POISON_FREE +#define poison_free(ptr) \ + do { \ + memset(ptr, 0x42, sizeof(*(ptr))); \ + free(ptr); \ + } while (0) +#else +#define poison_free(ptr) free(ptr) +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#else +#define debug_yield_read() +#endif +#include +#include +#include + +struct wr_count { + unsigned long update_ops; + unsigned long add; + unsigned long add_exist; + unsigned long remove; +}; + +extern DECLARE_URCU_TLS(unsigned int, rand_lookup); +extern DECLARE_URCU_TLS(unsigned long, nr_add); +extern DECLARE_URCU_TLS(unsigned long, nr_addexist); +extern DECLARE_URCU_TLS(unsigned long, nr_del); +extern DECLARE_URCU_TLS(unsigned long, nr_delnoent); +extern DECLARE_URCU_TLS(unsigned long, lookup_fail); +extern DECLARE_URCU_TLS(unsigned long, lookup_ok); + +extern struct cds_ja *test_ja; + +struct ja_test_node { + struct cds_ja_node node; + uint64_t key; /* for testing */ + struct rcu_head head; /* delayed reclaim */ +}; + +static inline struct ja_test_node * +to_test_node(struct cds_ja_node *node) +{ + return caa_container_of(node, struct ja_test_node, node); +} + +static inline +void ja_test_node_init(struct ja_test_node *node, uint64_t key) +{ + cds_ja_node_init(&node->node); + node->key = key; +} + +extern volatile int test_go, test_stop; + +extern unsigned long wdelay; + +extern unsigned long duration; + +/* read-side C.S. duration, in loops */ +extern unsigned long rduration; + +extern unsigned long init_populate; +extern int add_only; + +extern unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; +extern unsigned long init_pool_size, + lookup_pool_size, + write_pool_size; +extern int validate_lookup; + +extern int count_pipe[2]; + +static inline void loop_sleep(unsigned long l) +{ + while(l-- != 0) + caa_cpu_relax(); +} + +extern int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +extern unsigned int cpu_affinities[NR_CPUS]; +extern unsigned int next_aff; +extern int use_affinity; + +extern pthread_mutex_t affinity_mutex; + +#ifndef HAVE_CPU_SET_T +typedef unsigned long cpu_set_t; +# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) +# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) +#endif + +void set_affinity(void); + +/* + * returns 0 if test should end. + */ +static inline int test_duration_write(void) +{ + return !test_stop; +} + +static inline int test_duration_read(void) +{ + return !test_stop; +} + +extern DECLARE_URCU_TLS(unsigned long long, nr_writes); +extern DECLARE_URCU_TLS(unsigned long long, nr_reads); + +extern unsigned int nr_readers; +extern unsigned int nr_writers; + +void rcu_copy_mutex_lock(void); +void rcu_copy_mutex_unlock(void); + +#endif /* _TEST_URCU_JA_H */ diff --git a/tests/regression/test_urcu_ja_range.c b/tests/regression/test_urcu_ja_range.c new file mode 100644 index 0000000..93ac706 --- /dev/null +++ b/tests/regression/test_urcu_ja_range.c @@ -0,0 +1,680 @@ +/* + * test_urcu_ja_range.c + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include "test_urcu_ja_range.h" +#include +#include + +DEFINE_URCU_TLS(unsigned int, rand_lookup); +DEFINE_URCU_TLS(unsigned long, nr_add); +DEFINE_URCU_TLS(unsigned long, nr_addexist); +DEFINE_URCU_TLS(unsigned long, nr_del); +DEFINE_URCU_TLS(unsigned long, nr_delnoent); +DEFINE_URCU_TLS(unsigned long, lookup_fail); +DEFINE_URCU_TLS(unsigned long, lookup_ok); + +struct cds_ja *test_ja; + +volatile int test_go, test_stop; + +unsigned long wdelay; + +unsigned long duration; + +/* read-side C.S. duration, in loops */ +unsigned long rduration; + +unsigned long init_populate; +int add_only; + +unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; +unsigned long init_pool_size = DEFAULT_RAND_POOL, + lookup_pool_size = DEFAULT_RAND_POOL, + write_pool_size = DEFAULT_RAND_POOL; +int validate_lookup; +int sanity_test; +unsigned int key_bits = 32; + +int count_pipe[2]; + +int verbose_mode; + +unsigned int cpu_affinities[NR_CPUS]; +unsigned int next_aff = 0; +int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +DEFINE_URCU_TLS(unsigned long long, nr_writes); +DEFINE_URCU_TLS(unsigned long long, nr_reads); + +unsigned int nr_readers; +unsigned int nr_writers; + +static unsigned int add_ratio = 50, range_max_len = 0; +static uint64_t key_mul = 1ULL; + +static int add_unique, add_replace; + +static pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +static int leak_detection; +static unsigned long test_nodes_allocated, test_nodes_freed; + +void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + +#if HAVE_SCHED_SETAFFINITY + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); +#if SCHED_SETAFFINITY_ARGS == 2 + sched_setaffinity(0, &mask); +#else + sched_setaffinity(0, sizeof(mask), &mask); +#endif +#endif /* HAVE_SCHED_SETAFFINITY */ +} + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s)\n", argv[0]); +#ifdef DEBUG_YIELD + printf(" [-r] [-w] (yield reader and/or writer)\n"); +#endif + printf(" [-d delay] (writer period (us))\n"); + printf(" [-c duration] (reader C.S. duration (in loops))\n"); + printf(" [-v] (verbose output)\n"); + printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); + printf(" [-u] Add unique keys.\n"); + printf(" [-s] Replace existing keys.\n"); +printf(" [not -u nor -s] Add entries (supports redundant keys).\n"); + printf(" [-r ratio] Add ratio (in %% of add+removal).\n"); + printf(" [-k] Populate init nodes.\n"); + printf(" [-R offset] Lookup pool offset.\n"); + printf(" [-S offset] Write pool offset.\n"); + printf(" [-T offset] Init pool offset.\n"); + printf(" [-M size] Lookup pool size.\n"); + printf(" [-N size] Write pool size.\n"); + printf(" [-O size] Init pool size.\n"); + printf(" [-V] Validate lookups of init values (use with filled init pool, same lookup range, with different write range).\n"); + printf(" [-t] Do sanity test.\n"); + printf(" [-B] Key bits for multithread test (default: 32).\n"); + printf(" [-m factor] Key multiplication factor.\n"); + printf(" [-l] Memory leak detection.\n"); + printf(" [-L len] Range max len.\n"); + printf("\n\n"); +} + +enum urcu_ja_addremove { + AR_RANDOM = 0, + AR_ADD = 1, + AR_REMOVE = -1, +}; /* 1: add, -1 remove, 0: random */ + +static enum urcu_ja_addremove addremove; /* 1: add, -1 remove, 0: random */ + +static +void test_ja_rw_sigusr1_handler(int signo) +{ + switch (addremove) { + case AR_ADD: + printf("Add/Remove: random.\n"); + addremove = AR_RANDOM; + break; + case AR_RANDOM: + printf("Add/Remove: remove only.\n"); + addremove = AR_REMOVE; + break; + case AR_REMOVE: + printf("Add/Remove: add only.\n"); + addremove = AR_ADD; + break; + } +} + +static +void *test_ja_rw_thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct cds_ja_range *range; + uint64_t key; + + printf_verbose("thread_begin %s, tid %lu\n", + "reader", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = (unsigned int) urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + rcu_read_lock(); + + key = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); + key += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; + key = (key % lookup_pool_size) + lookup_pool_offset; + key *= key_mul; + + range = cds_ja_range_lookup(test_ja, key); + if (!range) { + if (validate_lookup) { + printf("[ERROR] Lookup cannot find initial node.\n"); + exit(-1); + } + URCU_TLS(lookup_fail)++; + } else { + range = cds_ja_range_lock(range); + if (!range) { + if (validate_lookup) { + printf("[ERROR] Lookup cannot find initial node.\n"); + exit(-1); + } + } else { + URCU_TLS(lookup_ok)++; + cds_ja_range_unlock(range); + } + } + rcu_debug_yield_read(); + if (caa_unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + URCU_TLS(nr_reads)++; + if (caa_unlikely(!test_duration_read())) + break; + if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + *count = URCU_TLS(nr_reads); + printf_verbose("thread_end %s, tid %lu\n", + "reader", urcu_get_thread_id()); + printf_verbose("read tid : %lu, lookupfail %lu, lookupok %lu\n", + urcu_get_thread_id(), URCU_TLS(lookup_fail), + URCU_TLS(lookup_ok)); + return ((void*)1); +} + +static +int is_add(void) +{ + return ((unsigned int) rand_r(&URCU_TLS(rand_lookup)) % 100) < add_ratio; +} + +static +void *test_ja_rw_thr_writer(void *_count) +{ + struct wr_count *count = _count; + int ret; + + printf_verbose("thread_begin %s, tid %lu\n", + "writer", urcu_get_thread_id()); + + URCU_TLS(rand_lookup) = (unsigned int) urcu_get_thread_id() ^ time(NULL); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + cmm_smp_mb(); + + for (;;) { + if ((addremove == AR_ADD) + || (addremove == AR_RANDOM && is_add())) { + struct cds_ja_range *range; + uint64_t start, end, tmp; + + start = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); + start += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; + start = (start % write_pool_size) + write_pool_offset; + + end = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); + end += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; + end = (end % write_pool_size) + write_pool_offset; + + start *= key_mul; + end *= key_mul; + if (start > end) { + tmp = start; + start = end; + end = tmp; + } + if (end - start > range_max_len) { + end = start + range_max_len; + } + rcu_read_lock(); + ret = cds_ja_range_add(test_ja, start, end, NULL); + if (ret) { + if (ret == -EEXIST) { + URCU_TLS(nr_addexist)++; + } else { + assert(0); + } + } else { + URCU_TLS(nr_add)++; + } + rcu_read_unlock(); + } else { + struct cds_ja_range *range; + uint64_t key; + + /* May delete */ + key = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); + key += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; + key = (key % write_pool_size) + write_pool_offset; + key *= key_mul; + + rcu_read_lock(); + + range = cds_ja_range_lookup(test_ja, key); + if (range) { + ret = cds_ja_range_del(test_ja, range); + if (!ret) { + URCU_TLS(nr_del)++; + } else { + URCU_TLS(nr_delnoent)++; + } + } else { + URCU_TLS(nr_delnoent)++; + } + rcu_read_unlock(); + } + + URCU_TLS(nr_writes)++; + if (caa_unlikely(!test_duration_write())) + break; + if (caa_unlikely(wdelay)) + loop_sleep(wdelay); + if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) + rcu_quiescent_state(); + } + + rcu_unregister_thread(); + + printf_verbose("thread_end %s, tid %lu\n", + "writer", urcu_get_thread_id()); + printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, " + "nr_delnoent %lu\n", urcu_get_thread_id(), + URCU_TLS(nr_add), + URCU_TLS(nr_addexist), URCU_TLS(nr_del), + URCU_TLS(nr_delnoent)); + count->update_ops = URCU_TLS(nr_writes); + count->add = URCU_TLS(nr_add); + count->add_exist = URCU_TLS(nr_addexist); + count->remove = URCU_TLS(nr_del); + return ((void*)2); +} + +static +int do_mt_populate_ja(void) +{ + uint64_t iter; + int ret; + + if (!init_populate) + return 0; + + printf("Starting rw test\n"); + + for (iter = init_pool_offset; iter < init_pool_offset + init_pool_size; iter++) { + struct cds_ja_range *range; + uint64_t key; + + /* note: only inserting ulong keys */ + key = (unsigned long) iter; + key *= key_mul; + rcu_read_lock(); + ret = cds_ja_range_add(test_ja, key, key, NULL); + URCU_TLS(nr_add)++; + URCU_TLS(nr_writes)++; + rcu_read_unlock(); + /* Hash table resize only occurs in call_rcu thread */ + if (!(iter % 100)) + rcu_quiescent_state(); + if (ret) { + fprintf(stderr, "Error (%d) adding range %" PRIu64 "\n", + ret, key); + assert(0); + } + } + return 0; +} + +static +int do_mt_test(void) +{ + pthread_t *tid_reader, *tid_writer; + void *tret; + int ret, i, err; + unsigned long long *count_reader; + struct wr_count *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0, + tot_add = 0, tot_add_exist = 0, tot_remove = 0; + unsigned int remain; + + tid_reader = malloc(sizeof(*tid_reader) * nr_readers); + tid_writer = malloc(sizeof(*tid_writer) * nr_writers); + count_reader = malloc(sizeof(*count_reader) * nr_readers); + count_writer = malloc(sizeof(*count_writer) * nr_writers); + + printf("Allocating %u-bit Judy Array for ranges\n", + key_bits); + test_ja = cds_ja_range_new(key_bits); + if (!test_ja) { + printf("Error allocating judy array.\n"); + ret = -1; + goto end; + } + + do_mt_populate_ja(); + + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], + NULL, test_ja_rw_thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], + NULL, test_ja_rw_thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + cmm_smp_mb(); + + test_go = 1; + + rcu_thread_offline_qsbr(); + + remain = duration; + do { + remain = sleep(remain); + } while (remain > 0); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i].update_ops; + tot_add += count_writer[i].add; + tot_add_exist += count_writer[i].add_exist; + tot_remove += count_writer[i].remove; + } + rcu_thread_online_qsbr(); + + ret = cds_ja_range_validate(test_ja); + assert(!ret); + + ret = cds_ja_range_destroy(test_ja, NULL); + if (ret) { + fprintf(stderr, "Error destroying judy array\n"); + goto end; + } + + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + ret = 0; +end: + return ret; +} + +int main(int argc, char **argv) +{ + int i, j, a, ret, err; + uint64_t key; + struct sigaction act; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { +#ifdef DEBUG_YIELD + case 'r': + yield_active |= YIELD_READ; + break; + case 'w': + yield_active |= YIELD_WRITE; + break; +#endif + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + case 'r': + add_ratio = atoi(argv[++i]); + break; + case 'k': + init_populate = 1; + break; + case 'R': + lookup_pool_offset = atol(argv[++i]); + break; + case 'S': + write_pool_offset = atol(argv[++i]); + break; + case 'T': + init_pool_offset = atol(argv[++i]); + break; + case 'M': + lookup_pool_size = atol(argv[++i]); + break; + case 'N': + write_pool_size = atol(argv[++i]); + break; + case 'O': + init_pool_size = atol(argv[++i]); + break; + case 'V': + validate_lookup = 1; + break; + case 't': + sanity_test = 1; + break; + case 'B': + key_bits = atol(argv[++i]); + break; + case 'm': + key_mul = atoll(argv[++i]); + break; + case 'u': + add_unique = 1; + break; + case 's': + add_replace = 1; + break; + case 'l': + leak_detection = 1; + break; + case 'L': + range_max_len = atol(argv[++i]); + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("Add ratio: %u%%.\n", add_ratio); + printf_verbose("Mode:%s%s.\n", + " add/remove", + add_unique ? " uniquify" : ( add_replace ? " replace" : " insert")); + printf_verbose("Key multiplication factor: %" PRIu64 ".\n", key_mul); + printf_verbose("Init pool size offset %lu size %lu.\n", + init_pool_offset, init_pool_size); + printf_verbose("Lookup pool size offset %lu size %lu.\n", + lookup_pool_offset, lookup_pool_size); + printf_verbose("Update pool size offset %lu size %lu.\n", + write_pool_offset, write_pool_size); + printf_verbose("Range max len: %lu.\n", + range_max_len); + if (validate_lookup) + printf_verbose("Validating lookups.\n"); + if (leak_detection) + printf_verbose("Memory leak dection activated.\n"); + printf_verbose("thread %-6s, tid %lu\n", + "main", urcu_get_thread_id()); + + memset(&act, 0, sizeof(act)); + ret = sigemptyset(&act.sa_mask); + if (ret == -1) { + perror("sigemptyset"); + return -1; + } + act.sa_handler = test_ja_rw_sigusr1_handler; + act.sa_flags = SA_RESTART; + ret = sigaction(SIGUSR1, &act, NULL); + if (ret == -1) { + perror("sigaction"); + return -1; + } + + err = create_all_cpu_call_rcu_data(0); + if (err) { + printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); + } + + rcu_register_thread(); + + ret = do_mt_test(); + + /* Wait for in-flight call_rcu free to complete for leak detection */ + rcu_barrier(); + + rcu_unregister_thread(); + free_all_cpu_call_rcu_data(); + + if (ret) { + printf("Test ended with error: %d\n", ret); + } + return ret; +} diff --git a/tests/regression/test_urcu_ja_range.h b/tests/regression/test_urcu_ja_range.h new file mode 100644 index 0000000..1f8c2a2 --- /dev/null +++ b/tests/regression/test_urcu_ja_range.h @@ -0,0 +1,157 @@ +#ifndef _TEST_URCU_JA_RANGE_H +#define _TEST_URCU_JA_RANGE_H + +/* + * test_urcu_ja.h + * + * Userspace RCU library - test program + * + * Copyright 2009-2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "thread-id.h" + +#define DEFAULT_RAND_POOL 1000000 + +/* Make this big enough to include the POWER5+ L3 cacheline size of 256B */ +#define CACHE_LINE_SIZE 4096 + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#ifdef POISON_FREE +#define poison_free(ptr) \ + do { \ + memset(ptr, 0x42, sizeof(*(ptr))); \ + free(ptr); \ + } while (0) +#else +#define poison_free(ptr) free(ptr) +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#else +#define debug_yield_read() +#endif +#include +#include +#include + +struct wr_count { + unsigned long update_ops; + unsigned long add; + unsigned long add_exist; + unsigned long remove; +}; + +extern DECLARE_URCU_TLS(unsigned int, rand_lookup); +extern DECLARE_URCU_TLS(unsigned long, nr_add); +extern DECLARE_URCU_TLS(unsigned long, nr_addexist); +extern DECLARE_URCU_TLS(unsigned long, nr_del); +extern DECLARE_URCU_TLS(unsigned long, nr_delnoent); +extern DECLARE_URCU_TLS(unsigned long, lookup_fail); +extern DECLARE_URCU_TLS(unsigned long, lookup_ok); + +extern struct cds_ja *test_ja; + +extern volatile int test_go, test_stop; + +extern unsigned long wdelay; + +extern unsigned long duration; + +/* read-side C.S. duration, in loops */ +extern unsigned long rduration; + +extern unsigned long init_populate; +extern int add_only; + +extern unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; +extern unsigned long init_pool_size, + lookup_pool_size, + write_pool_size; +extern int validate_lookup; + +extern int count_pipe[2]; + +static inline void loop_sleep(unsigned long l) +{ + while(l-- != 0) + caa_cpu_relax(); +} + +extern int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, ## args); \ + } while (0) + +extern unsigned int cpu_affinities[NR_CPUS]; +extern unsigned int next_aff; +extern int use_affinity; + +extern pthread_mutex_t affinity_mutex; + +#ifndef HAVE_CPU_SET_T +typedef unsigned long cpu_set_t; +# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) +# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) +#endif + +void set_affinity(void); + +/* + * returns 0 if test should end. + */ +static inline int test_duration_write(void) +{ + return !test_stop; +} + +static inline int test_duration_read(void) +{ + return !test_stop; +} + +extern DECLARE_URCU_TLS(unsigned long long, nr_writes); +extern DECLARE_URCU_TLS(unsigned long long, nr_reads); + +extern unsigned int nr_readers; +extern unsigned int nr_writers; + +void rcu_copy_mutex_lock(void); +void rcu_copy_mutex_unlock(void); + +#endif /* _TEST_URCU_JA_RANGE_H */ diff --git a/tests/regression/urcutorture.c b/tests/regression/urcutorture.c new file mode 100644 index 0000000..35096a6 --- /dev/null +++ b/tests/regression/urcutorture.c @@ -0,0 +1,30 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "api.h" +#define _LGPL_SOURCE + +#ifdef RCU_MEMBARRIER +#include +#endif +#ifdef RCU_SIGNAL +#include +#endif +#ifdef RCU_MB +#include +#endif +#ifdef RCU_QSBR +#include +#endif +#ifdef RCU_BP +#include +#endif + +#include +#include +#include "rcutorture.h" diff --git a/tests/runall.sh b/tests/runall.sh deleted file mode 100755 index 9577ede..0000000 --- a/tests/runall.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -NUM_CPUS=8 - -#extra options, e.g. for setting affinity on even CPUs : -#EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) - -#ppc64 striding, use with NUM_CPUS=8 - -#stride 1 -#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) -#stride 2 -#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) -#stride 4 -#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) -#stride 8 -#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) - -#Vary update fraction -#x: vary update fraction from 0 to 0.0001 - #fix number of readers and reader C.S. length, vary delay between updates -#y: ops/s - -rm -f runall.log -rm -fr runall.detail.log - - -echo Executing batch RCU test - -DURATION=10 -BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 - 131072 262144" -NR_WRITERS=$((${NUM_CPUS} / 2)) - -rm -f batch-rcu.log - -NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) -for BATCH_SIZE in ${BATCH_ARRAY}; do - echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log - (./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log) || exit 1 -done - -#setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON TEST ABOVE ** -EXTRA_OPTS="${EXTRA_OPTS} -b 32768" - -echo Executing update fraction test - -DURATION=10 -WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 - 65536 131072 262144 524288 1048576 2097152 4194304 8388608 - 16777216 33554432 67108864 134217728" -NR_WRITERS=$((${NUM_CPUS} / 2)) - -rm -f update-fraction.log - -NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) -for WDELAY in ${WDELAY_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log - (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log) || exit 1 -done - -#Test scalability : -# x: vary number of readers from 0 to num cpus -# y: ops/s -# 0 writer. - -echo Executing scalability test - -NR_WRITERS=0 -DURATION=10 - -rm -f scalability.log - -for NR_READERS in $(seq 1 ${NUM_CPUS}); do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log - (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log) || exit 1 -done - - -# x: Vary reader C.S. length from 0 to 100 us -# y: ops/s -# 8 readers -# 0 writers - -echo Executing reader C.S. length test - -NR_READERS=${NUM_CPUS} -NR_WRITERS=0 -DURATION=10 -#in loops. -READERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" - -rm -f readercslen.log - -for READERCSLEN in ${READERCSLEN_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log" >> runall.log - (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log) || exit 1 -done - -echo Executing multi-flavor RCU test -./test_urcu_multiflavor || exit 1 -./test_urcu_multiflavor_dynlink || exit 1 - -echo Executing Hash table test -./runhash.sh || exit 1 - -echo Executing Judy array test -./runja.sh || exit 1 diff --git a/tests/runhash.sh b/tests/runhash.sh deleted file mode 100755 index 0ba20ef..0000000 --- a/tests/runhash.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/sh - -# TODO: missing tests: -# - send kill signals during tests to change the behavior between -# add/remove/random -# - validate that "nr_leaked" is always 0 in SUMMARY for all tests - -# 30 seconds per test -TIME_UNITS=30 - -TESTPROG=./test_urcu_hash - -#thread multiplier -THREAD_MUL=1 - -EXTRA_PARAMS=-v - -# ** test update coherency with single-value table - -# rw test, single key, replace and del randomly, 4 threads, auto resize. -# key range: init, lookup, and update: 0 to 0 -${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 - -# rw test, single key, add unique and del randomly, 4 threads, auto resize. -# key range: init, lookup, and update: 0 to 0 -${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -u -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 - -# rw test, single key, replace and del randomly, 2 lookup threads, 2 update threads, auto resize. -# key range: init, lookup, and update: 0 to 0 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 - -# rw test, single key, add and del randomly, 2 lookup threads, 2 update threads, auto resize. -# key range: init, lookup, and update: 0 to 0 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 - - -# ** test updates vs lookups with default table - -# rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. -# max 1048576 buckets -# key range: init, lookup, and update: 0 to 999999 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. -# max 1048576 buckets -# key range: init, lookup, and update: 0 to 999999 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add_unique and del randomly, auto resize. -# max 1048576 buckets -# key range: init, lookup, and update: 0 to 999999 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -u ${EXTRA_PARAMS} || exit 1 - - -# test memory management backends - -# rw test, 2 lookup, 2 update threads, add only, auto resize. -# max buckets: 1048576 -# key range: init, lookup, and update: 0 to 99999999 -# mm backend: "order" -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ - -M 100000000 -N 100000000 -O 100000000 -B order ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add only, auto resize. -# max buckets: 1048576 -# key range: init, lookup, and update: 0 to 99999999 -# mm backend: "chunk" -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ - -M 100000000 -N 100000000 -O 100000000 -B chunk ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add only, auto resize. -# max buckets: 1048576 -# key range: init, lookup, and update: 0 to 99999999 -# mm backend: "mmap" -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ - -M 100000000 -N 100000000 -O 100000000 -B mmap ${EXTRA_PARAMS} || exit 1 - - -# ** key range tests - -# rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. -# max 1048576 buckets -# key range: init, and update: 0 to 999999 -# key range: lookup: 1000000 to 1999999 -# NOTE: reader threads in this test should never have a successful -# lookup. TODO -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -R 1000000 ${EXTRA_PARAMS} || exit 1 - -# ** small key range - -# rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. -# max 1048576 buckets -# key range: init, update, and lookups: 0 to 9 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add_unique and del randomly, auto resize. -# max 1048576 buckets -# key range: init, update, and lookups: 0 to 9 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 -u ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. -# max 1048576 buckets -# key range: init, update, and lookups: 0 to 9 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 -s ${EXTRA_PARAMS} || exit 1 - -# ** lookup for known keys - -# rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. -# max 1048576 buckets -# lookup range is entirely populated. -# key range: init, and lookups: 0 to 9 -# key range: updates: 10 to 19 -# NOTE: reader threads in this test should always have successful -# lookups. TODO -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 -R 0 -T 0 -S 10 -k 10 -s ${EXTRA_PARAMS} || exit 1 - -# ** Uniqueness test - -# rw test, 2 lookup, 2 update threads, add_unique, add_replace and del randomly, auto resize. -# max 1048576 buckets -# asserts that no duplicates are observed by reader threads -# standard length hash chains -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -U ${EXTRA_PARAMS} || exit 1 - -# rw test, 2 lookup, 2 update threads, add_unique, add_replace and del randomly, auto resize. -# max 1048576 buckets -# asserts that no duplicates are observed by reader threads -# create long hash chains: using modulo 4 on keys as hash -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -U -C 4 ${EXTRA_PARAMS} || exit 1 diff --git a/tests/runpaul-phase1.sh b/tests/runpaul-phase1.sh deleted file mode 100755 index d2c8649..0000000 --- a/tests/runpaul-phase1.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -NUM_CPUS=64 - -#extra options, e.g. for setting affinity on even CPUs : -EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) - -#ppc64 striding, use with NUM_CPUS=8 - -#stride 1 -#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) -#stride 2 -#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) -#stride 4 -#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) -#stride 8 -#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) - -#Vary update fraction -#x: vary update fraction from 0 to 0.0001 - #fix number of readers and reader C.S. length, vary delay between updates -#y: ops/s - -rm -f runall.log -rm -fr runall.detail.log - - -echo Executing batch RCU test - -DURATION=10 -BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 - 131072 262144" -NR_WRITERS=$((${NUM_CPUS} / 2)) - -rm -f batch-rcu.log - -NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) -for BATCH_SIZE in ${BATCH_ARRAY}; do - echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log - ./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log -done diff --git a/tests/runpaul-phase2.sh b/tests/runpaul-phase2.sh deleted file mode 100755 index bba9e3e..0000000 --- a/tests/runpaul-phase2.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -NUM_CPUS=64 - -#extra options, e.g. for setting affinity on even CPUs : -EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) - -#ppc64 striding, use with NUM_CPUS=8 - -#stride 1 -#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) -#stride 2 -#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) -#stride 4 -#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) -#stride 8 -#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) - -#Vary update fraction -#x: vary update fraction from 0 to 0.0001 - #fix number of readers and reader C.S. length, vary delay between updates -#y: ops/s - -rm -f runall.log -rm -fr runall.detail.log - -#setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON PHASE 1 RESULT ** -EXTRA_OPTS="${EXTRA_OPTS} -b 32768" - -echo Executing update fraction test - -DURATION=10 -WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 - 65536 131072 262144 524288 1048576 2097152 4194304 8388608 - 16777216 33554432 67108864 134217728" -NR_WRITERS=$((${NUM_CPUS} / 2)) - -rm -f update-fraction.log - -NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) -for WDELAY in ${WDELAY_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log - ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log -done diff --git a/tests/runpaul-phase3.sh b/tests/runpaul-phase3.sh deleted file mode 100755 index 7c5f055..0000000 --- a/tests/runpaul-phase3.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -NUM_CPUS=64 - -#extra options, e.g. for setting affinity on even CPUs : -EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) - -#ppc64 striding, use with NUM_CPUS=8 - -#stride 1 -#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) -#stride 2 -#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) -#stride 4 -#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) -#stride 8 -#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) - -#Vary update fraction -#x: vary update fraction from 0 to 0.0001 - #fix number of readers and reader C.S. length, vary delay between updates -#y: ops/s - -rm -f runall.log -rm -fr runall.detail.log - -#setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON PHASE 1 RESULT ** -EXTRA_OPTS="${EXTRA_OPTS} -b 32768" - -#Test scalability : -# x: vary number of readers from 0 to num cpus -# y: ops/s -# 0 writer. - -echo Executing scalability test - -NR_WRITERS=0 -DURATION=10 - -rm -f scalability.log - -for NR_READERS in $(seq 1 ${NUM_CPUS}); do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log - ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log -done - - diff --git a/tests/runpaul-phase4.sh b/tests/runpaul-phase4.sh deleted file mode 100755 index ede402c..0000000 --- a/tests/runpaul-phase4.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -export NUM_CPUS=8 - -#extra options, e.g. for setting affinity on even CPUs : -#EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) - -#ppc64 striding, use with NUM_CPUS=8 - -rm -f *.log - -#stride 1 -export EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) -sh subphase4.sh $* -mkdir ppc64-8cores-stride1 -mv *.log ppc64-8cores-stride1/ - - -#stride 2 -export EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) -sh subphase4.sh $* -mkdir ppc64-8cores-stride2 -mv *.log ppc64-8cores-stride2/ - - -#stride 4 -export EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) -sh subphase4.sh $* -mkdir ppc64-8cores-stride4 -mv *.log ppc64-8cores-stride4/ - - -#stride 8 -export EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) -sh subphase4.sh $* -mkdir ppc64-8cores-stride8 -mv *.log ppc64-8cores-stride8/ diff --git a/tests/runpaul-phase5.sh b/tests/runpaul-phase5.sh deleted file mode 100644 index bb4bfe7..0000000 --- a/tests/runpaul-phase5.sh +++ /dev/null @@ -1,11 +0,0 @@ -# test run after write-size update - -sh runpaul-phase1.sh -mkdir runpaul-phase1 -mv *.log runpaul-phase1/ - -sh runpaul-phase2.sh -mkdir runpaul-phase2 -mv *.log runpaul-phase2/ - -sh runpaul-phase4.sh diff --git a/tests/runpaul-phase6.sh b/tests/runpaul-phase6.sh deleted file mode 100644 index 5f65072..0000000 --- a/tests/runpaul-phase6.sh +++ /dev/null @@ -1,7 +0,0 @@ -sh runpaul-phase1.sh -mkdir runpaul-phase1 -mv *.log runpaul-phase1/ - -sh runpaul-phase2.sh -mkdir runpaul-phase2 -mv *.log runpaul-phase2/ diff --git a/tests/runpaul-phase7.sh b/tests/runpaul-phase7.sh deleted file mode 100755 index 4c301da..0000000 --- a/tests/runpaul-phase7.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -export NUM_CPUS=64 -#export NUM_CPUS=8 - -#extra options, e.g. for setting affinity on even CPUs : -EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) -#EXTRA_OPTS=$(for a in $(seq 0 1 7); do echo -n "-a ${a} "; done) - -rm -f *.log - -# x: Vary writer C.S. length from 0 to 100 us -# y: reads/s -# 4 readers -# 4 writers - -echo Executing writer C.S. length test - -NR_READERS=$((${NUM_CPUS} / 2)) -NR_WRITERS=$((${NUM_CPUS} / 2)) -DURATION=10 -WDELAY=10 -#in loops. -WRITERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" - -rm -f writercslen.log - -for WRITERCSLEN in ${WRITERCSLEN_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -d ${WDELAY} -e ${WRITERCSLEN} | tee -a writercslen.log" >> runall.log - ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -d ${WDELAY} -e ${WRITERCSLEN} | tee -a writercslen.log -done - - - -mkdir ppc64-writercslen -mv *.log ppc64-writercslen/ -#mkdir xeon-writercslen -#mv *.log xeon-writercslen/ diff --git a/tests/runtests-batch.sh b/tests/runtests-batch.sh deleted file mode 100755 index 2da1401..0000000 --- a/tests/runtests-batch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -. ./common.sh - -log_file="runall.detail.log" - -# Check if time bin is non-empty -if [ -n "$test_time_bin" ]; then - time_command="$test_time_bin -a -o $log_file" -else - time_command="" -fi - -#for a in test_urcu_gc test_urcu_gc_mb test_urcu_qsbr_gc; do -for a in test_urcu_gc; do - echo "./${a} $*" | tee -a "$log_file" - $time_command ./${a} $* -done - diff --git a/tests/runtests.sh b/tests/runtests.sh deleted file mode 100755 index 6001174..0000000 --- a/tests/runtests.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -. ./common.sh - -log_file="runall.detail.log" - -# Check if time bin is non-empty -if [ -n "$test_time_bin" ]; then - time_command="$test_time_bin -a -o $log_file" -else - time_command="" -fi - -for a in test_urcu_gc test_urcu_signal_gc test_urcu_mb_gc test_urcu_qsbr_gc \ - test_urcu_lgc test_urcu_signal_lgc test_urcu_mb_lgc test_urcu_qsbr_lgc \ - test_urcu test_urcu_signal test_urcu_mb test_urcu_qsbr \ - test_rwlock test_perthreadlock test_mutex; do - echo "./${a} $*" | tee -a "$log_file" - $time_command ./${a} $* -done - diff --git a/tests/subphase4.sh b/tests/subphase4.sh deleted file mode 100755 index d0d7587..0000000 --- a/tests/subphase4.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/sh - -#run all tests - -#set to number of active CPUS -#NUM_CPUS=8 - -#extra options, e.g. for setting affinity on even CPUs : -#EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) - -#ppc64 striding, use with NUM_CPUS=8 - -#stride 1 -#EXTRA_OPTS=$(for a in $(seq 0 2 15); do echo -n "-a ${a} "; done) -#stride 2 -#EXTRA_OPTS=$(for a in $(seq 0 4 31); do echo -n "-a ${a} "; done) -#stride 4 -#EXTRA_OPTS=$(for a in $(seq 0 8 63); do echo -n "-a ${a} "; done) -#stride 8 -#EXTRA_OPTS=$(for a in $(seq 0 16 127); do echo -n "-a ${a} "; done) - -#Vary update fraction -#x: vary update fraction from 0 to 0.0001 - #fix number of readers and reader C.S. length, vary delay between updates -#y: ops/s - -rm -f runall.log -rm -fr runall.detail.log - - -echo Executing batch RCU test - -DURATION=10 -BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 - 131072 262144" -NR_WRITERS=$((${NUM_CPUS} / 2)) - -rm -f batch-rcu.log - -NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) -for BATCH_SIZE in ${BATCH_ARRAY}; do - echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log - ./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log -done - -#setting gc each 4096. ** UPDATE FOR YOUR ARCHITECTURE BASED ON TEST ABOVE ** -EXTRA_OPTS="${EXTRA_OPTS} -b 32768" - -echo Executing update fraction test - -DURATION=10 -WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 - 65536 131072 262144 524288 1048576 2097152 4194304 8388608 - 16777216 33554432 67108864 134217728" -NR_WRITERS=$((${NUM_CPUS} / 2)) - -rm -f update-fraction.log - -NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) -for WDELAY in ${WDELAY_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log - ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log -done - -#Test scalability : -# x: vary number of readers from 0 to num cpus -# y: ops/s -# 0 writer. - -echo Executing scalability test - -NR_WRITERS=0 -DURATION=10 - -rm -f scalability.log - -for NR_READERS in $(seq 1 ${NUM_CPUS}); do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log - ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log -done - - -# x: Vary reader C.S. length from 0 to 100 us -# y: ops/s -# 8 readers -# 0 writers - -echo Executing reader C.S. length test - -NR_READERS=${NUM_CPUS} -NR_WRITERS=0 -DURATION=10 -#in loops. -READERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" - -rm -f readercslen.log - -for READERCSLEN in ${READERCSLEN_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log" >> runall.log - ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log -done diff --git a/tests/test_cycles_per_loop.c b/tests/test_cycles_per_loop.c deleted file mode 100644 index 6ff100b..0000000 --- a/tests/test_cycles_per_loop.c +++ /dev/null @@ -1,44 +0,0 @@ -/* - * test_cycles_per_loop.c - * - * Userspace RCU library - test cycles per loop - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include - -#define NR_LOOPS 1000000UL - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -int main() -{ - cycles_t time1, time2; - - time1 = caa_get_cycles(); - loop_sleep(NR_LOOPS); - time2 = caa_get_cycles(); - printf("CPU clock cycles per loop: %g\n", (time2 - time1) / - (double)NR_LOOPS); - return 0; -} diff --git a/tests/test_looplen.c b/tests/test_looplen.c deleted file mode 100644 index 16674e7..0000000 --- a/tests/test_looplen.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * test_looplen.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -#define LOOPS 1048576 -#define TESTS 10 - -int main(int argc, char **argv) -{ - unsigned long i; - cycles_t time1, time2; - cycles_t time_tot = 0; - double cpl; - - for (i = 0; i < TESTS; i++) { - time1 = caa_get_cycles(); - loop_sleep(LOOPS); - time2 = caa_get_cycles(); - time_tot += time2 - time1; - } - cpl = ((double)time_tot) / (double)TESTS / (double)LOOPS; - - printf("CALIBRATION : %g cycles per loop\n", cpl); - printf("time_tot = %llu, LOOPS = %d, TESTS = %d\n", - time_tot, LOOPS, TESTS); - - return 0; -} diff --git a/tests/test_mutex.c b/tests/test_mutex.c deleted file mode 100644 index 83adfdb..0000000 --- a/tests/test_mutex.c +++ /dev/null @@ -1,399 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include - -struct test_array { - int a; -}; - -static pthread_mutex_t lock; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static volatile struct test_array test_array = { 8 }; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_reads; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *data) -{ - unsigned long tidx = (unsigned long)data; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - - for (;;) { - pthread_mutex_lock(&lock); - assert(test_array.a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - pthread_mutex_unlock(&lock); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - tot_nr_reads[tidx] = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *data) -{ - unsigned long wtidx = (unsigned long)data; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - pthread_mutex_lock(&lock); - test_array.a = 0; - test_array.a = 8; - if (caa_unlikely(wduration)) - loop_sleep(wduration); - pthread_mutex_unlock(&lock); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - tot_nr_writes[wtidx] = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - cmm_smp_mb(); - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - tot_nr_reads = malloc(sizeof(*tot_nr_reads) * nr_readers); - tot_nr_writes = malloc(sizeof(*tot_nr_writes) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += tot_nr_reads[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += tot_nr_writes[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - free(tot_nr_reads); - free(tot_nr_writes); - return 0; -} diff --git a/tests/test_perthreadlock.c b/tests/test_perthreadlock.c deleted file mode 100644 index be6e59d..0000000 --- a/tests/test_perthreadlock.c +++ /dev/null @@ -1,410 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include - -struct test_array { - int a; -}; - -struct per_thread_lock { - pthread_mutex_t lock; -} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); /* cache-line aligned */ - -static struct per_thread_lock *per_thread_lock; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static volatile struct test_array test_array = { 8 }; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_reads; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *data) -{ - unsigned long tidx = (unsigned long)data; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - - for (;;) { - pthread_mutex_lock(&per_thread_lock[tidx].lock); - assert(test_array.a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - pthread_mutex_unlock(&per_thread_lock[tidx].lock); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - tot_nr_reads[tidx] = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *data) -{ - unsigned long wtidx = (unsigned long)data; - long tidx; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - for (tidx = 0; tidx < nr_readers; tidx++) { - pthread_mutex_lock(&per_thread_lock[tidx].lock); - } - test_array.a = 0; - test_array.a = 8; - if (caa_unlikely(wduration)) - loop_sleep(wduration); - for (tidx = (long)nr_readers - 1; tidx >= 0; tidx--) { - pthread_mutex_unlock(&per_thread_lock[tidx].lock); - } - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - tot_nr_writes[wtidx] = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - cmm_smp_mb(); - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - tot_nr_reads = malloc(sizeof(*tot_nr_reads) * nr_readers); - tot_nr_writes = malloc(sizeof(*tot_nr_writes) * nr_writers); - per_thread_lock = malloc(sizeof(*per_thread_lock) * nr_readers); - for (i = 0; i < nr_readers; i++) { - err = pthread_mutex_init(&per_thread_lock[i].lock, NULL); - if (err != 0) - exit(1); - } - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += tot_nr_reads[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += tot_nr_writes[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - - free(tid_reader); - free(tid_writer); - free(tot_nr_reads); - free(tot_nr_writes); - free(per_thread_lock); - return 0; -} diff --git a/tests/test_perthreadlock_timing.c b/tests/test_perthreadlock_timing.c deleted file mode 100644 index 9321f03..0000000 --- a/tests/test_perthreadlock_timing.c +++ /dev/null @@ -1,198 +0,0 @@ -/* - * test_perthreadloc_timing.c - * - * Per thread locks - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "thread-id.h" - -#include - -struct test_array { - int a; -}; - -static struct test_array test_array = { 8 }; - -struct per_thread_lock { - pthread_mutex_t lock; -} __attribute__((aligned(CAA_CACHE_LINE_SIZE))); /* cache-line aligned */ - -static struct per_thread_lock *per_thread_lock; - -#define OUTER_READ_LOOP 200U -#define INNER_READ_LOOP 100000U -#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) - -#define OUTER_WRITE_LOOP 10U -#define INNER_WRITE_LOOP 200U -#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) - -static int num_read; -static int num_write; - -#define NR_READ num_read -#define NR_WRITE num_write - -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; - -void *thr_reader(void *arg) -{ - int i, j; - cycles_t time1, time2; - long tidx = (long)arg; - - printf("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - sleep(2); - - time1 = caa_get_cycles(); - for (i = 0; i < OUTER_READ_LOOP; i++) { - for (j = 0; j < INNER_READ_LOOP; j++) { - pthread_mutex_lock(&per_thread_lock[tidx].lock); - assert(test_array.a == 8); - pthread_mutex_unlock(&per_thread_lock[tidx].lock); - } - } - time2 = caa_get_cycles(); - - reader_time[tidx] = time2 - time1; - - sleep(2); - printf("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *arg) -{ - int i, j; - long tidx; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - sleep(2); - - for (i = 0; i < OUTER_WRITE_LOOP; i++) { - for (j = 0; j < INNER_WRITE_LOOP; j++) { - time1 = caa_get_cycles(); - for (tidx = 0; tidx < NR_READ; tidx++) { - pthread_mutex_lock(&per_thread_lock[tidx].lock); - } - test_array.a = 8; - for (tidx = NR_READ - 1; tidx >= 0; tidx--) { - pthread_mutex_unlock(&per_thread_lock[tidx].lock); - } - time2 = caa_get_cycles(); - writer_time[(unsigned long)arg] += time2 - time1; - usleep(1); - } - } - - printf("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - return ((void*)2); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - int i; - cycles_t tot_rtime = 0; - cycles_t tot_wtime = 0; - - if (argc < 2) { - printf("Usage : %s nr_readers nr_writers\n", argv[0]); - exit(-1); - } - num_read = atoi(argv[1]); - num_write = atoi(argv[2]); - - reader_time = malloc(sizeof(*reader_time) * num_read); - writer_time = malloc(sizeof(*writer_time) * num_write); - tid_reader = malloc(sizeof(*tid_reader) * num_read); - tid_writer = malloc(sizeof(*tid_writer) * num_write); - - printf("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - per_thread_lock = malloc(sizeof(struct per_thread_lock) * NR_READ); - - for (i = 0; i < NR_READ; i++) { - pthread_mutex_init(&per_thread_lock[i].lock, NULL); - } - for (i = 0; i < NR_READ; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - sleep(10); - - for (i = 0; i < NR_READ; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_rtime += reader_time[i]; - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_wtime += writer_time[i]; - } - printf("Time per read : %g cycles\n", - (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); - printf("Time per write : %g cycles\n", - (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); - free(per_thread_lock); - - free(reader_time); - free(writer_time); - free(tid_reader); - free(tid_writer); - - return 0; -} diff --git a/tests/test_rwlock.c b/tests/test_rwlock.c deleted file mode 100644 index 287b14e..0000000 --- a/tests/test_rwlock.c +++ /dev/null @@ -1,391 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include - -struct test_array { - int a; -}; - -pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static volatile struct test_array test_array = { 8 }; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - - for (;;) { - pthread_rwlock_rdlock(&lock); - assert(test_array.a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - pthread_rwlock_unlock(&lock); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - pthread_rwlock_wrlock(&lock); - test_array.a = 0; - test_array.a = 8; - if (caa_unlikely(wduration)) - loop_sleep(wduration); - pthread_rwlock_unlock(&lock); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - *count = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - cmm_smp_mb(); - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_rwlock_timing.c b/tests/test_rwlock_timing.c deleted file mode 100644 index 3ff7ee4..0000000 --- a/tests/test_rwlock_timing.c +++ /dev/null @@ -1,183 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "thread-id.h" - -#include - -struct test_array { - int a; -}; - -pthread_rwlock_t lock = PTHREAD_RWLOCK_INITIALIZER; - -static struct test_array test_array = { 8 }; - -#define OUTER_READ_LOOP 200U -#define INNER_READ_LOOP 100000U -#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) - -#define OUTER_WRITE_LOOP 10U -#define INNER_WRITE_LOOP 200U -#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) - -static int num_read; -static int num_write; - -#define NR_READ num_read -#define NR_WRITE num_write - -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; - -void *thr_reader(void *arg) -{ - int i, j; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - sleep(2); - - time1 = caa_get_cycles(); - for (i = 0; i < OUTER_READ_LOOP; i++) { - for (j = 0; j < INNER_READ_LOOP; j++) { - pthread_rwlock_rdlock(&lock); - assert(test_array.a == 8); - pthread_rwlock_unlock(&lock); - } - } - time2 = caa_get_cycles(); - - reader_time[(unsigned long)arg] = time2 - time1; - - sleep(2); - printf("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *arg) -{ - int i, j; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - sleep(2); - - for (i = 0; i < OUTER_WRITE_LOOP; i++) { - for (j = 0; j < INNER_WRITE_LOOP; j++) { - time1 = caa_get_cycles(); - pthread_rwlock_wrlock(&lock); - test_array.a = 8; - pthread_rwlock_unlock(&lock); - time2 = caa_get_cycles(); - writer_time[(unsigned long)arg] += time2 - time1; - usleep(1); - } - } - - printf("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - return ((void*)2); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - int i; - cycles_t tot_rtime = 0; - cycles_t tot_wtime = 0; - - if (argc < 2) { - printf("Usage : %s nr_readers nr_writers\n", argv[0]); - exit(-1); - } - num_read = atoi(argv[1]); - num_write = atoi(argv[2]); - - reader_time = malloc(sizeof(*reader_time) * num_read); - writer_time = malloc(sizeof(*writer_time) * num_write); - tid_reader = malloc(sizeof(*tid_reader) * num_read); - tid_writer = malloc(sizeof(*tid_writer) * num_write); - - printf("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - for (i = 0; i < NR_READ; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - sleep(10); - - for (i = 0; i < NR_READ; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_rtime += reader_time[i]; - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_wtime += writer_time[i]; - } - printf("Time per read : %g cycles\n", - (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); - printf("Time per write : %g cycles\n", - (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); - - free(reader_time); - free(writer_time); - free(tid_reader); - free(tid_writer); - - return 0; -} diff --git a/tests/test_uatomic.c b/tests/test_uatomic.c deleted file mode 100644 index 804ce7b..0000000 --- a/tests/test_uatomic.c +++ /dev/null @@ -1,91 +0,0 @@ -/* - * test_uatomic.c - * - * Userspace RCU library - test atomic operations - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include - -struct testvals { -#ifdef UATOMIC_HAS_ATOMIC_BYTE - unsigned char c; -#endif -#ifdef UATOMIC_HAS_ATOMIC_SHORT - unsigned short s; -#endif - unsigned int i; - unsigned long l; -}; - -static struct testvals vals; - -#define do_test(ptr) \ -do { \ - __typeof__(*(ptr)) v; \ - \ - uatomic_add(ptr, 10); \ - assert(uatomic_read(ptr) == 10); \ - uatomic_add(ptr, -11UL); \ - assert(uatomic_read(ptr) == (__typeof__(*(ptr)))-1UL); \ - v = uatomic_cmpxchg(ptr, -1UL, 22); \ - assert(uatomic_read(ptr) == 22); \ - assert(v == (__typeof__(*(ptr)))-1UL); \ - v = uatomic_cmpxchg(ptr, 33, 44); \ - assert(uatomic_read(ptr) == 22); \ - assert(v == 22); \ - v = uatomic_xchg(ptr, 55); \ - assert(uatomic_read(ptr) == 55); \ - assert(v == 22); \ - uatomic_set(ptr, 22); \ - uatomic_inc(ptr); \ - assert(uatomic_read(ptr) == 23); \ - uatomic_dec(ptr); \ - assert(uatomic_read(ptr) == 22); \ - v = uatomic_add_return(ptr, 74); \ - assert(v == 96); \ - assert(uatomic_read(ptr) == 96); \ - uatomic_or(ptr, 58); \ - assert(uatomic_read(ptr) == 122); \ - v = uatomic_sub_return(ptr, 1); \ - assert(v == 121); \ - uatomic_sub(ptr, (unsigned int) 2); \ - assert(uatomic_read(ptr) == 119); \ - uatomic_inc(ptr); \ - uatomic_inc(ptr); \ - assert(uatomic_read(ptr) == 121); \ - uatomic_and(ptr, 129); \ - assert(uatomic_read(ptr) == 1); \ -} while (0) - -int main(int argc, char **argv) -{ -#ifdef UATOMIC_HAS_ATOMIC_BYTE - do_test(&vals.c); -#endif -#ifdef UATOMIC_HAS_ATOMIC_SHORT - do_test(&vals.s); -#endif - do_test(&vals.i); - do_test(&vals.l); - printf("Atomic ops test OK\n"); - - return 0; -} diff --git a/tests/test_urcu.c b/tests/test_urcu.c deleted file mode 100644 index 3017e81..0000000 --- a/tests/test_urcu.c +++ /dev/null @@ -1,404 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static int *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - int *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - assert(!rcu_read_ongoing()); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - assert(rcu_read_ongoing()); - local_ptr = rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(*local_ptr == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - /* test extra thread registration */ - rcu_register_thread(); - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - int *new, *old; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - new = malloc(sizeof(int)); - assert(new); - *new = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (caa_unlikely(wduration)) - loop_sleep(wduration); - synchronize_rcu(); - if (old) - *old = 0; - free(old); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - *count = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - free(test_rcu_pointer); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_urcu_assign.c b/tests/test_urcu_assign.c deleted file mode 100644 index 67253da..0000000 --- a/tests/test_urcu_assign.c +++ /dev/null @@ -1,439 +0,0 @@ -/* - * test_urcu_assign.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include - -struct test_array { - int a; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static struct test_array *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -/* - * malloc/free are reusing memory areas too quickly, which does not let us - * test races appropriately. Use a large circular array for allocations. - * ARRAY_SIZE is larger than nr_writers, and we keep the mutex across - * both alloc and free, which insures we never run over our tail. - */ -#define ARRAY_SIZE (1048576 * nr_writers) -#define ARRAY_POISON 0xDEADBEEF -static int array_index; -static struct test_array *test_array; - -static struct test_array *test_array_alloc(void) -{ - struct test_array *ret; - int index; - - index = array_index % ARRAY_SIZE; - assert(test_array[index].a == ARRAY_POISON || - test_array[index].a == 0); - ret = &test_array[index]; - array_index++; - if (array_index == ARRAY_SIZE) - array_index = 0; - return ret; -} - -static void test_array_free(struct test_array *ptr) -{ - if (!ptr) - return; - ptr->a = ARRAY_POISON; -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_array *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - local_ptr = rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(local_ptr->a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - struct test_array *new, *old; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_copy_mutex_lock(); - new = test_array_alloc(); - new->a = 8; - old = test_rcu_pointer; - rcu_assign_pointer(test_rcu_pointer, new); - if (caa_unlikely(wduration)) - loop_sleep(wduration); - synchronize_rcu(); - if (old) - old->a = 0; - test_array_free(old); - rcu_copy_mutex_unlock(); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - *count = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - test_array = calloc(1, sizeof(*test_array) * ARRAY_SIZE); - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - test_array_free(test_rcu_pointer); - free(test_array); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_urcu_bp.c b/tests/test_urcu_bp.c deleted file mode 100644 index a0d6f11..0000000 --- a/tests/test_urcu_bp.c +++ /dev/null @@ -1,399 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static int *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - int *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - assert(!rcu_read_ongoing()); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - assert(rcu_read_ongoing()); - local_ptr = rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(*local_ptr == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - int *new, *old; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - new = malloc(sizeof(int)); - *new = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (caa_unlikely(wduration)) - loop_sleep(wduration); - synchronize_rcu(); - if (old) - *old = 0; - free(old); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - *count = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - free(test_rcu_pointer); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_urcu_defer.c b/tests/test_urcu_defer.c deleted file mode 100644 index 395014e..0000000 --- a/tests/test_urcu_defer.c +++ /dev/null @@ -1,426 +0,0 @@ -/* - * test_urcu_defer.c - * - * Userspace RCU library - test program (with automatic reclamation) - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include -#include - -struct test_array { - int a; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static struct test_array *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_array *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - local_ptr = rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(local_ptr->a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -static void test_cb2(void *data) -{ -} - -static void test_cb1(void *data) -{ -} - -void *thr_writer(void *data) -{ - unsigned long wtidx = (unsigned long)data; - struct test_array *new, *old = NULL; - int ret; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - ret = rcu_defer_register_thread(); - if (ret) { - printf("Error in rcu_defer_register_thread\n"); - exit(-1); - } - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - new = malloc(sizeof(*new)); - new->a = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (caa_unlikely(wduration)) - loop_sleep(wduration); - defer_rcu(free, old); - defer_rcu(test_cb1, old); - defer_rcu(test_cb1, (void *)-2L); - defer_rcu(test_cb1, (void *)-2L); - defer_rcu(test_cb1, old); - defer_rcu(test_cb2, (void *)-2L); - defer_rcu(test_cb2, (void *)-4L); - defer_rcu(test_cb2, (void *)-2L); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - rcu_defer_unregister_thread(); - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - tot_nr_writes[wtidx] = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - tot_nr_writes = malloc(sizeof(*tot_nr_writes) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += tot_nr_writes[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(tot_nr_writes); - - return 0; -} diff --git a/tests/test_urcu_fork.c b/tests/test_urcu_fork.c deleted file mode 100644 index 6e454b5..0000000 --- a/tests/test_urcu_fork.c +++ /dev/null @@ -1,147 +0,0 @@ -/* - * test_urcu_fork.c - * - * Userspace RCU library - test program (fork) - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include - -struct test_node { - int somedata; - struct rcu_head head; -}; - -static void cb(struct rcu_head *head) -{ - struct test_node *node; - - fprintf(stderr, "rcu callback invoked in pid: %d\n", - (int) getpid()); - node = caa_container_of(head, struct test_node, head); - free(node); -} - -static void test_rcu(void) -{ - struct test_node *node; - - rcu_register_thread(); - - synchronize_rcu(); - - rcu_read_lock(); - rcu_read_unlock(); - - node = malloc(sizeof(*node)); - assert(node); - - call_rcu(&node->head, cb); - - synchronize_rcu(); - - rcu_unregister_thread(); -} - -int main(int argc, char **argv) -{ - pid_t pid; - int ret; - -#if 0 - /* pthread_atfork does not work with malloc/free in callbacks */ - ret = pthread_atfork(call_rcu_before_fork, - call_rcu_after_fork_parent, - call_rcu_after_fork_child); - if (ret) { - errno = ret; - perror("pthread_atfork"); - exit(EXIT_FAILURE); - } -#endif - - test_rcu(); - - synchronize_rcu(); - - fprintf(stderr, "%s parent pid: %d, before fork\n", - argv[0], (int) getpid()); - - call_rcu_before_fork(); - pid = fork(); - - if (pid == 0) { - /* child */ - call_rcu_after_fork_child(); - fprintf(stderr, "%s child pid: %d, after fork\n", - argv[0], (int) getpid()); - test_rcu(); - fprintf(stderr, "%s child pid: %d, after rcu test\n", - argv[0], (int) getpid()); - } else if (pid > 0) { - int status; - - /* parent */ - call_rcu_after_fork_parent(); - fprintf(stderr, "%s parent pid: %d, after fork\n", - argv[0], (int) getpid()); - test_rcu(); - fprintf(stderr, "%s parent pid: %d, after rcu test\n", - argv[0], (int) getpid()); - for (;;) { - pid = wait(&status); - if (WIFEXITED(status)) { - fprintf(stderr, "child %u exited normally with status %u\n", - pid, WEXITSTATUS(status)); - break; - } else if (WIFSIGNALED(status)) { - fprintf(stderr, "child %u was terminated by signal %u\n", - pid, WTERMSIG(status)); - break; - } else { - continue; - } - } - } else { - perror("fork"); - exit(EXIT_FAILURE); - } - exit(EXIT_SUCCESS); -} diff --git a/tests/test_urcu_gc.c b/tests/test_urcu_gc.c deleted file mode 100644 index 6e5cc1d..0000000 --- a/tests/test_urcu_gc.c +++ /dev/null @@ -1,470 +0,0 @@ -/* - * test_urcu_gc.c - * - * Userspace RCU library - test program (with batch reclamation) - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include - -struct test_array { - int a; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static struct test_array *test_rcu_pointer; - -static unsigned int reclaim_batch = 1; - -struct reclaim_queue { - void **queue; /* Beginning of queue */ - void **head; /* Insert position */ -}; - -static struct reclaim_queue *pending_reclaims; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_array *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - local_ptr = rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(local_ptr->a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -static void rcu_gc_clear_queue(unsigned long wtidx) -{ - void **p; - - /* Wait for Q.S and empty queue */ - synchronize_rcu(); - - for (p = pending_reclaims[wtidx].queue; - p < pending_reclaims[wtidx].head; p++) { - /* poison */ - if (*p) - ((struct test_array *)*p)->a = 0; - free(*p); - } - pending_reclaims[wtidx].head = pending_reclaims[wtidx].queue; -} - -/* Using per-thread queue */ -static void rcu_gc_reclaim(unsigned long wtidx, void *old) -{ - /* Queue pointer */ - *pending_reclaims[wtidx].head = old; - pending_reclaims[wtidx].head++; - - if (caa_likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue - < reclaim_batch)) - return; - - rcu_gc_clear_queue(wtidx); -} - -void *thr_writer(void *data) -{ - unsigned long wtidx = (unsigned long)data; -#ifdef TEST_LOCAL_GC - struct test_array *old = NULL; -#else - struct test_array *new, *old; -#endif - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { -#ifndef TEST_LOCAL_GC - new = malloc(sizeof(*new)); - new->a = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); -#endif - if (caa_unlikely(wduration)) - loop_sleep(wduration); - rcu_gc_reclaim(wtidx, old); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - tot_nr_writes[wtidx] = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'b': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - reclaim_batch = atol(argv[++i]); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - tot_nr_writes = malloc(sizeof(*tot_nr_writes) * nr_writers); - pending_reclaims = malloc(sizeof(*pending_reclaims) * nr_writers); - if (reclaim_batch * sizeof(*pending_reclaims[i].queue) - < CAA_CACHE_LINE_SIZE) - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].queue = calloc(1, CAA_CACHE_LINE_SIZE); - else - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].queue = calloc(reclaim_batch, - sizeof(*pending_reclaims[i].queue)); - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].head = pending_reclaims[i].queue; - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += tot_nr_writes[i]; - rcu_gc_clear_queue(i); - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " - "batch %u\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes, reclaim_batch); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(tot_nr_writes); - for (i = 0; i < nr_writers; i++) - free(pending_reclaims[i].queue); - free(pending_reclaims); - - return 0; -} diff --git a/tests/test_urcu_hash.c b/tests/test_urcu_hash.c deleted file mode 100644 index 9416224..0000000 --- a/tests/test_urcu_hash.c +++ /dev/null @@ -1,774 +0,0 @@ -/* - * test_urcu_hash.c - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "test_urcu_hash.h" - -enum test_hash { - TEST_HASH_RW, - TEST_HASH_UNIQUE, -}; - -struct test_hash_cb { - void (*sigusr1)(int signo); - void (*sigusr2)(int signo); - void *(*thr_reader)(void *_count); - void *(*thr_writer)(void *_count); - int (*populate_hash)(void); -}; - -static -struct test_hash_cb test_hash_cb[] = { - [TEST_HASH_RW] = { - test_hash_rw_sigusr1_handler, - test_hash_rw_sigusr2_handler, - test_hash_rw_thr_reader, - test_hash_rw_thr_writer, - test_hash_rw_populate_hash, - }, - [TEST_HASH_UNIQUE] = { - test_hash_unique_sigusr1_handler, - test_hash_unique_sigusr2_handler, - test_hash_unique_thr_reader, - test_hash_unique_thr_writer, - test_hash_unique_populate_hash, - }, - -}; - -static enum test_hash test_choice = TEST_HASH_RW; - -void (*get_sigusr1_cb(void))(int) -{ - return test_hash_cb[test_choice].sigusr1; -} - -void (*get_sigusr2_cb(void))(int) -{ - return test_hash_cb[test_choice].sigusr2; -} - -void *(*get_thr_reader_cb(void))(void *) -{ - return test_hash_cb[test_choice].thr_reader; -} - -void *(*get_thr_writer_cb(void))(void *) -{ - return test_hash_cb[test_choice].thr_writer; -} - -int (*get_populate_hash_cb(void))(void) -{ - return test_hash_cb[test_choice].populate_hash; -} - -DEFINE_URCU_TLS(unsigned int, rand_lookup); -DEFINE_URCU_TLS(unsigned long, nr_add); -DEFINE_URCU_TLS(unsigned long, nr_addexist); -DEFINE_URCU_TLS(unsigned long, nr_del); -DEFINE_URCU_TLS(unsigned long, nr_delnoent); -DEFINE_URCU_TLS(unsigned long, lookup_fail); -DEFINE_URCU_TLS(unsigned long, lookup_ok); - -struct cds_lfht *test_ht; - -volatile int test_go, test_stop; - -unsigned long wdelay; - -unsigned long duration; - -/* read-side C.S. duration, in loops */ -unsigned long rduration; - -unsigned long init_hash_size = DEFAULT_HASH_SIZE; -unsigned long min_hash_alloc_size = DEFAULT_MIN_ALLOC_SIZE; -unsigned long max_hash_buckets_size = (1UL << 20); -unsigned long init_populate; -int opt_auto_resize; -int add_only, add_unique, add_replace; -const struct cds_lfht_mm_type *memory_backend; - -unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; -unsigned long init_pool_size = DEFAULT_RAND_POOL, - lookup_pool_size = DEFAULT_RAND_POOL, - write_pool_size = DEFAULT_RAND_POOL; -int validate_lookup; -unsigned long nr_hash_chains; /* 0: normal table, other: number of hash chains */ - -int count_pipe[2]; - -int verbose_mode; - -unsigned int cpu_affinities[NR_CPUS]; -unsigned int next_aff = 0; -int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -DEFINE_URCU_TLS(unsigned long long, nr_writes); -DEFINE_URCU_TLS(unsigned long long, nr_reads); - -unsigned int nr_readers; -unsigned int nr_writers; - -static pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -unsigned long test_compare(const void *key1, size_t key1_len, - const void *key2, size_t key2_len) -{ - if (caa_unlikely(key1_len != key2_len)) - return -1; - assert(key1_len == sizeof(unsigned long)); - if (key1 == key2) - return 0; - else - return 1; -} - -void *thr_count(void *arg) -{ - printf_verbose("thread_begin %s, tid %lu\n", - "counter", urcu_get_thread_id()); - - rcu_register_thread(); - - for (;;) { - unsigned long count; - long approx_before, approx_after; - ssize_t len; - char buf[1]; - - rcu_thread_offline(); - len = read(count_pipe[0], buf, 1); - rcu_thread_online(); - if (caa_unlikely(!test_duration_read())) - break; - if (len != 1) - continue; - /* Accounting */ - printf("Counting nodes... "); - fflush(stdout); - rcu_read_lock(); - cds_lfht_count_nodes(test_ht, &approx_before, &count, - &approx_after); - rcu_read_unlock(); - printf("done.\n"); - printf("Approximation before node accounting: %ld nodes.\n", - approx_before); - printf("Accounting of nodes in the hash table: " - "%lu nodes.\n", - count); - printf("Approximation after node accounting: %ld nodes.\n", - approx_after); - } - rcu_unregister_thread(); - return NULL; -} - -void free_node_cb(struct rcu_head *head) -{ - struct lfht_test_node *node = - caa_container_of(head, struct lfht_test_node, head); - free(node); -} - -static -void test_delete_all_nodes(struct cds_lfht *ht) -{ - struct cds_lfht_iter iter; - struct lfht_test_node *node; - unsigned long count = 0; - - cds_lfht_for_each_entry(ht, &iter, node, node) { - int ret; - - ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); - assert(!ret); - call_rcu(&node->head, free_node_cb); - count++; - } - printf("deleted %lu nodes.\n", count); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf(" [-h size] (initial number of buckets)\n"); - printf(" [-m size] (minimum number of allocated buckets)\n"); - printf(" [-n size] (maximum number of buckets)\n"); -printf(" [not -u nor -s] Add entries (supports redundant keys).\n"); - printf(" [-u] Uniquify add (no redundant keys).\n"); - printf(" [-s] Replace (swap) entries.\n"); - printf(" [-i] Add only (no removal).\n"); - printf(" [-k nr_nodes] Number of nodes to insert initially.\n"); - printf(" [-A] Automatically resize hash table.\n"); - printf(" [-B order|chunk|mmap] Specify the memory backend.\n"); - printf(" [-R offset] Lookup pool offset.\n"); - printf(" [-S offset] Write pool offset.\n"); - printf(" [-T offset] Init pool offset.\n"); - printf(" [-M size] Lookup pool size.\n"); - printf(" [-N size] Write pool size.\n"); - printf(" [-O size] Init pool size.\n"); - printf(" [-V] Validate lookups of init values.\n"); - printf(" (use with filled init pool, same lookup range,\n"); - printf(" with different write range)\n"); - printf(" [-U] Uniqueness test.\n"); - printf(" [-C] Number of hash chains.\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - pthread_t *tid_reader, *tid_writer; - pthread_t tid_count; - void *tret; - unsigned long long *count_reader; - struct wr_count *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0, - tot_add = 0, tot_add_exist = 0, tot_remove = 0; - unsigned long count; - long approx_before, approx_after; - int i, a, ret, err, mainret = 0; - struct sigaction act; - unsigned int remain; - unsigned int nr_readers_created = 0, nr_writers_created = 0; - long long nr_leaked; - - if (argc < 4) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - case 'h': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - init_hash_size = atol(argv[++i]); - break; - case 'm': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - min_hash_alloc_size = atol(argv[++i]); - break; - case 'n': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - max_hash_buckets_size = atol(argv[++i]); - break; - case 'u': - if (add_replace) { - printf("Please specify at most one of -s or -u.\n"); - exit(-1); - } - add_unique = 1; - break; - case 's': - if (add_unique) { - printf("Please specify at most one of -s or -u.\n"); - exit(-1); - } - add_replace = 1; - break; - case 'i': - add_only = 1; - break; - case 'k': - init_populate = atol(argv[++i]); - break; - case 'A': - opt_auto_resize = 1; - break; - case 'B': - if (argc < i + 2) { - show_usage(argc, argv); - mainret = 1; - goto end; - } - i++; - if (!strcmp("order", argv[i])) - memory_backend = &cds_lfht_mm_order; - else if (!strcmp("chunk", argv[i])) - memory_backend = &cds_lfht_mm_chunk; - else if (!strcmp("mmap", argv[i])) - memory_backend = &cds_lfht_mm_mmap; - else { - printf("Please specify memory backend with order|chunk|mmap.\n"); - mainret = 1; - goto end; - } - break; - case 'R': - lookup_pool_offset = atol(argv[++i]); - break; - case 'S': - write_pool_offset = atol(argv[++i]); - break; - case 'T': - init_pool_offset = atol(argv[++i]); - break; - case 'M': - lookup_pool_size = atol(argv[++i]); - break; - case 'N': - write_pool_size = atol(argv[++i]); - break; - case 'O': - init_pool_size = atol(argv[++i]); - break; - case 'V': - validate_lookup = 1; - break; - case 'U': - test_choice = TEST_HASH_UNIQUE; - break; - case 'C': - nr_hash_chains = atol(argv[++i]); - break; - } - } - - /* Check if hash size is power of 2 */ - if (init_hash_size && init_hash_size & (init_hash_size - 1)) { - printf("Error: Initial number of buckets (%lu) is not a power of 2.\n", - init_hash_size); - mainret = 1; - goto end; - } - - if (min_hash_alloc_size && min_hash_alloc_size & (min_hash_alloc_size - 1)) { - printf("Error: Minimum number of allocated buckets (%lu) is not a power of 2.\n", - min_hash_alloc_size); - mainret = 1; - goto end; - } - - if (max_hash_buckets_size && max_hash_buckets_size & (max_hash_buckets_size - 1)) { - printf("Error: Maximum number of buckets (%lu) is not a power of 2.\n", - max_hash_buckets_size); - mainret = 1; - goto end; - } - - memset(&act, 0, sizeof(act)); - ret = sigemptyset(&act.sa_mask); - if (ret == -1) { - perror("sigemptyset"); - mainret = 1; - goto end; - } - act.sa_handler = get_sigusr1_cb(); - act.sa_flags = SA_RESTART; - ret = sigaction(SIGUSR1, &act, NULL); - if (ret == -1) { - perror("sigaction"); - mainret = 1; - goto end; - } - - act.sa_handler = get_sigusr2_cb(); - act.sa_flags = SA_RESTART; - ret = sigaction(SIGUSR2, &act, NULL); - if (ret == -1) { - perror("sigaction"); - mainret = 1; - goto end; - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("Mode:%s%s.\n", - add_only ? " add only" : " add/remove", - add_unique ? " uniquify" : ( add_replace ? " replace" : " insert")); - printf_verbose("Initial number of buckets: %lu buckets.\n", init_hash_size); - printf_verbose("Minimum number of allocated buckets: %lu buckets.\n", min_hash_alloc_size); - printf_verbose("Maximum number of buckets: %lu buckets.\n", max_hash_buckets_size); - printf_verbose("Init pool size offset %lu size %lu.\n", - init_pool_offset, init_pool_size); - printf_verbose("Lookup pool size offset %lu size %lu.\n", - lookup_pool_offset, lookup_pool_size); - printf_verbose("Update pool size offset %lu size %lu.\n", - write_pool_offset, write_pool_size); - printf_verbose("Number of hash chains: %lu.\n", - nr_hash_chains); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - if (!tid_reader) { - mainret = 1; - goto end; - } - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - if (!tid_writer) { - mainret = 1; - goto end_free_tid_reader; - } - count_reader = malloc(sizeof(*count_reader) * nr_readers); - if (!count_reader) { - mainret = 1; - goto end_free_tid_writer; - } - count_writer = malloc(sizeof(*count_writer) * nr_writers); - if (!count_writer) { - mainret = 1; - goto end_free_count_reader; - } - - err = create_all_cpu_call_rcu_data(0); - if (err) { - printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); - } - - if (memory_backend) { - test_ht = _cds_lfht_new(init_hash_size, min_hash_alloc_size, - max_hash_buckets_size, - (opt_auto_resize ? CDS_LFHT_AUTO_RESIZE : 0) | - CDS_LFHT_ACCOUNTING, memory_backend, - &rcu_flavor, NULL); - } else { - test_ht = cds_lfht_new(init_hash_size, min_hash_alloc_size, - max_hash_buckets_size, - (opt_auto_resize ? CDS_LFHT_AUTO_RESIZE : 0) | - CDS_LFHT_ACCOUNTING, NULL); - } - if (!test_ht) { - printf("Error allocating hash table.\n"); - mainret = 1; - goto end_free_call_rcu_data; - } - - /* - * Hash Population needs to be seen as a RCU reader - * thread from the point of view of resize. - */ - rcu_register_thread(); - ret = (get_populate_hash_cb())(); - assert(!ret); - - rcu_thread_offline(); - - next_aff = 0; - - ret = pipe(count_pipe); - if (ret == -1) { - perror("pipe"); - mainret = 1; - goto end_online; - } - - /* spawn counter thread */ - err = pthread_create(&tid_count, NULL, thr_count, - NULL); - if (err != 0) { - errno = err; - mainret = 1; - perror("pthread_create"); - goto end_close_pipe; - } - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], - NULL, get_thr_reader_cb(), - &count_reader[i]); - if (err != 0) { - errno = err; - mainret = 1; - perror("pthread_create"); - goto end_pthread_join; - } - nr_readers_created++; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], - NULL, get_thr_writer_cb(), - &count_writer[i]); - if (err != 0) { - errno = err; - mainret = 1; - perror("pthread_create"); - goto end_pthread_join; - } - nr_writers_created++; - } - - cmm_smp_mb(); - - test_go = 1; - - remain = duration; - do { - remain = sleep(remain); - } while (remain > 0); - - test_stop = 1; - -end_pthread_join: - for (i = 0; i < nr_readers_created; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) { - errno = err; - mainret = 1; - perror("pthread_join"); - } - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers_created; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) { - errno = err; - mainret = 1; - perror("pthread_join"); - } - tot_writes += count_writer[i].update_ops; - tot_add += count_writer[i].add; - tot_add_exist += count_writer[i].add_exist; - tot_remove += count_writer[i].remove; - } - - /* teardown counter thread */ - act.sa_handler = SIG_IGN; - act.sa_flags = SA_RESTART; - ret = sigaction(SIGUSR2, &act, NULL); - if (ret == -1) { - mainret = 1; - perror("sigaction"); - } - { - char msg[1] = { 0x42 }; - ssize_t ret; - - do { - ret = write(count_pipe[1], msg, 1); /* wakeup thread */ - } while (ret == -1L && errno == EINTR); - } - err = pthread_join(tid_count, &tret); - if (err != 0) { - errno = err; - mainret = 1; - perror("pthread_join"); - } - -end_close_pipe: - for (i = 0; i < 2; i++) { - err = close(count_pipe[i]); - if (err) { - mainret = 1; - perror("close pipe"); - } - } - fflush(stdout); -end_online: - rcu_thread_online(); - rcu_read_lock(); - printf("Counting nodes... "); - cds_lfht_count_nodes(test_ht, &approx_before, &count, &approx_after); - printf("done.\n"); - test_delete_all_nodes(test_ht); - rcu_read_unlock(); - rcu_thread_offline(); - if (count) { - printf("Approximation before node accounting: %ld nodes.\n", - approx_before); - printf("Nodes deleted from hash table before destroy: " - "%lu nodes.\n", - count); - printf("Approximation after node accounting: %ld nodes.\n", - approx_after); - } - - ret = cds_lfht_destroy(test_ht, NULL); - if (ret) { - printf_verbose("final delete aborted\n"); - mainret = 1; - } else { - printf_verbose("final delete success\n"); - } - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - nr_leaked = (long long) tot_add + init_populate - tot_remove - count; - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " - "nr_add %12llu nr_add_fail %12llu nr_remove %12llu nr_leaked %12lld\n", - argv[0], duration, nr_readers, rduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes, tot_add, tot_add_exist, tot_remove, - nr_leaked); - if (nr_leaked != 0) { - mainret = 1; - printf("WARNING: %lld nodes were leaked!\n", nr_leaked); - } - - rcu_unregister_thread(); -end_free_call_rcu_data: - free_all_cpu_call_rcu_data(); - free(count_writer); -end_free_count_reader: - free(count_reader); -end_free_tid_writer: - free(tid_writer); -end_free_tid_reader: - free(tid_reader); -end: - if (!mainret) - exit(EXIT_SUCCESS); - else - exit(EXIT_FAILURE); -} diff --git a/tests/test_urcu_hash.h b/tests/test_urcu_hash.h deleted file mode 100644 index cd064e6..0000000 --- a/tests/test_urcu_hash.h +++ /dev/null @@ -1,390 +0,0 @@ -#ifndef _TEST_URCU_HASH_H -#define _TEST_URCU_HASH_H - -/* - * test_urcu_hash.h - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "cpuset.h" -#include "thread-id.h" - -#define DEFAULT_HASH_SIZE 32 -#define DEFAULT_MIN_ALLOC_SIZE 1 -#define DEFAULT_RAND_POOL 1000000 - -/* - * Note: the hash seed should be a random value for hash tables - * targeting production environments to provide protection against - * denial of service attacks. We keep it a static value within this test - * program to compare identical benchmark runs. - */ -#define TEST_HASH_SEED 0x42UL - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifdef POISON_FREE -#define poison_free(ptr) \ - do { \ - memset(ptr, 0x42, sizeof(*(ptr))); \ - free(ptr); \ - } while (0) -#else -#define poison_free(ptr) free(ptr) -#endif - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include -#include -#include - -struct wr_count { - unsigned long update_ops; - unsigned long add; - unsigned long add_exist; - unsigned long remove; -}; - -extern DECLARE_URCU_TLS(unsigned int, rand_lookup); -extern DECLARE_URCU_TLS(unsigned long, nr_add); -extern DECLARE_URCU_TLS(unsigned long, nr_addexist); -extern DECLARE_URCU_TLS(unsigned long, nr_del); -extern DECLARE_URCU_TLS(unsigned long, nr_delnoent); -extern DECLARE_URCU_TLS(unsigned long, lookup_fail); -extern DECLARE_URCU_TLS(unsigned long, lookup_ok); - -extern struct cds_lfht *test_ht; - -struct test_data { - int a; - int b; -}; - -struct lfht_test_node { - struct cds_lfht_node node; - void *key; - unsigned int key_len; - /* cache-cold for iteration */ - struct rcu_head head; -}; - -static inline struct lfht_test_node * -to_test_node(struct cds_lfht_node *node) -{ - return caa_container_of(node, struct lfht_test_node, node); -} - -static inline -void lfht_test_node_init(struct lfht_test_node *node, void *key, - size_t key_len) -{ - cds_lfht_node_init(&node->node); - node->key = key; - node->key_len = key_len; -} - -static inline struct lfht_test_node * -cds_lfht_iter_get_test_node(struct cds_lfht_iter *iter) -{ - return to_test_node(cds_lfht_iter_get_node(iter)); -} - -extern volatile int test_go, test_stop; - -extern unsigned long wdelay; - -extern unsigned long duration; - -/* read-side C.S. duration, in loops */ -extern unsigned long rduration; - -extern unsigned long init_hash_size; -extern unsigned long min_hash_alloc_size; -extern unsigned long max_hash_buckets_size; -extern unsigned long init_populate; -extern int opt_auto_resize; -extern int add_only, add_unique, add_replace; -extern const struct cds_lfht_mm_type *memory_backend; - -extern unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; -extern unsigned long init_pool_size, - lookup_pool_size, - write_pool_size; -extern int validate_lookup; - -extern unsigned long nr_hash_chains; - -extern int count_pipe[2]; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -extern int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, ## args); \ - } while (0) - -extern unsigned int cpu_affinities[NR_CPUS]; -extern unsigned int next_aff; -extern int use_affinity; - -extern pthread_mutex_t affinity_mutex; - -void set_affinity(void); - -/* - * returns 0 if test should end. - */ -static inline int test_duration_write(void) -{ - return !test_stop; -} - -static inline int test_duration_read(void) -{ - return !test_stop; -} - -extern DECLARE_URCU_TLS(unsigned long long, nr_writes); -extern DECLARE_URCU_TLS(unsigned long long, nr_reads); - -extern unsigned int nr_readers; -extern unsigned int nr_writers; - -void rcu_copy_mutex_lock(void); -void rcu_copy_mutex_unlock(void); - -/* - * Hash function - * Source: http://burtleburtle.net/bob/c/lookup3.c - * Originally Public Domain - */ - -#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k)))) - -#define mix(a, b, c) \ -do { \ - a -= c; a ^= rot(c, 4); c += b; \ - b -= a; b ^= rot(a, 6); a += c; \ - c -= b; c ^= rot(b, 8); b += a; \ - a -= c; a ^= rot(c, 16); c += b; \ - b -= a; b ^= rot(a, 19); a += c; \ - c -= b; c ^= rot(b, 4); b += a; \ -} while (0) - -#define final(a, b, c) \ -{ \ - c ^= b; c -= rot(b, 14); \ - a ^= c; a -= rot(c, 11); \ - b ^= a; b -= rot(a, 25); \ - c ^= b; c -= rot(b, 16); \ - a ^= c; a -= rot(c, 4);\ - b ^= a; b -= rot(a, 14); \ - c ^= b; c -= rot(b, 24); \ -} - -static inline __attribute__((unused)) -uint32_t hash_u32( - const uint32_t *k, /* the key, an array of uint32_t values */ - size_t length, /* the length of the key, in uint32_ts */ - uint32_t initval) /* the previous hash, or an arbitrary value */ -{ - uint32_t a, b, c; - - /* Set up the internal state */ - a = b = c = 0xdeadbeef + (((uint32_t) length) << 2) + initval; - - /*----------------------------------------- handle most of the key */ - while (length > 3) { - a += k[0]; - b += k[1]; - c += k[2]; - mix(a, b, c); - length -= 3; - k += 3; - } - - /*----------------------------------- handle the last 3 uint32_t's */ - switch (length) { /* all the case statements fall through */ - case 3: c += k[2]; - case 2: b += k[1]; - case 1: a += k[0]; - final(a, b, c); - case 0: /* case 0: nothing left to add */ - break; - } - /*---------------------------------------------- report the result */ - return c; -} - -static inline -void hashword2( - const uint32_t *k, /* the key, an array of uint32_t values */ - size_t length, /* the length of the key, in uint32_ts */ - uint32_t *pc, /* IN: seed OUT: primary hash value */ - uint32_t *pb) /* IN: more seed OUT: secondary hash value */ -{ - uint32_t a, b, c; - - /* Set up the internal state */ - a = b = c = 0xdeadbeef + ((uint32_t) (length << 2)) + *pc; - c += *pb; - - /*----------------------------------------- handle most of the key */ - while (length > 3) { - a += k[0]; - b += k[1]; - c += k[2]; - mix(a, b, c); - length -= 3; - k += 3; - } - - /*----------------------------------- handle the last 3 uint32_t's */ - switch (length) { /* all the case statements fall through */ - case 3: c += k[2]; - case 2: b += k[1]; - case 1: a += k[0]; - final(a, b, c); - case 0: /* case 0: nothing left to add */ - break; - } - /*---------------------------------------------- report the result */ - *pc = c; - *pb = b; -} - -#if (CAA_BITS_PER_LONG == 32) -static inline -unsigned long test_hash_mix(const void *_key, size_t length, unsigned long seed) -{ - unsigned int key = (unsigned int) _key; - - assert(length == sizeof(unsigned int)); - return hash_u32(&key, 1, seed); -} -#else -static inline -unsigned long test_hash_mix(const void *_key, size_t length, unsigned long seed) -{ - union { - uint64_t v64; - uint32_t v32[2]; - } v; - union { - uint64_t v64; - uint32_t v32[2]; - } key; - - assert(length == sizeof(unsigned long)); - v.v64 = (uint64_t) seed; - key.v64 = (uint64_t) _key; - hashword2(key.v32, 2, &v.v32[0], &v.v32[1]); - return v.v64; -} -#endif - -/* - * Hash function with nr_hash_chains != 0 for testing purpose only! - * Creates very long hash chains, deteriorating the hash table into a - * few linked lists, depending on the nr_hash_chains value. The purpose - * of this test is to check how the hash table behaves with hash chains - * containing different values, which is a rare case in a normal hash - * table. - */ -static inline -unsigned long test_hash(const void *_key, size_t length, - unsigned long seed) -{ - if (nr_hash_chains == 0) { - return test_hash_mix(_key, length, seed); - } else { - unsigned long v; - - assert(length == sizeof(unsigned long)); - v = (unsigned long) _key; - return v % nr_hash_chains; - } -} - -unsigned long test_compare(const void *key1, size_t key1_len, - const void *key2, size_t key2_len); - -static inline -int test_match(struct cds_lfht_node *node, const void *key) -{ - struct lfht_test_node *test_node = to_test_node(node); - - return !test_compare(test_node->key, test_node->key_len, - key, sizeof(unsigned long)); -} - -static inline -void cds_lfht_test_lookup(struct cds_lfht *ht, void *key, size_t key_len, - struct cds_lfht_iter *iter) -{ - assert(key_len == sizeof(unsigned long)); - - cds_lfht_lookup(ht, test_hash(key, key_len, TEST_HASH_SEED), - test_match, key, iter); -} - -void free_node_cb(struct rcu_head *head); - -/* rw test */ -void test_hash_rw_sigusr1_handler(int signo); -void test_hash_rw_sigusr2_handler(int signo); -void *test_hash_rw_thr_reader(void *_count); -void *test_hash_rw_thr_writer(void *_count); -int test_hash_rw_populate_hash(void); - -/* unique test */ -void test_hash_unique_sigusr1_handler(int signo); -void test_hash_unique_sigusr2_handler(int signo); -void *test_hash_unique_thr_reader(void *_count); -void *test_hash_unique_thr_writer(void *_count); -int test_hash_unique_populate_hash(void); - -#endif /* _TEST_URCU_HASH_H */ diff --git a/tests/test_urcu_hash_rw.c b/tests/test_urcu_hash_rw.c deleted file mode 100644 index 8802b9c..0000000 --- a/tests/test_urcu_hash_rw.c +++ /dev/null @@ -1,284 +0,0 @@ -/* - * test_urcu_hash_rw.c - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "test_urcu_hash.h" - -enum urcu_hash_addremove { - AR_RANDOM = 0, - AR_ADD = 1, - AR_REMOVE = -1, -}; /* 1: add, -1 remove, 0: random */ - -static enum urcu_hash_addremove addremove; /* 1: add, -1 remove, 0: random */ - -void test_hash_rw_sigusr1_handler(int signo) -{ - switch (addremove) { - case AR_ADD: - printf("Add/Remove: random.\n"); - addremove = AR_RANDOM; - break; - case AR_RANDOM: - printf("Add/Remove: remove only.\n"); - addremove = AR_REMOVE; - break; - case AR_REMOVE: - printf("Add/Remove: add only.\n"); - addremove = AR_ADD; - break; - } -} - -void test_hash_rw_sigusr2_handler(int signo) -{ - char msg[1] = { 0x42 }; - ssize_t ret; - - do { - ret = write(count_pipe[1], msg, 1); /* wakeup thread */ - } while (ret == -1L && errno == EINTR); -} - -void *test_hash_rw_thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct lfht_test_node *node; - struct cds_lfht_iter iter; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - cds_lfht_test_lookup(test_ht, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % lookup_pool_size) + lookup_pool_offset), - sizeof(void *), &iter); - node = cds_lfht_iter_get_test_node(&iter); - if (node == NULL) { - if (validate_lookup) { - printf("[ERROR] Lookup cannot find initial node.\n"); - exit(-1); - } - URCU_TLS(lookup_fail)++; - } else { - URCU_TLS(lookup_ok)++; - } - rcu_debug_yield_read(); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - printf_verbose("read tid : %lx, lookupfail %lu, lookupok %lu\n", - urcu_get_thread_id(), - URCU_TLS(lookup_fail), - URCU_TLS(lookup_ok)); - return ((void*)1); - -} - -void *test_hash_rw_thr_writer(void *_count) -{ - struct lfht_test_node *node; - struct cds_lfht_node *ret_node; - struct cds_lfht_iter iter; - struct wr_count *count = _count; - int ret; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - if ((addremove == AR_ADD || add_only) - || (addremove == AR_RANDOM && rand_r(&URCU_TLS(rand_lookup)) & 1)) { - node = malloc(sizeof(struct lfht_test_node)); - lfht_test_node_init(node, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), - sizeof(void *)); - rcu_read_lock(); - if (add_unique) { - ret_node = cds_lfht_add_unique(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); - } else { - if (add_replace) - ret_node = cds_lfht_add_replace(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); - else - cds_lfht_add(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - &node->node); - } - rcu_read_unlock(); - if (add_unique && ret_node != &node->node) { - free(node); - URCU_TLS(nr_addexist)++; - } else { - if (add_replace && ret_node) { - call_rcu(&to_test_node(ret_node)->head, - free_node_cb); - URCU_TLS(nr_addexist)++; - } else { - URCU_TLS(nr_add)++; - } - } - } else { - /* May delete */ - rcu_read_lock(); - cds_lfht_test_lookup(test_ht, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), - sizeof(void *), &iter); - ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); - rcu_read_unlock(); - if (ret == 0) { - node = cds_lfht_iter_get_test_node(&iter); - call_rcu(&node->head, free_node_cb); - URCU_TLS(nr_del)++; - } else - URCU_TLS(nr_delnoent)++; - } -#if 0 - //if (URCU_TLS(nr_writes) % 100000 == 0) { - if (URCU_TLS(nr_writes) % 1000 == 0) { - rcu_read_lock(); - if (rand_r(&URCU_TLS(rand_lookup)) & 1) { - ht_resize(test_ht, 1); - } else { - ht_resize(test_ht, -1); - } - rcu_read_unlock(); - } -#endif //0 - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, " - "nr_delnoent %lu\n", urcu_get_thread_id(), - URCU_TLS(nr_add), - URCU_TLS(nr_addexist), - URCU_TLS(nr_del), - URCU_TLS(nr_delnoent)); - count->update_ops = URCU_TLS(nr_writes); - count->add = URCU_TLS(nr_add); - count->add_exist = URCU_TLS(nr_addexist); - count->remove = URCU_TLS(nr_del); - return ((void*)2); -} - -int test_hash_rw_populate_hash(void) -{ - struct lfht_test_node *node; - struct cds_lfht_node *ret_node; - - if (!init_populate) - return 0; - - printf("Starting rw test\n"); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - if ((add_unique || add_replace) && init_populate * 10 > init_pool_size) { - printf("WARNING: required to populate %lu nodes (-k), but random " -"pool is quite small (%lu values) and we are in add_unique (-u) or add_replace (-s) mode. Try with a " -"larger random pool (-p option). This may take a while...\n", init_populate, init_pool_size); - } - - while (URCU_TLS(nr_add) < init_populate) { - node = malloc(sizeof(struct lfht_test_node)); - lfht_test_node_init(node, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % init_pool_size) + init_pool_offset), - sizeof(void *)); - rcu_read_lock(); - if (add_unique) { - ret_node = cds_lfht_add_unique(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); - } else { - if (add_replace) - ret_node = cds_lfht_add_replace(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); - else - cds_lfht_add(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - &node->node); - } - rcu_read_unlock(); - if (add_unique && ret_node != &node->node) { - free(node); - URCU_TLS(nr_addexist)++; - } else { - if (add_replace && ret_node) { - call_rcu(&to_test_node(ret_node)->head, free_node_cb); - URCU_TLS(nr_addexist)++; - } else { - URCU_TLS(nr_add)++; - } - } - URCU_TLS(nr_writes)++; - } - return 0; -} diff --git a/tests/test_urcu_hash_unique.c b/tests/test_urcu_hash_unique.c deleted file mode 100644 index adbbde0..0000000 --- a/tests/test_urcu_hash_unique.c +++ /dev/null @@ -1,279 +0,0 @@ -/* - * test_urcu_hash_unique.c - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "test_urcu_hash.h" - -enum urcu_hash_addremove { - AR_RANDOM = 0, - AR_ADD = 1, - AR_REMOVE = -1, -}; /* 1: add, -1 remove, 0: random */ - -static enum urcu_hash_addremove addremove; /* 1: add, -1 remove, 0: random */ - -void test_hash_unique_sigusr1_handler(int signo) -{ - switch (addremove) { - case AR_ADD: - printf("Add/Remove: random.\n"); - addremove = AR_RANDOM; - break; - case AR_RANDOM: - printf("Add/Remove: remove only.\n"); - addremove = AR_REMOVE; - break; - case AR_REMOVE: - printf("Add/Remove: add only.\n"); - addremove = AR_ADD; - break; - } -} - -void test_hash_unique_sigusr2_handler(int signo) -{ - char msg[1] = { 0x42 }; - ssize_t ret; - - do { - ret = write(count_pipe[1], msg, 1); /* wakeup thread */ - } while (ret == -1L && errno == EINTR); -} - -void *test_hash_unique_thr_reader(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct lfht_test_node *node; - struct cds_lfht_iter iter; - /* - * iterate on whole table, ensuring that no duplicate is - * found. - */ - rcu_read_lock(); - cds_lfht_for_each_entry(test_ht, &iter, node, node) { - struct cds_lfht_iter dup_iter; - - dup_iter = iter; - cds_lfht_next_duplicate(test_ht, test_match, - node->key, &dup_iter); - if (dup_iter.node != NULL) { - printf("[ERROR] Duplicate key %p found\n", node->key); - } - } - rcu_read_unlock(); - - rcu_debug_yield_read(); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - printf_verbose("read tid : %lu, lookupfail %lu, lookupok %lu\n", - urcu_get_thread_id(), URCU_TLS(lookup_fail), - URCU_TLS(lookup_ok)); - return ((void*)1); - -} - -void *test_hash_unique_thr_writer(void *_count) -{ - struct lfht_test_node *node; - struct cds_lfht_node *ret_node; - struct cds_lfht_iter iter; - struct wr_count *count = _count; - int ret; - int loc_add_unique; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - /* - * add unique/add replace with new node key from range. - */ - if (1 || (addremove == AR_ADD || add_only) - || (addremove == AR_RANDOM && rand_r(&URCU_TLS(rand_lookup)) & 1)) { - node = malloc(sizeof(struct lfht_test_node)); - lfht_test_node_init(node, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), - sizeof(void *)); - rcu_read_lock(); - loc_add_unique = rand_r(&URCU_TLS(rand_lookup)) & 1; - if (loc_add_unique) { - ret_node = cds_lfht_add_unique(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); - } else { - ret_node = cds_lfht_add_replace(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); -#if 0 //generate an error on purpose - cds_lfht_add(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - &node->node); - ret_node = NULL; -#endif //0 - } - rcu_read_unlock(); - if (loc_add_unique) { - if (ret_node != &node->node) { - free(node); - URCU_TLS(nr_addexist)++; - } else { - URCU_TLS(nr_add)++; - } - } else { - if (ret_node) { - call_rcu(&to_test_node(ret_node)->head, - free_node_cb); - URCU_TLS(nr_addexist)++; - } else { - URCU_TLS(nr_add)++; - } - } - } else { - /* May delete */ - rcu_read_lock(); - cds_lfht_test_lookup(test_ht, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset), - sizeof(void *), &iter); - ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); - rcu_read_unlock(); - if (ret == 0) { - node = cds_lfht_iter_get_test_node(&iter); - call_rcu(&node->head, free_node_cb); - URCU_TLS(nr_del)++; - } else - URCU_TLS(nr_delnoent)++; - } -#if 0 - //if (URCU_TLS(nr_writes) % 100000 == 0) { - if (URCU_TLS(nr_writes) % 1000 == 0) { - rcu_read_lock(); - if (rand_r(&URCU_TLS(rand_lookup)) & 1) { - ht_resize(test_ht, 1); - } else { - ht_resize(test_ht, -1); - } - rcu_read_unlock(); - } -#endif //0 - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, " - "nr_delnoent %lu\n", urcu_get_thread_id(), - URCU_TLS(nr_add), - URCU_TLS(nr_addexist), - URCU_TLS(nr_del), - URCU_TLS(nr_delnoent)); - count->update_ops = URCU_TLS(nr_writes); - count->add = URCU_TLS(nr_add); - count->add_exist = URCU_TLS(nr_addexist); - count->remove = URCU_TLS(nr_del); - return ((void*)2); -} - -int test_hash_unique_populate_hash(void) -{ - struct lfht_test_node *node; - struct cds_lfht_node *ret_node; - - printf("Starting uniqueness test.\n"); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - if (!init_populate) - return 0; - - if (init_populate * 10 > init_pool_size) { - printf("WARNING: required to populate %lu nodes (-k), but random " -"pool is quite small (%lu values) and we are in add_unique (-u) or add_replace (-s) mode. Try with a " -"larger random pool (-p option). This may take a while...\n", init_populate, init_pool_size); - } - - while (URCU_TLS(nr_add) < init_populate) { - node = malloc(sizeof(struct lfht_test_node)); - lfht_test_node_init(node, - (void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % init_pool_size) + init_pool_offset), - sizeof(void *)); - rcu_read_lock(); - ret_node = cds_lfht_add_replace(test_ht, - test_hash(node->key, node->key_len, TEST_HASH_SEED), - test_match, node->key, &node->node); - rcu_read_unlock(); - if (ret_node) { - call_rcu(&to_test_node(ret_node)->head, free_node_cb); - URCU_TLS(nr_addexist)++; - } else { - URCU_TLS(nr_add)++; - } - URCU_TLS(nr_writes)++; - } - return 0; -} diff --git a/tests/test_urcu_ja.c b/tests/test_urcu_ja.c deleted file mode 100644 index 0572527..0000000 --- a/tests/test_urcu_ja.c +++ /dev/null @@ -1,1339 +0,0 @@ -/* - * test_urcu_ja.c - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "test_urcu_ja.h" -#include -#include - -DEFINE_URCU_TLS(unsigned int, rand_lookup); -DEFINE_URCU_TLS(unsigned long, nr_add); -DEFINE_URCU_TLS(unsigned long, nr_addexist); -DEFINE_URCU_TLS(unsigned long, nr_del); -DEFINE_URCU_TLS(unsigned long, nr_delnoent); -DEFINE_URCU_TLS(unsigned long, lookup_fail); -DEFINE_URCU_TLS(unsigned long, lookup_ok); - -struct cds_ja *test_ja; - -volatile int test_go, test_stop; - -unsigned long wdelay; - -unsigned long duration; - -/* read-side C.S. duration, in loops */ -unsigned long rduration; - -unsigned long init_populate; -int add_only; - -unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; -unsigned long init_pool_size = DEFAULT_RAND_POOL, - lookup_pool_size = DEFAULT_RAND_POOL, - write_pool_size = DEFAULT_RAND_POOL; -int validate_lookup; -int sanity_test; -unsigned int key_bits = 32; - -int count_pipe[2]; - -int verbose_mode; - -unsigned int cpu_affinities[NR_CPUS]; -unsigned int next_aff = 0; -int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -DEFINE_URCU_TLS(unsigned long long, nr_writes); -DEFINE_URCU_TLS(unsigned long long, nr_reads); - -unsigned int nr_readers; -unsigned int nr_writers; - -static unsigned int add_ratio = 50; -static uint64_t key_mul = 1ULL; - -static int add_unique, add_replace; - -static pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -static int leak_detection; -static unsigned long test_nodes_allocated, test_nodes_freed; - -void set_affinity(void) -{ - cpu_set_t mask; - int cpu; - int ret; - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -static -struct ja_test_node *node_alloc(void) -{ - struct ja_test_node *node; - - node = calloc(sizeof(*node), 1); - if (leak_detection && node) - uatomic_inc(&test_nodes_allocated); - return node; -} - -static -void free_test_node(struct ja_test_node *node) -{ - poison_free(node); - if (leak_detection) - uatomic_inc(&test_nodes_freed); -} - -static -void free_test_node_cb(struct rcu_head *head) -{ - struct ja_test_node *node = - caa_container_of(head, struct ja_test_node, head); - free_test_node(node); -} - -static -void rcu_free_test_node(struct ja_test_node *test_node) -{ - call_rcu(&test_node->head, free_test_node_cb); -} - -static -void free_node(struct cds_ja_node *node) -{ - struct ja_test_node *test_node = to_test_node(node); - - free_test_node(test_node); -} - -#if 0 -static -void test_delete_all_nodes(struct cds_lfht *ht) -{ - struct cds_lfht_iter iter; - struct lfht_test_node *node; - unsigned long count = 0; - - cds_lfht_for_each_entry(ht, &iter, node, node) { - int ret; - - ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter)); - assert(!ret); - call_rcu(&node->head, free_node_cb); - count++; - } - printf("deleted %lu nodes.\n", count); -} -#endif - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s)\n", argv[0]); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf(" [-u] Add unique keys.\n"); - printf(" [-s] Replace existing keys.\n"); -printf(" [not -u nor -s] Add entries (supports redundant keys).\n"); - printf(" [-r ratio] Add ratio (in %% of add+removal).\n"); - printf(" [-k] Populate init nodes.\n"); - printf(" [-R offset] Lookup pool offset.\n"); - printf(" [-S offset] Write pool offset.\n"); - printf(" [-T offset] Init pool offset.\n"); - printf(" [-M size] Lookup pool size.\n"); - printf(" [-N size] Write pool size.\n"); - printf(" [-O size] Init pool size.\n"); - printf(" [-V] Validate lookups of init values (use with filled init pool, same lookup range, with different write range).\n"); - printf(" [-t] Do sanity test.\n"); - printf(" [-B] Key bits for multithread test (default: 32).\n"); - printf(" [-m factor] Key multiplication factor.\n"); - printf(" [-l] Memory leak detection.\n"); - printf("\n\n"); -} - -static -int test_free_all_nodes(struct cds_ja *ja) -{ - uint64_t key; - struct cds_ja_node *ja_node; - int ret = 0; - - rcu_read_lock(); - cds_ja_for_each_key_rcu(test_ja, key, ja_node) { - struct cds_ja_node *tmp_node; - - cds_ja_for_each_duplicate_safe_rcu(ja_node, tmp_node) { - ret = cds_ja_del(test_ja, key, ja_node); - if (ret) { - fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); - goto end; - } - /* Alone using Judy array, OK to free now */ - free_node(ja_node); - } - } -end: - rcu_read_unlock(); - return ret; -} - -static -int test_8bit_key(void) -{ - int ret, i; - uint64_t key; - uint64_t ka[] = { 5, 17, 100, 222 }; - uint64_t ka_test_offset = 5; - struct cds_ja_node *ja_node; - - /* Test with 8-bit key */ - test_ja = cds_ja_new(8); - if (!test_ja) { - printf("Error allocating judy array.\n"); - return -1; - } - - /* Add keys */ - printf("Test #1: add keys (8-bit).\n"); - for (key = 0; key < 200; key++) { - struct ja_test_node *node = node_alloc(); - - ja_test_node_init(node, key); - rcu_read_lock(); - ret = cds_ja_add(test_ja, key, &node->node); - rcu_read_unlock(); - if (ret) { - fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", - ret, key); - assert(0); - } - } - printf("OK\n"); - - printf("Test #2: successful key lookup (8-bit).\n"); - for (key = 0; key < 200; key++) { - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (!ja_node) { - fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); - assert(0); - } - rcu_read_unlock(); - } - printf("OK\n"); - printf("Test #3: unsuccessful key lookup (8-bit).\n"); - for (key = 200; key < 240; key++) { - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (ja_node) { - fprintf(stderr, - "Error unexpected lookup node %" PRIu64 "\n", - key); - assert(0); - } - rcu_read_unlock(); - } - printf("OK\n"); - printf("Test #4: remove keys (8-bit).\n"); - for (key = 0; key < 200; key++) { - struct ja_test_node *node; - - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (!ja_node) { - fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - ret = cds_ja_del(test_ja, key, &node->node); - if (ret) { - fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); - assert(0); - } - rcu_free_test_node(node); - ja_node = cds_ja_lookup(test_ja, key); - if (ja_node) { - fprintf(stderr, "Error lookup %" PRIu64 ": %p (after delete) failed. Node is not expected.\n", key, ja_node); - assert(0); - } - rcu_read_unlock(); - } - printf("OK\n"); - - printf("Test #5: lookup below/above equal (8-bit).\n"); - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node = node_alloc(); - - key = ka[i]; - ja_test_node_init(node, key); - rcu_read_lock(); - ret = cds_ja_add(test_ja, key, &node->node); - rcu_read_unlock(); - if (ret) { - fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", - ret, key); - assert(0); - } - } - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node; - uint64_t result_key; - - key = ka[i] + ka_test_offset; - rcu_read_lock(); - ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - rcu_read_unlock(); - } - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node; - uint64_t result_key; - - key = ka[i] - ka_test_offset; - rcu_read_lock(); - ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - rcu_read_unlock(); - } - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node; - uint64_t result_key; - - key = ka[i]; /* without offset */ - rcu_read_lock(); - ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - - ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - rcu_read_unlock(); - } - - printf("OK\n"); - - ret = test_free_all_nodes(test_ja); - if (ret) { - fprintf(stderr, "Error freeing all nodes\n"); - return -1; - } - - ret = cds_ja_destroy(test_ja); - if (ret) { - fprintf(stderr, "Error destroying judy array\n"); - return -1; - } - return 0; -} - -static -int test_16bit_key(void) -{ - int ret, i; - uint64_t key; - uint64_t ka[] = { 105, 206, 4000, 4111, 59990, 65435 }; - uint64_t ka_test_offset = 100; - struct cds_ja_node *ja_node; - - /* Test with 16-bit key */ - test_ja = cds_ja_new(16); - if (!test_ja) { - printf("Error allocating judy array.\n"); - return -1; - } - - /* Add keys */ - printf("Test #1: add keys (16-bit).\n"); - for (key = 0; key < 10000; key++) { - //for (key = 0; key < 65536; key+=256) { - struct ja_test_node *node = node_alloc(); - - ja_test_node_init(node, key); - rcu_read_lock(); - ret = cds_ja_add(test_ja, key, &node->node); - rcu_read_unlock(); - if (ret) { - fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", - ret, key); - assert(0); - } - } - printf("OK\n"); - - printf("Test #2: successful key lookup (16-bit).\n"); - for (key = 0; key < 10000; key++) { - //for (key = 0; key < 65536; key+=256) { - struct cds_ja_node *ja_node; - - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (!ja_node) { - fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); - assert(0); - } - rcu_read_unlock(); - } - printf("OK\n"); - printf("Test #3: unsuccessful key lookup (16-bit).\n"); - for (key = 11000; key <= 11002; key++) { - struct cds_ja_node *ja_node; - - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (ja_node) { - fprintf(stderr, - "Error unexpected lookup node %" PRIu64 "\n", - key); - assert(0); - } - rcu_read_unlock(); - } - printf("OK\n"); - printf("Test #4: remove keys (16-bit).\n"); - for (key = 0; key < 10000; key++) { - //for (key = 0; key < 65536; key+=256) { - struct ja_test_node *node; - - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (!ja_node) { - fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - ret = cds_ja_del(test_ja, key, &node->node); - if (ret) { - fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); - assert(0); - } - rcu_free_test_node(node); - ja_node = cds_ja_lookup(test_ja, key); - if (ja_node) { - fprintf(stderr, "Error lookup %" PRIu64 ": %p (after delete) failed. Node is not expected.\n", key, ja_node); - assert(0); - } - rcu_read_unlock(); - } - printf("OK\n"); - - printf("Test #5: lookup below/above equal (16-bit).\n"); - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node = node_alloc(); - - key = ka[i]; - ja_test_node_init(node, key); - rcu_read_lock(); - ret = cds_ja_add(test_ja, key, &node->node); - rcu_read_unlock(); - if (ret) { - fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", - ret, key); - assert(0); - } - } - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node; - uint64_t result_key; - - key = ka[i] + ka_test_offset; - rcu_read_lock(); - ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - rcu_read_unlock(); - } - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node; - uint64_t result_key; - - key = ka[i] - ka_test_offset; - rcu_read_lock(); - ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" above or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " above or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - rcu_read_unlock(); - } - - for (i = 0; i < CAA_ARRAY_SIZE(ka); i++) { - struct ja_test_node *node; - uint64_t result_key; - - key = ka[i]; /* without offset */ - rcu_read_lock(); - ja_node = cds_ja_lookup_below_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup below equal. Cannot find expected key %" PRIu64" below or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup below equal. Expecting key %" PRIu64 " below or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - - ja_node = cds_ja_lookup_above_equal(test_ja, key, &result_key); - if (!ja_node) { - fprintf(stderr, "Error lookup above equal. Cannot find expected key %" PRIu64" above or equal to %" PRIu64 ".\n", - ka[i], key); - assert(0); - } - node = caa_container_of(ja_node, struct ja_test_node, node); - if (node->key != ka[i] || result_key != ka[i]) { - fprintf(stderr, "Error lookup above equal. Expecting key %" PRIu64 " above or equal to %" PRIu64 ", but found %" PRIu64 "/%" PRIu64" instead.\n", - ka[i], key, node->key, result_key); - assert(0); - } - rcu_read_unlock(); - } - - printf("OK\n"); - - ret = test_free_all_nodes(test_ja); - if (ret) { - fprintf(stderr, "Error freeing all nodes\n"); - return -1; - } - - ret = cds_ja_destroy(test_ja); - if (ret) { - fprintf(stderr, "Error destroying judy array\n"); - return -1; - } - return 0; -} - -/* - * nr_dup is number of nodes per key. - */ -static -int test_sparse_key(unsigned int bits, int nr_dup) -{ - uint64_t key, max_key; - int zerocount, i, ret; - struct cds_ja_node *ja_node; - - if (bits == 64) - max_key = UINT64_MAX; - else - max_key = (1ULL << bits) - 1; - - printf("Sparse key test begins for %u-bit keys\n", bits); - /* Test with 16-bit key */ - test_ja = cds_ja_new(bits); - if (!test_ja) { - printf("Error allocating judy array.\n"); - return -1; - } - - /* Add keys */ - printf("Test #1: add keys (%u-bit).\n", bits); - for (i = 0; i < nr_dup; i++) { - zerocount = 0; - for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { - struct ja_test_node *node = node_alloc(); - - ja_test_node_init(node, key); - rcu_read_lock(); - ret = cds_ja_add(test_ja, key, &node->node); - rcu_read_unlock(); - if (ret) { - fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", - ret, key); - assert(0); - } - if (key == 0) - zerocount++; - } - } - printf("OK\n"); - - printf("Test #2: successful key lookup (%u-bit).\n", bits); - zerocount = 0; - for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { - struct ja_test_node *node; - int count = 0; - - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - if (!ja_node) { - fprintf(stderr, "Error lookup node %" PRIu64 "\n", key); - assert(0); - } - cds_ja_for_each_duplicate_rcu(ja_node) { - count++; - } - if (count != nr_dup) { - fprintf(stderr, "Unexpected number of match for key %" PRIu64 ", expected %d, got %d.\n", key, nr_dup, count); - } - rcu_read_unlock(); - if (key == 0) - zerocount++; - } - printf("OK\n"); - if (bits > 8) { - printf("Test #3: unsuccessful key lookup (%u-bit).\n", bits); - zerocount = 0; - for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key + 42); - if (ja_node) { - fprintf(stderr, - "Error unexpected lookup node %" PRIu64 "\n", - key + 42); - assert(0); - } - rcu_read_unlock(); - if (key == 0) - zerocount++; - } - printf("OK\n"); - } - printf("Test #4: remove keys (%u-bit).\n", bits); - zerocount = 0; - for (key = 0; key <= max_key && (key != 0 || zerocount < 1); key += 1ULL << (bits - 8)) { - int count = 0; - - rcu_read_lock(); - ja_node = cds_ja_lookup(test_ja, key); - - cds_ja_for_each_duplicate_rcu(ja_node) { - struct cds_ja_node *test_ja_node; - struct ja_test_node *node; - - count++; - node = caa_container_of(ja_node, - struct ja_test_node, node); - ret = cds_ja_del(test_ja, key, &node->node); - if (ret) { - fprintf(stderr, "Error (%d) removing node %" PRIu64 "\n", ret, key); - assert(0); - } - rcu_free_test_node(node); - test_ja_node = cds_ja_lookup(test_ja, key); - if (count < nr_dup && !test_ja_node) { - fprintf(stderr, "Error: no node found after deletion of some nodes of a key\n"); - assert(0); - } - } - ja_node = cds_ja_lookup(test_ja, key); - if (ja_node) { - fprintf(stderr, "Error lookup %" PRIu64 ": %p (after delete) failed. Node is not expected.\n", key, ja_node); - assert(0); - } - rcu_read_unlock(); - if (key == 0) - zerocount++; - } - printf("OK\n"); - - ret = test_free_all_nodes(test_ja); - if (ret) { - fprintf(stderr, "Error freeing all nodes\n"); - return -1; - } - - ret = cds_ja_destroy(test_ja); - if (ret) { - fprintf(stderr, "Error destroying judy array\n"); - return -1; - } - printf("Test ends\n"); - - return 0; -} - -static -int do_sanity_test(void) -{ - int i, j, ret; - - printf("Sanity test start.\n"); - - for (i = 0; i < 3; i++) { - ret = test_8bit_key(); - if (ret) { - return ret; - } - rcu_quiescent_state(); - } - ret = test_16bit_key(); - if (ret) { - return ret; - } - rcu_quiescent_state(); - - /* key bits */ - for (i = 8; i <= 64; i *= 2) { - /* nr of nodes per key */ - for (j = 1; j < 4; j++) { - ret = test_sparse_key(i, j); - if (ret) { - return ret; - } - rcu_quiescent_state(); - } - } - printf("Sanity test end.\n"); - - return 0; -} - -enum urcu_ja_addremove { - AR_RANDOM = 0, - AR_ADD = 1, - AR_REMOVE = -1, -}; /* 1: add, -1 remove, 0: random */ - -static enum urcu_ja_addremove addremove; /* 1: add, -1 remove, 0: random */ - -static -void test_ja_rw_sigusr1_handler(int signo) -{ - switch (addremove) { - case AR_ADD: - printf("Add/Remove: random.\n"); - addremove = AR_RANDOM; - break; - case AR_RANDOM: - printf("Add/Remove: remove only.\n"); - addremove = AR_REMOVE; - break; - case AR_REMOVE: - printf("Add/Remove: add only.\n"); - addremove = AR_ADD; - break; - } -} - -static -void *test_ja_rw_thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct cds_ja_node *ja_node; - uint64_t key; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - - /* note: only looking up ulong keys */ - key = ((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % lookup_pool_size) + lookup_pool_offset; - key *= key_mul; - ja_node = cds_ja_lookup(test_ja, key); - if (!ja_node) { - if (validate_lookup) { - printf("[ERROR] Lookup cannot find initial node.\n"); - exit(-1); - } - URCU_TLS(lookup_fail)++; - } else { - URCU_TLS(lookup_ok)++; - } - rcu_debug_yield_read(); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - printf_verbose("readid : %lx, lookupfail %lu, lookupok %lu\n", - pthread_self(), URCU_TLS(lookup_fail), - URCU_TLS(lookup_ok)); - return ((void*)1); -} - -static -int is_add(void) -{ - return ((unsigned int) rand_r(&URCU_TLS(rand_lookup)) % 100) < add_ratio; -} - -static -void *test_ja_rw_thr_writer(void *_count) -{ - struct wr_count *count = _count; - uint64_t key; - int ret; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - if ((addremove == AR_ADD) - || (addremove == AR_RANDOM && is_add())) { - struct ja_test_node *node = node_alloc(); - struct cds_ja_node *ret_node; - - /* note: only inserting ulong keys */ - key = ((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset; - key *= key_mul; - ja_test_node_init(node, key); - rcu_read_lock(); - if (add_unique) { - ret_node = cds_ja_add_unique(test_ja, key, &node->node); - if (ret_node != &node->node) { - free_test_node(node); - URCU_TLS(nr_addexist)++; - } else { - URCU_TLS(nr_add)++; - } - } else if (add_replace) { - assert(0); /* not implemented yet. */ - } else { - ret = cds_ja_add(test_ja, key, &node->node); - if (ret) { - fprintf(stderr, "Error in cds_ja_add: %d\n", ret); - free_test_node(node); - } else { - URCU_TLS(nr_add)++; - } - } - rcu_read_unlock(); - } else { - struct cds_ja_node *ja_node; - struct ja_test_node *node; - - /* May delete */ - /* note: only deleting ulong keys */ - key = ((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset; - key *= key_mul; - - rcu_read_lock(); - - ja_node = cds_ja_lookup(test_ja, key); - /* Remove first entry */ - if (ja_node) { - node = caa_container_of(ja_node, - struct ja_test_node, node); - ret = cds_ja_del(test_ja, key, &node->node); - if (!ret) { - rcu_free_test_node(node); - URCU_TLS(nr_del)++; - } else { - URCU_TLS(nr_delnoent)++; - } - } else { - URCU_TLS(nr_delnoent)++; - } - rcu_read_unlock(); - } - - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - printf_verbose("info id %lx: nr_add %lu, nr_addexist %lu, nr_del %lu, " - "nr_delnoent %lu\n", pthread_self(), URCU_TLS(nr_add), - URCU_TLS(nr_addexist), URCU_TLS(nr_del), - URCU_TLS(nr_delnoent)); - count->update_ops = URCU_TLS(nr_writes); - count->add = URCU_TLS(nr_add); - count->add_exist = URCU_TLS(nr_addexist); - count->remove = URCU_TLS(nr_del); - return ((void*)2); -} - -static -int do_mt_populate_ja(void) -{ - uint64_t iter; - int ret; - - if (!init_populate) - return 0; - - printf("Starting rw test\n"); - - for (iter = init_pool_offset; iter < init_pool_offset + init_pool_size; iter++) { - struct ja_test_node *node = node_alloc(); - uint64_t key; - - /* note: only inserting ulong keys */ - key = (unsigned long) iter; - key *= key_mul; - ja_test_node_init(node, key); - rcu_read_lock(); - ret = cds_ja_add(test_ja, key, &node->node); - URCU_TLS(nr_add)++; - URCU_TLS(nr_writes)++; - rcu_read_unlock(); - /* Hash table resize only occurs in call_rcu thread */ - if (!(iter % 100)) - rcu_quiescent_state(); - if (ret) { - fprintf(stderr, "Error (%d) adding node %" PRIu64 "\n", - ret, key); - assert(0); - } - } - return 0; -} - -static -int do_mt_test(void) -{ - pthread_t *tid_reader, *tid_writer; - void *tret; - int ret, i, err; - unsigned long long *count_reader; - struct wr_count *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0, - tot_add = 0, tot_add_exist = 0, tot_remove = 0; - unsigned int remain; - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - printf("Allocating Judy Array for %u-bit keys\n", key_bits); - test_ja = cds_ja_new(key_bits); - if (!test_ja) { - printf("Error allocating judy array.\n"); - ret = -1; - goto end; - } - - do_mt_populate_ja(); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], - NULL, test_ja_rw_thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], - NULL, test_ja_rw_thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - rcu_thread_offline_qsbr(); - - remain = duration; - do { - remain = sleep(remain); - } while (remain > 0); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i].update_ops; - tot_add += count_writer[i].add; - tot_add_exist += count_writer[i].add_exist; - tot_remove += count_writer[i].remove; - } - rcu_thread_online_qsbr(); - - ret = test_free_all_nodes(test_ja); - if (ret) { - fprintf(stderr, "Error freeing all nodes\n"); - return -1; - } - - ret = cds_ja_destroy(test_ja); - if (ret) { - fprintf(stderr, "Error destroying judy array\n"); - goto end; - } - - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - ret = 0; -end: - return ret; -} - -static -int check_memory_leaks(void) -{ - unsigned long na, nf; - - na = uatomic_read(&test_nodes_allocated); - nf = uatomic_read(&test_nodes_freed); - if (na != nf) { - fprintf(stderr, "Memory leak of %ld test nodes detected. Allocated: %lu, freed: %lu\n", - na - nf, na, nf); - return -1; - } - return 0; -} - -int main(int argc, char **argv) -{ - int i, j, a, ret, err; - uint64_t key; - struct sigaction act; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - case 'r': - add_ratio = atoi(argv[++i]); - break; - case 'k': - init_populate = 1; - break; - case 'R': - lookup_pool_offset = atol(argv[++i]); - break; - case 'S': - write_pool_offset = atol(argv[++i]); - break; - case 'T': - init_pool_offset = atol(argv[++i]); - break; - case 'M': - lookup_pool_size = atol(argv[++i]); - break; - case 'N': - write_pool_size = atol(argv[++i]); - break; - case 'O': - init_pool_size = atol(argv[++i]); - break; - case 'V': - validate_lookup = 1; - break; - case 't': - sanity_test = 1; - break; - case 'B': - key_bits = atol(argv[++i]); - break; - case 'm': - key_mul = atoll(argv[++i]); - break; - case 'u': - add_unique = 1; - break; - case 's': - add_replace = 1; - break; - case 'l': - leak_detection = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("Add ratio: %u%%.\n", add_ratio); - printf_verbose("Mode:%s%s.\n", - " add/remove", - add_unique ? " uniquify" : ( add_replace ? " replace" : " insert")); - printf_verbose("Key multiplication factor: %" PRIu64 ".\n", key_mul); - printf_verbose("Init pool size offset %lu size %lu.\n", - init_pool_offset, init_pool_size); - printf_verbose("Lookup pool size offset %lu size %lu.\n", - lookup_pool_offset, lookup_pool_size); - printf_verbose("Update pool size offset %lu size %lu.\n", - write_pool_offset, write_pool_size); - if (validate_lookup) - printf_verbose("Validating lookups.\n"); - if (leak_detection) - printf_verbose("Memory leak dection activated.\n"); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - memset(&act, 0, sizeof(act)); - ret = sigemptyset(&act.sa_mask); - if (ret == -1) { - perror("sigemptyset"); - return -1; - } - act.sa_handler = test_ja_rw_sigusr1_handler; - act.sa_flags = SA_RESTART; - ret = sigaction(SIGUSR1, &act, NULL); - if (ret == -1) { - perror("sigaction"); - return -1; - } - - err = create_all_cpu_call_rcu_data(0); - if (err) { - printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); - } - - rcu_register_thread(); - - if (sanity_test) { - ret = do_sanity_test(); - } else { - ret = do_mt_test(); - } - - /* Wait for in-flight call_rcu free to complete for leak detection */ - rcu_barrier(); - - ret |= check_memory_leaks(); - - rcu_unregister_thread(); - free_all_cpu_call_rcu_data(); - - if (ret) { - printf("Test ended with error: %d\n", ret); - } - return ret; -} diff --git a/tests/test_urcu_ja.h b/tests/test_urcu_ja.h deleted file mode 100644 index 1dab429..0000000 --- a/tests/test_urcu_ja.h +++ /dev/null @@ -1,176 +0,0 @@ -#ifndef _TEST_URCU_JA_H -#define _TEST_URCU_JA_H - -/* - * test_urcu_ja.h - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "thread-id.h" - -#define DEFAULT_RAND_POOL 1000000 - -/* Make this big enough to include the POWER5+ L3 cacheline size of 256B */ -#define CACHE_LINE_SIZE 4096 - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifdef POISON_FREE -#define poison_free(ptr) \ - do { \ - memset(ptr, 0x42, sizeof(*(ptr))); \ - free(ptr); \ - } while (0) -#else -#define poison_free(ptr) free(ptr) -#endif - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include -#include -#include - -struct wr_count { - unsigned long update_ops; - unsigned long add; - unsigned long add_exist; - unsigned long remove; -}; - -extern DECLARE_URCU_TLS(unsigned int, rand_lookup); -extern DECLARE_URCU_TLS(unsigned long, nr_add); -extern DECLARE_URCU_TLS(unsigned long, nr_addexist); -extern DECLARE_URCU_TLS(unsigned long, nr_del); -extern DECLARE_URCU_TLS(unsigned long, nr_delnoent); -extern DECLARE_URCU_TLS(unsigned long, lookup_fail); -extern DECLARE_URCU_TLS(unsigned long, lookup_ok); - -extern struct cds_ja *test_ja; - -struct ja_test_node { - struct cds_ja_node node; - uint64_t key; /* for testing */ - struct rcu_head head; /* delayed reclaim */ -}; - -static inline struct ja_test_node * -to_test_node(struct cds_ja_node *node) -{ - return caa_container_of(node, struct ja_test_node, node); -} - -static inline -void ja_test_node_init(struct ja_test_node *node, uint64_t key) -{ - cds_ja_node_init(&node->node); - node->key = key; -} - -extern volatile int test_go, test_stop; - -extern unsigned long wdelay; - -extern unsigned long duration; - -/* read-side C.S. duration, in loops */ -extern unsigned long rduration; - -extern unsigned long init_populate; -extern int add_only; - -extern unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; -extern unsigned long init_pool_size, - lookup_pool_size, - write_pool_size; -extern int validate_lookup; - -extern int count_pipe[2]; - -static inline void loop_sleep(unsigned long l) -{ - while(l-- != 0) - caa_cpu_relax(); -} - -extern int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, ## args); \ - } while (0) - -extern unsigned int cpu_affinities[NR_CPUS]; -extern unsigned int next_aff; -extern int use_affinity; - -extern pthread_mutex_t affinity_mutex; - -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - -void set_affinity(void); - -/* - * returns 0 if test should end. - */ -static inline int test_duration_write(void) -{ - return !test_stop; -} - -static inline int test_duration_read(void) -{ - return !test_stop; -} - -extern DECLARE_URCU_TLS(unsigned long long, nr_writes); -extern DECLARE_URCU_TLS(unsigned long long, nr_reads); - -extern unsigned int nr_readers; -extern unsigned int nr_writers; - -void rcu_copy_mutex_lock(void); -void rcu_copy_mutex_unlock(void); - -#endif /* _TEST_URCU_JA_H */ diff --git a/tests/test_urcu_ja_range.c b/tests/test_urcu_ja_range.c deleted file mode 100644 index 93ac706..0000000 --- a/tests/test_urcu_ja_range.c +++ /dev/null @@ -1,680 +0,0 @@ -/* - * test_urcu_ja_range.c - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "test_urcu_ja_range.h" -#include -#include - -DEFINE_URCU_TLS(unsigned int, rand_lookup); -DEFINE_URCU_TLS(unsigned long, nr_add); -DEFINE_URCU_TLS(unsigned long, nr_addexist); -DEFINE_URCU_TLS(unsigned long, nr_del); -DEFINE_URCU_TLS(unsigned long, nr_delnoent); -DEFINE_URCU_TLS(unsigned long, lookup_fail); -DEFINE_URCU_TLS(unsigned long, lookup_ok); - -struct cds_ja *test_ja; - -volatile int test_go, test_stop; - -unsigned long wdelay; - -unsigned long duration; - -/* read-side C.S. duration, in loops */ -unsigned long rduration; - -unsigned long init_populate; -int add_only; - -unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; -unsigned long init_pool_size = DEFAULT_RAND_POOL, - lookup_pool_size = DEFAULT_RAND_POOL, - write_pool_size = DEFAULT_RAND_POOL; -int validate_lookup; -int sanity_test; -unsigned int key_bits = 32; - -int count_pipe[2]; - -int verbose_mode; - -unsigned int cpu_affinities[NR_CPUS]; -unsigned int next_aff = 0; -int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -DEFINE_URCU_TLS(unsigned long long, nr_writes); -DEFINE_URCU_TLS(unsigned long long, nr_reads); - -unsigned int nr_readers; -unsigned int nr_writers; - -static unsigned int add_ratio = 50, range_max_len = 0; -static uint64_t key_mul = 1ULL; - -static int add_unique, add_replace; - -static pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -static int leak_detection; -static unsigned long test_nodes_allocated, test_nodes_freed; - -void set_affinity(void) -{ - cpu_set_t mask; - int cpu; - int ret; - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s)\n", argv[0]); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf(" [-u] Add unique keys.\n"); - printf(" [-s] Replace existing keys.\n"); -printf(" [not -u nor -s] Add entries (supports redundant keys).\n"); - printf(" [-r ratio] Add ratio (in %% of add+removal).\n"); - printf(" [-k] Populate init nodes.\n"); - printf(" [-R offset] Lookup pool offset.\n"); - printf(" [-S offset] Write pool offset.\n"); - printf(" [-T offset] Init pool offset.\n"); - printf(" [-M size] Lookup pool size.\n"); - printf(" [-N size] Write pool size.\n"); - printf(" [-O size] Init pool size.\n"); - printf(" [-V] Validate lookups of init values (use with filled init pool, same lookup range, with different write range).\n"); - printf(" [-t] Do sanity test.\n"); - printf(" [-B] Key bits for multithread test (default: 32).\n"); - printf(" [-m factor] Key multiplication factor.\n"); - printf(" [-l] Memory leak detection.\n"); - printf(" [-L len] Range max len.\n"); - printf("\n\n"); -} - -enum urcu_ja_addremove { - AR_RANDOM = 0, - AR_ADD = 1, - AR_REMOVE = -1, -}; /* 1: add, -1 remove, 0: random */ - -static enum urcu_ja_addremove addremove; /* 1: add, -1 remove, 0: random */ - -static -void test_ja_rw_sigusr1_handler(int signo) -{ - switch (addremove) { - case AR_ADD: - printf("Add/Remove: random.\n"); - addremove = AR_RANDOM; - break; - case AR_RANDOM: - printf("Add/Remove: remove only.\n"); - addremove = AR_REMOVE; - break; - case AR_REMOVE: - printf("Add/Remove: add only.\n"); - addremove = AR_ADD; - break; - } -} - -static -void *test_ja_rw_thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct cds_ja_range *range; - uint64_t key; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = (unsigned int) urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - - key = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); - key += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; - key = (key % lookup_pool_size) + lookup_pool_offset; - key *= key_mul; - - range = cds_ja_range_lookup(test_ja, key); - if (!range) { - if (validate_lookup) { - printf("[ERROR] Lookup cannot find initial node.\n"); - exit(-1); - } - URCU_TLS(lookup_fail)++; - } else { - range = cds_ja_range_lock(range); - if (!range) { - if (validate_lookup) { - printf("[ERROR] Lookup cannot find initial node.\n"); - exit(-1); - } - } else { - URCU_TLS(lookup_ok)++; - cds_ja_range_unlock(range); - } - } - rcu_debug_yield_read(); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - if (caa_unlikely(!test_duration_read())) - break; - if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - printf_verbose("read tid : %lu, lookupfail %lu, lookupok %lu\n", - urcu_get_thread_id(), URCU_TLS(lookup_fail), - URCU_TLS(lookup_ok)); - return ((void*)1); -} - -static -int is_add(void) -{ - return ((unsigned int) rand_r(&URCU_TLS(rand_lookup)) % 100) < add_ratio; -} - -static -void *test_ja_rw_thr_writer(void *_count) -{ - struct wr_count *count = _count; - int ret; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - URCU_TLS(rand_lookup) = (unsigned int) urcu_get_thread_id() ^ time(NULL); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - if ((addremove == AR_ADD) - || (addremove == AR_RANDOM && is_add())) { - struct cds_ja_range *range; - uint64_t start, end, tmp; - - start = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); - start += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; - start = (start % write_pool_size) + write_pool_offset; - - end = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); - end += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; - end = (end % write_pool_size) + write_pool_offset; - - start *= key_mul; - end *= key_mul; - if (start > end) { - tmp = start; - start = end; - end = tmp; - } - if (end - start > range_max_len) { - end = start + range_max_len; - } - rcu_read_lock(); - ret = cds_ja_range_add(test_ja, start, end, NULL); - if (ret) { - if (ret == -EEXIST) { - URCU_TLS(nr_addexist)++; - } else { - assert(0); - } - } else { - URCU_TLS(nr_add)++; - } - rcu_read_unlock(); - } else { - struct cds_ja_range *range; - uint64_t key; - - /* May delete */ - key = (uint64_t) rand_r(&URCU_TLS(rand_lookup)); - key += (uint64_t) rand_r(&URCU_TLS(rand_lookup)) << 32ULL; - key = (key % write_pool_size) + write_pool_offset; - key *= key_mul; - - rcu_read_lock(); - - range = cds_ja_range_lookup(test_ja, key); - if (range) { - ret = cds_ja_range_del(test_ja, range); - if (!ret) { - URCU_TLS(nr_del)++; - } else { - URCU_TLS(nr_delnoent)++; - } - } else { - URCU_TLS(nr_delnoent)++; - } - rcu_read_unlock(); - } - - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - } - - rcu_unregister_thread(); - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, " - "nr_delnoent %lu\n", urcu_get_thread_id(), - URCU_TLS(nr_add), - URCU_TLS(nr_addexist), URCU_TLS(nr_del), - URCU_TLS(nr_delnoent)); - count->update_ops = URCU_TLS(nr_writes); - count->add = URCU_TLS(nr_add); - count->add_exist = URCU_TLS(nr_addexist); - count->remove = URCU_TLS(nr_del); - return ((void*)2); -} - -static -int do_mt_populate_ja(void) -{ - uint64_t iter; - int ret; - - if (!init_populate) - return 0; - - printf("Starting rw test\n"); - - for (iter = init_pool_offset; iter < init_pool_offset + init_pool_size; iter++) { - struct cds_ja_range *range; - uint64_t key; - - /* note: only inserting ulong keys */ - key = (unsigned long) iter; - key *= key_mul; - rcu_read_lock(); - ret = cds_ja_range_add(test_ja, key, key, NULL); - URCU_TLS(nr_add)++; - URCU_TLS(nr_writes)++; - rcu_read_unlock(); - /* Hash table resize only occurs in call_rcu thread */ - if (!(iter % 100)) - rcu_quiescent_state(); - if (ret) { - fprintf(stderr, "Error (%d) adding range %" PRIu64 "\n", - ret, key); - assert(0); - } - } - return 0; -} - -static -int do_mt_test(void) -{ - pthread_t *tid_reader, *tid_writer; - void *tret; - int ret, i, err; - unsigned long long *count_reader; - struct wr_count *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0, - tot_add = 0, tot_add_exist = 0, tot_remove = 0; - unsigned int remain; - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - printf("Allocating %u-bit Judy Array for ranges\n", - key_bits); - test_ja = cds_ja_range_new(key_bits); - if (!test_ja) { - printf("Error allocating judy array.\n"); - ret = -1; - goto end; - } - - do_mt_populate_ja(); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], - NULL, test_ja_rw_thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], - NULL, test_ja_rw_thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - rcu_thread_offline_qsbr(); - - remain = duration; - do { - remain = sleep(remain); - } while (remain > 0); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i].update_ops; - tot_add += count_writer[i].add; - tot_add_exist += count_writer[i].add_exist; - tot_remove += count_writer[i].remove; - } - rcu_thread_online_qsbr(); - - ret = cds_ja_range_validate(test_ja); - assert(!ret); - - ret = cds_ja_range_destroy(test_ja, NULL); - if (ret) { - fprintf(stderr, "Error destroying judy array\n"); - goto end; - } - - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - ret = 0; -end: - return ret; -} - -int main(int argc, char **argv) -{ - int i, j, a, ret, err; - uint64_t key; - struct sigaction act; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - case 'r': - add_ratio = atoi(argv[++i]); - break; - case 'k': - init_populate = 1; - break; - case 'R': - lookup_pool_offset = atol(argv[++i]); - break; - case 'S': - write_pool_offset = atol(argv[++i]); - break; - case 'T': - init_pool_offset = atol(argv[++i]); - break; - case 'M': - lookup_pool_size = atol(argv[++i]); - break; - case 'N': - write_pool_size = atol(argv[++i]); - break; - case 'O': - init_pool_size = atol(argv[++i]); - break; - case 'V': - validate_lookup = 1; - break; - case 't': - sanity_test = 1; - break; - case 'B': - key_bits = atol(argv[++i]); - break; - case 'm': - key_mul = atoll(argv[++i]); - break; - case 'u': - add_unique = 1; - break; - case 's': - add_replace = 1; - break; - case 'l': - leak_detection = 1; - break; - case 'L': - range_max_len = atol(argv[++i]); - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("Add ratio: %u%%.\n", add_ratio); - printf_verbose("Mode:%s%s.\n", - " add/remove", - add_unique ? " uniquify" : ( add_replace ? " replace" : " insert")); - printf_verbose("Key multiplication factor: %" PRIu64 ".\n", key_mul); - printf_verbose("Init pool size offset %lu size %lu.\n", - init_pool_offset, init_pool_size); - printf_verbose("Lookup pool size offset %lu size %lu.\n", - lookup_pool_offset, lookup_pool_size); - printf_verbose("Update pool size offset %lu size %lu.\n", - write_pool_offset, write_pool_size); - printf_verbose("Range max len: %lu.\n", - range_max_len); - if (validate_lookup) - printf_verbose("Validating lookups.\n"); - if (leak_detection) - printf_verbose("Memory leak dection activated.\n"); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - memset(&act, 0, sizeof(act)); - ret = sigemptyset(&act.sa_mask); - if (ret == -1) { - perror("sigemptyset"); - return -1; - } - act.sa_handler = test_ja_rw_sigusr1_handler; - act.sa_flags = SA_RESTART; - ret = sigaction(SIGUSR1, &act, NULL); - if (ret == -1) { - perror("sigaction"); - return -1; - } - - err = create_all_cpu_call_rcu_data(0); - if (err) { - printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); - } - - rcu_register_thread(); - - ret = do_mt_test(); - - /* Wait for in-flight call_rcu free to complete for leak detection */ - rcu_barrier(); - - rcu_unregister_thread(); - free_all_cpu_call_rcu_data(); - - if (ret) { - printf("Test ended with error: %d\n", ret); - } - return ret; -} diff --git a/tests/test_urcu_ja_range.h b/tests/test_urcu_ja_range.h deleted file mode 100644 index 1f8c2a2..0000000 --- a/tests/test_urcu_ja_range.h +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef _TEST_URCU_JA_RANGE_H -#define _TEST_URCU_JA_RANGE_H - -/* - * test_urcu_ja.h - * - * Userspace RCU library - test program - * - * Copyright 2009-2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "thread-id.h" - -#define DEFAULT_RAND_POOL 1000000 - -/* Make this big enough to include the POWER5+ L3 cacheline size of 256B */ -#define CACHE_LINE_SIZE 4096 - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifdef POISON_FREE -#define poison_free(ptr) \ - do { \ - memset(ptr, 0x42, sizeof(*(ptr))); \ - free(ptr); \ - } while (0) -#else -#define poison_free(ptr) free(ptr) -#endif - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include -#include -#include - -struct wr_count { - unsigned long update_ops; - unsigned long add; - unsigned long add_exist; - unsigned long remove; -}; - -extern DECLARE_URCU_TLS(unsigned int, rand_lookup); -extern DECLARE_URCU_TLS(unsigned long, nr_add); -extern DECLARE_URCU_TLS(unsigned long, nr_addexist); -extern DECLARE_URCU_TLS(unsigned long, nr_del); -extern DECLARE_URCU_TLS(unsigned long, nr_delnoent); -extern DECLARE_URCU_TLS(unsigned long, lookup_fail); -extern DECLARE_URCU_TLS(unsigned long, lookup_ok); - -extern struct cds_ja *test_ja; - -extern volatile int test_go, test_stop; - -extern unsigned long wdelay; - -extern unsigned long duration; - -/* read-side C.S. duration, in loops */ -extern unsigned long rduration; - -extern unsigned long init_populate; -extern int add_only; - -extern unsigned long init_pool_offset, lookup_pool_offset, write_pool_offset; -extern unsigned long init_pool_size, - lookup_pool_size, - write_pool_size; -extern int validate_lookup; - -extern int count_pipe[2]; - -static inline void loop_sleep(unsigned long l) -{ - while(l-- != 0) - caa_cpu_relax(); -} - -extern int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, ## args); \ - } while (0) - -extern unsigned int cpu_affinities[NR_CPUS]; -extern unsigned int next_aff; -extern int use_affinity; - -extern pthread_mutex_t affinity_mutex; - -#ifndef HAVE_CPU_SET_T -typedef unsigned long cpu_set_t; -# define CPU_ZERO(cpuset) do { *(cpuset) = 0; } while(0) -# define CPU_SET(cpu, cpuset) do { *(cpuset) |= (1UL << (cpu)); } while(0) -#endif - -void set_affinity(void); - -/* - * returns 0 if test should end. - */ -static inline int test_duration_write(void) -{ - return !test_stop; -} - -static inline int test_duration_read(void) -{ - return !test_stop; -} - -extern DECLARE_URCU_TLS(unsigned long long, nr_writes); -extern DECLARE_URCU_TLS(unsigned long long, nr_reads); - -extern unsigned int nr_readers; -extern unsigned int nr_writers; - -void rcu_copy_mutex_lock(void); -void rcu_copy_mutex_unlock(void); - -#endif /* _TEST_URCU_JA_RANGE_H */ diff --git a/tests/test_urcu_lfq.c b/tests/test_urcu_lfq.c deleted file mode 100644 index 52ca225..0000000 --- a/tests/test_urcu_lfq.c +++ /dev/null @@ -1,436 +0,0 @@ -/* - * test_urcu_lfq.c - * - * Userspace RCU library - example RCU-based lock-free queue - * - * Copyright February 2010 - Mathieu Desnoyers - * Copyright February 2010 - Paolo Bonzini - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif -#include -#include - -static volatile int test_go, test_stop; - -static unsigned long rduration; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long wdelay; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_dequeue(void) -{ - return !test_stop; -} - -static int test_duration_enqueue(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); - -static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); - -static unsigned int nr_enqueuers; -static unsigned int nr_dequeuers; - -struct test { - struct cds_lfq_node_rcu list; - struct rcu_head rcu; -}; - -static struct cds_lfq_queue_rcu q; - -void *thr_enqueuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "enqueuer", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct test *node = malloc(sizeof(*node)); - if (!node) - goto fail; - cds_lfq_node_init_rcu(&node->list); - rcu_read_lock(); - cds_lfq_enqueue_rcu(&q, &node->list); - rcu_read_unlock(); - URCU_TLS(nr_successful_enqueues)++; - - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); -fail: - URCU_TLS(nr_enqueues)++; - if (caa_unlikely(!test_duration_enqueue())) - break; - } - - rcu_unregister_thread(); - - count[0] = URCU_TLS(nr_enqueues); - count[1] = URCU_TLS(nr_successful_enqueues); - printf_verbose("enqueuer thread_end, tid %lu, " - "enqueues %llu successful_enqueues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_enqueues), - URCU_TLS(nr_successful_enqueues)); - return ((void*)1); - -} - -static -void free_node_cb(struct rcu_head *head) -{ - struct test *node = - caa_container_of(head, struct test, rcu); - free(node); -} - -void *thr_dequeuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "dequeuer", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct cds_lfq_node_rcu *qnode; - - rcu_read_lock(); - qnode = cds_lfq_dequeue_rcu(&q); - rcu_read_unlock(); - - if (qnode) { - struct test *node; - - node = caa_container_of(qnode, struct test, list); - call_rcu(&node->rcu, free_node_cb); - URCU_TLS(nr_successful_dequeues)++; - } - - URCU_TLS(nr_dequeues)++; - if (caa_unlikely(!test_duration_dequeue())) - break; - if (caa_unlikely(rduration)) - loop_sleep(rduration); - } - - rcu_unregister_thread(); - printf_verbose("dequeuer thread_end, tid %lu, " - "dequeues %llu, successful_dequeues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_dequeues), - URCU_TLS(nr_successful_dequeues)); - count[0] = URCU_TLS(nr_dequeues); - count[1] = URCU_TLS(nr_successful_dequeues); - return ((void*)2); -} - -void test_end(struct cds_lfq_queue_rcu *q, unsigned long long *nr_dequeues) -{ - struct cds_lfq_node_rcu *snode; - - do { - snode = cds_lfq_dequeue_rcu(q); - if (snode) { - struct test *node; - - node = caa_container_of(snode, struct test, list); - free(node); /* no more concurrent access */ - (*nr_dequeues)++; - } - } while (snode); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); - printf(" [-d delay] (enqueuer period (in loops))\n"); - printf(" [-c duration] (dequeuer period (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_enqueuer, *tid_dequeuer; - void *tret; - unsigned long long *count_enqueuer, *count_dequeuer; - unsigned long long tot_enqueues = 0, tot_dequeues = 0; - unsigned long long tot_successful_enqueues = 0, - tot_successful_dequeues = 0; - unsigned long long end_dequeues = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_dequeuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_enqueuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u enqueuers, " - "%u dequeuers.\n", - duration, nr_enqueuers, nr_dequeuers); - printf_verbose("Writer delay : %lu loops.\n", rduration); - printf_verbose("Reader duration : %lu loops.\n", wdelay); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); - tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); - count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); - count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); - cds_lfq_init_rcu(&q, call_rcu); - err = create_all_cpu_call_rcu_data(0); - if (err) { - printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); - } - - next_aff = 0; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, - &count_enqueuer[2 * i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, - &count_dequeuer[2 * i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - for (i = 0; i < duration; i++) { - sleep(1); - if (verbose_mode) - write (1, ".", 1); - } - - test_stop = 1; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_join(tid_enqueuer[i], &tret); - if (err != 0) - exit(1); - tot_enqueues += count_enqueuer[2 * i]; - tot_successful_enqueues += count_enqueuer[2 * i + 1]; - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_join(tid_dequeuer[i], &tret); - if (err != 0) - exit(1); - tot_dequeues += count_dequeuer[2 * i]; - tot_successful_dequeues += count_dequeuer[2 * i + 1]; - } - - test_end(&q, &end_dequeues); - err = cds_lfq_destroy_rcu(&q); - assert(!err); - - printf_verbose("total number of enqueues : %llu, dequeues %llu\n", - tot_enqueues, tot_dequeues); - printf_verbose("total number of successful enqueues : %llu, " - "successful dequeues %llu\n", - tot_successful_enqueues, tot_successful_dequeues); - printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " - "nr_dequeuers %3u " - "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " - "successful enqueues %12llu successful dequeues %12llu " - "end_dequeues %llu nr_ops %12llu\n", - argv[0], duration, nr_enqueuers, wdelay, - nr_dequeuers, rduration, tot_enqueues, tot_dequeues, - tot_successful_enqueues, - tot_successful_dequeues, end_dequeues, - tot_enqueues + tot_dequeues); - if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) - printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " - "succ. dequeues + end dequeues %llu.\n", - tot_successful_enqueues, - tot_successful_dequeues + end_dequeues); - - free_all_cpu_call_rcu_data(); - free(count_enqueuer); - free(count_dequeuer); - free(tid_enqueuer); - free(tid_dequeuer); - return 0; -} diff --git a/tests/test_urcu_lfs.c b/tests/test_urcu_lfs.c deleted file mode 100644 index b2a3371..0000000 --- a/tests/test_urcu_lfs.c +++ /dev/null @@ -1,518 +0,0 @@ -/* - * test_urcu_lfs.c - * - * Userspace RCU library - example lock-free stack - * - * Copyright 2010-2012 - Mathieu Desnoyers - * Copyright February 2010 - Paolo Bonzini - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif -#include -#include - -/* - * External synchronization used. - */ -enum test_sync { - TEST_SYNC_NONE = 0, - TEST_SYNC_RCU, -}; - -static enum test_sync test_sync; - -static volatile int test_go, test_stop; - -static unsigned long rduration; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long wdelay; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -static int test_pop, test_pop_all; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, ## args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_dequeue(void) -{ - return !test_stop; -} - -static int test_duration_enqueue(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); - -static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); - -static unsigned int nr_enqueuers; -static unsigned int nr_dequeuers; - -struct test { - struct cds_lfs_node list; - struct rcu_head rcu; -}; - -static struct cds_lfs_stack s; - -static void *thr_enqueuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "enqueuer", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct test *node = malloc(sizeof(*node)); - if (!node) - goto fail; - cds_lfs_node_init(&node->list); - cds_lfs_push(&s, &node->list); - URCU_TLS(nr_successful_enqueues)++; - - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); -fail: - URCU_TLS(nr_enqueues)++; - if (caa_unlikely(!test_duration_enqueue())) - break; - } - - rcu_unregister_thread(); - - count[0] = URCU_TLS(nr_enqueues); - count[1] = URCU_TLS(nr_successful_enqueues); - printf_verbose("enqueuer thread_end, tid %lu, " - "enqueues %llu successful_enqueues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_enqueues), - URCU_TLS(nr_successful_enqueues)); - return ((void*)1); - -} - -static -void free_node_cb(struct rcu_head *head) -{ - struct test *node = - caa_container_of(head, struct test, rcu); - free(node); -} - -static -void do_test_pop(enum test_sync sync) -{ - struct cds_lfs_node *snode; - - if (sync == TEST_SYNC_RCU) - rcu_read_lock(); - snode = __cds_lfs_pop(&s); - if (sync == TEST_SYNC_RCU) - rcu_read_unlock(); - if (snode) { - struct test *node; - - node = caa_container_of(snode, - struct test, list); - if (sync == TEST_SYNC_RCU) - call_rcu(&node->rcu, free_node_cb); - else - free(node); - URCU_TLS(nr_successful_dequeues)++; - } - URCU_TLS(nr_dequeues)++; -} - -static -void do_test_pop_all(enum test_sync sync) -{ - struct cds_lfs_node *snode; - struct cds_lfs_head *head; - struct cds_lfs_node *n; - - head = __cds_lfs_pop_all(&s); - cds_lfs_for_each_safe(head, snode, n) { - struct test *node; - - node = caa_container_of(snode, struct test, list); - if (sync == TEST_SYNC_RCU) - call_rcu(&node->rcu, free_node_cb); - else - free(node); - URCU_TLS(nr_successful_dequeues)++; - URCU_TLS(nr_dequeues)++; - } - -} - -static void *thr_dequeuer(void *_count) -{ - unsigned long long *count = _count; - unsigned int counter = 0; - - printf_verbose("thread_begin %s, tid %lu\n", - "dequeuer", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - assert(test_pop || test_pop_all); - - for (;;) { - if (test_pop && test_pop_all) { - /* both pop and pop all */ - if (counter & 1) - do_test_pop(test_sync); - else - do_test_pop_all(test_sync); - counter++; - } else { - if (test_pop) - do_test_pop(test_sync); - else - do_test_pop_all(test_sync); - } - - if (caa_unlikely(!test_duration_dequeue())) - break; - if (caa_unlikely(rduration)) - loop_sleep(rduration); - } - - rcu_unregister_thread(); - - printf_verbose("dequeuer thread_end, tid %lu, " - "dequeues %llu, successful_dequeues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_dequeues), - URCU_TLS(nr_successful_dequeues)); - count[0] = URCU_TLS(nr_dequeues); - count[1] = URCU_TLS(nr_successful_dequeues); - return ((void*)2); -} - -static void test_end(struct cds_lfs_stack *s, unsigned long long *nr_dequeues) -{ - struct cds_lfs_node *snode; - - do { - snode = __cds_lfs_pop(s); - if (snode) { - struct test *node; - - node = caa_container_of(snode, struct test, list); - free(node); - (*nr_dequeues)++; - } - } while (snode); -} - -static void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); - printf(" [-d delay] (enqueuer period (in loops))\n"); - printf(" [-c duration] (dequeuer period (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf(" [-p] (test pop)\n"); - printf(" [-P] (test pop_all, enabled by default)\n"); - printf(" [-R] (use RCU external synchronization)\n"); - printf(" Note: default: no external synchronization used.\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_enqueuer, *tid_dequeuer; - void *tret; - unsigned long long *count_enqueuer, *count_dequeuer; - unsigned long long tot_enqueues = 0, tot_dequeues = 0; - unsigned long long tot_successful_enqueues = 0, - tot_successful_dequeues = 0; - unsigned long long end_dequeues = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_dequeuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_enqueuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - case 'p': - test_pop = 1; - break; - case 'P': - test_pop_all = 1; - break; - case 'R': - test_sync = TEST_SYNC_RCU; - break; - } - } - - /* activate pop_all test by default */ - if (!test_pop && !test_pop_all) - test_pop_all = 1; - - printf_verbose("running test for %lu seconds, %u enqueuers, " - "%u dequeuers.\n", - duration, nr_enqueuers, nr_dequeuers); - if (test_pop) - printf_verbose("pop test activated.\n"); - if (test_pop_all) - printf_verbose("pop_all test activated.\n"); - if (test_sync == TEST_SYNC_RCU) - printf_verbose("External sync: RCU.\n"); - else - printf_verbose("External sync: none.\n"); - printf_verbose("Writer delay : %lu loops.\n", rduration); - printf_verbose("Reader duration : %lu loops.\n", wdelay); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); - tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); - count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); - count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); - cds_lfs_init(&s); - err = create_all_cpu_call_rcu_data(0); - if (err) { - printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); - } - - next_aff = 0; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, - &count_enqueuer[2 * i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, - &count_dequeuer[2 * i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - for (i = 0; i < duration; i++) { - sleep(1); - if (verbose_mode) - write (1, ".", 1); - } - - test_stop = 1; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_join(tid_enqueuer[i], &tret); - if (err != 0) - exit(1); - tot_enqueues += count_enqueuer[2 * i]; - tot_successful_enqueues += count_enqueuer[2 * i + 1]; - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_join(tid_dequeuer[i], &tret); - if (err != 0) - exit(1); - tot_dequeues += count_dequeuer[2 * i]; - tot_successful_dequeues += count_dequeuer[2 * i + 1]; - } - - test_end(&s, &end_dequeues); - - printf_verbose("total number of enqueues : %llu, dequeues %llu\n", - tot_enqueues, tot_dequeues); - printf_verbose("total number of successful enqueues : %llu, " - "successful dequeues %llu\n", - tot_successful_enqueues, tot_successful_dequeues); - printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " - "nr_dequeuers %3u " - "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " - "successful enqueues %12llu successful dequeues %12llu " - "end_dequeues %llu nr_ops %12llu\n", - argv[0], duration, nr_enqueuers, wdelay, - nr_dequeuers, rduration, tot_enqueues, tot_dequeues, - tot_successful_enqueues, - tot_successful_dequeues, end_dequeues, - tot_enqueues + tot_dequeues); - if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) - printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " - "succ. dequeues + end dequeues %llu.\n", - tot_successful_enqueues, - tot_successful_dequeues + end_dequeues); - - free_all_cpu_call_rcu_data(); - free(count_enqueuer); - free(count_dequeuer); - free(tid_enqueuer); - free(tid_dequeuer); - return 0; -} diff --git a/tests/test_urcu_lfs_rcu.c b/tests/test_urcu_lfs_rcu.c deleted file mode 100644 index 89077dc..0000000 --- a/tests/test_urcu_lfs_rcu.c +++ /dev/null @@ -1,435 +0,0 @@ -/* - * test_urcu_lfs_rcu.c - * - * Userspace RCU library - example RCU-based lock-free stack - * - * Copyright February 2010 - Mathieu Desnoyers - * Copyright February 2010 - Paolo Bonzini - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif -#include - -/* Remove deprecation warnings from test build. */ -#define CDS_LFS_RCU_DEPRECATED - -#include - -static volatile int test_go, test_stop; - -static unsigned long rduration; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long wdelay; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ - cpu_set_t mask; - int cpu; - int ret; - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_dequeue(void) -{ - return !test_stop; -} - -static int test_duration_enqueue(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); - -static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); - -static unsigned int nr_enqueuers; -static unsigned int nr_dequeuers; - -struct test { - struct cds_lfs_node_rcu list; - struct rcu_head rcu; -}; - -static struct cds_lfs_stack_rcu s; - -void *thr_enqueuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "enqueuer", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct test *node = malloc(sizeof(*node)); - if (!node) - goto fail; - cds_lfs_node_init_rcu(&node->list); - /* No rcu read-side is needed for push */ - cds_lfs_push_rcu(&s, &node->list); - URCU_TLS(nr_successful_enqueues)++; - - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); -fail: - URCU_TLS(nr_enqueues)++; - if (caa_unlikely(!test_duration_enqueue())) - break; - } - - rcu_unregister_thread(); - - count[0] = URCU_TLS(nr_enqueues); - count[1] = URCU_TLS(nr_successful_enqueues); - printf_verbose("enqueuer thread_end, tid %lu, " - "enqueues %llu successful_enqueues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_enqueues), - URCU_TLS(nr_successful_enqueues)); - return ((void*)1); - -} - -static -void free_node_cb(struct rcu_head *head) -{ - struct test *node = - caa_container_of(head, struct test, rcu); - free(node); -} - -void *thr_dequeuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "dequeuer", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct cds_lfs_node_rcu *snode; - - rcu_read_lock(); - snode = cds_lfs_pop_rcu(&s); - rcu_read_unlock(); - if (snode) { - struct test *node; - - node = caa_container_of(snode, struct test, list); - call_rcu(&node->rcu, free_node_cb); - URCU_TLS(nr_successful_dequeues)++; - } - URCU_TLS(nr_dequeues)++; - if (caa_unlikely(!test_duration_dequeue())) - break; - if (caa_unlikely(rduration)) - loop_sleep(rduration); - } - - rcu_unregister_thread(); - - printf_verbose("dequeuer thread_end, tid %lu, " - "dequeues %llu, successful_dequeues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_dequeues), - URCU_TLS(nr_successful_dequeues)); - count[0] = URCU_TLS(nr_dequeues); - count[1] = URCU_TLS(nr_successful_dequeues); - return ((void*)2); -} - -void test_end(struct cds_lfs_stack_rcu *s, unsigned long long *nr_dequeues) -{ - struct cds_lfs_node_rcu *snode; - - do { - snode = cds_lfs_pop_rcu(s); - if (snode) { - struct test *node; - - node = caa_container_of(snode, struct test, list); - free(node); - (*nr_dequeues)++; - } - } while (snode); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); - printf(" [-d delay] (enqueuer period (in loops))\n"); - printf(" [-c duration] (dequeuer period (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_enqueuer, *tid_dequeuer; - void *tret; - unsigned long long *count_enqueuer, *count_dequeuer; - unsigned long long tot_enqueues = 0, tot_dequeues = 0; - unsigned long long tot_successful_enqueues = 0, - tot_successful_dequeues = 0; - unsigned long long end_dequeues = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_dequeuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_enqueuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u enqueuers, " - "%u dequeuers.\n", - duration, nr_enqueuers, nr_dequeuers); - printf_verbose("Writer delay : %lu loops.\n", rduration); - printf_verbose("Reader duration : %lu loops.\n", wdelay); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); - tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); - count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); - count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); - cds_lfs_init_rcu(&s); - err = create_all_cpu_call_rcu_data(0); - if (err) { - printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n"); - } - - next_aff = 0; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, - &count_enqueuer[2 * i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, - &count_dequeuer[2 * i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - for (i = 0; i < duration; i++) { - sleep(1); - if (verbose_mode) - write (1, ".", 1); - } - - test_stop = 1; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_join(tid_enqueuer[i], &tret); - if (err != 0) - exit(1); - tot_enqueues += count_enqueuer[2 * i]; - tot_successful_enqueues += count_enqueuer[2 * i + 1]; - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_join(tid_dequeuer[i], &tret); - if (err != 0) - exit(1); - tot_dequeues += count_dequeuer[2 * i]; - tot_successful_dequeues += count_dequeuer[2 * i + 1]; - } - - test_end(&s, &end_dequeues); - - printf_verbose("total number of enqueues : %llu, dequeues %llu\n", - tot_enqueues, tot_dequeues); - printf_verbose("total number of successful enqueues : %llu, " - "successful dequeues %llu\n", - tot_successful_enqueues, tot_successful_dequeues); - printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " - "nr_dequeuers %3u " - "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " - "successful enqueues %12llu successful dequeues %12llu " - "end_dequeues %llu nr_ops %12llu\n", - argv[0], duration, nr_enqueuers, wdelay, - nr_dequeuers, rduration, tot_enqueues, tot_dequeues, - tot_successful_enqueues, - tot_successful_dequeues, end_dequeues, - tot_enqueues + tot_dequeues); - if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) - printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " - "succ. dequeues + end dequeues %llu.\n", - tot_successful_enqueues, - tot_successful_dequeues + end_dequeues); - - free_all_cpu_call_rcu_data(); - free(count_enqueuer); - free(count_dequeuer); - free(tid_enqueuer); - free(tid_dequeuer); - return 0; -} diff --git a/tests/test_urcu_multiflavor-bp.c b/tests/test_urcu_multiflavor-bp.c deleted file mode 100644 index 199818b..0000000 --- a/tests/test_urcu_multiflavor-bp.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * test_urcu_multiflavor-bp.c - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif - -#define RCU_SIGNAL -#include -#include "test_urcu_multiflavor.h" - -int test_mf_bp(void) -{ - rcu_register_thread(); - rcu_read_lock(); - rcu_read_unlock(); - synchronize_rcu(); - rcu_unregister_thread(); - return 0; -} diff --git a/tests/test_urcu_multiflavor-mb.c b/tests/test_urcu_multiflavor-mb.c deleted file mode 100644 index e4ba6a4..0000000 --- a/tests/test_urcu_multiflavor-mb.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * test_urcu_multiflavor-mb.c - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif - -#define RCU_MB -#include -#include "test_urcu_multiflavor.h" - -int test_mf_mb(void) -{ - rcu_register_thread(); - rcu_read_lock(); - rcu_read_unlock(); - synchronize_rcu(); - rcu_unregister_thread(); - return 0; -} diff --git a/tests/test_urcu_multiflavor-memb.c b/tests/test_urcu_multiflavor-memb.c deleted file mode 100644 index 583f2e1..0000000 --- a/tests/test_urcu_multiflavor-memb.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * test_urcu_multiflavor-memb.c - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif - -#include -#include "test_urcu_multiflavor.h" - -int test_mf_memb(void) -{ - rcu_register_thread(); - rcu_read_lock(); - rcu_read_unlock(); - synchronize_rcu(); - rcu_unregister_thread(); - return 0; -} diff --git a/tests/test_urcu_multiflavor-qsbr.c b/tests/test_urcu_multiflavor-qsbr.c deleted file mode 100644 index 64f32f4..0000000 --- a/tests/test_urcu_multiflavor-qsbr.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * test_urcu_multiflavor-qsbr.c - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif - -#include -#include "test_urcu_multiflavor.h" - -int test_mf_qsbr(void) -{ - rcu_register_thread(); - rcu_read_lock(); - rcu_read_unlock(); - synchronize_rcu(); - rcu_unregister_thread(); - return 0; -} diff --git a/tests/test_urcu_multiflavor-signal.c b/tests/test_urcu_multiflavor-signal.c deleted file mode 100644 index 816c615..0000000 --- a/tests/test_urcu_multiflavor-signal.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * test_urcu_multiflavor-signal.c - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif - -#define RCU_SIGNAL -#include -#include "test_urcu_multiflavor.h" - -int test_mf_signal(void) -{ - rcu_register_thread(); - rcu_read_lock(); - rcu_read_unlock(); - synchronize_rcu(); - rcu_unregister_thread(); - return 0; -} diff --git a/tests/test_urcu_multiflavor.c b/tests/test_urcu_multiflavor.c deleted file mode 100644 index cd573ca..0000000 --- a/tests/test_urcu_multiflavor.c +++ /dev/null @@ -1,50 +0,0 @@ -/* - * test_urcu_multiflavor.c - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include "test_urcu_multiflavor.h" - -int main(int argc, char **argv) -{ - int ret; - - ret = test_mf_memb(); - if (ret) - goto failure; - ret = test_mf_mb(); - if (ret) - goto failure; - ret = test_mf_signal(); - if (ret) - goto failure; - ret = test_mf_qsbr(); - if (ret) - goto failure; - ret = test_mf_bp(); - if (ret) - goto failure; - - exit(EXIT_SUCCESS); - -failure: - exit(EXIT_FAILURE); -} diff --git a/tests/test_urcu_multiflavor.h b/tests/test_urcu_multiflavor.h deleted file mode 100644 index 419ad5d..0000000 --- a/tests/test_urcu_multiflavor.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * test_urcu_multiflavor.h - * - * Userspace RCU library - test multiple RCU flavors into one program - * - * Copyright February 2012 - Mathieu Desnoyers - * Copyright February 2012 - Lai Jiangshan - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -extern int test_mf_memb(void); -extern int test_mf_mb(void); -extern int test_mf_signal(void); -extern int test_mf_qsbr(void); -extern int test_mf_bp(void); - diff --git a/tests/test_urcu_qsbr.c b/tests/test_urcu_qsbr.c deleted file mode 100644 index 2e9e2b2..0000000 --- a/tests/test_urcu_qsbr.c +++ /dev/null @@ -1,410 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define rcu_debug_yield_read() -#endif -#include "urcu-qsbr.h" - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static int *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - int *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - assert(rcu_read_ongoing()); - rcu_thread_offline(); - assert(!rcu_read_ongoing()); - rcu_thread_online(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - rcu_read_lock(); - assert(rcu_read_ongoing()); - local_ptr = rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(*local_ptr == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - URCU_TLS(nr_reads)++; - /* QS each 1024 reads */ - if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) - rcu_quiescent_state(); - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - /* test extra thread registration */ - rcu_register_thread(); - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - int *new, *old; - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - new = malloc(sizeof(int)); - assert(new); - *new = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - if (caa_unlikely(wduration)) - loop_sleep(wduration); - synchronize_rcu(); - if (old) - *old = 0; - free(old); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - *count = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - free(test_rcu_pointer); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_urcu_qsbr_gc.c b/tests/test_urcu_qsbr_gc.c deleted file mode 100644 index 64d2e24..0000000 --- a/tests/test_urcu_qsbr_gc.c +++ /dev/null @@ -1,470 +0,0 @@ -/* - * test_urcu_gc.c - * - * Userspace RCU library - test program (with baatch reclamation) - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#define _LGPL_SOURCE -#include - -struct test_array { - int a; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static struct test_array *test_rcu_pointer; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; -static unsigned int reclaim_batch = 1; - -struct reclaim_queue { - void **queue; /* Beginning of queue */ - void **head; /* Insert position */ -}; - -static struct reclaim_queue *pending_reclaims; - - -/* write-side C.S. duration, in loops */ -static unsigned long wduration; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_writes); -static DEFINE_URCU_TLS(unsigned long long, nr_reads); - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; -static -unsigned long long __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *tot_nr_writes; - - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_array *local_ptr; - - printf_verbose("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - _rcu_read_lock(); - local_ptr = _rcu_dereference(test_rcu_pointer); - rcu_debug_yield_read(); - if (local_ptr) - assert(local_ptr->a == 8); - if (caa_unlikely(rduration)) - loop_sleep(rduration); - _rcu_read_unlock(); - URCU_TLS(nr_reads)++; - /* QS each 1024 reads */ - if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0)) - _rcu_quiescent_state(); - if (caa_unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = URCU_TLS(nr_reads); - printf_verbose("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -static void rcu_gc_clear_queue(unsigned long wtidx) -{ - void **p; - - /* Wait for Q.S and empty queue */ - synchronize_rcu(); - - for (p = pending_reclaims[wtidx].queue; - p < pending_reclaims[wtidx].head; p++) { - /* poison */ - if (*p) - ((struct test_array *)*p)->a = 0; - free(*p); - } - pending_reclaims[wtidx].head = pending_reclaims[wtidx].queue; -} - -/* Using per-thread queue */ -static void rcu_gc_reclaim(unsigned long wtidx, void *old) -{ - /* Queue pointer */ - *pending_reclaims[wtidx].head = old; - pending_reclaims[wtidx].head++; - - if (caa_likely(pending_reclaims[wtidx].head - pending_reclaims[wtidx].queue - < reclaim_batch)) - return; - - rcu_gc_clear_queue(wtidx); -} - -void *thr_writer(void *data) -{ - unsigned long wtidx = (unsigned long)data; -#ifdef TEST_LOCAL_GC - struct test_array *old = NULL; -#else - struct test_array *new, *old; -#endif - - printf_verbose("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { -#ifndef TEST_LOCAL_GC - new = malloc(sizeof(*new)); - new->a = 8; - old = _rcu_xchg_pointer(&test_rcu_pointer, new); -#endif - if (caa_unlikely(wduration)) - loop_sleep(wduration); - rcu_gc_reclaim(wtidx, old); - URCU_TLS(nr_writes)++; - if (caa_unlikely(!test_duration_write())) - break; - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); - } - - printf_verbose("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - tot_nr_writes[wtidx] = URCU_TLS(nr_writes); - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)\n"); -#endif - printf(" [-b batch] (batch reclaim)\n"); - printf(" [-d delay] (writer period (us))\n"); - printf(" [-c duration] (reader C.S. duration (in loops))\n"); - printf(" [-e duration] (writer C.S. duration (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - rcu_yield_active |= RCU_YIELD_READ; - break; - case 'w': - rcu_yield_active |= RCU_YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'b': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - reclaim_batch = atol(argv[++i]); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'e': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wduration = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - tot_nr_writes = malloc(sizeof(*tot_nr_writes) * nr_writers); - pending_reclaims = malloc(sizeof(*pending_reclaims) * nr_writers); - if (reclaim_batch * sizeof(*pending_reclaims[i].queue) - < CAA_CACHE_LINE_SIZE) - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].queue = calloc(1, CAA_CACHE_LINE_SIZE); - else - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].queue = calloc(reclaim_batch, - sizeof(*pending_reclaims[i].queue)); - for (i = 0; i < nr_writers; i++) - pending_reclaims[i].head = pending_reclaims[i].queue; - - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += tot_nr_writes[i]; - rcu_gc_clear_queue(i); - } - - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu " - "batch %u\n", - argv[0], duration, nr_readers, rduration, wduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes, reclaim_batch); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(tot_nr_writes); - for (i = 0; i < nr_writers; i++) - free(pending_reclaims[i].queue); - free(pending_reclaims); - - return 0; -} diff --git a/tests/test_urcu_qsbr_timing.c b/tests/test_urcu_qsbr_timing.c deleted file mode 100644 index bb29301..0000000 --- a/tests/test_urcu_qsbr_timing.c +++ /dev/null @@ -1,225 +0,0 @@ -/* - * test_qsbr_timing.c - * - * Userspace QSBR - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "thread-id.h" - -#define _LGPL_SOURCE -#include - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -struct test_array { - int a; -}; - -static struct test_array *test_rcu_pointer; - -#define OUTER_READ_LOOP 2000U -#define INNER_READ_LOOP 100000U -#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) - -#define OUTER_WRITE_LOOP 10U -#define INNER_WRITE_LOOP 200U -#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) - -static int num_read; -static int num_write; - -#define NR_READ num_read -#define NR_WRITE num_write - -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; - -void *thr_reader(void *arg) -{ - int i, j; - struct test_array *local_ptr; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - sleep(2); - - rcu_register_thread(); - - time1 = caa_get_cycles(); - for (i = 0; i < OUTER_READ_LOOP; i++) { - for (j = 0; j < INNER_READ_LOOP; j++) { - _rcu_read_lock(); - local_ptr = _rcu_dereference(test_rcu_pointer); - if (local_ptr) { - assert(local_ptr->a == 8); - } - _rcu_read_unlock(); - } - _rcu_quiescent_state(); - } - time2 = caa_get_cycles(); - - rcu_unregister_thread(); - - reader_time[(unsigned long)arg] = time2 - time1; - - sleep(2); - printf("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *arg) -{ - int i, j; - struct test_array *new, *old; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - sleep(2); - - for (i = 0; i < OUTER_WRITE_LOOP; i++) { - for (j = 0; j < INNER_WRITE_LOOP; j++) { - time1 = caa_get_cycles(); - new = malloc(sizeof(struct test_array)); - rcu_copy_mutex_lock(); - old = test_rcu_pointer; - if (old) { - assert(old->a == 8); - } - new->a = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - rcu_copy_mutex_unlock(); - synchronize_rcu(); - /* can be done after unlock */ - if (old) { - old->a = 0; - } - free(old); - time2 = caa_get_cycles(); - writer_time[(unsigned long)arg] += time2 - time1; - usleep(1); - } - } - - printf("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - return ((void*)2); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - int i; - cycles_t tot_rtime = 0; - cycles_t tot_wtime = 0; - - if (argc < 2) { - printf("Usage : %s nr_readers nr_writers\n", argv[0]); - exit(-1); - } - num_read = atoi(argv[1]); - num_write = atoi(argv[2]); - - reader_time = malloc(sizeof(*reader_time) * num_read); - writer_time = malloc(sizeof(*writer_time) * num_write); - tid_reader = malloc(sizeof(*tid_reader) * num_read); - tid_writer = malloc(sizeof(*tid_writer) * num_write); - - printf("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - for (i = 0; i < NR_READ; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - sleep(10); - - for (i = 0; i < NR_READ; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_rtime += reader_time[i]; - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_wtime += writer_time[i]; - } - free(test_rcu_pointer); - printf("Time per read : %g cycles\n", - (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); - printf("Time per write : %g cycles\n", - (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); - - free(reader_time); - free(writer_time); - free(tid_reader); - free(tid_writer); - - return 0; -} diff --git a/tests/test_urcu_timing.c b/tests/test_urcu_timing.c deleted file mode 100644 index 4fbdcca..0000000 --- a/tests/test_urcu_timing.c +++ /dev/null @@ -1,224 +0,0 @@ -/* - * test_urcu.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "thread-id.h" - -#define _LGPL_SOURCE -#include - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -struct test_array { - int a; -}; - -static struct test_array *test_rcu_pointer; - -#define OUTER_READ_LOOP 2000U -#define INNER_READ_LOOP 100000U -#define READ_LOOP ((unsigned long long)OUTER_READ_LOOP * INNER_READ_LOOP) - -#define OUTER_WRITE_LOOP 10U -#define INNER_WRITE_LOOP 200U -#define WRITE_LOOP ((unsigned long long)OUTER_WRITE_LOOP * INNER_WRITE_LOOP) - -static int num_read; -static int num_write; - -#define NR_READ num_read -#define NR_WRITE num_write - -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *reader_time; -static cycles_t __attribute__((aligned(CAA_CACHE_LINE_SIZE))) *writer_time; - -void *thr_reader(void *arg) -{ - int i, j; - struct test_array *local_ptr; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "reader", urcu_get_thread_id()); - sleep(2); - - rcu_register_thread(); - - time1 = caa_get_cycles(); - for (i = 0; i < OUTER_READ_LOOP; i++) { - for (j = 0; j < INNER_READ_LOOP; j++) { - rcu_read_lock(); - local_ptr = rcu_dereference(test_rcu_pointer); - if (local_ptr) { - assert(local_ptr->a == 8); - } - rcu_read_unlock(); - } - } - time2 = caa_get_cycles(); - - rcu_unregister_thread(); - - reader_time[(unsigned long)arg] = time2 - time1; - - sleep(2); - printf("thread_end %s, tid %lu\n", - "reader", urcu_get_thread_id()); - return ((void*)1); - -} - -void *thr_writer(void *arg) -{ - int i, j; - struct test_array *new, *old; - cycles_t time1, time2; - - printf("thread_begin %s, tid %lu\n", - "writer", urcu_get_thread_id()); - sleep(2); - - for (i = 0; i < OUTER_WRITE_LOOP; i++) { - for (j = 0; j < INNER_WRITE_LOOP; j++) { - time1 = caa_get_cycles(); - new = malloc(sizeof(struct test_array)); - rcu_copy_mutex_lock(); - old = test_rcu_pointer; - if (old) { - assert(old->a == 8); - } - new->a = 8; - old = rcu_xchg_pointer(&test_rcu_pointer, new); - rcu_copy_mutex_unlock(); - synchronize_rcu(); - /* can be done after unlock */ - if (old) { - old->a = 0; - } - free(old); - time2 = caa_get_cycles(); - writer_time[(unsigned long)arg] += time2 - time1; - usleep(1); - } - } - - printf("thread_end %s, tid %lu\n", - "writer", urcu_get_thread_id()); - return ((void*)2); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - int i; - cycles_t tot_rtime = 0; - cycles_t tot_wtime = 0; - - if (argc < 2) { - printf("Usage : %s nr_readers nr_writers\n", argv[0]); - exit(-1); - } - num_read = atoi(argv[1]); - num_write = atoi(argv[2]); - - reader_time = malloc(sizeof(*reader_time) * num_read); - writer_time = malloc(sizeof(*writer_time) * num_write); - tid_reader = malloc(sizeof(*tid_reader) * num_read); - tid_writer = malloc(sizeof(*tid_writer) * num_write); - - printf("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - for (i = 0; i < NR_READ; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - (void *)(long)i); - if (err != 0) - exit(1); - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - (void *)(long)i); - if (err != 0) - exit(1); - } - - sleep(10); - - for (i = 0; i < NR_READ; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_rtime += reader_time[i]; - } - for (i = 0; i < NR_WRITE; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_wtime += writer_time[i]; - } - free(test_rcu_pointer); - printf("Time per read : %g cycles\n", - (double)tot_rtime / ((double)NR_READ * (double)READ_LOOP)); - printf("Time per write : %g cycles\n", - (double)tot_wtime / ((double)NR_WRITE * (double)WRITE_LOOP)); - - free(reader_time); - free(writer_time); - free(tid_reader); - free(tid_writer); - - return 0; -} diff --git a/tests/test_urcu_wfcq.c b/tests/test_urcu_wfcq.c deleted file mode 100644 index 5a36c76..0000000 --- a/tests/test_urcu_wfcq.c +++ /dev/null @@ -1,581 +0,0 @@ -/* - * test_urcu_wfcq.c - * - * Userspace RCU library - example RCU-based lock-free concurrent queue - * - * Copyright February 2010 - Mathieu Desnoyers - * Copyright February 2010 - Paolo Bonzini - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif -#include - -enum test_sync { - TEST_SYNC_NONE = 0, - TEST_SYNC_MUTEX, -}; - -static enum test_sync test_sync; - -static int test_force_sync; - -static volatile int test_go, test_stop_enqueue, test_stop_dequeue; - -static unsigned long rduration; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long wdelay; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -static int test_dequeue, test_splice, test_wait_empty; -static int test_enqueue_stopped; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, ## args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_dequeue(void) -{ - return !test_stop_dequeue; -} - -static int test_duration_enqueue(void) -{ - return !test_stop_enqueue; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); - -static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); -static DEFINE_URCU_TLS(unsigned long long, nr_empty_dest_enqueues); -static DEFINE_URCU_TLS(unsigned long long, nr_splice); -static DEFINE_URCU_TLS(unsigned long long, nr_dequeue_last); - -static unsigned int nr_enqueuers; -static unsigned int nr_dequeuers; - -static struct cds_wfcq_head __attribute__((aligned(CAA_CACHE_LINE_SIZE))) head; -static struct cds_wfcq_tail __attribute__((aligned(CAA_CACHE_LINE_SIZE))) tail; - -static void *thr_enqueuer(void *_count) -{ - unsigned long long *count = _count; - bool was_nonempty; - - printf_verbose("thread_begin %s, tid %lu\n", - "enqueuer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct cds_wfcq_node *node = malloc(sizeof(*node)); - if (!node) - goto fail; - cds_wfcq_node_init(node); - was_nonempty = cds_wfcq_enqueue(&head, &tail, node); - URCU_TLS(nr_successful_enqueues)++; - if (!was_nonempty) - URCU_TLS(nr_empty_dest_enqueues)++; - - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); -fail: - URCU_TLS(nr_enqueues)++; - if (caa_unlikely(!test_duration_enqueue())) - break; - } - - uatomic_inc(&test_enqueue_stopped); - count[0] = URCU_TLS(nr_enqueues); - count[1] = URCU_TLS(nr_successful_enqueues); - count[2] = URCU_TLS(nr_empty_dest_enqueues); - printf_verbose("enqueuer thread_end, tid %lu, " - "enqueues %llu successful_enqueues %llu, " - "empty_dest_enqueues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_enqueues), - URCU_TLS(nr_successful_enqueues), - URCU_TLS(nr_empty_dest_enqueues)); - return ((void*)1); - -} - -static void do_test_dequeue(enum test_sync sync) -{ - struct cds_wfcq_node *node; - int state; - - if (sync == TEST_SYNC_MUTEX) - node = cds_wfcq_dequeue_with_state_blocking(&head, &tail, - &state); - else - node = __cds_wfcq_dequeue_with_state_blocking(&head, &tail, - &state); - - if (state & CDS_WFCQ_STATE_LAST) - URCU_TLS(nr_dequeue_last)++; - - if (node) { - free(node); - URCU_TLS(nr_successful_dequeues)++; - } - URCU_TLS(nr_dequeues)++; -} - -static void do_test_splice(enum test_sync sync) -{ - struct cds_wfcq_head tmp_head; - struct cds_wfcq_tail tmp_tail; - struct cds_wfcq_node *node, *n; - enum cds_wfcq_ret ret; - - cds_wfcq_init(&tmp_head, &tmp_tail); - - if (sync == TEST_SYNC_MUTEX) - ret = cds_wfcq_splice_blocking(&tmp_head, &tmp_tail, - &head, &tail); - else - ret = __cds_wfcq_splice_blocking(&tmp_head, &tmp_tail, - &head, &tail); - - switch (ret) { - case CDS_WFCQ_RET_WOULDBLOCK: - assert(0); /* blocking call */ - break; - case CDS_WFCQ_RET_DEST_EMPTY: - URCU_TLS(nr_splice)++; - URCU_TLS(nr_dequeue_last)++; - /* ok */ - break; - case CDS_WFCQ_RET_DEST_NON_EMPTY: - assert(0); /* entirely unexpected */ - break; - case CDS_WFCQ_RET_SRC_EMPTY: - /* ok, we could even skip iteration on dest if we wanted */ - break; - } - - __cds_wfcq_for_each_blocking_safe(&tmp_head, &tmp_tail, node, n) { - free(node); - URCU_TLS(nr_successful_dequeues)++; - URCU_TLS(nr_dequeues)++; - } -} - -static void *thr_dequeuer(void *_count) -{ - unsigned long long *count = _count; - unsigned int counter = 0; - - printf_verbose("thread_begin %s, tid %lu\n", - "dequeuer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - if (test_dequeue && test_splice) { - if (counter & 1) - do_test_dequeue(test_sync); - else - do_test_splice(test_sync); - counter++; - } else { - if (test_dequeue) - do_test_dequeue(test_sync); - else - do_test_splice(test_sync); - } - if (caa_unlikely(!test_duration_dequeue())) - break; - if (caa_unlikely(rduration)) - loop_sleep(rduration); - } - - printf_verbose("dequeuer thread_end, tid %lu, " - "dequeues %llu, successful_dequeues %llu, " - "nr_splice %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues), - URCU_TLS(nr_splice)); - count[0] = URCU_TLS(nr_dequeues); - count[1] = URCU_TLS(nr_successful_dequeues); - count[2] = URCU_TLS(nr_splice); - count[3] = URCU_TLS(nr_dequeue_last); - return ((void*)2); -} - -static void test_end(unsigned long long *nr_dequeues, - unsigned long long *nr_dequeue_last) -{ - struct cds_wfcq_node *node; - int state; - - do { - node = cds_wfcq_dequeue_with_state_blocking(&head, &tail, - &state); - if (node) { - if (state & CDS_WFCQ_STATE_LAST) - (*nr_dequeue_last)++; - free(node); - (*nr_dequeues)++; - } - } while (node); -} - -static void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); - printf(" [-d delay] (enqueuer period (in loops))\n"); - printf(" [-c duration] (dequeuer period (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf(" [-q] (test dequeue)\n"); - printf(" [-s] (test splice, enabled by default)\n"); - printf(" [-M] (use mutex external synchronization)\n"); - printf(" Note: default: no external synchronization used.\n"); - printf(" [-f] (force user-provided synchronization)\n"); - printf(" [-w] Wait for dequeuer to empty queue\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_enqueuer, *tid_dequeuer; - void *tret; - unsigned long long *count_enqueuer, *count_dequeuer; - unsigned long long tot_enqueues = 0, tot_dequeues = 0; - unsigned long long tot_successful_enqueues = 0, - tot_successful_dequeues = 0, - tot_empty_dest_enqueues = 0, - tot_splice = 0, tot_dequeue_last = 0; - unsigned long long end_dequeues = 0; - int i, a, retval = 0; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_dequeuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_enqueuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - case 'q': - test_dequeue = 1; - break; - case 's': - test_splice = 1; - break; - case 'M': - test_sync = TEST_SYNC_MUTEX; - break; - case 'w': - test_wait_empty = 1; - break; - case 'f': - test_force_sync = 1; - break; - } - } - - /* activate splice test by default */ - if (!test_dequeue && !test_splice) - test_splice = 1; - - if (test_sync == TEST_SYNC_NONE && nr_dequeuers > 1 && test_dequeue) { - if (test_force_sync) { - fprintf(stderr, "[WARNING] Using dequeue concurrently " - "with other dequeue or splice without external " - "synchronization. Expect run-time failure.\n"); - } else { - printf("Enforcing mutex synchronization\n"); - test_sync = TEST_SYNC_MUTEX; - } - } - - printf_verbose("running test for %lu seconds, %u enqueuers, " - "%u dequeuers.\n", - duration, nr_enqueuers, nr_dequeuers); - if (test_dequeue) - printf_verbose("dequeue test activated.\n"); - else - printf_verbose("splice test activated.\n"); - if (test_sync == TEST_SYNC_MUTEX) - printf_verbose("External sync: mutex.\n"); - else - printf_verbose("External sync: none.\n"); - if (test_wait_empty) - printf_verbose("Wait for dequeuers to empty queue.\n"); - printf_verbose("Writer delay : %lu loops.\n", rduration); - printf_verbose("Reader duration : %lu loops.\n", wdelay); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); - tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); - count_enqueuer = malloc(3 * sizeof(*count_enqueuer) * nr_enqueuers); - count_dequeuer = malloc(4 * sizeof(*count_dequeuer) * nr_dequeuers); - cds_wfcq_init(&head, &tail); - - next_aff = 0; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, - &count_enqueuer[3 * i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, - &count_dequeuer[4 * i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - for (i = 0; i < duration; i++) { - sleep(1); - if (verbose_mode) - write (1, ".", 1); - } - - test_stop_enqueue = 1; - - if (test_wait_empty) { - while (nr_enqueuers != uatomic_read(&test_enqueue_stopped)) { - sleep(1); - } - while (!cds_wfcq_empty(&head, &tail)) { - sleep(1); - } - } - - test_stop_dequeue = 1; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_join(tid_enqueuer[i], &tret); - if (err != 0) - exit(1); - tot_enqueues += count_enqueuer[3 * i]; - tot_successful_enqueues += count_enqueuer[3 * i + 1]; - tot_empty_dest_enqueues += count_enqueuer[3 * i + 2]; - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_join(tid_dequeuer[i], &tret); - if (err != 0) - exit(1); - tot_dequeues += count_dequeuer[4 * i]; - tot_successful_dequeues += count_dequeuer[4 * i + 1]; - tot_splice += count_dequeuer[4 * i + 2]; - tot_dequeue_last += count_dequeuer[4 * i + 3]; - } - - test_end(&end_dequeues, &tot_dequeue_last); - - printf_verbose("total number of enqueues : %llu, dequeues %llu\n", - tot_enqueues, tot_dequeues); - printf_verbose("total number of successful enqueues : %llu, " - "enqueues to empty dest : %llu, " - "successful dequeues %llu, " - "splice : %llu, dequeue_last : %llu\n", - tot_successful_enqueues, - tot_empty_dest_enqueues, - tot_successful_dequeues, - tot_splice, tot_dequeue_last); - printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " - "nr_dequeuers %3u " - "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " - "successful enqueues %12llu enqueues to empty dest %12llu " - "successful dequeues %12llu splice %12llu " - "dequeue_last %llu " - "end_dequeues %llu nr_ops %12llu\n", - argv[0], duration, nr_enqueuers, wdelay, - nr_dequeuers, rduration, tot_enqueues, tot_dequeues, - tot_successful_enqueues, - tot_empty_dest_enqueues, - tot_successful_dequeues, tot_splice, tot_dequeue_last, - end_dequeues, - tot_enqueues + tot_dequeues); - - if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) { - printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " - "succ. dequeues + end dequeues %llu.\n", - tot_successful_enqueues, - tot_successful_dequeues + end_dequeues); - retval = 1; - } - - /* - * If only using splice to dequeue, the enqueuer should see - * exactly as many empty queues than the number of non-empty - * src splice. - */ - if (tot_empty_dest_enqueues != tot_dequeue_last) { - printf("WARNING! Discrepancy between empty enqueue (%llu) and " - "number of dequeue of last element (%llu)\n", - tot_empty_dest_enqueues, - tot_dequeue_last); - retval = 1; - } - free(count_enqueuer); - free(count_dequeuer); - free(tid_enqueuer); - free(tid_dequeuer); - return retval; -} diff --git a/tests/test_urcu_wfq.c b/tests/test_urcu_wfq.c deleted file mode 100644 index db00d8c..0000000 --- a/tests/test_urcu_wfq.c +++ /dev/null @@ -1,401 +0,0 @@ -/* - * test_urcu_wfq.c - * - * Userspace RCU library - example RCU-based lock-free queue - * - * Copyright February 2010 - Mathieu Desnoyers - * Copyright February 2010 - Paolo Bonzini - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif - -/* Remove deprecation warnings from test build. */ -#define CDS_WFQ_DEPRECATED - -#include -#include - -static volatile int test_go, test_stop; - -static unsigned long rduration; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long wdelay; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_dequeue(void) -{ - return !test_stop; -} - -static int test_duration_enqueue(void) -{ - return !test_stop; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); - -static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); - -static unsigned int nr_enqueuers; -static unsigned int nr_dequeuers; - -static struct cds_wfq_queue q; - -void *thr_enqueuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "enqueuer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct cds_wfq_node *node = malloc(sizeof(*node)); - if (!node) - goto fail; - cds_wfq_node_init(node); - cds_wfq_enqueue(&q, node); - URCU_TLS(nr_successful_enqueues)++; - - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); -fail: - URCU_TLS(nr_enqueues)++; - if (caa_unlikely(!test_duration_enqueue())) - break; - } - - count[0] = URCU_TLS(nr_enqueues); - count[1] = URCU_TLS(nr_successful_enqueues); - printf_verbose("enqueuer thread_end, tid %lu, " - "enqueues %llu successful_enqueues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_enqueues), - URCU_TLS(nr_successful_enqueues)); - return ((void*)1); - -} - -void *thr_dequeuer(void *_count) -{ - unsigned long long *count = _count; - - printf_verbose("thread_begin %s, tid %lu\n", - "dequeuer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct cds_wfq_node *node = cds_wfq_dequeue_blocking(&q); - - if (node) { - free(node); - URCU_TLS(nr_successful_dequeues)++; - } - - URCU_TLS(nr_dequeues)++; - if (caa_unlikely(!test_duration_dequeue())) - break; - if (caa_unlikely(rduration)) - loop_sleep(rduration); - } - - printf_verbose("dequeuer thread_end, tid %lu, " - "dequeues %llu, successful_dequeues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_dequeues), - URCU_TLS(nr_successful_dequeues)); - count[0] = URCU_TLS(nr_dequeues); - count[1] = URCU_TLS(nr_successful_dequeues); - return ((void*)2); -} - -void test_end(struct cds_wfq_queue *q, unsigned long long *nr_dequeues) -{ - struct cds_wfq_node *node; - - do { - node = cds_wfq_dequeue_blocking(q); - if (node) { - free(node); - (*nr_dequeues)++; - } - } while (node); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); - printf(" [-d delay] (enqueuer period (in loops))\n"); - printf(" [-c duration] (dequeuer period (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_enqueuer, *tid_dequeuer; - void *tret; - unsigned long long *count_enqueuer, *count_dequeuer; - unsigned long long tot_enqueues = 0, tot_dequeues = 0; - unsigned long long tot_successful_enqueues = 0, - tot_successful_dequeues = 0; - unsigned long long end_dequeues = 0; - int i, a; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_dequeuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_enqueuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u enqueuers, " - "%u dequeuers.\n", - duration, nr_enqueuers, nr_dequeuers); - printf_verbose("Writer delay : %lu loops.\n", rduration); - printf_verbose("Reader duration : %lu loops.\n", wdelay); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); - tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); - count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers); - count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers); - cds_wfq_init(&q); - - next_aff = 0; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, - &count_enqueuer[2 * i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, - &count_dequeuer[2 * i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - for (i = 0; i < duration; i++) { - sleep(1); - if (verbose_mode) - write (1, ".", 1); - } - - test_stop = 1; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_join(tid_enqueuer[i], &tret); - if (err != 0) - exit(1); - tot_enqueues += count_enqueuer[2 * i]; - tot_successful_enqueues += count_enqueuer[2 * i + 1]; - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_join(tid_dequeuer[i], &tret); - if (err != 0) - exit(1); - tot_dequeues += count_dequeuer[2 * i]; - tot_successful_dequeues += count_dequeuer[2 * i + 1]; - } - - test_end(&q, &end_dequeues); - - printf_verbose("total number of enqueues : %llu, dequeues %llu\n", - tot_enqueues, tot_dequeues); - printf_verbose("total number of successful enqueues : %llu, " - "successful dequeues %llu\n", - tot_successful_enqueues, tot_successful_dequeues); - printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " - "nr_dequeuers %3u " - "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " - "successful enqueues %12llu successful dequeues %12llu " - "end_dequeues %llu nr_ops %12llu\n", - argv[0], duration, nr_enqueuers, wdelay, - nr_dequeuers, rduration, tot_enqueues, tot_dequeues, - tot_successful_enqueues, - tot_successful_dequeues, end_dequeues, - tot_enqueues + tot_dequeues); - if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) - printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " - "succ. dequeues + end dequeues %llu.\n", - tot_successful_enqueues, - tot_successful_dequeues + end_dequeues); - - free(count_enqueuer); - free(count_dequeuer); - free(tid_enqueuer); - free(tid_dequeuer); - return 0; -} diff --git a/tests/test_urcu_wfs.c b/tests/test_urcu_wfs.c deleted file mode 100644 index 2a031fe..0000000 --- a/tests/test_urcu_wfs.c +++ /dev/null @@ -1,565 +0,0 @@ -/* - * test_urcu_wfs.c - * - * Userspace RCU library - example RCU-based lock-free stack - * - * Copyright February 2010 - Mathieu Desnoyers - * Copyright February 2010 - Paolo Bonzini - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include "../config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include "cpuset.h" -#include "thread-id.h" - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#endif -#include - -/* - * External synchronization used. - */ -enum test_sync { - TEST_SYNC_NONE = 0, - TEST_SYNC_MUTEX, -}; - -static enum test_sync test_sync; - -static int test_force_sync; - -static volatile int test_go, test_stop_enqueue, test_stop_dequeue; - -static unsigned long rduration; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long wdelay; - -static inline void loop_sleep(unsigned long loops) -{ - while (loops-- != 0) - caa_cpu_relax(); -} - -static int verbose_mode; - -static int test_pop, test_pop_all, test_wait_empty; -static int test_enqueue_stopped; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, ## args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ -#if HAVE_SCHED_SETAFFINITY - cpu_set_t mask; - int cpu, ret; -#endif /* HAVE_SCHED_SETAFFINITY */ - - if (!use_affinity) - return; - -#if HAVE_SCHED_SETAFFINITY - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); -#if SCHED_SETAFFINITY_ARGS == 2 - sched_setaffinity(0, &mask); -#else - sched_setaffinity(0, sizeof(mask), &mask); -#endif -#endif /* HAVE_SCHED_SETAFFINITY */ -} - -/* - * returns 0 if test should end. - */ -static int test_duration_dequeue(void) -{ - return !test_stop_dequeue; -} - -static int test_duration_enqueue(void) -{ - return !test_stop_enqueue; -} - -static DEFINE_URCU_TLS(unsigned long long, nr_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_enqueues); - -static DEFINE_URCU_TLS(unsigned long long, nr_successful_dequeues); -static DEFINE_URCU_TLS(unsigned long long, nr_successful_enqueues); -static DEFINE_URCU_TLS(unsigned long long, nr_empty_dest_enqueues); -static DEFINE_URCU_TLS(unsigned long long, nr_pop_all); -static DEFINE_URCU_TLS(unsigned long long, nr_pop_last); - -static unsigned int nr_enqueuers; -static unsigned int nr_dequeuers; - -static struct cds_wfs_stack s; - -static void *thr_enqueuer(void *_count) -{ - unsigned long long *count = _count; - bool was_nonempty; - - printf_verbose("thread_begin %s, tid %lu\n", - "enqueuer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - for (;;) { - struct cds_wfs_node *node = malloc(sizeof(*node)); - if (!node) - goto fail; - cds_wfs_node_init(node); - was_nonempty = cds_wfs_push(&s, node); - URCU_TLS(nr_successful_enqueues)++; - if (!was_nonempty) - URCU_TLS(nr_empty_dest_enqueues)++; - - if (caa_unlikely(wdelay)) - loop_sleep(wdelay); -fail: - URCU_TLS(nr_enqueues)++; - if (caa_unlikely(!test_duration_enqueue())) - break; - } - - uatomic_inc(&test_enqueue_stopped); - count[0] = URCU_TLS(nr_enqueues); - count[1] = URCU_TLS(nr_successful_enqueues); - count[2] = URCU_TLS(nr_empty_dest_enqueues); - printf_verbose("enqueuer thread_end, tid %lu, " - "enqueues %llu successful_enqueues %llu, " - "empty_dest_enqueues %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_enqueues), - URCU_TLS(nr_successful_enqueues), - URCU_TLS(nr_empty_dest_enqueues)); - return ((void*)1); - -} - -static void do_test_pop(enum test_sync sync) -{ - struct cds_wfs_node *node; - int state; - - if (sync == TEST_SYNC_MUTEX) - cds_wfs_pop_lock(&s); - node = __cds_wfs_pop_with_state_blocking(&s, &state); - if (sync == TEST_SYNC_MUTEX) - cds_wfs_pop_unlock(&s); - - if (node) { - if (state & CDS_WFS_STATE_LAST) - URCU_TLS(nr_pop_last)++; - free(node); - URCU_TLS(nr_successful_dequeues)++; - } - URCU_TLS(nr_dequeues)++; -} - -static void do_test_pop_all(enum test_sync sync) -{ - struct cds_wfs_head *head; - struct cds_wfs_node *node, *n; - - if (sync == TEST_SYNC_MUTEX) - cds_wfs_pop_lock(&s); - head = __cds_wfs_pop_all(&s); - if (sync == TEST_SYNC_MUTEX) - cds_wfs_pop_unlock(&s); - - /* Check if empty */ - if (cds_wfs_first(head) == NULL) - return; - - URCU_TLS(nr_pop_all)++; - URCU_TLS(nr_pop_last)++; - - cds_wfs_for_each_blocking_safe(head, node, n) { - free(node); - URCU_TLS(nr_successful_dequeues)++; - URCU_TLS(nr_dequeues)++; - } -} - -static void *thr_dequeuer(void *_count) -{ - unsigned long long *count = _count; - unsigned int counter = 0; - - printf_verbose("thread_begin %s, tid %lu\n", - "dequeuer", urcu_get_thread_id()); - - set_affinity(); - - while (!test_go) - { - } - cmm_smp_mb(); - - assert(test_pop || test_pop_all); - - for (;;) { - if (test_pop && test_pop_all) { - if (counter & 1) - do_test_pop(test_sync); - else - do_test_pop_all(test_sync); - counter++; - } else { - if (test_pop) - do_test_pop(test_sync); - else - do_test_pop_all(test_sync); - } - - if (caa_unlikely(!test_duration_dequeue())) - break; - if (caa_unlikely(rduration)) - loop_sleep(rduration); - } - - printf_verbose("dequeuer thread_end, tid %lu, " - "dequeues %llu, successful_dequeues %llu " - "pop_all %llu pop_last %llu\n", - urcu_get_thread_id(), - URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues), - URCU_TLS(nr_pop_all), - URCU_TLS(nr_pop_last)); - count[0] = URCU_TLS(nr_dequeues); - count[1] = URCU_TLS(nr_successful_dequeues); - count[2] = URCU_TLS(nr_pop_all); - count[3] = URCU_TLS(nr_pop_last); - return ((void*)2); -} - -static void test_end(struct cds_wfs_stack *s, unsigned long long *nr_dequeues, - unsigned long long *nr_pop_last) -{ - struct cds_wfs_node *node; - int state; - - do { - node = cds_wfs_pop_with_state_blocking(s, &state); - if (node) { - if (state & CDS_WFS_STATE_LAST) - (*nr_pop_last)++; - free(node); - (*nr_dequeues)++; - } - } while (node); -} - -static void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_dequeuers nr_enqueuers duration (s) \n", - argv[0]); - printf("OPTIONS:\n"); - printf(" [-d delay] (enqueuer period (in loops))\n"); - printf(" [-c duration] (dequeuer period (in loops))\n"); - printf(" [-v] (verbose output)\n"); - printf(" [-a cpu#] [-a cpu#]... (affinity)\n"); - printf(" [-p] (test pop)\n"); - printf(" [-P] (test pop_all, enabled by default)\n"); - printf(" [-M] (use mutex external synchronization)\n"); - printf(" Note: default: no external synchronization used.\n"); - printf(" [-f] (force user-provided synchronization)\n"); - printf(" [-w] Wait for dequeuer to empty stack\n"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_enqueuer, *tid_dequeuer; - void *tret; - unsigned long long *count_enqueuer, *count_dequeuer; - unsigned long long tot_enqueues = 0, tot_dequeues = 0; - unsigned long long tot_successful_enqueues = 0, - tot_successful_dequeues = 0, - tot_empty_dest_enqueues = 0, - tot_pop_all = 0, tot_pop_last = 0; - unsigned long long end_dequeues = 0; - int i, a, retval = 0; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_dequeuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_enqueuers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - case 'p': - test_pop = 1; - break; - case 'P': - test_pop_all = 1; - break; - case 'M': - test_sync = TEST_SYNC_MUTEX; - break; - case 'w': - test_wait_empty = 1; - break; - case 'f': - test_force_sync = 1; - break; - } - } - - /* activate pop_all test by default */ - if (!test_pop && !test_pop_all) - test_pop_all = 1; - - if (test_sync == TEST_SYNC_NONE && nr_dequeuers > 1 && test_pop) { - if (test_force_sync) { - fprintf(stderr, "[WARNING] Using pop concurrently " - "with other pop or pop_all without external " - "synchronization. Expect run-time failure.\n"); - } else { - printf("Enforcing mutex synchronization\n"); - test_sync = TEST_SYNC_MUTEX; - } - } - - printf_verbose("running test for %lu seconds, %u enqueuers, " - "%u dequeuers.\n", - duration, nr_enqueuers, nr_dequeuers); - if (test_pop) - printf_verbose("pop test activated.\n"); - if (test_pop_all) - printf_verbose("pop_all test activated.\n"); - if (test_sync == TEST_SYNC_MUTEX) - printf_verbose("External sync: mutex.\n"); - else - printf_verbose("External sync: none.\n"); - if (test_wait_empty) - printf_verbose("Wait for dequeuers to empty stack.\n"); - printf_verbose("Writer delay : %lu loops.\n", rduration); - printf_verbose("Reader duration : %lu loops.\n", wdelay); - printf_verbose("thread %-6s, tid %lu\n", - "main", urcu_get_thread_id()); - - tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers); - tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers); - count_enqueuer = malloc(3 * sizeof(*count_enqueuer) * nr_enqueuers); - count_dequeuer = malloc(4 * sizeof(*count_dequeuer) * nr_dequeuers); - cds_wfs_init(&s); - - next_aff = 0; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer, - &count_enqueuer[3 * i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer, - &count_dequeuer[4 * i]); - if (err != 0) - exit(1); - } - - cmm_smp_mb(); - - test_go = 1; - - for (i = 0; i < duration; i++) { - sleep(1); - if (verbose_mode) - write (1, ".", 1); - } - - test_stop_enqueue = 1; - - if (test_wait_empty) { - while (nr_enqueuers != uatomic_read(&test_enqueue_stopped)) { - sleep(1); - } - while (!cds_wfs_empty(&s)) { - sleep(1); - } - } - - test_stop_dequeue = 1; - - for (i = 0; i < nr_enqueuers; i++) { - err = pthread_join(tid_enqueuer[i], &tret); - if (err != 0) - exit(1); - tot_enqueues += count_enqueuer[3 * i]; - tot_successful_enqueues += count_enqueuer[3 * i + 1]; - tot_empty_dest_enqueues += count_enqueuer[3 * i + 2]; - } - for (i = 0; i < nr_dequeuers; i++) { - err = pthread_join(tid_dequeuer[i], &tret); - if (err != 0) - exit(1); - tot_dequeues += count_dequeuer[4 * i]; - tot_successful_dequeues += count_dequeuer[4 * i + 1]; - tot_pop_all += count_dequeuer[4 * i + 2]; - tot_pop_last += count_dequeuer[4 * i + 3]; - } - - test_end(&s, &end_dequeues, &tot_pop_last); - - printf_verbose("total number of enqueues : %llu, dequeues %llu\n", - tot_enqueues, tot_dequeues); - printf_verbose("total number of successful enqueues : %llu, " - "enqueues to empty dest : %llu, " - "successful dequeues %llu, " - "pop_all : %llu, pop_last : %llu\n", - tot_successful_enqueues, - tot_empty_dest_enqueues, - tot_successful_dequeues, - tot_pop_all, tot_pop_last); - printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu " - "nr_dequeuers %3u " - "rdur %6lu nr_enqueues %12llu nr_dequeues %12llu " - "successful enqueues %12llu enqueues to empty dest %12llu " - "successful dequeues %12llu pop_all %12llu " - "pop_last %llu end_dequeues %llu nr_ops %12llu\n", - argv[0], duration, nr_enqueuers, wdelay, - nr_dequeuers, rduration, tot_enqueues, tot_dequeues, - tot_successful_enqueues, - tot_empty_dest_enqueues, - tot_successful_dequeues, tot_pop_all, tot_pop_last, - end_dequeues, - tot_enqueues + tot_dequeues); - if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues) { - printf("WARNING! Discrepancy between nr succ. enqueues %llu vs " - "succ. dequeues + end dequeues %llu.\n", - tot_successful_enqueues, - tot_successful_dequeues + end_dequeues); - retval = 1; - } - /* - * The enqueuer should see exactly as many empty queues than the - * number of non-empty stacks dequeued. - */ - if (tot_empty_dest_enqueues != tot_pop_last) { - printf("WARNING! Discrepancy between empty enqueue (%llu) and " - "number of pop last (%llu)\n", - tot_empty_dest_enqueues, - tot_pop_last); - retval = 1; - } - free(count_enqueuer); - free(count_dequeuer); - free(tid_enqueuer); - free(tid_dequeuer); - return retval; -} diff --git a/tests/thread-id.h b/tests/thread-id.h deleted file mode 100644 index 9378edc..0000000 --- a/tests/thread-id.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef _TEST_THREAD_ID_H -#define _TEST_THREAD_ID_H - -/* - * thread-id.h - * - * Userspace RCU library - thread ID - * - * Copyright 2013 - Mathieu Desnoyers - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef __linux__ -# include - -# if defined(_syscall0) -_syscall0(pid_t, gettid) -# elif defined(__NR_gettid) -static inline pid_t gettid(void) -{ - return syscall(__NR_gettid); -} -# endif - -static inline -unsigned long urcu_get_thread_id(void) -{ - return (unsigned long) gettid(); -} -#elif defined(__FreeBSD__) -# include - -static inline -unsigned long urcu_get_thread_id(void) -{ - return (unsigned long) pthread_getthreadid_np(); -} -#else -# warning "use pid as thread ID" -static inline -unsigned long urcu_get_thread_id(void) -{ - return (unsigned long) getpid(); -} -#endif - -#endif /* _TEST_THREAD_ID_H */ diff --git a/tests/unit/Makefile.am b/tests/unit/Makefile.am new file mode 100644 index 0000000..8cc4acb --- /dev/null +++ b/tests/unit/Makefile.am @@ -0,0 +1,45 @@ +if !LIBC_INCLUDES_PTHREAD +AM_LDFLAGS=-lpthread +endif +AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/common -g + +noinst_PROGRAMS = test_uatomic \ + test_urcu_multiflavor \ + test_urcu_multiflavor_dynlink + +noinst_HEADERS = test_urcu_multiflavor.h + +URCU_COMMON_LIB=$(top_builddir)/liburcu-common.la +URCU_LIB=$(top_builddir)/liburcu.la +URCU_QSBR_LIB=$(top_builddir)/liburcu-qsbr.la +URCU_MB_LIB=$(top_builddir)/liburcu-mb.la +URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la +URCU_BP_LIB=$(top_builddir)/liburcu-bp.la +URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la + +test_uatomic_SOURCES = test_uatomic.c +test_uatomic_LDADD = $(URCU_COMMON_LIB) + +test_urcu_multiflavor_SOURCES = test_urcu_multiflavor.c \ + test_urcu_multiflavor-memb.c \ + test_urcu_multiflavor-mb.c \ + test_urcu_multiflavor-signal.c \ + test_urcu_multiflavor-qsbr.c \ + test_urcu_multiflavor-bp.c +test_urcu_multiflavor_LDADD = $(URCU_LIB) $(URCU_MB_LIB) \ + $(URCU_SIGNAL_LIB) $(URCU_QSBR_LIB) $(URCU_BP_LIB) + +test_urcu_multiflavor_dynlink_SOURCES = test_urcu_multiflavor.c \ + test_urcu_multiflavor-memb.c \ + test_urcu_multiflavor-mb.c \ + test_urcu_multiflavor-signal.c \ + test_urcu_multiflavor-qsbr.c \ + test_urcu_multiflavor-bp.c +test_urcu_multiflavor_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) +test_urcu_multiflavor_dynlink_LDADD = $(URCU_LIB) $(URCU_MB_LIB) \ + $(URCU_SIGNAL_LIB) $(URCU_QSBR_LIB) $(URCU_BP_LIB) + +check-am: + ./test_uatomic + ./test_urcu_multiflavor + ./test_urcu_multiflavor_dynlink diff --git a/tests/unit/test_uatomic.c b/tests/unit/test_uatomic.c new file mode 100644 index 0000000..804ce7b --- /dev/null +++ b/tests/unit/test_uatomic.c @@ -0,0 +1,91 @@ +/* + * test_uatomic.c + * + * Userspace RCU library - test atomic operations + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include +#include + +struct testvals { +#ifdef UATOMIC_HAS_ATOMIC_BYTE + unsigned char c; +#endif +#ifdef UATOMIC_HAS_ATOMIC_SHORT + unsigned short s; +#endif + unsigned int i; + unsigned long l; +}; + +static struct testvals vals; + +#define do_test(ptr) \ +do { \ + __typeof__(*(ptr)) v; \ + \ + uatomic_add(ptr, 10); \ + assert(uatomic_read(ptr) == 10); \ + uatomic_add(ptr, -11UL); \ + assert(uatomic_read(ptr) == (__typeof__(*(ptr)))-1UL); \ + v = uatomic_cmpxchg(ptr, -1UL, 22); \ + assert(uatomic_read(ptr) == 22); \ + assert(v == (__typeof__(*(ptr)))-1UL); \ + v = uatomic_cmpxchg(ptr, 33, 44); \ + assert(uatomic_read(ptr) == 22); \ + assert(v == 22); \ + v = uatomic_xchg(ptr, 55); \ + assert(uatomic_read(ptr) == 55); \ + assert(v == 22); \ + uatomic_set(ptr, 22); \ + uatomic_inc(ptr); \ + assert(uatomic_read(ptr) == 23); \ + uatomic_dec(ptr); \ + assert(uatomic_read(ptr) == 22); \ + v = uatomic_add_return(ptr, 74); \ + assert(v == 96); \ + assert(uatomic_read(ptr) == 96); \ + uatomic_or(ptr, 58); \ + assert(uatomic_read(ptr) == 122); \ + v = uatomic_sub_return(ptr, 1); \ + assert(v == 121); \ + uatomic_sub(ptr, (unsigned int) 2); \ + assert(uatomic_read(ptr) == 119); \ + uatomic_inc(ptr); \ + uatomic_inc(ptr); \ + assert(uatomic_read(ptr) == 121); \ + uatomic_and(ptr, 129); \ + assert(uatomic_read(ptr) == 1); \ +} while (0) + +int main(int argc, char **argv) +{ +#ifdef UATOMIC_HAS_ATOMIC_BYTE + do_test(&vals.c); +#endif +#ifdef UATOMIC_HAS_ATOMIC_SHORT + do_test(&vals.s); +#endif + do_test(&vals.i); + do_test(&vals.l); + printf("Atomic ops test OK\n"); + + return 0; +} diff --git a/tests/unit/test_urcu_multiflavor-bp.c b/tests/unit/test_urcu_multiflavor-bp.c new file mode 100644 index 0000000..199818b --- /dev/null +++ b/tests/unit/test_urcu_multiflavor-bp.c @@ -0,0 +1,39 @@ +/* + * test_urcu_multiflavor-bp.c + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif + +#define RCU_SIGNAL +#include +#include "test_urcu_multiflavor.h" + +int test_mf_bp(void) +{ + rcu_register_thread(); + rcu_read_lock(); + rcu_read_unlock(); + synchronize_rcu(); + rcu_unregister_thread(); + return 0; +} diff --git a/tests/unit/test_urcu_multiflavor-mb.c b/tests/unit/test_urcu_multiflavor-mb.c new file mode 100644 index 0000000..e4ba6a4 --- /dev/null +++ b/tests/unit/test_urcu_multiflavor-mb.c @@ -0,0 +1,39 @@ +/* + * test_urcu_multiflavor-mb.c + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif + +#define RCU_MB +#include +#include "test_urcu_multiflavor.h" + +int test_mf_mb(void) +{ + rcu_register_thread(); + rcu_read_lock(); + rcu_read_unlock(); + synchronize_rcu(); + rcu_unregister_thread(); + return 0; +} diff --git a/tests/unit/test_urcu_multiflavor-memb.c b/tests/unit/test_urcu_multiflavor-memb.c new file mode 100644 index 0000000..583f2e1 --- /dev/null +++ b/tests/unit/test_urcu_multiflavor-memb.c @@ -0,0 +1,38 @@ +/* + * test_urcu_multiflavor-memb.c + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif + +#include +#include "test_urcu_multiflavor.h" + +int test_mf_memb(void) +{ + rcu_register_thread(); + rcu_read_lock(); + rcu_read_unlock(); + synchronize_rcu(); + rcu_unregister_thread(); + return 0; +} diff --git a/tests/unit/test_urcu_multiflavor-qsbr.c b/tests/unit/test_urcu_multiflavor-qsbr.c new file mode 100644 index 0000000..64f32f4 --- /dev/null +++ b/tests/unit/test_urcu_multiflavor-qsbr.c @@ -0,0 +1,38 @@ +/* + * test_urcu_multiflavor-qsbr.c + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif + +#include +#include "test_urcu_multiflavor.h" + +int test_mf_qsbr(void) +{ + rcu_register_thread(); + rcu_read_lock(); + rcu_read_unlock(); + synchronize_rcu(); + rcu_unregister_thread(); + return 0; +} diff --git a/tests/unit/test_urcu_multiflavor-signal.c b/tests/unit/test_urcu_multiflavor-signal.c new file mode 100644 index 0000000..816c615 --- /dev/null +++ b/tests/unit/test_urcu_multiflavor-signal.c @@ -0,0 +1,39 @@ +/* + * test_urcu_multiflavor-signal.c + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#endif + +#define RCU_SIGNAL +#include +#include "test_urcu_multiflavor.h" + +int test_mf_signal(void) +{ + rcu_register_thread(); + rcu_read_lock(); + rcu_read_unlock(); + synchronize_rcu(); + rcu_unregister_thread(); + return 0; +} diff --git a/tests/unit/test_urcu_multiflavor.c b/tests/unit/test_urcu_multiflavor.c new file mode 100644 index 0000000..cd573ca --- /dev/null +++ b/tests/unit/test_urcu_multiflavor.c @@ -0,0 +1,50 @@ +/* + * test_urcu_multiflavor.c + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include +#include "test_urcu_multiflavor.h" + +int main(int argc, char **argv) +{ + int ret; + + ret = test_mf_memb(); + if (ret) + goto failure; + ret = test_mf_mb(); + if (ret) + goto failure; + ret = test_mf_signal(); + if (ret) + goto failure; + ret = test_mf_qsbr(); + if (ret) + goto failure; + ret = test_mf_bp(); + if (ret) + goto failure; + + exit(EXIT_SUCCESS); + +failure: + exit(EXIT_FAILURE); +} diff --git a/tests/unit/test_urcu_multiflavor.h b/tests/unit/test_urcu_multiflavor.h new file mode 100644 index 0000000..419ad5d --- /dev/null +++ b/tests/unit/test_urcu_multiflavor.h @@ -0,0 +1,29 @@ +/* + * test_urcu_multiflavor.h + * + * Userspace RCU library - test multiple RCU flavors into one program + * + * Copyright February 2012 - Mathieu Desnoyers + * Copyright February 2012 - Lai Jiangshan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +extern int test_mf_memb(void); +extern int test_mf_mb(void); +extern int test_mf_signal(void); +extern int test_mf_qsbr(void); +extern int test_mf_bp(void); + diff --git a/tests/unit/urcu-asm.c b/tests/unit/urcu-asm.c new file mode 100644 index 0000000..b616d34 --- /dev/null +++ b/tests/unit/urcu-asm.c @@ -0,0 +1,37 @@ +/* + * urcu-asm.c + * + * Userspace RCU library - assembly dump of primitives + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include + +void show_read_lock(void) +{ + asm volatile ("/* start */"); + rcu_read_lock(); + asm volatile ("/* end */"); +} + +void show_read_unlock(void) +{ + asm volatile ("/* start */"); + rcu_read_unlock(); + asm volatile ("/* end */"); +} diff --git a/tests/urcu-asm.c b/tests/urcu-asm.c deleted file mode 100644 index b616d34..0000000 --- a/tests/urcu-asm.c +++ /dev/null @@ -1,37 +0,0 @@ -/* - * urcu-asm.c - * - * Userspace RCU library - assembly dump of primitives - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include - -void show_read_lock(void) -{ - asm volatile ("/* start */"); - rcu_read_lock(); - asm volatile ("/* end */"); -} - -void show_read_unlock(void) -{ - asm volatile ("/* start */"); - rcu_read_unlock(); - asm volatile ("/* end */"); -} diff --git a/tests/urcutorture.c b/tests/urcutorture.c deleted file mode 100644 index 35096a6..0000000 --- a/tests/urcutorture.c +++ /dev/null @@ -1,30 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include "api.h" -#define _LGPL_SOURCE - -#ifdef RCU_MEMBARRIER -#include -#endif -#ifdef RCU_SIGNAL -#include -#endif -#ifdef RCU_MB -#include -#endif -#ifdef RCU_QSBR -#include -#endif -#ifdef RCU_BP -#include -#endif - -#include -#include -#include "rcutorture.h" diff --git a/urcu-bp.c b/urcu-bp.c index a823659..4dc4028 100644 --- a/urcu-bp.c +++ b/urcu-bp.c @@ -66,48 +66,59 @@ void *mremap_wrapper(void *old_address, size_t old_size, #define MREMAP_FIXED 2 /* - * mremap wrapper for non-Linux systems. Maps a RW, anonymous private mapping. + * mremap wrapper for non-Linux systems not allowing MAYMOVE. * This is not generic. */ static void *mremap_wrapper(void *old_address, size_t old_size, size_t new_size, int flags) { - void *new_address; - - assert(flags & MREMAP_MAYMOVE); - assert(!(flags & MREMAP_FIXED)); - new_address = mmap(old_address, new_size, - PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, - -1, 0); - if (new_address == MAP_FAILED) - return MAP_FAILED; - if (old_address) { - memcpy(new_address, old_address, old_size); - munmap(old_address, old_size); - } - return new_address; + assert(!(flags & MREMAP_MAYMOVE)); + + return MAP_FAILED; } #endif -/* Sleep delay in us */ -#define RCU_SLEEP_DELAY 1000 -#define ARENA_INIT_ALLOC 16 +/* Sleep delay in ms */ +#define RCU_SLEEP_DELAY_MS 10 +#define INIT_NR_THREADS 8 +#define ARENA_INIT_ALLOC \ + sizeof(struct registry_chunk) \ + + INIT_NR_THREADS * sizeof(struct rcu_reader) /* * Active attempts to check for reader Q.S. before calling sleep(). */ #define RCU_QS_ACTIVE_ATTEMPTS 100 +static +int rcu_bp_refcount; + +static +void __attribute__((constructor)) rcu_bp_init(void); +static void __attribute__((destructor)) rcu_bp_exit(void); +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; -#ifdef DEBUG_YIELD -unsigned int rcu_yield_active; -DEFINE_URCU_TLS(unsigned int, rcu_rand_yield); -#endif +static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; +static int initialized; + +static pthread_key_t urcu_bp_key; struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT }; @@ -119,19 +130,24 @@ DEFINE_URCU_TLS(struct rcu_reader *, rcu_reader); static CDS_LIST_HEAD(registry); +struct registry_chunk { + size_t data_len; /* data length */ + size_t used; /* amount of data used */ + struct cds_list_head node; /* chunk_list node */ + char data[]; +}; + struct registry_arena { - void *p; - size_t len; - size_t used; + struct cds_list_head chunk_list; }; -static struct registry_arena registry_arena; +static struct registry_arena registry_arena = { + .chunk_list = CDS_LIST_HEAD_INIT(registry_arena.chunk_list), +}; /* Saved fork signal mask, protected by rcu_gp_lock */ static sigset_t saved_fork_signal_mask; -static void rcu_gc_registry(void); - static void mutex_lock(pthread_mutex_t *mutex) { int ret; @@ -158,11 +174,15 @@ static void mutex_unlock(pthread_mutex_t *mutex) urcu_die(ret); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, struct cds_list_head *qsreaders) { - int wait_loops = 0; + unsigned int wait_loops = 0; struct rcu_reader *index, *tmp; /* @@ -171,7 +191,9 @@ static void wait_for_readers(struct cds_list_head *input_readers, * rcu_gp.ctr value. */ for (;;) { - wait_loops++; + if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS) + wait_loops++; + cds_list_for_each_entry_safe(index, tmp, input_readers, node) { switch (rcu_reader_state(&index->ctr)) { case RCU_READER_ACTIVE_CURRENT: @@ -198,10 +220,14 @@ static void wait_for_readers(struct cds_list_head *input_readers, if (cds_list_empty(input_readers)) { break; } else { - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) - usleep(RCU_SLEEP_DELAY); + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) + (void) poll(NULL, 0, RCU_SLEEP_DELAY_MS); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } } @@ -220,6 +246,8 @@ void synchronize_rcu(void) mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; @@ -228,11 +256,10 @@ void synchronize_rcu(void) /* Write new ptr before changing the qparity */ cmm_smp_mb(); - /* Remove old registry elements */ - rcu_gc_registry(); - /* * Wait for readers to observe original parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, &cur_snap_readers, &qsreaders); @@ -262,6 +289,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(&cur_snap_readers, NULL, &qsreaders); @@ -276,6 +305,7 @@ void synchronize_rcu(void) */ cmm_smp_mb(); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); @@ -301,82 +331,162 @@ int rcu_read_ongoing(void) } /* - * only grow for now. + * Only grow for now. If empty, allocate a ARENA_INIT_ALLOC sized chunk. + * Else, try expanding the last chunk. If this fails, allocate a new + * chunk twice as big as the last chunk. + * Memory used by chunks _never_ moves. A chunk could theoretically be + * freed when all "used" slots are released, but we don't do it at this + * point. */ -static void resize_arena(struct registry_arena *arena, size_t len) +static +void expand_arena(struct registry_arena *arena) { - void *new_arena; - - if (!arena->p) - new_arena = mmap(arena->p, len, - PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, - -1, 0); - else - new_arena = mremap_wrapper(arena->p, arena->len, - len, MREMAP_MAYMOVE); - assert(new_arena != MAP_FAILED); + struct registry_chunk *new_chunk, *last_chunk; + size_t old_chunk_len, new_chunk_len; + + /* No chunk. */ + if (cds_list_empty(&arena->chunk_list)) { + assert(ARENA_INIT_ALLOC >= + sizeof(struct registry_chunk) + + sizeof(struct rcu_reader)); + new_chunk_len = ARENA_INIT_ALLOC; + new_chunk = mmap(NULL, new_chunk_len, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + if (new_chunk == MAP_FAILED) + abort(); + bzero(new_chunk, new_chunk_len); + new_chunk->data_len = + new_chunk_len - sizeof(struct registry_chunk); + cds_list_add_tail(&new_chunk->node, &arena->chunk_list); + return; /* We're done. */ + } - /* - * re-used the same region ? - */ - if (new_arena == arena->p) - return; + /* Try expanding last chunk. */ + last_chunk = cds_list_entry(arena->chunk_list.prev, + struct registry_chunk, node); + old_chunk_len = + last_chunk->data_len + sizeof(struct registry_chunk); + new_chunk_len = old_chunk_len << 1; + + /* Don't allow memory mapping to move, just expand. */ + new_chunk = mremap_wrapper(last_chunk, old_chunk_len, + new_chunk_len, 0); + if (new_chunk != MAP_FAILED) { + /* Should not have moved. */ + assert(new_chunk == last_chunk); + bzero((char *) last_chunk + old_chunk_len, + new_chunk_len - old_chunk_len); + last_chunk->data_len = + new_chunk_len - sizeof(struct registry_chunk); + return; /* We're done. */ + } - bzero(new_arena + arena->len, len - arena->len); - arena->p = new_arena; + /* Remap did not succeed, we need to add a new chunk. */ + new_chunk = mmap(NULL, new_chunk_len, + PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); + if (new_chunk == MAP_FAILED) + abort(); + bzero(new_chunk, new_chunk_len); + new_chunk->data_len = + new_chunk_len - sizeof(struct registry_chunk); + cds_list_add_tail(&new_chunk->node, &arena->chunk_list); } -/* Called with signals off and mutex locked */ -static void add_thread(void) +static +struct rcu_reader *arena_alloc(struct registry_arena *arena) { + struct registry_chunk *chunk; struct rcu_reader *rcu_reader_reg; + int expand_done = 0; /* Only allow to expand once per alloc */ + size_t len = sizeof(struct rcu_reader); - if (registry_arena.len - < registry_arena.used + sizeof(struct rcu_reader)) - resize_arena(®istry_arena, - caa_max(registry_arena.len << 1, ARENA_INIT_ALLOC)); - /* - * Find a free spot. - */ - for (rcu_reader_reg = registry_arena.p; - (void *)rcu_reader_reg < registry_arena.p + registry_arena.len; - rcu_reader_reg++) { - if (!rcu_reader_reg->alloc) - break; +retry: + cds_list_for_each_entry(chunk, &arena->chunk_list, node) { + if (chunk->data_len - chunk->used < len) + continue; + /* Find spot */ + for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0]; + rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len]; + rcu_reader_reg++) { + if (!rcu_reader_reg->alloc) { + rcu_reader_reg->alloc = 1; + chunk->used += len; + return rcu_reader_reg; + } + } + } + + if (!expand_done) { + expand_arena(arena); + expand_done = 1; + goto retry; } - rcu_reader_reg->alloc = 1; - registry_arena.used += sizeof(struct rcu_reader); + + return NULL; +} + +/* Called with signals off and mutex locked */ +static +void add_thread(void) +{ + struct rcu_reader *rcu_reader_reg; + int ret; + + rcu_reader_reg = arena_alloc(®istry_arena); + if (!rcu_reader_reg) + abort(); + ret = pthread_setspecific(urcu_bp_key, rcu_reader_reg); + if (ret) + abort(); /* Add to registry */ rcu_reader_reg->tid = pthread_self(); assert(rcu_reader_reg->ctr == 0); cds_list_add(&rcu_reader_reg->node, ®istry); + /* + * Reader threads are pointing to the reader registry. This is + * why its memory should never be relocated. + */ URCU_TLS(rcu_reader) = rcu_reader_reg; } -/* Called with signals off and mutex locked */ -static void rcu_gc_registry(void) +/* Called with mutex locked */ +static +void cleanup_thread(struct registry_chunk *chunk, + struct rcu_reader *rcu_reader_reg) { - struct rcu_reader *rcu_reader_reg; - pthread_t tid; - int ret; + rcu_reader_reg->ctr = 0; + cds_list_del(&rcu_reader_reg->node); + rcu_reader_reg->tid = 0; + rcu_reader_reg->alloc = 0; + chunk->used -= sizeof(struct rcu_reader); +} - for (rcu_reader_reg = registry_arena.p; - (void *)rcu_reader_reg < registry_arena.p + registry_arena.len; - rcu_reader_reg++) { - if (!rcu_reader_reg->alloc) +static +struct registry_chunk *find_chunk(struct rcu_reader *rcu_reader_reg) +{ + struct registry_chunk *chunk; + + cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) { + if (rcu_reader_reg < (struct rcu_reader *) &chunk->data[0]) continue; - tid = rcu_reader_reg->tid; - ret = pthread_kill(tid, 0); - assert(ret != EINVAL); - if (ret == ESRCH) { - cds_list_del(&rcu_reader_reg->node); - rcu_reader_reg->ctr = 0; - rcu_reader_reg->alloc = 0; - registry_arena.used -= sizeof(struct rcu_reader); - } + if (rcu_reader_reg >= (struct rcu_reader *) &chunk->data[chunk->data_len]) + continue; + return chunk; } + return NULL; +} + +/* Called with signals off and mutex locked */ +static +void remove_thread(struct rcu_reader *rcu_reader_reg) +{ + cleanup_thread(find_chunk(rcu_reader_reg), rcu_reader_reg); + URCU_TLS(rcu_reader) = NULL; } /* Disable signals, take mutex, add to registry */ @@ -386,34 +496,107 @@ void rcu_bp_register(void) int ret; ret = sigfillset(&newmask); - assert(!ret); + if (ret) + abort(); ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); - assert(!ret); + if (ret) + abort(); /* * Check if a signal concurrently registered our thread since - * the check in rcu_read_lock(). */ + * the check in rcu_read_lock(). + */ if (URCU_TLS(rcu_reader)) goto end; - mutex_lock(&rcu_gp_lock); + /* + * Take care of early registration before urcu_bp constructor. + */ + rcu_bp_init(); + + mutex_lock(&rcu_registry_lock); add_thread(); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); end: ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); - assert(!ret); + if (ret) + abort(); +} + +/* Disable signals, take mutex, remove from registry */ +static +void rcu_bp_unregister(struct rcu_reader *rcu_reader_reg) +{ + sigset_t newmask, oldmask; + int ret; + + ret = sigfillset(&newmask); + if (ret) + abort(); + ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); + if (ret) + abort(); + + mutex_lock(&rcu_registry_lock); + remove_thread(rcu_reader_reg); + mutex_unlock(&rcu_registry_lock); + ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); + if (ret) + abort(); + rcu_bp_exit(); +} + +/* + * Remove thread from the registry when it exits, and flag it as + * destroyed so garbage collection can take care of it. + */ +static +void urcu_bp_thread_exit_notifier(void *rcu_key) +{ + rcu_bp_unregister(rcu_key); +} + +static +void rcu_bp_init(void) +{ + mutex_lock(&init_lock); + if (!rcu_bp_refcount++) { + int ret; + + ret = pthread_key_create(&urcu_bp_key, + urcu_bp_thread_exit_notifier); + if (ret) + abort(); + initialized = 1; + } + mutex_unlock(&init_lock); } +static void rcu_bp_exit(void) { - if (registry_arena.p) - munmap(registry_arena.p, registry_arena.len); + mutex_lock(&init_lock); + if (!--rcu_bp_refcount) { + struct registry_chunk *chunk, *tmp; + int ret; + + cds_list_for_each_entry_safe(chunk, tmp, + ®istry_arena.chunk_list, node) { + munmap(chunk, chunk->data_len + + sizeof(struct registry_chunk)); + } + ret = pthread_key_delete(urcu_bp_key); + if (ret) + abort(); + } + mutex_unlock(&init_lock); } /* - * Holding the rcu_gp_lock across fork will make sure we fork() don't race with - * a concurrent thread executing with this same lock held. This ensures that the - * registry is in a coherent state in the child. + * Holding the rcu_gp_lock and rcu_registry_lock across fork will make + * sure we fork() don't race with a concurrent thread executing with + * any of those locks held. This ensures that the registry and data + * protected by rcu_gp_lock are in a coherent state in the child. */ void rcu_bp_before_fork(void) { @@ -425,6 +608,7 @@ void rcu_bp_before_fork(void) ret = pthread_sigmask(SIG_BLOCK, &newmask, &oldmask); assert(!ret); mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); saved_fork_signal_mask = oldmask; } @@ -434,18 +618,43 @@ void rcu_bp_after_fork_parent(void) int ret; oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); } +/* + * Prune all entries from registry except our own thread. Fits the Linux + * fork behavior. Called with rcu_gp_lock and rcu_registry_lock held. + */ +static +void urcu_bp_prune_registry(void) +{ + struct registry_chunk *chunk; + struct rcu_reader *rcu_reader_reg; + + cds_list_for_each_entry(chunk, ®istry_arena.chunk_list, node) { + for (rcu_reader_reg = (struct rcu_reader *) &chunk->data[0]; + rcu_reader_reg < (struct rcu_reader *) &chunk->data[chunk->data_len]; + rcu_reader_reg++) { + if (!rcu_reader_reg->alloc) + continue; + if (rcu_reader_reg->tid == pthread_self()) + continue; + cleanup_thread(chunk, rcu_reader_reg); + } + } +} + void rcu_bp_after_fork_child(void) { sigset_t oldmask; int ret; - rcu_gc_registry(); + urcu_bp_prune_registry(); oldmask = saved_fork_signal_mask; + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); ret = pthread_sigmask(SIG_SETMASK, &oldmask, NULL); assert(!ret); diff --git a/urcu-bp.h b/urcu-bp.h index 4718f3b..b40b3b6 100644 --- a/urcu-bp.h +++ b/urcu-bp.h @@ -94,6 +94,7 @@ extern int rcu_read_ongoing(void); extern void *rcu_dereference_sym_bp(void *p); #define rcu_dereference_bp(p) \ + __extension__ \ ({ \ __typeof__(p) _________p1 = URCU_FORCE_CAST(__typeof__(p), \ rcu_dereference_sym_bp(URCU_FORCE_CAST(void *, p))); \ @@ -102,6 +103,7 @@ extern void *rcu_dereference_sym_bp(void *p); extern void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new); #define rcu_cmpxchg_pointer_bp(p, old, _new) \ + __extension__ \ ({ \ __typeof__(*(p)) _________pold = (old); \ __typeof__(*(p)) _________pnew = (_new); \ @@ -114,6 +116,7 @@ extern void *rcu_cmpxchg_pointer_sym_bp(void **p, void *old, void *_new); extern void *rcu_xchg_pointer_sym_bp(void **p, void *v); #define rcu_xchg_pointer_bp(p, v) \ + __extension__ \ ({ \ __typeof__(*(p)) _________pv = (v); \ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)),\ @@ -124,6 +127,7 @@ extern void *rcu_xchg_pointer_sym_bp(void **p, void *v); extern void *rcu_set_pointer_sym_bp(void **p, void *v); #define rcu_set_pointer_bp(p, v) \ + __extension__ \ ({ \ __typeof__(*(p)) _________pv = (v); \ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \ diff --git a/urcu-call-rcu-impl.h b/urcu-call-rcu-impl.h index 10c4f3e..c4a7875 100644 --- a/urcu-call-rcu-impl.h +++ b/urcu-call-rcu-impl.h @@ -42,6 +42,7 @@ #include "urcu/list.h" #include "urcu/futex.h" #include "urcu/tls-compat.h" +#include "urcu/ref.h" #include "urcu-die.h" /* Data structure that identifies a call_rcu thread. */ @@ -67,6 +68,7 @@ struct call_rcu_data { struct call_rcu_completion { int barrier_count; int32_t futex; + struct urcu_ref ref; }; struct call_rcu_completion_work { @@ -308,6 +310,8 @@ static void *call_rcu_thread(void *arg) uatomic_or(&crdp->flags, URCU_CALL_RCU_PAUSED); while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSE) != 0) poll(NULL, 0, 1); + uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSED); + cmm_smp_mb__after_uatomic_and(); rcu_register_thread(); } @@ -665,10 +669,10 @@ void call_rcu(struct rcu_head *head, struct call_rcu_data *crdp; /* Holding rcu read-side lock across use of per-cpu crdp */ - rcu_read_lock(); + _rcu_read_lock(); crdp = get_call_rcu_data(); _call_rcu(head, func, crdp); - rcu_read_unlock(); + _rcu_read_unlock(); } /* @@ -766,6 +770,15 @@ void free_all_cpu_call_rcu_data(void) free(crdp); } +static +void free_completion(struct urcu_ref *ref) +{ + struct call_rcu_completion *completion; + + completion = caa_container_of(ref, struct call_rcu_completion, ref); + free(completion); +} + static void _rcu_barrier_complete(struct rcu_head *head) { @@ -774,8 +787,9 @@ void _rcu_barrier_complete(struct rcu_head *head) work = caa_container_of(head, struct call_rcu_completion_work, head); completion = work->completion; - uatomic_dec(&completion->barrier_count); - call_rcu_completion_wake_up(completion); + if (!uatomic_sub_return(&completion->barrier_count, 1)) + call_rcu_completion_wake_up(completion); + urcu_ref_put(&completion->ref, free_completion); free(work); } @@ -785,19 +799,19 @@ void _rcu_barrier_complete(struct rcu_head *head) void rcu_barrier(void) { struct call_rcu_data *crdp; - struct call_rcu_completion completion; + struct call_rcu_completion *completion; int count = 0; int was_online; /* Put in offline state in QSBR. */ - was_online = rcu_read_ongoing(); + was_online = _rcu_read_ongoing(); if (was_online) rcu_thread_offline(); /* * Calling a rcu_barrier() within a RCU read-side critical * section is an error. */ - if (rcu_read_ongoing()) { + if (_rcu_read_ongoing()) { static int warned = 0; if (!warned) { @@ -807,11 +821,17 @@ void rcu_barrier(void) goto online; } + completion = calloc(sizeof(*completion), 1); + if (!completion) + urcu_die(errno); + call_rcu_lock(&call_rcu_mutex); cds_list_for_each_entry(crdp, &call_rcu_data_list, list) count++; - completion.barrier_count = count; + /* Referenced by rcu_barrier() and each call_rcu thread. */ + urcu_ref_set(&completion->ref, count + 1); + completion->barrier_count = count; cds_list_for_each_entry(crdp, &call_rcu_data_list, list) { struct call_rcu_completion_work *work; @@ -819,20 +839,23 @@ void rcu_barrier(void) work = calloc(sizeof(*work), 1); if (!work) urcu_die(errno); - work->completion = &completion; + work->completion = completion; _call_rcu(&work->head, _rcu_barrier_complete, crdp); } call_rcu_unlock(&call_rcu_mutex); /* Wait for them */ for (;;) { - uatomic_dec(&completion.futex); + uatomic_dec(&completion->futex); /* Decrement futex before reading barrier_count */ cmm_smp_mb(); - if (!uatomic_read(&completion.barrier_count)) + if (!uatomic_read(&completion->barrier_count)) break; - call_rcu_completion_wait(&completion); + call_rcu_completion_wait(completion); } + + urcu_ref_put(&completion->ref, free_completion); + online: if (was_online) rcu_thread_online(); @@ -872,6 +895,10 @@ void call_rcu_after_fork_parent(void) cds_list_for_each_entry(crdp, &call_rcu_data_list, list) uatomic_and(&crdp->flags, ~URCU_CALL_RCU_PAUSE); + cds_list_for_each_entry(crdp, &call_rcu_data_list, list) { + while ((uatomic_read(&crdp->flags) & URCU_CALL_RCU_PAUSED) != 0) + poll(NULL, 0, 1); + } call_rcu_unlock(&call_rcu_mutex); } diff --git a/urcu-call-rcu.h b/urcu-call-rcu.h index 30388c5..98807ea 100644 --- a/urcu-call-rcu.h +++ b/urcu-call-rcu.h @@ -64,7 +64,7 @@ struct rcu_head { /* * Exported functions * - * Important: see rcu-api.txt in userspace-rcu documentation for + * Important: see rcu-api.md in userspace-rcu documentation for * call_rcu family of functions usage detail, including the surrounding * RCU usage required when using these primitives. */ diff --git a/urcu-pointer.h b/urcu-pointer.h index 03bfe79..18ea99e 100644 --- a/urcu-pointer.h +++ b/urcu-pointer.h @@ -34,7 +34,7 @@ extern "C" { #endif -#ifdef _LGPL_SOURCE +#if defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS) #include @@ -62,10 +62,11 @@ extern "C" { #define rcu_xchg_pointer _rcu_xchg_pointer #define rcu_set_pointer _rcu_set_pointer -#else /* !_LGPL_SOURCE */ +#else /* !(defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)) */ extern void *rcu_dereference_sym(void *p); #define rcu_dereference(p) \ + __extension__ \ ({ \ __typeof__(p) _________p1 = URCU_FORCE_CAST(__typeof__(p), \ rcu_dereference_sym(URCU_FORCE_CAST(void *, p))); \ @@ -74,6 +75,7 @@ extern void *rcu_dereference_sym(void *p); extern void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new); #define rcu_cmpxchg_pointer(p, old, _new) \ + __extension__ \ ({ \ __typeof__(*(p)) _________pold = (old); \ __typeof__(*(p)) _________pnew = (_new); \ @@ -86,6 +88,7 @@ extern void *rcu_cmpxchg_pointer_sym(void **p, void *old, void *_new); extern void *rcu_xchg_pointer_sym(void **p, void *v); #define rcu_xchg_pointer(p, v) \ + __extension__ \ ({ \ __typeof__(*(p)) _________pv = (v); \ __typeof__(*(p)) _________p1 = URCU_FORCE_CAST(__typeof__(*(p)), \ @@ -108,7 +111,7 @@ extern void *rcu_set_pointer_sym(void **p, void *v); _________pv); \ } while (0) -#endif /* !_LGPL_SOURCE */ +#endif /* !(defined(_LGPL_SOURCE) || defined(URCU_INLINE_SMALL_FUNCTIONS)) */ /* * void rcu_assign_pointer(type *ptr, type *new) diff --git a/urcu-qsbr.c b/urcu-qsbr.c index d6adc5b..685efb5 100644 --- a/urcu-qsbr.c +++ b/urcu-qsbr.c @@ -52,7 +52,21 @@ void __attribute__((destructor)) rcu_exit(void); +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; struct rcu_gp rcu_gp = { .ctr = RCU_GP_ONLINE }; /* @@ -66,11 +80,6 @@ struct rcu_gp rcu_gp = { .ctr = RCU_GP_ONLINE }; */ DEFINE_URCU_TLS(struct rcu_reader, rcu_reader); -#ifdef DEBUG_YIELD -unsigned int rcu_yield_active; -DEFINE_URCU_TLS(unsigned int, rcu_rand_yield); -#endif - static CDS_LIST_HEAD(registry); /* @@ -117,11 +126,15 @@ static void wait_gp(void) NULL, NULL, 0); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, struct cds_list_head *qsreaders) { - int wait_loops = 0; + unsigned int wait_loops = 0; struct rcu_reader *index, *tmp; /* @@ -130,7 +143,8 @@ static void wait_for_readers(struct cds_list_head *input_readers, * current rcu_gp.ctr value. */ for (;;) { - wait_loops++; + if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS) + wait_loops++; if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { uatomic_set(&rcu_gp.futex, -1); /* @@ -175,6 +189,8 @@ static void wait_for_readers(struct cds_list_head *input_readers, } break; } else { + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { wait_gp(); } else { @@ -184,6 +200,8 @@ static void wait_for_readers(struct cds_list_head *input_readers, cmm_smp_mb(); #endif /* #else #ifndef HAS_INCOHERENT_CACHES */ } + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } } } @@ -237,11 +255,15 @@ void synchronize_rcu(void) */ urcu_move_waiters(&waiters, &gp_waiters); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; /* * Wait for readers to observe original parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, &cur_snap_readers, &qsreaders); @@ -283,6 +305,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(&cur_snap_readers, NULL, &qsreaders); @@ -291,6 +315,7 @@ void synchronize_rcu(void) */ cds_list_splice(&qsreaders, ®istry); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); urcu_wake_all_waiters(&waiters); gp_end: @@ -343,6 +368,8 @@ void synchronize_rcu(void) */ urcu_move_waiters(&waiters, &gp_waiters); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; @@ -367,6 +394,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new count of be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, NULL, &qsreaders); @@ -375,6 +404,7 @@ void synchronize_rcu(void) */ cds_list_splice(&qsreaders, ®istry); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); urcu_wake_all_waiters(&waiters); gp_end: @@ -424,9 +454,9 @@ void rcu_register_thread(void) URCU_TLS(rcu_reader).tid = pthread_self(); assert(URCU_TLS(rcu_reader).ctr == 0); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); cds_list_add(&URCU_TLS(rcu_reader).node, ®istry); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); _rcu_thread_online(); } @@ -437,9 +467,9 @@ void rcu_unregister_thread(void) * with a waiting writer. */ _rcu_thread_offline(); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); cds_list_del(&URCU_TLS(rcu_reader).node); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); } void rcu_exit(void) diff --git a/urcu-qsbr.h b/urcu-qsbr.h index bb0523c..82a045c 100644 --- a/urcu-qsbr.h +++ b/urcu-qsbr.h @@ -43,6 +43,10 @@ extern "C" { #include +#ifdef RCU_DEBUG /* For backward compatibility */ +#define DEBUG_RCU +#endif + /* * Important ! * @@ -86,12 +90,12 @@ extern "C" { * QSBR read lock/unlock are guaranteed to be no-ops. Therefore, we expose them * in the LGPL header for any code to use. However, the debug version is not * nops and may contain sanity checks. To activate it, applications must be - * recompiled with -DRCU_DEBUG (even non-LGPL/GPL applications). This is the + * recompiled with -DDEBUG_RCU (even non-LGPL/GPL applications). This is the * best trade-off between license/performance/code triviality and * library debugging & tracing features we could come up with. */ -#if (!defined(BUILD_QSBR_LIB) && defined(RCU_DEBUG)) +#if (!defined(BUILD_QSBR_LIB) && !defined(DEBUG_RCU)) static inline void rcu_read_lock(void) { @@ -101,12 +105,12 @@ static inline void rcu_read_unlock(void) { } -#else /* !RCU_DEBUG */ +#else /* !DEBUG_RCU */ extern void rcu_read_lock(void); extern void rcu_read_unlock(void); -#endif /* !RCU_DEBUG */ +#endif /* !DEBUG_RCU */ extern int rcu_read_ongoing(void); extern void rcu_quiescent_state(void); diff --git a/urcu.c b/urcu.c index 759b94b..1429f6d 100644 --- a/urcu.c +++ b/urcu.c @@ -53,9 +53,9 @@ /* * If a reader is really non-cooperative and refuses to commit its * rcu_active_readers count to memory (there is no barrier in the reader - * per-se), kick it after a few loops waiting for it. + * per-se), kick it after 10 loops waiting for it. */ -#define KICK_READER_LOOPS 10000 +#define KICK_READER_LOOPS 10 /* * Active attempts to check for reader Q.S. before calling futex(). @@ -66,7 +66,7 @@ * RCU_MEMBARRIER is only possibly available on Linux. */ #if defined(RCU_MEMBARRIER) && defined(__linux__) -#include +#include #endif /* If the headers do not support SYS_membarrier, fall back on RCU_MB */ @@ -100,7 +100,21 @@ void __attribute__((constructor)) rcu_init(void); void __attribute__((destructor)) rcu_exit(void); #endif +/* + * rcu_gp_lock ensures mutual exclusion between threads calling + * synchronize_rcu(). + */ static pthread_mutex_t rcu_gp_lock = PTHREAD_MUTEX_INITIALIZER; +/* + * rcu_registry_lock ensures mutual exclusion between threads + * registering and unregistering themselves to/from the registry, and + * with threads reading that registry from synchronize_rcu(). However, + * this lock is not held all the way through the completion of awaiting + * for the grace period. It is sporadically released between iterations + * on the registry. + * rcu_registry_lock may nest inside rcu_gp_lock. + */ +static pthread_mutex_t rcu_registry_lock = PTHREAD_MUTEX_INITIALIZER; struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT }; /* @@ -109,11 +123,6 @@ struct rcu_gp rcu_gp = { .ctr = RCU_GP_COUNT }; */ DEFINE_URCU_TLS(struct rcu_reader, rcu_reader); -#ifdef DEBUG_YIELD -unsigned int rcu_yield_active; -DEFINE_URCU_TLS(unsigned int, rcu_rand_yield); -#endif - static CDS_LIST_HEAD(registry); /* @@ -231,12 +240,19 @@ static void wait_gp(void) NULL, NULL, 0); } +/* + * Always called with rcu_registry lock held. Releases this lock between + * iterations and grabs it again. Holds the lock when it returns. + */ static void wait_for_readers(struct cds_list_head *input_readers, struct cds_list_head *cur_snap_readers, struct cds_list_head *qsreaders) { - int wait_loops = 0; + unsigned int wait_loops = 0; struct rcu_reader *index, *tmp; +#ifdef HAS_INCOHERENT_CACHES + unsigned int wait_gp_loops = 0; +#endif /* HAS_INCOHERENT_CACHES */ /* * Wait for each thread URCU_TLS(rcu_reader).ctr to either @@ -244,8 +260,9 @@ static void wait_for_readers(struct cds_list_head *input_readers, * rcu_gp.ctr value. */ for (;;) { - wait_loops++; - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) { + if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS) + wait_loops++; + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { uatomic_dec(&rcu_gp.futex); /* Write futex before read reader_gp */ smp_mb_master(RCU_MB_GROUP); @@ -276,17 +293,21 @@ static void wait_for_readers(struct cds_list_head *input_readers, #ifndef HAS_INCOHERENT_CACHES if (cds_list_empty(input_readers)) { - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) { + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ smp_mb_master(RCU_MB_GROUP); uatomic_set(&rcu_gp.futex, 0); } break; } else { - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) wait_gp(); else caa_cpu_relax(); + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } #else /* #ifndef HAS_INCOHERENT_CACHES */ /* @@ -295,24 +316,27 @@ static void wait_for_readers(struct cds_list_head *input_readers, * for too long. */ if (cds_list_empty(input_readers)) { - if (wait_loops == RCU_QS_ACTIVE_ATTEMPTS) { + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { /* Read reader_gp before write futex */ smp_mb_master(RCU_MB_GROUP); uatomic_set(&rcu_gp.futex, 0); } break; } else { - switch (wait_loops) { - case RCU_QS_ACTIVE_ATTEMPTS: - wait_gp(); - break; /* only escape switch */ - case KICK_READER_LOOPS: + if (wait_gp_loops == KICK_READER_LOOPS) { smp_mb_master(RCU_MB_GROUP); - wait_loops = 0; - break; /* only escape switch */ - default: + wait_gp_loops = 0; + } + /* Temporarily unlock the registry lock. */ + mutex_unlock(&rcu_registry_lock); + if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) { + wait_gp(); + wait_gp_loops++; + } else { caa_cpu_relax(); } + /* Re-lock the registry lock before the next loop. */ + mutex_lock(&rcu_registry_lock); } #endif /* #else #ifndef HAS_INCOHERENT_CACHES */ } @@ -350,17 +374,23 @@ void synchronize_rcu(void) */ urcu_move_waiters(&waiters, &gp_waiters); + mutex_lock(&rcu_registry_lock); + if (cds_list_empty(®istry)) goto out; - /* All threads should read qparity before accessing data structure - * where new ptr points to. Must be done within rcu_gp_lock because it - * iterates on reader threads.*/ + /* + * All threads should read qparity before accessing data structure + * where new ptr points to. Must be done within rcu_registry_lock + * because it iterates on reader threads. + */ /* Write new ptr before changing the qparity */ smp_mb_master(RCU_MB_GROUP); /* * Wait for readers to observe original parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(®istry, &cur_snap_readers, &qsreaders); @@ -401,6 +431,8 @@ void synchronize_rcu(void) /* * Wait for readers to observe new parity or be quiescent. + * wait_for_readers() can release and grab again rcu_registry_lock + * interally. */ wait_for_readers(&cur_snap_readers, NULL, &qsreaders); @@ -409,11 +441,14 @@ void synchronize_rcu(void) */ cds_list_splice(&qsreaders, ®istry); - /* Finish waiting for reader threads before letting the old ptr being - * freed. Must be done within rcu_gp_lock because it iterates on reader - * threads. */ + /* + * Finish waiting for reader threads before letting the old ptr + * being freed. Must be done within rcu_registry_lock because it + * iterates on reader threads. + */ smp_mb_master(RCU_MB_GROUP); out: + mutex_unlock(&rcu_registry_lock); mutex_unlock(&rcu_gp_lock); /* @@ -449,17 +484,17 @@ void rcu_register_thread(void) assert(URCU_TLS(rcu_reader).need_mb == 0); assert(!(URCU_TLS(rcu_reader).ctr & RCU_GP_CTR_NEST_MASK)); - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); rcu_init(); /* In case gcc does not support constructor attribute */ cds_list_add(&URCU_TLS(rcu_reader).node, ®istry); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); } void rcu_unregister_thread(void) { - mutex_lock(&rcu_gp_lock); + mutex_lock(&rcu_registry_lock); cds_list_del(&URCU_TLS(rcu_reader).node); - mutex_unlock(&rcu_gp_lock); + mutex_unlock(&rcu_registry_lock); } #ifdef RCU_MEMBARRIER @@ -490,9 +525,9 @@ static void sigrcu_handler(int signo, siginfo_t *siginfo, void *context) * rcu_init constructor. Called when the library is linked, but also when * reader threads are calling rcu_register_thread(). * Should only be called by a single thread at a given time. This is ensured by - * holing the rcu_gp_lock from rcu_register_thread() or by running at library - * load time, which should not be executed by multiple threads nor concurrently - * with rcu_register_thread() anyway. + * holing the rcu_registry_lock from rcu_register_thread() or by running + * at library load time, which should not be executed by multiple + * threads nor concurrently with rcu_register_thread() anyway. */ void rcu_init(void) { diff --git a/urcu/arch/hppa.h b/urcu/arch/hppa.h new file mode 100644 index 0000000..b9b9494 --- /dev/null +++ b/urcu/arch/hppa.h @@ -0,0 +1,30 @@ +#ifndef _URCU_ARCH_HPPA_H +#define _URCU_ARCH_HPPA_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +typedef unsigned long cycles_t; + +static inline cycles_t caa_get_cycles(void) +{ + cycles_t cycles; + + asm volatile("mfctl 16, %0" : "=r" (cycles)); + return cycles; +} + +#ifdef __cplusplus +} +#endif + +#include + +#endif /* _URCU_ARCH_HPPA_H */ diff --git a/urcu/arch/ppc.h b/urcu/arch/ppc.h index 95393ea..1068b28 100644 --- a/urcu/arch/ppc.h +++ b/urcu/arch/ppc.h @@ -58,6 +58,7 @@ extern "C" { #define cmm_smp_wmb() __asm__ __volatile__ (LWSYNC_OPCODE:::"memory") #define mftbl() \ + __extension__ \ ({ \ unsigned long rval; \ __asm__ __volatile__ ("mftbl %0" : "=r" (rval)); \ @@ -65,6 +66,7 @@ extern "C" { }) #define mftbu() \ + __extension__ \ ({ \ unsigned long rval; \ __asm__ __volatile__ ("mftbu %0" : "=r" (rval)); \ @@ -72,6 +74,7 @@ extern "C" { }) #define mftb() \ + __extension__ \ ({ \ unsigned long long rval; \ __asm__ __volatile__ ("mftb %0" : "=r" (rval)); \ diff --git a/urcu/arch/x86.h b/urcu/arch/x86.h index 5853604..ccc4ce8 100644 --- a/urcu/arch/x86.h +++ b/urcu/arch/x86.h @@ -55,12 +55,18 @@ extern "C" { * IDT WinChip supports weak store ordering, and the kernel may enable it * under our feet; cmm_smp_wmb() ceases to be a nop for these processors. */ +#if (CAA_BITS_PER_LONG == 32) #define cmm_mb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory") -#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory") -#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)"::: "memory") +#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory") +#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%esp)":::"memory") +#else +#define cmm_mb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory") +#define cmm_rmb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory") +#define cmm_wmb() __asm__ __volatile__ ("lock; addl $0,0(%%rsp)":::"memory") +#endif #endif -#define caa_cpu_relax() __asm__ __volatile__ ("rep; nop" : : : "memory"); +#define caa_cpu_relax() __asm__ __volatile__ ("rep; nop" : : : "memory") #define rdtscll(val) \ do { \ diff --git a/urcu/compiler.h b/urcu/compiler.h index c4ade90..511dbdf 100644 --- a/urcu/compiler.h +++ b/urcu/compiler.h @@ -64,6 +64,7 @@ * @member: name of the field within the object. */ #define caa_container_of(ptr, type, member) \ + __extension__ \ ({ \ const __typeof__(((type *) NULL)->member) * __ptr = (ptr); \ (type *)((char *)__ptr - offsetof(type, member)); \ @@ -86,14 +87,18 @@ #define URCU_FORCE_CAST(type, arg) ((type) (arg)) #endif -#define caa_is_signed_type(type) (((type) (-1)) < 0) +#define caa_is_signed_type(type) ((type) -1 < (type) 0) -#define caa_cast_long_keep_sign(v) \ - (caa_is_signed_type(__typeof__(v)) ? (long) (v) : (unsigned long) (v)) +/* + * Cast to unsigned long, sign-extending if @v is signed. + * Note: casting to a larger type or to same type size keeps the sign of + * the expression being cast (see C99 6.3.1.3). + */ +#define caa_cast_long_keep_sign(v) ((unsigned long) (v)) #if defined (__GNUC__) \ - && ((__GNUC_MAJOR__ == 4) && (__GNUC_MINOR__ >= 5) \ - || __GNUC_MAJOR__ >= 5) + && ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 5) \ + || __GNUC__ >= 5) #define CDS_DEPRECATED(msg) \ __attribute__((deprecated(msg))) #else @@ -103,4 +108,23 @@ #define CAA_ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +/* + * Don't allow compiling with buggy compiler. + */ + +#ifdef __GNUC__ +# define URCU_GCC_VERSION (__GNUC__ * 10000 \ + + __GNUC_MINOR__ * 100 \ + + __GNUC_PATCHLEVEL__) + +/* + * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58854 + */ +# ifdef __ARMEL__ +# if URCU_GCC_VERSION >= 40800 && URCU_GCC_VERSION <= 40802 +# error Your gcc version produces clobbered frame accesses +# endif +# endif +#endif + #endif /* _URCU_COMPILER_H */ diff --git a/urcu/futex.h b/urcu/futex.h index cdaa430..bb270c2 100644 --- a/urcu/futex.h +++ b/urcu/futex.h @@ -45,7 +45,7 @@ extern "C" { */ #ifdef CONFIG_RCU_HAVE_FUTEX -#include +#include #define futex(...) syscall(__NR_futex, __VA_ARGS__) #define futex_noasync(uaddr, op, val, timeout, uaddr2, val3) \ futex(uaddr, op, val, timeout, uaddr2, val3) diff --git a/urcu/hlist.h b/urcu/hlist.h index d2e2ec3..f8624b1 100644 --- a/urcu/hlist.h +++ b/urcu/hlist.h @@ -103,14 +103,18 @@ void cds_hlist_del(struct cds_hlist_node *elem) entry = cds_hlist_entry(pos, __typeof__(*entry), member)) #define cds_hlist_for_each_entry_2(entry, head, member) \ - for (entry = cds_hlist_entry((head)->next, __typeof__(*entry), member); \ - &entry->member != NULL; \ - entry = cds_hlist_entry(entry->member.next, __typeof__(*entry), member)) + for (entry = ((head)->next == NULL ? NULL \ + : cds_hlist_entry((head)->next, __typeof__(*entry), member)); \ + entry != NULL; \ + entry = (entry->member.next == NULL ? NULL \ + : cds_hlist_entry(entry->member.next, __typeof__(*entry), member))) #define cds_hlist_for_each_entry_safe_2(entry, e, head, member) \ - for (entry = cds_hlist_entry((head)->next, __typeof__(*entry), member); \ - (&entry->member != NULL) && (e = cds_hlist_entry(entry->member.next, \ - __typeof__(*entry), member), 1); \ + for (entry = ((head)->next == NULL ? NULL \ + : cds_hlist_entry((head)->next, __typeof__(*entry), member)); \ + (entry != NULL) && (e = (entry->member.next == NULL ? NULL \ + : cds_hlist_entry(entry->member.next, \ + __typeof__(*entry), member)), 1); \ entry = e) #endif /* _KCOMPAT_HLIST_H */ diff --git a/urcu/lfstack.h b/urcu/lfstack.h index eddff0e..fa58054 100644 --- a/urcu/lfstack.h +++ b/urcu/lfstack.h @@ -70,17 +70,32 @@ struct cds_lfs_head { struct cds_lfs_node node; }; +struct __cds_lfs_stack { + struct cds_lfs_head *head; +}; + struct cds_lfs_stack { struct cds_lfs_head *head; pthread_mutex_t lock; }; +/* + * The transparent union allows calling functions that work on both + * struct cds_lfs_stack and struct __cds_lfs_stack on any of those two + * types. + */ +typedef union { + struct __cds_lfs_stack *_s; + struct cds_lfs_stack *s; +} __attribute__((__transparent_union__)) cds_lfs_stack_ptr_t; + #ifdef _LGPL_SOURCE #include #define cds_lfs_node_init _cds_lfs_node_init #define cds_lfs_init _cds_lfs_init +#define __cds_lfs_init ___cds_lfs_init #define cds_lfs_empty _cds_lfs_empty #define cds_lfs_push _cds_lfs_push @@ -108,12 +123,17 @@ extern void cds_lfs_node_init(struct cds_lfs_node *node); */ extern void cds_lfs_init(struct cds_lfs_stack *s); +/* + * __cds_lfs_init: initialize lock-free stack. + */ +extern void __cds_lfs_init(struct __cds_lfs_stack *s); + /* * cds_lfs_empty: return whether lock-free stack is empty. * * No memory barrier is issued. No mutual exclusion is required. */ -extern bool cds_lfs_empty(struct cds_lfs_stack *s); +extern bool cds_lfs_empty(cds_lfs_stack_ptr_t s); /* * cds_lfs_push: push a node into the stack. @@ -123,7 +143,7 @@ extern bool cds_lfs_empty(struct cds_lfs_stack *s); * Returns 0 if the stack was empty prior to adding the node. * Returns non-zero otherwise. */ -extern bool cds_lfs_push(struct cds_lfs_stack *s, +extern bool cds_lfs_push(cds_lfs_stack_ptr_t s, struct cds_lfs_node *node); /* @@ -166,7 +186,7 @@ extern void cds_lfs_pop_unlock(struct cds_lfs_stack *s); * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme). */ -extern struct cds_lfs_node *__cds_lfs_pop(struct cds_lfs_stack *s); +extern struct cds_lfs_node *__cds_lfs_pop(cds_lfs_stack_ptr_t s); /* * __cds_lfs_pop_all: pop all nodes from a stack. @@ -185,7 +205,7 @@ extern struct cds_lfs_node *__cds_lfs_pop(struct cds_lfs_stack *s); * 3) Ensuring that only ONE thread can call __cds_lfs_pop() and * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme). */ -extern struct cds_lfs_head *__cds_lfs_pop_all(struct cds_lfs_stack *s); +extern struct cds_lfs_head *__cds_lfs_pop_all(cds_lfs_stack_ptr_t s); #endif /* !_LGPL_SOURCE */ diff --git a/urcu/map/urcu-qsbr.h b/urcu/map/urcu-qsbr.h index b89dd24..9e90e3c 100644 --- a/urcu/map/urcu-qsbr.h +++ b/urcu/map/urcu-qsbr.h @@ -76,7 +76,4 @@ #define rcu_flavor rcu_flavor_qsbr -#define rcu_yield_active rcu_yield_active_memb_qsbr -#define rcu_rand_yield rcu_rand_yield_memb_qsbr - #endif /* _URCU_QSBR_MAP_H */ diff --git a/urcu/map/urcu.h b/urcu/map/urcu.h index 8f04caf..9a4bb1a 100644 --- a/urcu/map/urcu.h +++ b/urcu/map/urcu.h @@ -80,9 +80,6 @@ #define rcu_flavor rcu_flavor_memb -#define rcu_yield_active rcu_yield_active_memb -#define rcu_rand_yield rcu_rand_yield_memb - /* Specific to MEMBARRIER flavor */ #define rcu_has_sys_membarrier rcu_has_sys_membarrier_memb @@ -128,9 +125,6 @@ #define rcu_flavor rcu_flavor_sig -#define rcu_yield_active rcu_yield_active_sig -#define rcu_rand_yield rcu_rand_yield_sig - #elif defined(RCU_MB) #define rcu_read_lock rcu_read_lock_mb @@ -173,9 +167,6 @@ #define rcu_flavor rcu_flavor_mb -#define rcu_yield_active rcu_yield_active_mb -#define rcu_rand_yield rcu_rand_yield_mb - #else #error "Undefined selection" diff --git a/urcu/rand-compat.h b/urcu/rand-compat.h new file mode 100644 index 0000000..2c57751 --- /dev/null +++ b/urcu/rand-compat.h @@ -0,0 +1,63 @@ +#ifndef _URCU_RAND_COMPAT_H +#define _URCU_RAND_COMPAT_H + +/* + * urcu/rand-compat.h + * + * Userspace RCU library - rand/rand_r Compatibility Header + * + * Copyright 1996 - Ulrich Drepper + * Copyright 2013 - Pierre-Luc St-Charles + * + * Note: this file is only used to simplify the code required to + * use the 'rand_r(...)' system function across multiple platforms, + * which might not always be referenced the same way. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifdef __ANDROID__ +/* + * Reentrant random function from POSIX.1c. + * Copyright (C) 1996, 1999 Free Software Foundation, Inc. + * This file is part of the GNU C Library. + * Contributed by Ulrich Drepper >, 1996. + */ +static inline int rand_r(unsigned int *seed) +{ + unsigned int next = *seed; + int result; + + next *= 1103515245; + next += 12345; + result = (unsigned int) (next / 65536) % 2048; + + next *= 1103515245; + next += 12345; + result <<= 10; + result ^= (unsigned int) (next / 65536) % 1024; + + next *= 1103515245; + next += 12345; + result <<= 10; + result ^= (unsigned int) (next / 65536) % 1024; + + *seed = next; + + return result; +} +#endif /* __ANDROID__ */ + +#endif /* _URCU_RAND_COMPAT_H */ diff --git a/urcu/rculfhash.h b/urcu/rculfhash.h index 4c400ce..eca0471 100644 --- a/urcu/rculfhash.h +++ b/urcu/rculfhash.h @@ -133,7 +133,8 @@ struct cds_lfht *_cds_lfht_new(unsigned long init_size, * @min_nr_alloc_buckets: the minimum number of allocated buckets. * (must be power of two) * @max_nr_buckets: the maximum number of hash table buckets allowed. - * (must be power of two) + * (must be power of two, 0 is accepted, means + * "infinite") * @flags: hash table creation flags (can be combined with bitwise or: '|'). * 0: no flags. * CDS_LFHT_AUTO_RESIZE: automatically resize hash table. @@ -470,7 +471,7 @@ void cds_lfht_resize(struct cds_lfht *ht, unsigned long new_size); for (cds_lfht_first(ht, iter), \ pos = caa_container_of(cds_lfht_iter_get_node(iter), \ __typeof__(*(pos)), member); \ - &(pos)->member != NULL; \ + cds_lfht_iter_get_node(iter) != NULL; \ cds_lfht_next(ht, iter), \ pos = caa_container_of(cds_lfht_iter_get_node(iter), \ __typeof__(*(pos)), member)) @@ -480,7 +481,7 @@ void cds_lfht_resize(struct cds_lfht *ht, unsigned long new_size); for (cds_lfht_lookup(ht, hash, match, key, iter), \ pos = caa_container_of(cds_lfht_iter_get_node(iter), \ __typeof__(*(pos)), member); \ - &(pos)->member != NULL; \ + cds_lfht_iter_get_node(iter) != NULL; \ cds_lfht_next_duplicate(ht, match, key, iter), \ pos = caa_container_of(cds_lfht_iter_get_node(iter), \ __typeof__(*(pos)), member)) diff --git a/urcu/static/lfstack.h b/urcu/static/lfstack.h index 0be9594..41895a6 100644 --- a/urcu/static/lfstack.h +++ b/urcu/static/lfstack.h @@ -78,6 +78,15 @@ void _cds_lfs_init(struct cds_lfs_stack *s) assert(!ret); } +/* + * ___cds_lfs_init: initialize lock-free stack. + */ +static inline +void ___cds_lfs_init(struct __cds_lfs_stack *s) +{ + s->head = NULL; +} + static inline bool ___cds_lfs_empty_head(struct cds_lfs_head *head) { @@ -90,9 +99,9 @@ bool ___cds_lfs_empty_head(struct cds_lfs_head *head) * No memory barrier is issued. No mutual exclusion is required. */ static inline -bool _cds_lfs_empty(struct cds_lfs_stack *s) +bool _cds_lfs_empty(cds_lfs_stack_ptr_t s) { - return ___cds_lfs_empty_head(CMM_LOAD_SHARED(s->head)); + return ___cds_lfs_empty_head(CMM_LOAD_SHARED(s._s->head)); } /* @@ -125,9 +134,10 @@ bool _cds_lfs_empty(struct cds_lfs_stack *s) * Returns non-zero otherwise. */ static inline -bool _cds_lfs_push(struct cds_lfs_stack *s, +bool _cds_lfs_push(cds_lfs_stack_ptr_t u_s, struct cds_lfs_node *node) { + struct __cds_lfs_stack *s = u_s._s; struct cds_lfs_head *head = NULL; struct cds_lfs_head *new_head = caa_container_of(node, struct cds_lfs_head, node); @@ -148,7 +158,7 @@ bool _cds_lfs_push(struct cds_lfs_stack *s, if (old_head == head) break; } - return ___cds_lfs_empty_head(head); + return !___cds_lfs_empty_head(head); } /* @@ -168,8 +178,10 @@ bool _cds_lfs_push(struct cds_lfs_stack *s, * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme). */ static inline -struct cds_lfs_node *___cds_lfs_pop(struct cds_lfs_stack *s) +struct cds_lfs_node *___cds_lfs_pop(cds_lfs_stack_ptr_t u_s) { + struct __cds_lfs_stack *s = u_s._s; + for (;;) { struct cds_lfs_head *head, *next_head; struct cds_lfs_node *next; @@ -211,8 +223,10 @@ struct cds_lfs_node *___cds_lfs_pop(struct cds_lfs_stack *s) * __cds_lfs_pop_all(). (multi-provider/single-consumer scheme). */ static inline -struct cds_lfs_head *___cds_lfs_pop_all(struct cds_lfs_stack *s) +struct cds_lfs_head *___cds_lfs_pop_all(cds_lfs_stack_ptr_t u_s) { + struct __cds_lfs_stack *s = u_s._s; + /* * Implicit memory barrier after uatomic_xchg() matches implicit * memory barrier before uatomic_cmpxchg() in cds_lfs_push. It diff --git a/urcu/static/urcu-bp.h b/urcu/static/urcu-bp.h index 1052c44..b6d5f13 100644 --- a/urcu/static/urcu-bp.h +++ b/urcu/static/urcu-bp.h @@ -64,58 +64,6 @@ enum rcu_state { RCU_READER_INACTIVE, }; -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define RCU_YIELD_READ (1 << 0) -#define RCU_YIELD_WRITE (1 << 1) - -/* - * Updates without RCU_MB are much slower. Account this in - * the delay. - */ -/* maximum sleep delay, in us */ -#define MAX_SLEEP 50 - -extern unsigned int rcu_yield_active; -extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield); - -static inline void rcu_debug_yield_read(void) -{ - if (rcu_yield_active & RCU_YIELD_READ) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_write(void) -{ - if (rcu_yield_active & RCU_YIELD_WRITE) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_init(void) -{ - URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self(); -} -#else -static inline void rcu_debug_yield_read(void) -{ -} - -static inline void rcu_debug_yield_write(void) -{ -} - -static inline void rcu_debug_yield_init(void) -{ - -} -#endif - /* * The trick here is that RCU_GP_CTR_PHASE must be a multiple of 8 so we can use a * full 8-bits, 16-bits or 32-bits bitmask for the lower order bits. diff --git a/urcu/static/urcu-pointer.h b/urcu/static/urcu-pointer.h index 4361156..06371e3 100644 --- a/urcu/static/urcu-pointer.h +++ b/urcu/static/urcu-pointer.h @@ -64,7 +64,9 @@ extern "C" { * meets the 10-line criterion in LGPL, allowing this function to be * expanded directly in non-LGPL code. */ -#define _rcu_dereference(p) ({ \ +#define _rcu_dereference(p) \ + __extension__ \ + ({ \ __typeof__(p) _________p1 = CMM_LOAD_SHARED(p); \ cmm_smp_read_barrier_depends(); \ (_________p1); \ @@ -82,6 +84,7 @@ extern "C" { * expanded directly in non-LGPL code. */ #define _rcu_cmpxchg_pointer(p, old, _new) \ + __extension__ \ ({ \ __typeof__(*p) _________pold = (old); \ __typeof__(*p) _________pnew = (_new); \ @@ -101,6 +104,7 @@ extern "C" { * expanded directly in non-LGPL code. */ #define _rcu_xchg_pointer(p, v) \ + __extension__ \ ({ \ __typeof__(*p) _________pv = (v); \ if (!__builtin_constant_p(v) || \ @@ -129,7 +133,7 @@ extern "C" { * them. It also makes sure the compiler does not reorder code initializing the * data structure before its publication. * - * Should match rcu_dereference_pointer(). + * Should match rcu_dereference(). * * This macro is less than 10 lines long. The intent is that this macro * meets the 10-line criterion in LGPL, allowing this function to be diff --git a/urcu/static/urcu-qsbr.h b/urcu/static/urcu-qsbr.h index 690b77d..8f2ca32 100644 --- a/urcu/static/urcu-qsbr.h +++ b/urcu/static/urcu-qsbr.h @@ -68,54 +68,6 @@ enum rcu_state { RCU_READER_INACTIVE, }; -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define RCU_YIELD_READ (1 << 0) -#define RCU_YIELD_WRITE (1 << 1) - -/* maximum sleep delay, in us */ -#define MAX_SLEEP 50 - -extern unsigned int rcu_yield_active; -extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield); - -static inline void rcu_debug_yield_read(void) -{ - if (rcu_yield_active & RCU_YIELD_READ) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_write(void) -{ - if (rcu_yield_active & RCU_YIELD_WRITE) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_init(void) -{ - URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self(); -} -#else -static inline void rcu_debug_yield_read(void) -{ -} - -static inline void rcu_debug_yield_write(void) -{ -} - -static inline void rcu_debug_yield_init(void) -{ - -} -#endif - #define RCU_GP_ONLINE (1UL << 0) #define RCU_GP_CTR (1UL << 1) diff --git a/urcu/static/urcu.h b/urcu/static/urcu.h index 53d2610..b5fc09f 100644 --- a/urcu/static/urcu.h +++ b/urcu/static/urcu.h @@ -41,6 +41,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -84,61 +85,6 @@ enum rcu_state { #define rcu_assert(args...) #endif -#ifdef DEBUG_YIELD -#include -#include -#include -#include - -#define RCU_YIELD_READ (1 << 0) -#define RCU_YIELD_WRITE (1 << 1) - -/* - * Updates with RCU_SIGNAL are much slower. Account this in the delay. - */ -#ifdef RCU_SIGNAL -/* maximum sleep delay, in us */ -#define MAX_SLEEP 30000 -#else -#define MAX_SLEEP 50 -#endif - -extern unsigned int rcu_yield_active; -extern DECLARE_URCU_TLS(unsigned int, rcu_rand_yield); - -static inline void rcu_debug_yield_read(void) -{ - if (rcu_yield_active & RCU_YIELD_READ) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_write(void) -{ - if (rcu_yield_active & RCU_YIELD_WRITE) - if (rand_r(&URCU_TLS(rcu_rand_yield)) & 0x1) - usleep(rand_r(&URCU_TLS(rcu_rand_yield)) % MAX_SLEEP); -} - -static inline void rcu_debug_yield_init(void) -{ - URCU_TLS(rcu_rand_yield) = time(NULL) ^ (unsigned long) pthread_self(); -} -#else -static inline void rcu_debug_yield_read(void) -{ -} - -static inline void rcu_debug_yield_write(void) -{ -} - -static inline void rcu_debug_yield_init(void) -{ - -} -#endif - /* * RCU memory barrier broadcast group. Currently, only broadcast to all process * threads is supported (group 0). diff --git a/urcu/static/wfcqueue.h b/urcu/static/wfcqueue.h index 62e003f..48b2625 100644 --- a/urcu/static/wfcqueue.h +++ b/urcu/static/wfcqueue.h @@ -106,6 +106,17 @@ static inline void _cds_wfcq_init(struct cds_wfcq_head *head, assert(!ret); } +/* + * __cds_wfcq_init: initialize wait-free queue. + */ +static inline void ___cds_wfcq_init(struct __cds_wfcq_head *head, + struct cds_wfcq_tail *tail) +{ + /* Set queue head and tail */ + _cds_wfcq_node_init(&head->node); + tail->p = &head->node; +} + /* * cds_wfcq_empty: return whether wait-free queue is empty. * @@ -118,9 +129,10 @@ static inline void _cds_wfcq_init(struct cds_wfcq_head *head, * make a queue appear empty if an enqueuer is preempted for a long time * between xchg() and setting the previous node's next pointer. */ -static inline bool _cds_wfcq_empty(struct cds_wfcq_head *head, +static inline bool _cds_wfcq_empty(cds_wfcq_head_ptr_t u_head, struct cds_wfcq_tail *tail) { + struct __cds_wfcq_head *head = u_head._h; /* * Queue is empty if no node is pointed by head->node.next nor * tail->p. Even though the tail->p check is sufficient to find @@ -150,11 +162,12 @@ static inline void _cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head, assert(!ret); } -static inline bool ___cds_wfcq_append(struct cds_wfcq_head *head, +static inline bool ___cds_wfcq_append(cds_wfcq_head_ptr_t u_head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *new_head, struct cds_wfcq_node *new_tail) { + struct __cds_wfcq_head *head = u_head._h; struct cds_wfcq_node *old_tail; /* @@ -190,7 +203,7 @@ static inline bool ___cds_wfcq_append(struct cds_wfcq_head *head, * Returns false if the queue was empty prior to adding the node. * Returns true otherwise. */ -static inline bool _cds_wfcq_enqueue(struct cds_wfcq_head *head, +static inline bool _cds_wfcq_enqueue(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *new_tail) { @@ -237,10 +250,11 @@ ___cds_wfcq_node_sync_next(struct cds_wfcq_node *node, int blocking) } static inline struct cds_wfcq_node * -___cds_wfcq_first(struct cds_wfcq_head *head, +___cds_wfcq_first(cds_wfcq_head_ptr_t u_head, struct cds_wfcq_tail *tail, int blocking) { + struct __cds_wfcq_head *head = u_head._h; struct cds_wfcq_node *node; if (_cds_wfcq_empty(head, tail)) @@ -266,7 +280,7 @@ ___cds_wfcq_first(struct cds_wfcq_head *head, * Returns NULL if queue is empty, first node otherwise. */ static inline struct cds_wfcq_node * -___cds_wfcq_first_blocking(struct cds_wfcq_head *head, +___cds_wfcq_first_blocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_first(head, tail, 1); @@ -280,14 +294,14 @@ ___cds_wfcq_first_blocking(struct cds_wfcq_head *head, * it needs to block. */ static inline struct cds_wfcq_node * -___cds_wfcq_first_nonblocking(struct cds_wfcq_head *head, +___cds_wfcq_first_nonblocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_first(head, tail, 0); } static inline struct cds_wfcq_node * -___cds_wfcq_next(struct cds_wfcq_head *head, +___cds_wfcq_next(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node, int blocking) @@ -328,7 +342,7 @@ ___cds_wfcq_next(struct cds_wfcq_head *head, * otherwise. */ static inline struct cds_wfcq_node * -___cds_wfcq_next_blocking(struct cds_wfcq_head *head, +___cds_wfcq_next_blocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node) { @@ -342,7 +356,7 @@ ___cds_wfcq_next_blocking(struct cds_wfcq_head *head, * it needs to block. */ static inline struct cds_wfcq_node * -___cds_wfcq_next_nonblocking(struct cds_wfcq_head *head, +___cds_wfcq_next_nonblocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node) { @@ -350,11 +364,12 @@ ___cds_wfcq_next_nonblocking(struct cds_wfcq_head *head, } static inline struct cds_wfcq_node * -___cds_wfcq_dequeue_with_state(struct cds_wfcq_head *head, +___cds_wfcq_dequeue_with_state(cds_wfcq_head_ptr_t u_head, struct cds_wfcq_tail *tail, int *state, int blocking) { + struct __cds_wfcq_head *head = u_head._h; struct cds_wfcq_node *node, *next; if (state) @@ -422,7 +437,7 @@ ___cds_wfcq_dequeue_with_state(struct cds_wfcq_head *head, * caller. */ static inline struct cds_wfcq_node * -___cds_wfcq_dequeue_with_state_blocking(struct cds_wfcq_head *head, +___cds_wfcq_dequeue_with_state_blocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, int *state) { return ___cds_wfcq_dequeue_with_state(head, tail, state, 1); @@ -435,7 +450,7 @@ ___cds_wfcq_dequeue_with_state_blocking(struct cds_wfcq_head *head, * state. */ static inline struct cds_wfcq_node * -___cds_wfcq_dequeue_blocking(struct cds_wfcq_head *head, +___cds_wfcq_dequeue_blocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_dequeue_with_state_blocking(head, tail, NULL); @@ -448,7 +463,7 @@ ___cds_wfcq_dequeue_blocking(struct cds_wfcq_head *head, * if it needs to block. */ static inline struct cds_wfcq_node * -___cds_wfcq_dequeue_with_state_nonblocking(struct cds_wfcq_head *head, +___cds_wfcq_dequeue_with_state_nonblocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, int *state) { return ___cds_wfcq_dequeue_with_state(head, tail, state, 0); @@ -461,7 +476,7 @@ ___cds_wfcq_dequeue_with_state_nonblocking(struct cds_wfcq_head *head, * state. */ static inline struct cds_wfcq_node * -___cds_wfcq_dequeue_nonblocking(struct cds_wfcq_head *head, +___cds_wfcq_dequeue_nonblocking(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_dequeue_with_state_nonblocking(head, tail, NULL); @@ -479,12 +494,14 @@ ___cds_wfcq_dequeue_nonblocking(struct cds_wfcq_head *head, */ static inline enum cds_wfcq_ret ___cds_wfcq_splice( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t u_dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t u_src_q_head, struct cds_wfcq_tail *src_q_tail, int blocking) { + struct __cds_wfcq_head *dest_q_head = u_dest_q_head._h; + struct __cds_wfcq_head *src_q_head = u_src_q_head._h; struct cds_wfcq_node *head, *tail; int attempt = 0; @@ -540,9 +557,9 @@ ___cds_wfcq_splice( */ static inline enum cds_wfcq_ret ___cds_wfcq_splice_blocking( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t src_q_head, struct cds_wfcq_tail *src_q_tail) { return ___cds_wfcq_splice(dest_q_head, dest_q_tail, @@ -557,9 +574,9 @@ ___cds_wfcq_splice_blocking( */ static inline enum cds_wfcq_ret ___cds_wfcq_splice_nonblocking( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t src_q_head, struct cds_wfcq_tail *src_q_tail) { return ___cds_wfcq_splice(dest_q_head, dest_q_tail, diff --git a/urcu/static/wfstack.h b/urcu/static/wfstack.h index db0d5b8..261ff2c 100644 --- a/urcu/static/wfstack.h +++ b/urcu/static/wfstack.h @@ -76,6 +76,14 @@ void _cds_wfs_node_init(struct cds_wfs_node *node) node->next = NULL; } +/* + * __cds_wfs_init: initialize wait-free stack. + */ +static inline void ___cds_wfs_init(struct __cds_wfs_stack *s) +{ + s->head = CDS_WFS_END; +} + /* * cds_wfs_init: initialize wait-free stack. */ @@ -99,8 +107,10 @@ static inline bool ___cds_wfs_end(void *node) * * No memory barrier is issued. No mutual exclusion is required. */ -static inline bool _cds_wfs_empty(struct cds_wfs_stack *s) +static inline bool _cds_wfs_empty(cds_wfs_stack_ptr_t u_stack) { + struct __cds_wfs_stack *s = u_stack._s; + return ___cds_wfs_end(CMM_LOAD_SHARED(s->head)); } @@ -114,8 +124,9 @@ static inline bool _cds_wfs_empty(struct cds_wfs_stack *s) * Returns non-zero otherwise. */ static inline -int _cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node) +int _cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node) { + struct __cds_wfs_stack *s = u_stack._s; struct cds_wfs_head *old_head, *new_head; assert(node->next == NULL); @@ -161,10 +172,11 @@ ___cds_wfs_node_sync_next(struct cds_wfs_node *node, int blocking) static inline struct cds_wfs_node * -___cds_wfs_pop(struct cds_wfs_stack *s, int *state, int blocking) +___cds_wfs_pop(cds_wfs_stack_ptr_t u_stack, int *state, int blocking) { struct cds_wfs_head *head, *new_head; struct cds_wfs_node *next; + struct __cds_wfs_stack *s = u_stack._s; if (state) *state = 0; @@ -210,16 +222,16 @@ ___cds_wfs_pop(struct cds_wfs_stack *s, int *state, int blocking) */ static inline struct cds_wfs_node * -___cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state) +___cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, int *state) { - return ___cds_wfs_pop(s, state, 1); + return ___cds_wfs_pop(u_stack, state, 1); } static inline struct cds_wfs_node * -___cds_wfs_pop_blocking(struct cds_wfs_stack *s) +___cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack) { - return ___cds_wfs_pop_with_state_blocking(s, NULL); + return ___cds_wfs_pop_with_state_blocking(u_stack, NULL); } /* @@ -232,9 +244,9 @@ ___cds_wfs_pop_blocking(struct cds_wfs_stack *s) */ static inline struct cds_wfs_node * -___cds_wfs_pop_with_state_nonblocking(struct cds_wfs_stack *s, int *state) +___cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state) { - return ___cds_wfs_pop(s, state, 0); + return ___cds_wfs_pop(u_stack, state, 0); } /* @@ -245,9 +257,9 @@ ___cds_wfs_pop_with_state_nonblocking(struct cds_wfs_stack *s, int *state) */ static inline struct cds_wfs_node * -___cds_wfs_pop_nonblocking(struct cds_wfs_stack *s) +___cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack) { - return ___cds_wfs_pop_with_state_nonblocking(s, NULL); + return ___cds_wfs_pop_with_state_nonblocking(u_stack, NULL); } /* @@ -269,8 +281,9 @@ ___cds_wfs_pop_nonblocking(struct cds_wfs_stack *s) */ static inline struct cds_wfs_head * -___cds_wfs_pop_all(struct cds_wfs_stack *s) +___cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack) { + struct __cds_wfs_stack *s = u_stack._s; struct cds_wfs_head *head; /* diff --git a/urcu/syscall-compat.h b/urcu/syscall-compat.h new file mode 100644 index 0000000..55576f0 --- /dev/null +++ b/urcu/syscall-compat.h @@ -0,0 +1,36 @@ +#ifndef _URCU_SYSCALL_COMPAT_H +#define _URCU_SYSCALL_COMPAT_H + +/* + * urcu/syscall-compat.h + * + * Userspace RCU library - Syscall Compatibility Header + * + * Copyright 2013 - Pierre-Luc St-Charles + * + * Note: this file is only used to simplify the code required to + * include the 'syscall.h' system header across multiple platforms, + * which might not always be located at the same place (or needed at all). + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#if defined(__ANDROID__) +#include +#elif defined(__linux__) +#include +#endif + +#endif /* _URCU_SYSCALL_COMPAT_H */ diff --git a/urcu/system.h b/urcu/system.h index 6f31459..faae390 100644 --- a/urcu/system.h +++ b/urcu/system.h @@ -32,6 +32,7 @@ * Load a data from shared memory, doing a cache flush if required. */ #define CMM_LOAD_SHARED(p) \ + __extension__ \ ({ \ cmm_smp_rmc(); \ _CMM_LOAD_SHARED(p); \ @@ -41,13 +42,14 @@ * Identify a shared store. A cmm_smp_wmc() or cmm_smp_mc() should * follow the store. */ -#define _CMM_STORE_SHARED(x, v) ({ CMM_ACCESS_ONCE(x) = (v); }) +#define _CMM_STORE_SHARED(x, v) __extension__ ({ CMM_ACCESS_ONCE(x) = (v); }) /* * Store v into x, where x is located in shared memory. Performs the * required cache flush after writing. Returns v. */ #define CMM_STORE_SHARED(x, v) \ + __extension__ \ ({ \ __typeof__(x) _v = _CMM_STORE_SHARED(x, v); \ cmm_smp_wmc(); \ diff --git a/urcu/tls-compat.h b/urcu/tls-compat.h index a44b88d..8ac1ea0 100644 --- a/urcu/tls-compat.h +++ b/urcu/tls-compat.h @@ -74,6 +74,10 @@ extern "C" { #else /* #ifndef CONFIG_RCU_TLS */ +/* + * The *_1() macros ensure macro parameters are expanded. + */ + # include struct urcu_tls { @@ -82,14 +86,16 @@ struct urcu_tls { int init_done; }; -# define DECLARE_URCU_TLS(type, name) \ +# define DECLARE_URCU_TLS_1(type, name) \ type *__tls_access_ ## name(void) +# define DECLARE_URCU_TLS(type, name) \ + DECLARE_URCU_TLS_1(type, name) /* * Note: we don't free memory at process exit, since it will be dealt * with by the OS. */ -# define DEFINE_URCU_TLS(type, name) \ +# define DEFINE_URCU_TLS_1(type, name) \ type *__tls_access_ ## name(void) \ { \ static struct urcu_tls __tls_ ## name = { \ @@ -118,7 +124,12 @@ struct urcu_tls { return __tls_p; \ } -# define URCU_TLS(name) (*__tls_access_ ## name()) +# define DEFINE_URCU_TLS(type, name) \ + DEFINE_URCU_TLS_1(type, name) + +# define URCU_TLS_1(name) (*__tls_access_ ## name()) + +# define URCU_TLS(name) URCU_TLS_1(name) #endif /* #else #ifndef CONFIG_RCU_TLS */ diff --git a/urcu/uatomic/hppa.h b/urcu/uatomic/hppa.h new file mode 100644 index 0000000..2102153 --- /dev/null +++ b/urcu/uatomic/hppa.h @@ -0,0 +1,10 @@ +#ifndef _URCU_ARCH_UATOMIC_HPPA_H +#define _URCU_ARCH_UATOMIC_HPPA_H + +#include +#include + +#define UATOMIC_HAS_ATOMIC_SHORT +#include + +#endif /* _URCU_ARCH_UATOMIC_HPPA_H */ diff --git a/urcu/wfcqueue.h b/urcu/wfcqueue.h index 652b42d..83ec219 100644 --- a/urcu/wfcqueue.h +++ b/urcu/wfcqueue.h @@ -65,11 +65,25 @@ struct cds_wfcq_node { * enqueue/dequeue are expected from many CPUs. This eliminates * false-sharing between enqueue and dequeue. */ +struct __cds_wfcq_head { + struct cds_wfcq_node node; +}; + struct cds_wfcq_head { struct cds_wfcq_node node; pthread_mutex_t lock; }; +/* + * The transparent union allows calling functions that work on both + * struct cds_wfcq_head and struct __cds_wfcq_head on any of those two + * types. + */ +typedef union { + struct __cds_wfcq_head *_h; + struct cds_wfcq_head *h; +} __attribute__((__transparent_union__)) cds_wfcq_head_ptr_t; + struct cds_wfcq_tail { struct cds_wfcq_node *p; }; @@ -163,12 +177,18 @@ extern void cds_wfcq_node_init(struct cds_wfcq_node *node); extern void cds_wfcq_init(struct cds_wfcq_head *head, struct cds_wfcq_tail *tail); +/* + * __cds_wfcq_init: initialize wait-free queue. + */ +extern void __cds_wfcq_init(struct __cds_wfcq_head *head, + struct cds_wfcq_tail *tail); + /* * cds_wfcq_empty: return whether wait-free queue is empty. * * No memory barrier is issued. No mutual exclusion is required. */ -extern bool cds_wfcq_empty(struct cds_wfcq_head *head, +extern bool cds_wfcq_empty(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail); /* @@ -192,7 +212,7 @@ extern void cds_wfcq_dequeue_unlock(struct cds_wfcq_head *head, * Returns false if the queue was empty prior to adding the node. * Returns true otherwise. */ -extern bool cds_wfcq_enqueue(struct cds_wfcq_head *head, +extern bool cds_wfcq_enqueue(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node); @@ -231,7 +251,7 @@ extern struct cds_wfcq_node *cds_wfcq_dequeue_with_state_blocking( * ensured. * * Returns enum cds_wfcq_ret which indicates the state of the src or - * dest queue. Cannot block. + * dest queue. */ extern enum cds_wfcq_ret cds_wfcq_splice_blocking( struct cds_wfcq_head *dest_q_head, @@ -249,7 +269,7 @@ extern enum cds_wfcq_ret cds_wfcq_splice_blocking( * caller. */ extern struct cds_wfcq_node *__cds_wfcq_dequeue_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail); /* @@ -259,7 +279,7 @@ extern struct cds_wfcq_node *__cds_wfcq_dequeue_blocking( * last node of the queue into state (CDS_WFCQ_STATE_LAST). */ extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, int *state); @@ -270,7 +290,7 @@ extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking( * if it needs to block. */ extern struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail); /* @@ -280,7 +300,7 @@ extern struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking( * the last node of the queue into state (CDS_WFCQ_STATE_LAST). */ extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, int *state); @@ -295,9 +315,9 @@ extern struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking( * dest queue. Never returns CDS_WFCQ_RET_WOULDBLOCK. */ extern enum cds_wfcq_ret __cds_wfcq_splice_blocking( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t src_q_head, struct cds_wfcq_tail *src_q_tail); /* @@ -307,9 +327,9 @@ extern enum cds_wfcq_ret __cds_wfcq_splice_blocking( * CDS_WFCQ_RET_WOULDBLOCK if it needs to block. */ extern enum cds_wfcq_ret __cds_wfcq_splice_nonblocking( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t src_q_head, struct cds_wfcq_tail *src_q_tail); /* @@ -327,7 +347,7 @@ extern enum cds_wfcq_ret __cds_wfcq_splice_nonblocking( * Returns NULL if queue is empty, first node otherwise. */ extern struct cds_wfcq_node *__cds_wfcq_first_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail); /* @@ -337,7 +357,7 @@ extern struct cds_wfcq_node *__cds_wfcq_first_blocking( * it needs to block. */ extern struct cds_wfcq_node *__cds_wfcq_first_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail); /* @@ -356,7 +376,7 @@ extern struct cds_wfcq_node *__cds_wfcq_first_nonblocking( * otherwise. */ extern struct cds_wfcq_node *__cds_wfcq_next_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node); @@ -367,7 +387,7 @@ extern struct cds_wfcq_node *__cds_wfcq_next_blocking( * it needs to block. */ extern struct cds_wfcq_node *__cds_wfcq_next_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node); @@ -376,7 +396,7 @@ extern struct cds_wfcq_node *__cds_wfcq_next_nonblocking( /* * __cds_wfcq_for_each_blocking: Iterate over all nodes in a queue, * without dequeuing them. - * @head: head of the queue (struct cds_wfcq_head pointer). + * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer). * @tail: tail of the queue (struct cds_wfcq_tail pointer). * @node: iterator on the queue (struct cds_wfcq_node pointer). * @@ -393,7 +413,7 @@ extern struct cds_wfcq_node *__cds_wfcq_next_nonblocking( /* * __cds_wfcq_for_each_blocking_safe: Iterate over all nodes in a queue, * without dequeuing them. Safe against deletion. - * @head: head of the queue (struct cds_wfcq_head pointer). + * @head: head of the queue (struct cds_wfcq_head or __cds_wfcq_head pointer). * @tail: tail of the queue (struct cds_wfcq_tail pointer). * @node: iterator on the queue (struct cds_wfcq_node pointer). * @n: struct cds_wfcq_node pointer holding the next pointer (used diff --git a/urcu/wfstack.h b/urcu/wfstack.h index fc0b44b..28f6162 100644 --- a/urcu/wfstack.h +++ b/urcu/wfstack.h @@ -83,17 +83,32 @@ struct cds_wfs_head { struct cds_wfs_node node; }; +struct __cds_wfs_stack { + struct cds_wfs_head *head; +}; + struct cds_wfs_stack { struct cds_wfs_head *head; pthread_mutex_t lock; }; +/* + * The transparent union allows calling functions that work on both + * struct cds_wfs_stack and struct __cds_wfs_stack on any of those two + * types. + */ +typedef union { + struct __cds_wfs_stack *_s; + struct cds_wfs_stack *s; +} __attribute__((__transparent_union__)) cds_wfs_stack_ptr_t; + #ifdef _LGPL_SOURCE #include #define cds_wfs_node_init _cds_wfs_node_init #define cds_wfs_init _cds_wfs_init +#define __cds_wfs_init ___cds_wfs_init #define cds_wfs_empty _cds_wfs_empty #define cds_wfs_push _cds_wfs_push @@ -135,12 +150,17 @@ extern void cds_wfs_node_init(struct cds_wfs_node *node); */ extern void cds_wfs_init(struct cds_wfs_stack *s); +/* + * __cds_wfs_init: initialize wait-free stack. + */ +extern void __cds_wfs_init(struct __cds_wfs_stack *s); + /* * cds_wfs_empty: return whether wait-free stack is empty. * * No memory barrier is issued. No mutual exclusion is required. */ -extern bool cds_wfs_empty(struct cds_wfs_stack *s); +extern bool cds_wfs_empty(cds_wfs_stack_ptr_t u_stack); /* * cds_wfs_push: push a node into the stack. @@ -151,7 +171,7 @@ extern bool cds_wfs_empty(struct cds_wfs_stack *s); * Returns 0 if the stack was empty prior to adding the node. * Returns non-zero otherwise. */ -extern int cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node); +extern int cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node); /* * cds_wfs_pop_blocking: pop a node from the stack. @@ -239,7 +259,7 @@ extern void cds_wfs_pop_unlock(struct cds_wfs_stack *s); * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking() * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme). */ -extern struct cds_wfs_node *__cds_wfs_pop_blocking(struct cds_wfs_stack *s); +extern struct cds_wfs_node *__cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack); /* * __cds_wfs_pop_with_state_blocking: pop a node from the stack, with state. @@ -248,7 +268,8 @@ extern struct cds_wfs_node *__cds_wfs_pop_blocking(struct cds_wfs_stack *s); * empty into state (CDS_WFS_STATE_LAST). */ extern struct cds_wfs_node * - __cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state); + __cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, + int *state); /* * __cds_wfs_pop_nonblocking: pop a node from the stack. @@ -256,7 +277,7 @@ extern struct cds_wfs_node * * Same as __cds_wfs_pop_blocking, but returns CDS_WFS_WOULDBLOCK if * it needs to block. */ -extern struct cds_wfs_node *__cds_wfs_pop_nonblocking(struct cds_wfs_stack *s); +extern struct cds_wfs_node *__cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack); /* * __cds_wfs_pop_with_state_nonblocking: pop a node from the stack, with state. @@ -265,7 +286,7 @@ extern struct cds_wfs_node *__cds_wfs_pop_nonblocking(struct cds_wfs_stack *s); * empty into state (CDS_WFS_STATE_LAST). */ extern struct cds_wfs_node * - __cds_wfs_pop_with_state_nonblocking(struct cds_wfs_stack *s, + __cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state); /* @@ -285,7 +306,7 @@ extern struct cds_wfs_node * * 3) Ensuring that only ONE thread can call __cds_wfs_pop_blocking() * and __cds_wfs_pop_all(). (multi-provider/single-consumer scheme). */ -extern struct cds_wfs_head *__cds_wfs_pop_all(struct cds_wfs_stack *s); +extern struct cds_wfs_head *__cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack); #endif /* !_LGPL_SOURCE */ diff --git a/wfcqueue.c b/wfcqueue.c index 4950c10..e28d51a 100644 --- a/wfcqueue.c +++ b/wfcqueue.c @@ -40,14 +40,20 @@ void cds_wfcq_init(struct cds_wfcq_head *head, _cds_wfcq_init(head, tail); } -bool cds_wfcq_empty(struct cds_wfcq_head *head, +void __cds_wfcq_init(struct __cds_wfcq_head *head, + struct cds_wfcq_tail *tail) +{ + ___cds_wfcq_init(head, tail); +} + +bool cds_wfcq_empty(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return _cds_wfcq_empty(head, tail); } -bool cds_wfcq_enqueue(struct cds_wfcq_head *head, +bool cds_wfcq_enqueue(cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node) { @@ -92,14 +98,14 @@ enum cds_wfcq_ret cds_wfcq_splice_blocking( } struct cds_wfcq_node *__cds_wfcq_dequeue_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_dequeue_blocking(head, tail); } struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, int *state) { @@ -107,14 +113,14 @@ struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_blocking( } struct cds_wfcq_node *__cds_wfcq_dequeue_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_dequeue_nonblocking(head, tail); } struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, int *state) { @@ -122,9 +128,9 @@ struct cds_wfcq_node *__cds_wfcq_dequeue_with_state_nonblocking( } enum cds_wfcq_ret __cds_wfcq_splice_blocking( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t src_q_head, struct cds_wfcq_tail *src_q_tail) { return ___cds_wfcq_splice_blocking(dest_q_head, dest_q_tail, @@ -132,9 +138,9 @@ enum cds_wfcq_ret __cds_wfcq_splice_blocking( } enum cds_wfcq_ret __cds_wfcq_splice_nonblocking( - struct cds_wfcq_head *dest_q_head, + cds_wfcq_head_ptr_t dest_q_head, struct cds_wfcq_tail *dest_q_tail, - struct cds_wfcq_head *src_q_head, + cds_wfcq_head_ptr_t src_q_head, struct cds_wfcq_tail *src_q_tail) { return ___cds_wfcq_splice_nonblocking(dest_q_head, dest_q_tail, @@ -142,21 +148,21 @@ enum cds_wfcq_ret __cds_wfcq_splice_nonblocking( } struct cds_wfcq_node *__cds_wfcq_first_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_first_blocking(head, tail); } struct cds_wfcq_node *__cds_wfcq_first_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail) { return ___cds_wfcq_first_nonblocking(head, tail); } struct cds_wfcq_node *__cds_wfcq_next_blocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node) { @@ -164,7 +170,7 @@ struct cds_wfcq_node *__cds_wfcq_next_blocking( } struct cds_wfcq_node *__cds_wfcq_next_nonblocking( - struct cds_wfcq_head *head, + cds_wfcq_head_ptr_t head, struct cds_wfcq_tail *tail, struct cds_wfcq_node *node) { diff --git a/wfstack.c b/wfstack.c index c8bd7e6..7618be3 100644 --- a/wfstack.c +++ b/wfstack.c @@ -38,14 +38,19 @@ void cds_wfs_init(struct cds_wfs_stack *s) _cds_wfs_init(s); } -bool cds_wfs_empty(struct cds_wfs_stack *s) +void __cds_wfs_init(struct __cds_wfs_stack *s) { - return _cds_wfs_empty(s); + ___cds_wfs_init(s); } -int cds_wfs_push(struct cds_wfs_stack *s, struct cds_wfs_node *node) +bool cds_wfs_empty(cds_wfs_stack_ptr_t u_stack) { - return _cds_wfs_push(s, node); + return _cds_wfs_empty(u_stack); +} + +int cds_wfs_push(cds_wfs_stack_ptr_t u_stack, struct cds_wfs_node *node) +{ + return _cds_wfs_push(u_stack, node); } struct cds_wfs_node *cds_wfs_pop_blocking(struct cds_wfs_stack *s) @@ -89,30 +94,31 @@ void cds_wfs_pop_unlock(struct cds_wfs_stack *s) _cds_wfs_pop_unlock(s); } -struct cds_wfs_node *__cds_wfs_pop_blocking(struct cds_wfs_stack *s) +struct cds_wfs_node *__cds_wfs_pop_blocking(cds_wfs_stack_ptr_t u_stack) { - return ___cds_wfs_pop_blocking(s); + return ___cds_wfs_pop_blocking(u_stack); } struct cds_wfs_node * - __cds_wfs_pop_with_state_blocking(struct cds_wfs_stack *s, int *state) + __cds_wfs_pop_with_state_blocking(cds_wfs_stack_ptr_t u_stack, + int *state) { - return ___cds_wfs_pop_with_state_blocking(s, state); + return ___cds_wfs_pop_with_state_blocking(u_stack, state); } -struct cds_wfs_node *__cds_wfs_pop_nonblocking(struct cds_wfs_stack *s) +struct cds_wfs_node *__cds_wfs_pop_nonblocking(cds_wfs_stack_ptr_t u_stack) { - return ___cds_wfs_pop_nonblocking(s); + return ___cds_wfs_pop_nonblocking(u_stack); } struct cds_wfs_node * - __cds_wfs_pop_with_state_nonblocking(struct cds_wfs_stack *s, + __cds_wfs_pop_with_state_nonblocking(cds_wfs_stack_ptr_t u_stack, int *state) { - return ___cds_wfs_pop_with_state_nonblocking(s, state); + return ___cds_wfs_pop_with_state_nonblocking(u_stack, state); } -struct cds_wfs_head *__cds_wfs_pop_all(struct cds_wfs_stack *s) +struct cds_wfs_head *__cds_wfs_pop_all(cds_wfs_stack_ptr_t u_stack) { - return ___cds_wfs_pop_all(s); + return ___cds_wfs_pop_all(u_stack); }