[urcu.git] / tests / regression / rcutorture.h

// SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation.
//
// SPDX-License-Identifier: GPL-2.0-or-later

/*
 * rcutorture.h: simple user-level performance/stress test of RCU.
 *
 * Usage:
 * 	./rcu <nreaders> rperf [ <cpustride> ]
 * 		Run a read-side performance test with the specified
 * 		number of readers spaced by <cpustride>.
 * 		Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
 * 		CPUs from 0 to 30.
 * 	./rcu <nupdaters> uperf [ <cpustride> ]
 * 		Run an update-side performance test with the specified
 * 		number of updaters and specified CPU spacing.
 * 	./rcu <nreaders> perf [ <cpustride> ]
 * 		Run a combined read/update performance test with the specified
 * 		number of readers and one updater and specified CPU spacing.
 * 		The readers run on the low-numbered CPUs and the updater
 * 		of the highest-numbered CPU.
 *
 * The above tests produce output as follows:
 *
 * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
 * ns/read: 43.4707  ns/update: 6848.1
 *
 * The first line lists the total number of RCU reads and updates executed
 * during the test, the number of reader threads, the number of updater
 * threads, and the duration of the test in seconds.  The second line
 * lists the average duration of each type of operation in nanoseconds,
 * or "nan" if the corresponding type of operation was not performed.
 *
 * 	./rcu <nreaders> stress
 * 		Run a stress test with the specified number of readers and
 * 		one updater.  None of the threads are affinitied to any
 * 		particular CPU.
 *
 * This test produces output as follows:
 *
 * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
 *
 * The first line lists the number of RCU read and update operations
 * executed, followed by the number of memory-ordering violations
 * (which will be zero in a correct RCU implementation).  The second
 * line lists the number of readers observing progressively more stale
 * data.  A correct RCU implementation will have all but the first two
 * numbers non-zero.
 */

/*
 * Test variables.
 */

#include <stdlib.h>
#include "tap.h"

#define NR_TESTS	1

DEFINE_PER_THREAD(long long, n_reads_pt);
DEFINE_PER_THREAD(long long, n_updates_pt);

enum callrcu_type {
	CALLRCU_GLOBAL,
	CALLRCU_PERCPU,
	CALLRCU_PERTHREAD,
};

enum writer_state {
	WRITER_STATE_SYNC_RCU,
	WRITER_STATE_CALL_RCU,
	WRITER_STATE_POLL_RCU,
};

static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;

long long n_reads = 0LL;
long n_updates = 0L;
int nthreadsrunning;
char argsbuf[64];

#define GOFLAG_INIT 0
#define GOFLAG_RUN  1
#define GOFLAG_STOP 2

volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
        = GOFLAG_INIT;

#define RCU_READ_RUN 1000

//MD
#define RCU_READ_NESTABLE

#ifdef RCU_READ_NESTABLE
#define rcu_read_lock_nest() rcu_read_lock()
#define rcu_read_unlock_nest() rcu_read_unlock()
#else /* #ifdef RCU_READ_NESTABLE */
#define rcu_read_lock_nest()
#define rcu_read_unlock_nest()
#endif /* #else #ifdef RCU_READ_NESTABLE */

#ifdef TORTURE_QSBR
#define mark_rcu_quiescent_state	rcu_quiescent_state
#define put_thread_offline		rcu_thread_offline
#define put_thread_online		rcu_thread_online
#endif

#ifndef mark_rcu_quiescent_state
#define mark_rcu_quiescent_state() do {} while (0)
#endif /* #ifdef mark_rcu_quiescent_state */

#ifndef put_thread_offline
#define put_thread_offline()		do {} while (0)
#define put_thread_online()		do {} while (0)
#define put_thread_online_delay()	do {} while (0)
#else /* #ifndef put_thread_offline */
#define put_thread_online_delay()	synchronize_rcu()
#endif /* #else #ifndef put_thread_offline */

/*
 * Performance test.
 */

static
void *rcu_read_perf_test(void *arg)
{
	int i;
	int me = (long)arg;
	long long n_reads_local = 0;

	rcu_register_thread();
	run_on(me);
	uatomic_inc(&nthreadsrunning);
	put_thread_offline();
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	put_thread_online();
	while (goflag == GOFLAG_RUN) {
		for (i = 0; i < RCU_READ_RUN; i++) {
			rcu_read_lock();
			/* rcu_read_lock_nest(); */
			/* rcu_read_unlock_nest(); */
			rcu_read_unlock();
		}
		n_reads_local += RCU_READ_RUN;
		mark_rcu_quiescent_state();
	}
	__get_thread_var(n_reads_pt) += n_reads_local;
	put_thread_offline();
	rcu_unregister_thread();

	return (NULL);
}

static
void *rcu_update_perf_test(void *arg __attribute__((unused)))
{
	long long n_updates_local = 0;

	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = create_call_rcu_data(0, -1);
		if (crdp != NULL) {
			diag("Successfully using per-thread call_rcu() worker.");
			set_thread_call_rcu_data(crdp);
		}
	}
	uatomic_inc(&nthreadsrunning);
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		synchronize_rcu();
		n_updates_local++;
	}
	__get_thread_var(n_updates_pt) += n_updates_local;
	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = get_thread_call_rcu_data();
		set_thread_call_rcu_data(NULL);
		call_rcu_data_free(crdp);
	}
	return NULL;
}

static
void perftestinit(void)
{
	init_per_thread(n_reads_pt, 0LL);
	init_per_thread(n_updates_pt, 0LL);
	uatomic_set(&nthreadsrunning, 0);
}

static
int perftestrun(int nthreads, int nreaders, int nupdaters)
{
	int t;
	int duration = 1;

	cmm_smp_mb();
	while (uatomic_read(&nthreadsrunning) < nthreads)
		(void) poll(NULL, 0, 1);
	goflag = GOFLAG_RUN;
	cmm_smp_mb();
	sleep(duration);
	cmm_smp_mb();
	goflag = GOFLAG_STOP;
	cmm_smp_mb();
	wait_all_threads();
	for_each_thread(t) {
		n_reads += per_thread(n_reads_pt, t);
		n_updates += per_thread(n_updates_pt, t);
	}
	diag("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d",
	       n_reads, n_updates, nreaders, nupdaters, duration);
	diag("ns/read: %g  ns/update: %g",
	       ((duration * 1000*1000*1000.*(double)nreaders) /
	        (double)n_reads),
	       ((duration * 1000*1000*1000.*(double)nupdaters) /
	        (double)n_updates));
	if (get_cpu_call_rcu_data(0)) {
		diag("Deallocating per-CPU call_rcu threads.\n");
		free_all_cpu_call_rcu_data();
	}
	return 0;
}

static
int perftest(int nreaders, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	for (i = 0; i < nreaders; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_read_perf_test, (void *)arg);
	}
	arg = (long)(i * cpustride);
	create_thread(rcu_update_perf_test, (void *)arg);
	return perftestrun(i + 1, nreaders, 1);
}

static
int rperftest(int nreaders, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	init_per_thread(n_reads_pt, 0LL);
	for (i = 0; i < nreaders; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_read_perf_test, (void *)arg);
	}
	return perftestrun(i, nreaders, 0);
}

static
int uperftest(int nupdaters, int cpustride)
{
	int i;
	long arg;

	perftestinit();
	init_per_thread(n_reads_pt, 0LL);
	for (i = 0; i < nupdaters; i++) {
		arg = (long)(i * cpustride);
		create_thread(rcu_update_perf_test, (void *)arg);
	}
	return perftestrun(i, 0, nupdaters);
}

/*
 * Stress test.
 */

#define RCU_STRESS_PIPE_LEN 10

struct rcu_stress {
	int pipe_count;
	int mbtest;
};

struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0, 0 } };
struct rcu_stress *rcu_stress_current;
int rcu_stress_idx = 0;

int n_mberror = 0;
DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);

int garbage = 0;

static
void *rcu_read_stress_test(void *arg __attribute__((unused)))
{
	int i;
	int itercnt = 0;
	struct rcu_stress *p;
	int pc;

	rcu_register_thread();
	put_thread_offline();
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	put_thread_online();
	while (goflag == GOFLAG_RUN) {
		rcu_read_lock();
		p = rcu_dereference(rcu_stress_current);
		if (p->mbtest == 0)
			n_mberror++;
		rcu_read_lock_nest();
		for (i = 0; i < 100; i++)
			garbage++;
		rcu_read_unlock_nest();
		pc = p->pipe_count;
		rcu_read_unlock();
		if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
			pc = RCU_STRESS_PIPE_LEN;
		__get_thread_var(rcu_stress_count)[pc]++;
		__get_thread_var(n_reads_pt)++;
		mark_rcu_quiescent_state();
		if ((++itercnt % 0x1000) == 0) {
			put_thread_offline();
			put_thread_online_delay();
			put_thread_online();
		}
	}
	put_thread_offline();
	rcu_unregister_thread();

	return (NULL);
}

static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
static bool call_rcu_wait;

static
void rcu_update_stress_test_rcu(struct rcu_head *head __attribute__((unused)))
{
	int ret;

	ret = pthread_mutex_lock(&call_rcu_test_mutex);
	if (ret) {
		errno = ret;
		diag("pthread_mutex_lock: %s",
			strerror(errno));
		abort();
	}
	ret = pthread_cond_signal(&call_rcu_test_cond);
	if (ret) {
		errno = ret;
		diag("pthread_cond_signal: %s",
			strerror(errno));
		abort();
	}
	call_rcu_wait = false;
	ret = pthread_mutex_unlock(&call_rcu_test_mutex);
	if (ret) {
		errno = ret;
		diag("pthread_mutex_unlock: %s",
			strerror(errno));
		abort();
	}
}

static
void advance_writer_state(enum writer_state *state)
{
	switch (*state) {
	case WRITER_STATE_SYNC_RCU:
		*state = WRITER_STATE_CALL_RCU;
		break;
	case WRITER_STATE_CALL_RCU:
		*state = WRITER_STATE_POLL_RCU;
		break;
	case WRITER_STATE_POLL_RCU:
		*state = WRITER_STATE_SYNC_RCU;
		break;
	}
}

static
void *rcu_update_stress_test(void *arg __attribute__((unused)))
{
	int i;
	struct rcu_stress *p;
	struct rcu_head rh;
	enum writer_state writer_state = WRITER_STATE_SYNC_RCU;

	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		i = rcu_stress_idx + 1;
		if (i >= RCU_STRESS_PIPE_LEN)
			i = 0;
		p = &rcu_stress_array[i];
		p->mbtest = 0;
		cmm_smp_mb();
		p->pipe_count = 0;
		p->mbtest = 1;
		rcu_assign_pointer(rcu_stress_current, p);
		rcu_stress_idx = i;
		for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
			if (i != rcu_stress_idx)
				rcu_stress_array[i].pipe_count++;
		switch (writer_state) {
		case WRITER_STATE_SYNC_RCU:
			synchronize_rcu();
			break;
		case WRITER_STATE_CALL_RCU:
		{
			int ret;

			ret = pthread_mutex_lock(&call_rcu_test_mutex);
			if (ret) {
				errno = ret;
				diag("pthread_mutex_lock: %s",
					strerror(errno));
				abort();
			}
			rcu_register_thread();
			call_rcu(&rh, rcu_update_stress_test_rcu);
			rcu_unregister_thread();
			/*
			 * Our MacOS X test machine with the following
			 * config:
			 * 15.6.0 Darwin Kernel Version 15.6.0
			 * root:xnu-3248.60.10~1/RELEASE_X86_64
			 * appears to have issues with liburcu-signal
			 * signal being delivered on top of
			 * pthread_cond_wait. It seems to make the
			 * thread continue, and therefore corrupt the
			 * rcu_head. Work around this issue by
			 * unregistering the RCU read-side thread
			 * immediately after call_rcu (call_rcu needs
			 * us to be registered RCU readers).
			 */
			call_rcu_wait = true;
			do {
				ret = pthread_cond_wait(&call_rcu_test_cond,
							&call_rcu_test_mutex);
			} while (call_rcu_wait);
			if (ret) {
				errno = ret;
				diag("pthread_cond_signal: %s",
					strerror(errno));
				abort();
			}
			ret = pthread_mutex_unlock(&call_rcu_test_mutex);
			if (ret) {
				errno = ret;
				diag("pthread_mutex_unlock: %s",
					strerror(errno));
				abort();
			}
			break;
		}
		case WRITER_STATE_POLL_RCU:
		{
			struct urcu_gp_poll_state poll_state;

			rcu_register_thread();
			poll_state = start_poll_synchronize_rcu();
			rcu_unregister_thread();
			while (!poll_state_synchronize_rcu(poll_state))
				(void) poll(NULL, 0, 1);	/* Wait for 1ms */
			break;
		}
		}
		n_updates++;
		advance_writer_state(&writer_state);
	}

	return NULL;
}

static
void *rcu_fake_update_stress_test(void *arg __attribute__((unused)))
{
	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = create_call_rcu_data(0, -1);
		if (crdp != NULL) {
			diag("Successfully using per-thread call_rcu() worker.");
			set_thread_call_rcu_data(crdp);
		}
	}
	while (goflag == GOFLAG_INIT)
		(void) poll(NULL, 0, 1);
	while (goflag == GOFLAG_RUN) {
		synchronize_rcu();
		(void) poll(NULL, 0, 1);
	}
	if (callrcu_type == CALLRCU_PERTHREAD) {
		struct call_rcu_data *crdp;

		crdp = get_thread_call_rcu_data();
		set_thread_call_rcu_data(NULL);
		call_rcu_data_free(crdp);
	}
	return NULL;
}

static
int stresstest(int nreaders)
{
	int i;
	int t;
	long long *p;
	long long sum;

	init_per_thread(n_reads_pt, 0LL);
	for_each_thread(t) {
		p = &per_thread(rcu_stress_count,t)[0];
		for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
			p[i] = 0LL;
	}
	rcu_stress_current = &rcu_stress_array[0];
	rcu_stress_current->pipe_count = 0;
	rcu_stress_current->mbtest = 1;
	for (i = 0; i < nreaders; i++)
		create_thread(rcu_read_stress_test, NULL);
	create_thread(rcu_update_stress_test, NULL);
	for (i = 0; i < 5; i++)
		create_thread(rcu_fake_update_stress_test, NULL);
	cmm_smp_mb();
	goflag = GOFLAG_RUN;
	cmm_smp_mb();
	sleep(10);
	cmm_smp_mb();
	goflag = GOFLAG_STOP;
	cmm_smp_mb();
	wait_all_threads();
	for_each_thread(t)
		n_reads += per_thread(n_reads_pt, t);
	diag("n_reads: %lld  n_updates: %ld  n_mberror: %d",
	       n_reads, n_updates, n_mberror);
	rdiag_start();
	rdiag("rcu_stress_count:");
	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
		sum = 0LL;
		for_each_thread(t) {
			sum += per_thread(rcu_stress_count, t)[i];
		}
		rdiag(" %lld", sum);
	}
	rdiag_end();
	if (get_cpu_call_rcu_data(0)) {
		diag("Deallocating per-CPU call_rcu threads.");
		free_all_cpu_call_rcu_data();
	}
	if (!n_mberror)
		return 0;
	else
		return -1;
}

/*
 * Mainprogram.
 */

static
void usage(char *argv[]) __attribute__((__noreturn__));

static
void usage(char *argv[])
{
	diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
	exit(-1);
}

int main(int argc, char *argv[])
{
	int nreaders = 1;
	int cpustride = 1;

	plan_tests(NR_TESTS);

	smp_init();
	//rcu_init();
	if (argc > 4) {
		const char *callrcu_str = argv[4];;

		if (strcmp(callrcu_str, "callrcu_global") == 0) {
			callrcu_type = CALLRCU_GLOBAL;
		} else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
			callrcu_type = CALLRCU_PERCPU;
		} else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
			callrcu_type = CALLRCU_PERTHREAD;
		} else {
			usage(argv);
			goto end;
		}
	}

	switch (callrcu_type) {
	case CALLRCU_GLOBAL:
		diag("Using global per-process call_rcu thread.");
		break;
	case CALLRCU_PERCPU:
		diag("Using per-CPU call_rcu threads.");
		if (create_all_cpu_call_rcu_data(0))
			diag("create_all_cpu_call_rcu_data: %s",
				strerror(errno));
		break;
	case CALLRCU_PERTHREAD:
		diag("Using per-thread call_rcu() worker.");
		break;
	default:
		abort();
	}

#ifdef DEBUG_YIELD
	yield_active |= YIELD_READ;
	yield_active |= YIELD_WRITE;
#endif

	if (argc > 1) {
		if (strcmp(argv[1], "-h") == 0
				|| strcmp(argv[1], "--help") == 0) {
			usage(argv);
			goto end;
		}
		nreaders = strtoul(argv[1], NULL, 0);
		if (argc == 2) {
			ok(!perftest(nreaders, cpustride),
				"perftest readers: %d, stride: %d",
				nreaders, cpustride);
			goto end;
		}
		if (argc > 3)
			cpustride = strtoul(argv[3], NULL, 0);
		if (strcmp(argv[2], "perf") == 0)
			ok(!perftest(nreaders, cpustride),
				"perftest readers: %d, stride: %d",
				nreaders, cpustride);
		else if (strcmp(argv[2], "rperf") == 0)
			ok(!rperftest(nreaders, cpustride),
				"rperftest readers: %d, stride: %d",
				nreaders, cpustride);
		else if (strcmp(argv[2], "uperf") == 0)
			ok(!uperftest(nreaders, cpustride),
				"uperftest readers: %d, stride: %d",
				nreaders, cpustride);
		else if (strcmp(argv[2], "stress") == 0)
			ok(!stresstest(nreaders),
				"stresstest readers: %d, stride: %d",
				nreaders, cpustride);
		else
			usage(argv);
	} else {
		usage(argv);
	}
end:
	return exit_status();
}
Commit	Line	Data
ce29b371 MJ	1	// SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation.
	2	//
	3	// SPDX-License-Identifier: GPL-2.0-or-later
	4
8a953620 MD	5	/*
	6	* rcutorture.h: simple user-level performance/stress test of RCU.
	7	*
	8	* Usage:
	9	* ./rcu <nreaders> rperf [ <cpustride> ]
	10	* Run a read-side performance test with the specified
	11	* number of readers spaced by <cpustride>.
	12	* Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
	13	* CPUs from 0 to 30.
	14	* ./rcu <nupdaters> uperf [ <cpustride> ]
	15	* Run an update-side performance test with the specified
	16	* number of updaters and specified CPU spacing.
	17	* ./rcu <nreaders> perf [ <cpustride> ]
	18	* Run a combined read/update performance test with the specified
	19	* number of readers and one updater and specified CPU spacing.
	20	* The readers run on the low-numbered CPUs and the updater
	21	* of the highest-numbered CPU.
	22	*
	23	* The above tests produce output as follows:
	24	*
	25	* n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
	26	* ns/read: 43.4707 ns/update: 6848.1
	27	*
	28	* The first line lists the total number of RCU reads and updates executed
	29	* during the test, the number of reader threads, the number of updater
	30	* threads, and the duration of the test in seconds. The second line
	31	* lists the average duration of each type of operation in nanoseconds,
	32	* or "nan" if the corresponding type of operation was not performed.
	33	*
	34	* ./rcu <nreaders> stress
	35	* Run a stress test with the specified number of readers and
	36	* one updater. None of the threads are affinitied to any
	37	* particular CPU.
	38	*
	39	* This test produces output as follows:
	40	*
	41	* n_reads: 114633217 n_updates: 3903415 n_mberror: 0
	42	* rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
	43	*
	44	* The first line lists the number of RCU read and update operations
	45	* executed, followed by the number of memory-ordering violations
	46	* (which will be zero in a correct RCU implementation). The second
	47	* line lists the number of readers observing progressively more stale
	48	* data. A correct RCU implementation will have all but the first two
	49	* numbers non-zero.
8a953620 MD	50	*/
	51
	52	/*
	53	* Test variables.
	54	*/
	55
b57aee66	56	#include <stdlib.h>
ad460058 MD	57	#include "tap.h"
	58
	59	#define NR_TESTS 1
b57aee66	60
8a953620 MD	61	DEFINE_PER_THREAD(long long, n_reads_pt);
	62	DEFINE_PER_THREAD(long long, n_updates_pt);
	63
26b5a74b MD	64	enum callrcu_type {
	65	CALLRCU_GLOBAL,
	66	CALLRCU_PERCPU,
	67	CALLRCU_PERTHREAD,
	68	};
	69
eb218d4f MD	70	enum writer_state {
	71	WRITER_STATE_SYNC_RCU,
	72	WRITER_STATE_CALL_RCU,
	73	WRITER_STATE_POLL_RCU,
	74	};
	75
26b5a74b MD	76	static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
26b5a74b MD	77
8a953620 MD	78	long long n_reads = 0LL;
8a953620 MD	79	long n_updates = 0L;
6ee91d83	80	int nthreadsrunning;
8a953620 MD	81	char argsbuf[64];
	82
	83	#define GOFLAG_INIT 0
	84	#define GOFLAG_RUN 1
	85	#define GOFLAG_STOP 2
	86
4967f005 PB	87	volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
4967f005 PB	88	= GOFLAG_INIT;
8a953620 MD	89
	90	#define RCU_READ_RUN 1000
	91
	92	//MD
	93	#define RCU_READ_NESTABLE
	94
	95	#ifdef RCU_READ_NESTABLE
	96	#define rcu_read_lock_nest() rcu_read_lock()
	97	#define rcu_read_unlock_nest() rcu_read_unlock()
	98	#else /* #ifdef RCU_READ_NESTABLE */
	99	#define rcu_read_lock_nest()
	100	#define rcu_read_unlock_nest()
	101	#endif /* #else #ifdef RCU_READ_NESTABLE */
	102
1a43bbd8 MD	103	#ifdef TORTURE_QSBR
	104	#define mark_rcu_quiescent_state rcu_quiescent_state
	105	#define put_thread_offline rcu_thread_offline
	106	#define put_thread_online rcu_thread_online
	107	#endif
	108
8a953620	109	#ifndef mark_rcu_quiescent_state
447c9339	110	#define mark_rcu_quiescent_state() do {} while (0)
8a953620 MD	111	#endif /* #ifdef mark_rcu_quiescent_state */
	112
	113	#ifndef put_thread_offline
447c9339 MJ	114	#define put_thread_offline() do {} while (0)
	115	#define put_thread_online() do {} while (0)
	116	#define put_thread_online_delay() do {} while (0)
8a953620 MD	117	#else /* #ifndef put_thread_offline */
	118	#define put_thread_online_delay() synchronize_rcu()
	119	#endif /* #else #ifndef put_thread_offline */
	120
	121	/*
	122	* Performance test.
	123	*/
	124
61c3fb60	125	static
8a953620 MD	126	void rcu_read_perf_test(void arg)
	127	{
	128	int i;
	129	int me = (long)arg;
8a953620 MD	130	long long n_reads_local = 0;
8a953620 MD	131
121a5d44	132	rcu_register_thread();
8a953620	133	run_on(me);
ec4e58a3	134	uatomic_inc(&nthreadsrunning);
3bf02b5b	135	put_thread_offline();
8a953620	136	while (goflag == GOFLAG_INIT)
775aff2e	137	(void) poll(NULL, 0, 1);
3bf02b5b	138	put_thread_online();
8a953620 MD	139	while (goflag == GOFLAG_RUN) {
	140	for (i = 0; i < RCU_READ_RUN; i++) {
	141	rcu_read_lock();
	142	/* rcu_read_lock_nest(); */
	143	/* rcu_read_unlock_nest(); */
	144	rcu_read_unlock();
	145	}
	146	n_reads_local += RCU_READ_RUN;
	147	mark_rcu_quiescent_state();
	148	}
	149	__get_thread_var(n_reads_pt) += n_reads_local;
	150	put_thread_offline();
121a5d44	151	rcu_unregister_thread();
8a953620 MD	152
	153	return (NULL);
	154	}
	155
61c3fb60	156	static
70469b43	157	void rcu_update_perf_test(void arg __attribute__((unused)))
8a953620 MD	158	{
	159	long long n_updates_local = 0;
	160
26b5a74b	161	if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66 PM	162	struct call_rcu_data *crdp;
b57aee66 PM	163
c1d2c60b	164	crdp = create_call_rcu_data(0, -1);
b57aee66	165	if (crdp != NULL) {
26b5a74b	166	diag("Successfully using per-thread call_rcu() worker.");
b57aee66 PM	167	set_thread_call_rcu_data(crdp);
	168	}
	169	}
ec4e58a3	170	uatomic_inc(&nthreadsrunning);
8a953620	171	while (goflag == GOFLAG_INIT)
775aff2e	172	(void) poll(NULL, 0, 1);
8a953620 MD	173	while (goflag == GOFLAG_RUN) {
	174	synchronize_rcu();
	175	n_updates_local++;
	176	}
	177	__get_thread_var(n_updates_pt) += n_updates_local;
26b5a74b MD	178	if (callrcu_type == CALLRCU_PERTHREAD) {
	179	struct call_rcu_data *crdp;
	180
	181	crdp = get_thread_call_rcu_data();
	182	set_thread_call_rcu_data(NULL);
	183	call_rcu_data_free(crdp);
	184	}
b0b31506	185	return NULL;
8a953620 MD	186	}
8a953620 MD	187
61c3fb60	188	static
8a953620 MD	189	void perftestinit(void)
	190	{
	191	init_per_thread(n_reads_pt, 0LL);
	192	init_per_thread(n_updates_pt, 0LL);
ec4e58a3	193	uatomic_set(&nthreadsrunning, 0);
8a953620 MD	194	}
8a953620 MD	195
61c3fb60	196	static
ad460058	197	int perftestrun(int nthreads, int nreaders, int nupdaters)
8a953620 MD	198	{
	199	int t;
	200	int duration = 1;
	201
5481ddb3	202	cmm_smp_mb();
ec4e58a3	203	while (uatomic_read(&nthreadsrunning) < nthreads)
775aff2e	204	(void) poll(NULL, 0, 1);
8a953620	205	goflag = GOFLAG_RUN;
5481ddb3	206	cmm_smp_mb();
8a953620	207	sleep(duration);
5481ddb3	208	cmm_smp_mb();
8a953620	209	goflag = GOFLAG_STOP;
5481ddb3	210	cmm_smp_mb();
8a953620 MD	211	wait_all_threads();
	212	for_each_thread(t) {
	213	n_reads += per_thread(n_reads_pt, t);
	214	n_updates += per_thread(n_updates_pt, t);
	215	}
ad460058	216	diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
8a953620	217	n_reads, n_updates, nreaders, nupdaters, duration);
ad460058	218	diag("ns/read: %g ns/update: %g",
8a953620 MD	219	((duration * 100010001000.*(double)nreaders) /
	220	(double)n_reads),
	221	((duration * 100010001000.*(double)nupdaters) /
	222	(double)n_updates));
7106ddf8	223	if (get_cpu_call_rcu_data(0)) {
ad460058	224	diag("Deallocating per-CPU call_rcu threads.\n");
7106ddf8 PM	225	free_all_cpu_call_rcu_data();
7106ddf8 PM	226	}
ad460058	227	return 0;
8a953620 MD	228	}
8a953620 MD	229
61c3fb60	230	static
ad460058	231	int perftest(int nreaders, int cpustride)
8a953620 MD	232	{
	233	int i;
	234	long arg;
	235
	236	perftestinit();
	237	for (i = 0; i < nreaders; i++) {
	238	arg = (long)(i * cpustride);
	239	create_thread(rcu_read_perf_test, (void *)arg);
	240	}
	241	arg = (long)(i * cpustride);
	242	create_thread(rcu_update_perf_test, (void *)arg);
ad460058	243	return perftestrun(i + 1, nreaders, 1);
8a953620 MD	244	}
8a953620 MD	245
61c3fb60	246	static
ad460058	247	int rperftest(int nreaders, int cpustride)
8a953620 MD	248	{
	249	int i;
	250	long arg;
	251
	252	perftestinit();
	253	init_per_thread(n_reads_pt, 0LL);
	254	for (i = 0; i < nreaders; i++) {
	255	arg = (long)(i * cpustride);
	256	create_thread(rcu_read_perf_test, (void *)arg);
	257	}
ad460058	258	return perftestrun(i, nreaders, 0);
8a953620 MD	259	}
8a953620 MD	260
61c3fb60	261	static
ad460058	262	int uperftest(int nupdaters, int cpustride)
8a953620 MD	263	{
	264	int i;
	265	long arg;
	266
	267	perftestinit();
	268	init_per_thread(n_reads_pt, 0LL);
	269	for (i = 0; i < nupdaters; i++) {
	270	arg = (long)(i * cpustride);
	271	create_thread(rcu_update_perf_test, (void *)arg);
	272	}
ad460058	273	return perftestrun(i, 0, nupdaters);
8a953620 MD	274	}
	275
	276	/*
	277	* Stress test.
	278	*/
	279
	280	#define RCU_STRESS_PIPE_LEN 10
	281
	282	struct rcu_stress {
	283	int pipe_count;
	284	int mbtest;
	285	};
	286
153b081a	287	struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0, 0 } };
8a953620 MD	288	struct rcu_stress *rcu_stress_current;
	289	int rcu_stress_idx = 0;
	290
	291	int n_mberror = 0;
	292	DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
	293
	294	int garbage = 0;
	295
61c3fb60	296	static
70469b43	297	void rcu_read_stress_test(void arg __attribute__((unused)))
8a953620 MD	298	{
	299	int i;
	300	int itercnt = 0;
	301	struct rcu_stress *p;
	302	int pc;
	303
121a5d44	304	rcu_register_thread();
3bf02b5b	305	put_thread_offline();
8a953620	306	while (goflag == GOFLAG_INIT)
775aff2e	307	(void) poll(NULL, 0, 1);
3bf02b5b	308	put_thread_online();
8a953620 MD	309	while (goflag == GOFLAG_RUN) {
	310	rcu_read_lock();
	311	p = rcu_dereference(rcu_stress_current);
	312	if (p->mbtest == 0)
	313	n_mberror++;
	314	rcu_read_lock_nest();
	315	for (i = 0; i < 100; i++)
	316	garbage++;
	317	rcu_read_unlock_nest();
	318	pc = p->pipe_count;
	319	rcu_read_unlock();
	320	if ((pc > RCU_STRESS_PIPE_LEN) \|\| (pc < 0))
	321	pc = RCU_STRESS_PIPE_LEN;
	322	__get_thread_var(rcu_stress_count)[pc]++;
	323	__get_thread_var(n_reads_pt)++;
	324	mark_rcu_quiescent_state();
	325	if ((++itercnt % 0x1000) == 0) {
	326	put_thread_offline();
	327	put_thread_online_delay();
	328	put_thread_online();
	329	}
	330	}
	331	put_thread_offline();
121a5d44	332	rcu_unregister_thread();
8a953620 MD	333
	334	return (NULL);
	335	}
	336
b57aee66 PM	337	static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
b57aee66 PM	338	static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
a6ca497d	339	static bool call_rcu_wait;
b57aee66	340
61c3fb60	341	static
70469b43	342	void rcu_update_stress_test_rcu(struct rcu_head *head __attribute__((unused)))
b57aee66	343	{
ad460058 MD	344	int ret;
	345
	346	ret = pthread_mutex_lock(&call_rcu_test_mutex);
	347	if (ret) {
	348	errno = ret;
	349	diag("pthread_mutex_lock: %s",
	350	strerror(errno));
	351	abort();
b57aee66	352	}
ad460058 MD	353	ret = pthread_cond_signal(&call_rcu_test_cond);
	354	if (ret) {
	355	errno = ret;
	356	diag("pthread_cond_signal: %s",
	357	strerror(errno));
	358	abort();
b57aee66	359	}
a6ca497d	360	call_rcu_wait = false;
ad460058 MD	361	ret = pthread_mutex_unlock(&call_rcu_test_mutex);
	362	if (ret) {
	363	errno = ret;
	364	diag("pthread_mutex_unlock: %s",
	365	strerror(errno));
	366	abort();
b57aee66 PM	367	}
	368	}
	369
eb218d4f MD	370	static
	371	void advance_writer_state(enum writer_state *state)
	372	{
	373	switch (*state) {
	374	case WRITER_STATE_SYNC_RCU:
	375	*state = WRITER_STATE_CALL_RCU;
	376	break;
	377	case WRITER_STATE_CALL_RCU:
	378	*state = WRITER_STATE_POLL_RCU;
	379	break;
	380	case WRITER_STATE_POLL_RCU:
	381	*state = WRITER_STATE_SYNC_RCU;
	382	break;
	383	}
	384	}
	385
61c3fb60	386	static
70469b43	387	void rcu_update_stress_test(void arg __attribute__((unused)))
8a953620 MD	388	{
	389	int i;
	390	struct rcu_stress *p;
b57aee66	391	struct rcu_head rh;
eb218d4f	392	enum writer_state writer_state = WRITER_STATE_SYNC_RCU;
8a953620 MD	393
8a953620 MD	394	while (goflag == GOFLAG_INIT)
775aff2e	395	(void) poll(NULL, 0, 1);
8a953620 MD	396	while (goflag == GOFLAG_RUN) {
	397	i = rcu_stress_idx + 1;
	398	if (i >= RCU_STRESS_PIPE_LEN)
	399	i = 0;
	400	p = &rcu_stress_array[i];
	401	p->mbtest = 0;
5481ddb3	402	cmm_smp_mb();
8a953620 MD	403	p->pipe_count = 0;
	404	p->mbtest = 1;
	405	rcu_assign_pointer(rcu_stress_current, p);
	406	rcu_stress_idx = i;
	407	for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
	408	if (i != rcu_stress_idx)
	409	rcu_stress_array[i].pipe_count++;
eb218d4f MD	410	switch (writer_state) {
eb218d4f MD	411	case WRITER_STATE_SYNC_RCU:
b57aee66	412	synchronize_rcu();
eb218d4f MD	413	break;
	414	case WRITER_STATE_CALL_RCU:
	415	{
ad460058 MD	416	int ret;
	417
	418	ret = pthread_mutex_lock(&call_rcu_test_mutex);
	419	if (ret) {
	420	errno = ret;
	421	diag("pthread_mutex_lock: %s",
	422	strerror(errno));
	423	abort();
b57aee66	424	}
0b9c513b	425	rcu_register_thread();
b57aee66	426	call_rcu(&rh, rcu_update_stress_test_rcu);
0b9c513b MD	427	rcu_unregister_thread();
	428	/*
	429	* Our MacOS X test machine with the following
	430	* config:
	431	* 15.6.0 Darwin Kernel Version 15.6.0
	432	* root:xnu-3248.60.10~1/RELEASE_X86_64
	433	* appears to have issues with liburcu-signal
	434	* signal being delivered on top of
	435	* pthread_cond_wait. It seems to make the
	436	* thread continue, and therefore corrupt the
	437	* rcu_head. Work around this issue by
	438	* unregistering the RCU read-side thread
	439	* immediately after call_rcu (call_rcu needs
	440	* us to be registered RCU readers).
	441	*/
a6ca497d OD	442	call_rcu_wait = true;
	443	do {
	444	ret = pthread_cond_wait(&call_rcu_test_cond,
	445	&call_rcu_test_mutex);
	446	} while (call_rcu_wait);
ad460058 MD	447	if (ret) {
	448	errno = ret;
	449	diag("pthread_cond_signal: %s",
	450	strerror(errno));
	451	abort();
b57aee66	452	}
ad460058 MD	453	ret = pthread_mutex_unlock(&call_rcu_test_mutex);
	454	if (ret) {
	455	errno = ret;
	456	diag("pthread_mutex_unlock: %s",
	457	strerror(errno));
	458	abort();
b57aee66	459	}
eb218d4f MD	460	break;
	461	}
	462	case WRITER_STATE_POLL_RCU:
	463	{
	464	struct urcu_gp_poll_state poll_state;
	465
	466	rcu_register_thread();
	467	poll_state = start_poll_synchronize_rcu();
	468	rcu_unregister_thread();
	469	while (!poll_state_synchronize_rcu(poll_state))
	470	(void) poll(NULL, 0, 1); /* Wait for 1ms */
	471	break;
	472	}
b57aee66	473	}
8a953620	474	n_updates++;
eb218d4f	475	advance_writer_state(&writer_state);
8a953620	476	}
a13ef613	477
b0b31506	478	return NULL;
8a953620 MD	479	}
8a953620 MD	480
61c3fb60	481	static
70469b43	482	void rcu_fake_update_stress_test(void arg __attribute__((unused)))
8a953620	483	{
26b5a74b	484	if (callrcu_type == CALLRCU_PERTHREAD) {
b57aee66 PM	485	struct call_rcu_data *crdp;
b57aee66 PM	486
c1d2c60b	487	crdp = create_call_rcu_data(0, -1);
b57aee66	488	if (crdp != NULL) {
26b5a74b	489	diag("Successfully using per-thread call_rcu() worker.");
b57aee66 PM	490	set_thread_call_rcu_data(crdp);
	491	}
	492	}
8a953620	493	while (goflag == GOFLAG_INIT)
775aff2e	494	(void) poll(NULL, 0, 1);
8a953620 MD	495	while (goflag == GOFLAG_RUN) {
8a953620 MD	496	synchronize_rcu();
775aff2e	497	(void) poll(NULL, 0, 1);
8a953620	498	}
26b5a74b MD	499	if (callrcu_type == CALLRCU_PERTHREAD) {
	500	struct call_rcu_data *crdp;
	501
	502	crdp = get_thread_call_rcu_data();
	503	set_thread_call_rcu_data(NULL);
	504	call_rcu_data_free(crdp);
	505	}
b0b31506	506	return NULL;
8a953620 MD	507	}
8a953620 MD	508
61c3fb60	509	static
ad460058	510	int stresstest(int nreaders)
8a953620 MD	511	{
	512	int i;
	513	int t;
	514	long long *p;
	515	long long sum;
	516
	517	init_per_thread(n_reads_pt, 0LL);
	518	for_each_thread(t) {
	519	p = &per_thread(rcu_stress_count,t)[0];
	520	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
	521	p[i] = 0LL;
	522	}
	523	rcu_stress_current = &rcu_stress_array[0];
	524	rcu_stress_current->pipe_count = 0;
	525	rcu_stress_current->mbtest = 1;
	526	for (i = 0; i < nreaders; i++)
	527	create_thread(rcu_read_stress_test, NULL);
	528	create_thread(rcu_update_stress_test, NULL);
	529	for (i = 0; i < 5; i++)
	530	create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3	531	cmm_smp_mb();
8a953620	532	goflag = GOFLAG_RUN;
5481ddb3	533	cmm_smp_mb();
8a953620	534	sleep(10);
5481ddb3	535	cmm_smp_mb();
8a953620	536	goflag = GOFLAG_STOP;
5481ddb3	537	cmm_smp_mb();
8a953620 MD	538	wait_all_threads();
	539	for_each_thread(t)
	540	n_reads += per_thread(n_reads_pt, t);
ad460058	541	diag("n_reads: %lld n_updates: %ld n_mberror: %d",
8a953620	542	n_reads, n_updates, n_mberror);
ad460058 MD	543	rdiag_start();
ad460058 MD	544	rdiag("rcu_stress_count:");
8a953620 MD	545	for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
	546	sum = 0LL;
	547	for_each_thread(t) {
	548	sum += per_thread(rcu_stress_count, t)[i];
	549	}
ad460058	550	rdiag(" %lld", sum);
8a953620	551	}
ad460058	552	rdiag_end();
7106ddf8	553	if (get_cpu_call_rcu_data(0)) {
ad460058	554	diag("Deallocating per-CPU call_rcu threads.");
7106ddf8 PM	555	free_all_cpu_call_rcu_data();
7106ddf8 PM	556	}
ad460058 MD	557	if (!n_mberror)
	558	return 0;
	559	else
	560	return -1;
8a953620 MD	561	}
	562
	563	/*
	564	* Mainprogram.
	565	*/
	566
a142df4e	567	static
6fa8b4f8	568	void usage(char *argv[]) __attribute__((__noreturn__));
a142df4e	569
61c3fb60	570	static
70469b43	571	void usage(char *argv[])
8a953620	572	{
26b5a74b	573	diag("Usage: %s nreaders [ perf \| rperf \| uperf \| stress ] [ stride ] [ callrcu_global \| callrcu_percpu \| callrcu_perthread ]\n", argv[0]);
8a953620 MD	574	exit(-1);
	575	}
	576
	577	int main(int argc, char *argv[])
	578	{
	579	int nreaders = 1;
	580	int cpustride = 1;
	581
ad460058 MD	582	plan_tests(NR_TESTS);
ad460058 MD	583
8a953620 MD	584	smp_init();
8a953620 MD	585	//rcu_init();
26b5a74b MD	586	if (argc > 4) {
	587	const char *callrcu_str = argv[4];;
	588
	589	if (strcmp(callrcu_str, "callrcu_global") == 0) {
	590	callrcu_type = CALLRCU_GLOBAL;
	591	} else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
	592	callrcu_type = CALLRCU_PERCPU;
	593	} else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
	594	callrcu_type = CALLRCU_PERTHREAD;
	595	} else {
70469b43	596	usage(argv);
26b5a74b MD	597	goto end;
	598	}
	599	}
929cfaff	600
26b5a74b MD	601	switch (callrcu_type) {
	602	case CALLRCU_GLOBAL:
	603	diag("Using global per-process call_rcu thread.");
	604	break;
	605	case CALLRCU_PERCPU:
	606	diag("Using per-CPU call_rcu threads.");
b57aee66	607	if (create_all_cpu_call_rcu_data(0))
ad460058 MD	608	diag("create_all_cpu_call_rcu_data: %s",
ad460058 MD	609	strerror(errno));
26b5a74b MD	610	break;
	611	case CALLRCU_PERTHREAD:
	612	diag("Using per-thread call_rcu() worker.");
	613	break;
	614	default:
	615	abort();
b57aee66	616	}
8a953620	617
9b171f46 MD	618	#ifdef DEBUG_YIELD
	619	yield_active \|= YIELD_READ;
	620	yield_active \|= YIELD_WRITE;
	621	#endif
	622
8a953620	623	if (argc > 1) {
26b5a74b MD	624	if (strcmp(argv[1], "-h") == 0
26b5a74b MD	625	\|\| strcmp(argv[1], "--help") == 0) {
70469b43	626	usage(argv);
26b5a74b MD	627	goto end;
26b5a74b MD	628	}
8a953620	629	nreaders = strtoul(argv[1], NULL, 0);
ad460058 MD	630	if (argc == 2) {
	631	ok(!perftest(nreaders, cpustride),
	632	"perftest readers: %d, stride: %d",
	633	nreaders, cpustride);
	634	goto end;
	635	}
8a953620 MD	636	if (argc > 3)
	637	cpustride = strtoul(argv[3], NULL, 0);
	638	if (strcmp(argv[2], "perf") == 0)
ad460058 MD	639	ok(!perftest(nreaders, cpustride),
	640	"perftest readers: %d, stride: %d",
	641	nreaders, cpustride);
8a953620	642	else if (strcmp(argv[2], "rperf") == 0)
ad460058 MD	643	ok(!rperftest(nreaders, cpustride),
	644	"rperftest readers: %d, stride: %d",
	645	nreaders, cpustride);
8a953620	646	else if (strcmp(argv[2], "uperf") == 0)
ad460058 MD	647	ok(!uperftest(nreaders, cpustride),
	648	"uperftest readers: %d, stride: %d",
	649	nreaders, cpustride);
8a953620	650	else if (strcmp(argv[2], "stress") == 0)
ad460058 MD	651	ok(!stresstest(nreaders),
	652	"stresstest readers: %d, stride: %d",
	653	nreaders, cpustride);
	654	else
70469b43	655	usage(argv);
ad460058	656	} else {
70469b43	657	usage(argv);
8a953620	658	}
ad460058 MD	659	end:
ad460058 MD	660	return exit_status();
8a953620	661	}