X-Git-Url: http://git.liburcu.org/?a=blobdiff_plain;f=tests%2Fregression%2Frcutorture.h;h=441ff79bf349fa301b60e7ee61935e0caa58aba2;hb=HEAD;hp=f8548d811ed8980061716cc797a63524d97524a4;hpb=f5ab766ee2c8300cb00ca5878b1cb464f960a66d;p=urcu.git diff --git a/tests/regression/rcutorture.h b/tests/regression/rcutorture.h index f8548d8..441ff79 100644 --- a/tests/regression/rcutorture.h +++ b/tests/regression/rcutorture.h @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation. +// +// SPDX-License-Identifier: GPL-2.0-or-later + /* * rcutorture.h: simple user-level performance/stress test of RCU. * @@ -44,6 +48,14 @@ * data. A correct RCU implementation will have all but the first two * numbers non-zero. * + * rcu_stress_count: Histogram of "ages" of structures seen by readers. If any + * entries past the first two are non-zero, RCU is broken. The age of a newly + * allocated structure is zero, it becomes one when removed from reader + * visibility, and is incremented once per grace period subsequently -- and is + * freed after passing through (RCU_STRESS_PIPE_LEN-2) grace periods. Since + * this tests only has one true writer (there are fake writers), only buckets at + * indexes 0 and 1 should be none-zero. + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -66,10 +78,31 @@ */ #include +#include "tap.h" + +#include + +#include "urcu-wait.h" + +#define NR_TESTS 1 DEFINE_PER_THREAD(long long, n_reads_pt); DEFINE_PER_THREAD(long long, n_updates_pt); +enum callrcu_type { + CALLRCU_GLOBAL, + CALLRCU_PERCPU, + CALLRCU_PERTHREAD, +}; + +enum writer_state { + WRITER_STATE_SYNC_RCU, + WRITER_STATE_CALL_RCU, + WRITER_STATE_POLL_RCU, +}; + +static enum callrcu_type callrcu_type = CALLRCU_GLOBAL; + long long n_reads = 0LL; long n_updates = 0L; int nthreadsrunning; @@ -102,13 +135,13 @@ volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) #endif #ifndef mark_rcu_quiescent_state -#define mark_rcu_quiescent_state() do ; while (0) +#define mark_rcu_quiescent_state() do {} while (0) #endif /* #ifdef mark_rcu_quiescent_state */ #ifndef put_thread_offline -#define put_thread_offline() do ; while (0) -#define put_thread_online() do ; while (0) -#define put_thread_online_delay() do ; while (0) +#define put_thread_offline() do {} while (0) +#define put_thread_online() do {} while (0) +#define put_thread_online_delay() do {} while (0) #else /* #ifndef put_thread_offline */ #define put_thread_online_delay() synchronize_rcu() #endif /* #else #ifndef put_thread_offline */ @@ -117,9 +150,9 @@ volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) * Performance test. */ +static void *rcu_read_perf_test(void *arg) { - struct call_rcu_data *crdp; int i; int me = (long)arg; long long n_reads_local = 0; @@ -128,10 +161,10 @@ void *rcu_read_perf_test(void *arg) run_on(me); uatomic_inc(&nthreadsrunning); put_thread_offline(); - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); + while (uatomic_read(&goflag) == GOFLAG_INIT) + (void) poll(NULL, 0, 1); put_thread_online(); - while (goflag == GOFLAG_RUN) { + while (uatomic_read(&goflag) == GOFLAG_RUN) { for (i = 0; i < RCU_READ_RUN; i++) { rcu_read_lock(); /* rcu_read_lock_nest(); */ @@ -143,39 +176,44 @@ void *rcu_read_perf_test(void *arg) } __get_thread_var(n_reads_pt) += n_reads_local; put_thread_offline(); - crdp = get_thread_call_rcu_data(); - set_thread_call_rcu_data(NULL); - call_rcu_data_free(crdp); rcu_unregister_thread(); return (NULL); } -void *rcu_update_perf_test(void *arg) +static +void *rcu_update_perf_test(void *arg __attribute__((unused))) { long long n_updates_local = 0; - if ((random() & 0xf00) == 0) { + if (callrcu_type == CALLRCU_PERTHREAD) { struct call_rcu_data *crdp; crdp = create_call_rcu_data(0, -1); if (crdp != NULL) { - fprintf(stderr, - "Using per-thread call_rcu() worker.\n"); + diag("Successfully using per-thread call_rcu() worker."); set_thread_call_rcu_data(crdp); } } uatomic_inc(&nthreadsrunning); - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - while (goflag == GOFLAG_RUN) { + while (uatomic_read(&goflag) == GOFLAG_INIT) + (void) poll(NULL, 0, 1); + while (uatomic_read(&goflag) == GOFLAG_RUN) { synchronize_rcu(); n_updates_local++; } __get_thread_var(n_updates_pt) += n_updates_local; + if (callrcu_type == CALLRCU_PERTHREAD) { + struct call_rcu_data *crdp; + + crdp = get_thread_call_rcu_data(); + set_thread_call_rcu_data(NULL); + call_rcu_data_free(crdp); + } return NULL; } +static void perftestinit(void) { init_per_thread(n_reads_pt, 0LL); @@ -183,40 +221,38 @@ void perftestinit(void) uatomic_set(&nthreadsrunning, 0); } -void perftestrun(int nthreads, int nreaders, int nupdaters) +static +int perftestrun(int nthreads, int nreaders, int nupdaters) { int t; int duration = 1; - cmm_smp_mb(); while (uatomic_read(&nthreadsrunning) < nthreads) - poll(NULL, 0, 1); - goflag = GOFLAG_RUN; - cmm_smp_mb(); + (void) poll(NULL, 0, 1); + uatomic_set(&goflag, GOFLAG_RUN); sleep(duration); - cmm_smp_mb(); - goflag = GOFLAG_STOP; - cmm_smp_mb(); + uatomic_set(&goflag, GOFLAG_STOP); wait_all_threads(); for_each_thread(t) { n_reads += per_thread(n_reads_pt, t); n_updates += per_thread(n_updates_pt, t); } - printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n", + diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d", n_reads, n_updates, nreaders, nupdaters, duration); - printf("ns/read: %g ns/update: %g\n", + diag("ns/read: %g ns/update: %g", ((duration * 1000*1000*1000.*(double)nreaders) / (double)n_reads), ((duration * 1000*1000*1000.*(double)nupdaters) / (double)n_updates)); if (get_cpu_call_rcu_data(0)) { - fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); + diag("Deallocating per-CPU call_rcu threads.\n"); free_all_cpu_call_rcu_data(); } - exit(0); + return 0; } -void perftest(int nreaders, int cpustride) +static +int perftest(int nreaders, int cpustride) { int i; long arg; @@ -228,10 +264,11 @@ void perftest(int nreaders, int cpustride) } arg = (long)(i * cpustride); create_thread(rcu_update_perf_test, (void *)arg); - perftestrun(i + 1, nreaders, 1); + return perftestrun(i + 1, nreaders, 1); } -void rperftest(int nreaders, int cpustride) +static +int rperftest(int nreaders, int cpustride) { int i; long arg; @@ -242,10 +279,11 @@ void rperftest(int nreaders, int cpustride) arg = (long)(i * cpustride); create_thread(rcu_read_perf_test, (void *)arg); } - perftestrun(i, nreaders, 0); + return perftestrun(i, nreaders, 0); } -void uperftest(int nupdaters, int cpustride) +static +int uperftest(int nupdaters, int cpustride) { int i; long arg; @@ -256,7 +294,7 @@ void uperftest(int nupdaters, int cpustride) arg = (long)(i * cpustride); create_thread(rcu_update_perf_test, (void *)arg); } - perftestrun(i, 0, nupdaters); + return perftestrun(i, 0, nupdaters); } /* @@ -270,16 +308,24 @@ struct rcu_stress { int mbtest; }; -struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } }; +struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0, 0 } }; struct rcu_stress *rcu_stress_current; int rcu_stress_idx = 0; +/* + * How many time a reader has seen something that should not be visible. It is + * an error if this value is different than zero at the end of the stress test. + * + * Here, the something that should not be visibile is an old pipe that has been + * freed (mbtest = 0). + */ int n_mberror = 0; DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count); int garbage = 0; -void *rcu_read_stress_test(void *arg) +static +void *rcu_read_stress_test(void *arg __attribute__((unused))) { int i; int itercnt = 0; @@ -288,19 +334,25 @@ void *rcu_read_stress_test(void *arg) rcu_register_thread(); put_thread_offline(); - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); + while (uatomic_read(&goflag) == GOFLAG_INIT) + (void) poll(NULL, 0, 1); put_thread_online(); - while (goflag == GOFLAG_RUN) { + while (uatomic_read(&goflag) == GOFLAG_RUN) { rcu_read_lock(); p = rcu_dereference(rcu_stress_current); if (p->mbtest == 0) - n_mberror++; + uatomic_inc_mo(&n_mberror, CMM_RELAXED); rcu_read_lock_nest(); + /* + * The value of garbage is nothing important. This is + * essentially a busy loop. The atomic operation -- while not + * important here -- helps tools such as TSAN to not flag this + * as a race condition. + */ for (i = 0; i < 100; i++) - garbage++; + uatomic_inc(&garbage); rcu_read_unlock_nest(); - pc = p->pipe_count; + pc = uatomic_read(&p->pipe_count); rcu_read_unlock(); if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0)) pc = RCU_STRESS_PIPE_LEN; @@ -319,97 +371,163 @@ void *rcu_read_stress_test(void *arg) return (NULL); } -static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER; +static DEFINE_URCU_WAIT_QUEUE(call_rcu_waiters); -void rcu_update_stress_test_rcu(struct rcu_head *head) +static +void rcu_update_stress_test_rcu(struct rcu_head *head __attribute__((unused))) { - if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_lock"); - exit(-1); - } - if (pthread_cond_signal(&call_rcu_test_cond) != 0) { - perror("pthread_cond_signal"); - exit(-1); - } - if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_unlock"); - exit(-1); + struct urcu_waiters waiters; + + urcu_move_waiters(&waiters, &call_rcu_waiters); + urcu_wake_all_waiters(&waiters); +} + +static +void advance_writer_state(enum writer_state *state) +{ + switch (*state) { + case WRITER_STATE_SYNC_RCU: + *state = WRITER_STATE_CALL_RCU; + break; + case WRITER_STATE_CALL_RCU: + *state = WRITER_STATE_POLL_RCU; + break; + case WRITER_STATE_POLL_RCU: + *state = WRITER_STATE_SYNC_RCU; + break; } } -void *rcu_update_stress_test(void *arg) +static +void *rcu_update_stress_test(void *arg __attribute__((unused))) { int i; - struct rcu_stress *p; + struct rcu_stress *p, *old_p; struct rcu_head rh; + enum writer_state writer_state = WRITER_STATE_SYNC_RCU; - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - while (goflag == GOFLAG_RUN) { + rcu_register_thread(); + + /* Offline for poll. */ + put_thread_offline(); + while (uatomic_read(&goflag) == GOFLAG_INIT) + (void) poll(NULL, 0, 1); + put_thread_online(); + + old_p = NULL; + while (uatomic_read(&goflag) == GOFLAG_RUN) { i = rcu_stress_idx + 1; if (i >= RCU_STRESS_PIPE_LEN) i = 0; + + rcu_read_lock(); + old_p = rcu_dereference(rcu_stress_current); + rcu_read_unlock(); + + /* + * Allocate a new pipe. + */ p = &rcu_stress_array[i]; - p->mbtest = 0; - cmm_smp_mb(); p->pipe_count = 0; p->mbtest = 1; + rcu_assign_pointer(rcu_stress_current, p); rcu_stress_idx = i; + + /* + * Increment every pipe except the freshly allocated one. A + * reader should only see either the old pipe or the new + * pipe. This is reflected in the rcu_stress_count histogram. + */ for (i = 0; i < RCU_STRESS_PIPE_LEN; i++) if (i != rcu_stress_idx) - rcu_stress_array[i].pipe_count++; - if (n_updates & 0x1) + uatomic_inc(&rcu_stress_array[i].pipe_count); + + switch (writer_state) { + case WRITER_STATE_SYNC_RCU: synchronize_rcu(); - else { - if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_lock"); - exit(-1); - } + break; + case WRITER_STATE_CALL_RCU: + { + DEFINE_URCU_WAIT_NODE(wait, URCU_WAIT_WAITING); + + urcu_wait_add(&call_rcu_waiters, &wait); + call_rcu(&rh, rcu_update_stress_test_rcu); - if (pthread_cond_wait(&call_rcu_test_cond, - &call_rcu_test_mutex) != 0) { - perror("pthread_cond_wait"); - exit(-1); - } - if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_unlock"); - exit(-1); - } + + /* Offline for busy-wait. */ + put_thread_offline(); + urcu_adaptative_busy_wait(&wait); + put_thread_online(); + break; + } + case WRITER_STATE_POLL_RCU: + { + struct urcu_gp_poll_state poll_state; + + poll_state = start_poll_synchronize_rcu(); + + /* Offline for poll. */ + put_thread_offline(); + while (!poll_state_synchronize_rcu(poll_state)) + (void) poll(NULL, 0, 1); /* Wait for 1ms */ + put_thread_online(); + break; + } + } + /* + * No readers should see that old pipe now. Setting mbtest to 0 + * to mark it as "freed". + */ + if (old_p) { + old_p->mbtest = 0; } + old_p = p; n_updates++; + advance_writer_state(&writer_state); } + + rcu_unregister_thread(); + return NULL; } -void *rcu_fake_update_stress_test(void *arg) +static +void *rcu_fake_update_stress_test(void *arg __attribute__((unused))) { - if ((random() & 0xf00) == 0) { + if (callrcu_type == CALLRCU_PERTHREAD) { struct call_rcu_data *crdp; crdp = create_call_rcu_data(0, -1); if (crdp != NULL) { - fprintf(stderr, - "Using per-thread call_rcu() worker.\n"); + diag("Successfully using per-thread call_rcu() worker."); set_thread_call_rcu_data(crdp); } } - while (goflag == GOFLAG_INIT) - poll(NULL, 0, 1); - while (goflag == GOFLAG_RUN) { + while (uatomic_read(&goflag) == GOFLAG_INIT) + (void) poll(NULL, 0, 1); + while (uatomic_read(&goflag) == GOFLAG_RUN) { synchronize_rcu(); - poll(NULL, 0, 1); + (void) poll(NULL, 0, 1); + } + if (callrcu_type == CALLRCU_PERTHREAD) { + struct call_rcu_data *crdp; + + crdp = get_thread_call_rcu_data(); + set_thread_call_rcu_data(NULL); + call_rcu_data_free(crdp); } return NULL; } -void stresstest(int nreaders) +static +int stresstest(int nreaders) { int i; int t; long long *p; long long sum; + int ret; init_per_thread(n_reads_pt, 0LL); for_each_thread(t) { @@ -425,41 +543,52 @@ void stresstest(int nreaders) create_thread(rcu_update_stress_test, NULL); for (i = 0; i < 5; i++) create_thread(rcu_fake_update_stress_test, NULL); - cmm_smp_mb(); - goflag = GOFLAG_RUN; - cmm_smp_mb(); + uatomic_set(&goflag, GOFLAG_RUN); sleep(10); - cmm_smp_mb(); - goflag = GOFLAG_STOP; - cmm_smp_mb(); + uatomic_set(&goflag, GOFLAG_STOP); wait_all_threads(); for_each_thread(t) n_reads += per_thread(n_reads_pt, t); - printf("n_reads: %lld n_updates: %ld n_mberror: %d\n", + diag("n_reads: %lld n_updates: %ld n_mberror: %d", n_reads, n_updates, n_mberror); - printf("rcu_stress_count:"); + rdiag_start(); + rdiag("rcu_stress_count:"); + ret = 0; for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) { sum = 0LL; for_each_thread(t) { sum += per_thread(rcu_stress_count, t)[i]; } - printf(" %lld", sum); + /* + * If any entries past the first two are non-zero, RCU is + * broken. See details above about rcu_stress_count. + */ + if (i > 1 && sum != 0) { + ret = -1; + } + rdiag(" %lld", sum); } - printf("\n"); + rdiag_end(); if (get_cpu_call_rcu_data(0)) { - fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); + diag("Deallocating per-CPU call_rcu threads."); free_all_cpu_call_rcu_data(); } - exit(0); + if (n_mberror) + ret = -1; + return ret; } /* * Mainprogram. */ -void usage(int argc, char *argv[]) +static +void usage(char *argv[]) __attribute__((__noreturn__)); + +static +void usage(char *argv[]) { - fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]); + diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]); exit(-1); } @@ -468,13 +597,40 @@ int main(int argc, char *argv[]) int nreaders = 1; int cpustride = 1; + plan_tests(NR_TESTS); + smp_init(); //rcu_init(); - srandom(time(NULL)); - if (random() & 0x100) { - fprintf(stderr, "Allocating per-CPU call_rcu threads.\n"); + if (argc > 4) { + const char *callrcu_str = argv[4];; + + if (strcmp(callrcu_str, "callrcu_global") == 0) { + callrcu_type = CALLRCU_GLOBAL; + } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) { + callrcu_type = CALLRCU_PERCPU; + } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) { + callrcu_type = CALLRCU_PERTHREAD; + } else { + usage(argv); + goto end; + } + } + + switch (callrcu_type) { + case CALLRCU_GLOBAL: + diag("Using global per-process call_rcu thread."); + break; + case CALLRCU_PERCPU: + diag("Using per-CPU call_rcu threads."); if (create_all_cpu_call_rcu_data(0)) - perror("create_all_cpu_call_rcu_data"); + diag("create_all_cpu_call_rcu_data: %s", + strerror(errno)); + break; + case CALLRCU_PERTHREAD: + diag("Using per-thread call_rcu() worker."); + break; + default: + abort(); } #ifdef DEBUG_YIELD @@ -483,21 +639,41 @@ int main(int argc, char *argv[]) #endif if (argc > 1) { + if (strcmp(argv[1], "-h") == 0 + || strcmp(argv[1], "--help") == 0) { + usage(argv); + goto end; + } nreaders = strtoul(argv[1], NULL, 0); - if (argc == 2) - perftest(nreaders, cpustride); + if (argc == 2) { + ok(!perftest(nreaders, cpustride), + "perftest readers: %d, stride: %d", + nreaders, cpustride); + goto end; + } if (argc > 3) cpustride = strtoul(argv[3], NULL, 0); if (strcmp(argv[2], "perf") == 0) - perftest(nreaders, cpustride); + ok(!perftest(nreaders, cpustride), + "perftest readers: %d, stride: %d", + nreaders, cpustride); else if (strcmp(argv[2], "rperf") == 0) - rperftest(nreaders, cpustride); + ok(!rperftest(nreaders, cpustride), + "rperftest readers: %d, stride: %d", + nreaders, cpustride); else if (strcmp(argv[2], "uperf") == 0) - uperftest(nreaders, cpustride); + ok(!uperftest(nreaders, cpustride), + "uperftest readers: %d, stride: %d", + nreaders, cpustride); else if (strcmp(argv[2], "stress") == 0) - stresstest(nreaders); - usage(argc, argv); + ok(!stresstest(nreaders), + "stresstest readers: %d, stride: %d", + nreaders, cpustride); + else + usage(argv); + } else { + usage(argv); } - perftest(nreaders, cpustride); - return 0; +end: + return exit_status(); }