Fix: only define membarrier system call on Linux
[urcu.git] / tests / regression / rcutorture.h
CommitLineData
8a953620
MD
1/*
2 * rcutorture.h: simple user-level performance/stress test of RCU.
3 *
4 * Usage:
5 * ./rcu <nreaders> rperf [ <cpustride> ]
6 * Run a read-side performance test with the specified
7 * number of readers spaced by <cpustride>.
8 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
9 * CPUs from 0 to 30.
10 * ./rcu <nupdaters> uperf [ <cpustride> ]
11 * Run an update-side performance test with the specified
12 * number of updaters and specified CPU spacing.
13 * ./rcu <nreaders> perf [ <cpustride> ]
14 * Run a combined read/update performance test with the specified
15 * number of readers and one updater and specified CPU spacing.
16 * The readers run on the low-numbered CPUs and the updater
17 * of the highest-numbered CPU.
18 *
19 * The above tests produce output as follows:
20 *
21 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
22 * ns/read: 43.4707 ns/update: 6848.1
23 *
24 * The first line lists the total number of RCU reads and updates executed
25 * during the test, the number of reader threads, the number of updater
26 * threads, and the duration of the test in seconds. The second line
27 * lists the average duration of each type of operation in nanoseconds,
28 * or "nan" if the corresponding type of operation was not performed.
29 *
30 * ./rcu <nreaders> stress
31 * Run a stress test with the specified number of readers and
32 * one updater. None of the threads are affinitied to any
33 * particular CPU.
34 *
35 * This test produces output as follows:
36 *
37 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
38 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
39 *
40 * The first line lists the number of RCU read and update operations
41 * executed, followed by the number of memory-ordering violations
42 * (which will be zero in a correct RCU implementation). The second
43 * line lists the number of readers observing progressively more stale
44 * data. A correct RCU implementation will have all but the first two
45 * numbers non-zero.
46 *
47 * This program is free software; you can redistribute it and/or modify
48 * it under the terms of the GNU General Public License as published by
49 * the Free Software Foundation; either version 2 of the License, or
50 * (at your option) any later version.
51 *
52 * This program is distributed in the hope that it will be useful,
53 * but WITHOUT ANY WARRANTY; without even the implied warranty of
54 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
55 * GNU General Public License for more details.
56 *
57 * You should have received a copy of the GNU General Public License
58 * along with this program; if not, write to the Free Software
3282a76b 59 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
8a953620
MD
60 *
61 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
62 */
63
64/*
65 * Test variables.
66 */
67
b57aee66 68#include <stdlib.h>
ad460058
MD
69#include "tap.h"
70
71#define NR_TESTS 1
b57aee66 72
8a953620
MD
73DEFINE_PER_THREAD(long long, n_reads_pt);
74DEFINE_PER_THREAD(long long, n_updates_pt);
75
76long long n_reads = 0LL;
77long n_updates = 0L;
6ee91d83 78int nthreadsrunning;
8a953620
MD
79char argsbuf[64];
80
81#define GOFLAG_INIT 0
82#define GOFLAG_RUN 1
83#define GOFLAG_STOP 2
84
4967f005
PB
85volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
86 = GOFLAG_INIT;
8a953620
MD
87
88#define RCU_READ_RUN 1000
89
90//MD
91#define RCU_READ_NESTABLE
92
93#ifdef RCU_READ_NESTABLE
94#define rcu_read_lock_nest() rcu_read_lock()
95#define rcu_read_unlock_nest() rcu_read_unlock()
96#else /* #ifdef RCU_READ_NESTABLE */
97#define rcu_read_lock_nest()
98#define rcu_read_unlock_nest()
99#endif /* #else #ifdef RCU_READ_NESTABLE */
100
1a43bbd8
MD
101#ifdef TORTURE_QSBR
102#define mark_rcu_quiescent_state rcu_quiescent_state
103#define put_thread_offline rcu_thread_offline
104#define put_thread_online rcu_thread_online
105#endif
106
8a953620
MD
107#ifndef mark_rcu_quiescent_state
108#define mark_rcu_quiescent_state() do ; while (0)
109#endif /* #ifdef mark_rcu_quiescent_state */
110
111#ifndef put_thread_offline
112#define put_thread_offline() do ; while (0)
113#define put_thread_online() do ; while (0)
114#define put_thread_online_delay() do ; while (0)
115#else /* #ifndef put_thread_offline */
116#define put_thread_online_delay() synchronize_rcu()
117#endif /* #else #ifndef put_thread_offline */
118
119/*
120 * Performance test.
121 */
122
123void *rcu_read_perf_test(void *arg)
124{
7106ddf8 125 struct call_rcu_data *crdp;
8a953620
MD
126 int i;
127 int me = (long)arg;
8a953620
MD
128 long long n_reads_local = 0;
129
121a5d44 130 rcu_register_thread();
8a953620 131 run_on(me);
ec4e58a3 132 uatomic_inc(&nthreadsrunning);
3bf02b5b 133 put_thread_offline();
8a953620 134 while (goflag == GOFLAG_INIT)
775aff2e 135 (void) poll(NULL, 0, 1);
3bf02b5b 136 put_thread_online();
8a953620
MD
137 while (goflag == GOFLAG_RUN) {
138 for (i = 0; i < RCU_READ_RUN; i++) {
139 rcu_read_lock();
140 /* rcu_read_lock_nest(); */
141 /* rcu_read_unlock_nest(); */
142 rcu_read_unlock();
143 }
144 n_reads_local += RCU_READ_RUN;
145 mark_rcu_quiescent_state();
146 }
147 __get_thread_var(n_reads_pt) += n_reads_local;
148 put_thread_offline();
7106ddf8
PM
149 crdp = get_thread_call_rcu_data();
150 set_thread_call_rcu_data(NULL);
151 call_rcu_data_free(crdp);
121a5d44 152 rcu_unregister_thread();
8a953620
MD
153
154 return (NULL);
155}
156
157void *rcu_update_perf_test(void *arg)
158{
159 long long n_updates_local = 0;
160
b57aee66
PM
161 if ((random() & 0xf00) == 0) {
162 struct call_rcu_data *crdp;
163
c1d2c60b 164 crdp = create_call_rcu_data(0, -1);
b57aee66 165 if (crdp != NULL) {
ad460058 166 diag("Using per-thread call_rcu() worker.");
b57aee66
PM
167 set_thread_call_rcu_data(crdp);
168 }
169 }
ec4e58a3 170 uatomic_inc(&nthreadsrunning);
8a953620 171 while (goflag == GOFLAG_INIT)
775aff2e 172 (void) poll(NULL, 0, 1);
8a953620
MD
173 while (goflag == GOFLAG_RUN) {
174 synchronize_rcu();
175 n_updates_local++;
176 }
177 __get_thread_var(n_updates_pt) += n_updates_local;
b0b31506 178 return NULL;
8a953620
MD
179}
180
181void perftestinit(void)
182{
183 init_per_thread(n_reads_pt, 0LL);
184 init_per_thread(n_updates_pt, 0LL);
ec4e58a3 185 uatomic_set(&nthreadsrunning, 0);
8a953620
MD
186}
187
ad460058 188int perftestrun(int nthreads, int nreaders, int nupdaters)
8a953620
MD
189{
190 int t;
191 int duration = 1;
192
5481ddb3 193 cmm_smp_mb();
ec4e58a3 194 while (uatomic_read(&nthreadsrunning) < nthreads)
775aff2e 195 (void) poll(NULL, 0, 1);
8a953620 196 goflag = GOFLAG_RUN;
5481ddb3 197 cmm_smp_mb();
8a953620 198 sleep(duration);
5481ddb3 199 cmm_smp_mb();
8a953620 200 goflag = GOFLAG_STOP;
5481ddb3 201 cmm_smp_mb();
8a953620
MD
202 wait_all_threads();
203 for_each_thread(t) {
204 n_reads += per_thread(n_reads_pt, t);
205 n_updates += per_thread(n_updates_pt, t);
206 }
ad460058 207 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
8a953620 208 n_reads, n_updates, nreaders, nupdaters, duration);
ad460058 209 diag("ns/read: %g ns/update: %g",
8a953620
MD
210 ((duration * 1000*1000*1000.*(double)nreaders) /
211 (double)n_reads),
212 ((duration * 1000*1000*1000.*(double)nupdaters) /
213 (double)n_updates));
7106ddf8 214 if (get_cpu_call_rcu_data(0)) {
ad460058 215 diag("Deallocating per-CPU call_rcu threads.\n");
7106ddf8
PM
216 free_all_cpu_call_rcu_data();
217 }
ad460058 218 return 0;
8a953620
MD
219}
220
ad460058 221int perftest(int nreaders, int cpustride)
8a953620
MD
222{
223 int i;
224 long arg;
225
226 perftestinit();
227 for (i = 0; i < nreaders; i++) {
228 arg = (long)(i * cpustride);
229 create_thread(rcu_read_perf_test, (void *)arg);
230 }
231 arg = (long)(i * cpustride);
232 create_thread(rcu_update_perf_test, (void *)arg);
ad460058 233 return perftestrun(i + 1, nreaders, 1);
8a953620
MD
234}
235
ad460058 236int rperftest(int nreaders, int cpustride)
8a953620
MD
237{
238 int i;
239 long arg;
240
241 perftestinit();
242 init_per_thread(n_reads_pt, 0LL);
243 for (i = 0; i < nreaders; i++) {
244 arg = (long)(i * cpustride);
245 create_thread(rcu_read_perf_test, (void *)arg);
246 }
ad460058 247 return perftestrun(i, nreaders, 0);
8a953620
MD
248}
249
ad460058 250int uperftest(int nupdaters, int cpustride)
8a953620
MD
251{
252 int i;
253 long arg;
254
255 perftestinit();
256 init_per_thread(n_reads_pt, 0LL);
257 for (i = 0; i < nupdaters; i++) {
258 arg = (long)(i * cpustride);
259 create_thread(rcu_update_perf_test, (void *)arg);
260 }
ad460058 261 return perftestrun(i, 0, nupdaters);
8a953620
MD
262}
263
264/*
265 * Stress test.
266 */
267
268#define RCU_STRESS_PIPE_LEN 10
269
270struct rcu_stress {
271 int pipe_count;
272 int mbtest;
273};
274
b0b31506 275struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
8a953620
MD
276struct rcu_stress *rcu_stress_current;
277int rcu_stress_idx = 0;
278
279int n_mberror = 0;
280DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
281
282int garbage = 0;
283
284void *rcu_read_stress_test(void *arg)
285{
286 int i;
287 int itercnt = 0;
288 struct rcu_stress *p;
289 int pc;
290
121a5d44 291 rcu_register_thread();
3bf02b5b 292 put_thread_offline();
8a953620 293 while (goflag == GOFLAG_INIT)
775aff2e 294 (void) poll(NULL, 0, 1);
3bf02b5b 295 put_thread_online();
8a953620
MD
296 while (goflag == GOFLAG_RUN) {
297 rcu_read_lock();
298 p = rcu_dereference(rcu_stress_current);
299 if (p->mbtest == 0)
300 n_mberror++;
301 rcu_read_lock_nest();
302 for (i = 0; i < 100; i++)
303 garbage++;
304 rcu_read_unlock_nest();
305 pc = p->pipe_count;
306 rcu_read_unlock();
307 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
308 pc = RCU_STRESS_PIPE_LEN;
309 __get_thread_var(rcu_stress_count)[pc]++;
310 __get_thread_var(n_reads_pt)++;
311 mark_rcu_quiescent_state();
312 if ((++itercnt % 0x1000) == 0) {
313 put_thread_offline();
314 put_thread_online_delay();
315 put_thread_online();
316 }
317 }
318 put_thread_offline();
121a5d44 319 rcu_unregister_thread();
8a953620
MD
320
321 return (NULL);
322}
323
b57aee66
PM
324static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
325static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
326
327void rcu_update_stress_test_rcu(struct rcu_head *head)
328{
ad460058
MD
329 int ret;
330
331 ret = pthread_mutex_lock(&call_rcu_test_mutex);
332 if (ret) {
333 errno = ret;
334 diag("pthread_mutex_lock: %s",
335 strerror(errno));
336 abort();
b57aee66 337 }
ad460058
MD
338 ret = pthread_cond_signal(&call_rcu_test_cond);
339 if (ret) {
340 errno = ret;
341 diag("pthread_cond_signal: %s",
342 strerror(errno));
343 abort();
b57aee66 344 }
ad460058
MD
345 ret = pthread_mutex_unlock(&call_rcu_test_mutex);
346 if (ret) {
347 errno = ret;
348 diag("pthread_mutex_unlock: %s",
349 strerror(errno));
350 abort();
b57aee66
PM
351 }
352}
353
8a953620
MD
354void *rcu_update_stress_test(void *arg)
355{
356 int i;
357 struct rcu_stress *p;
b57aee66 358 struct rcu_head rh;
8a953620
MD
359
360 while (goflag == GOFLAG_INIT)
775aff2e 361 (void) poll(NULL, 0, 1);
8a953620
MD
362 while (goflag == GOFLAG_RUN) {
363 i = rcu_stress_idx + 1;
364 if (i >= RCU_STRESS_PIPE_LEN)
365 i = 0;
366 p = &rcu_stress_array[i];
367 p->mbtest = 0;
5481ddb3 368 cmm_smp_mb();
8a953620
MD
369 p->pipe_count = 0;
370 p->mbtest = 1;
371 rcu_assign_pointer(rcu_stress_current, p);
372 rcu_stress_idx = i;
373 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
374 if (i != rcu_stress_idx)
375 rcu_stress_array[i].pipe_count++;
b57aee66
PM
376 if (n_updates & 0x1)
377 synchronize_rcu();
378 else {
ad460058
MD
379 int ret;
380
381 ret = pthread_mutex_lock(&call_rcu_test_mutex);
382 if (ret) {
383 errno = ret;
384 diag("pthread_mutex_lock: %s",
385 strerror(errno));
386 abort();
b57aee66
PM
387 }
388 call_rcu(&rh, rcu_update_stress_test_rcu);
ad460058
MD
389 ret = pthread_cond_wait(&call_rcu_test_cond,
390 &call_rcu_test_mutex);
391 if (ret) {
392 errno = ret;
393 diag("pthread_cond_signal: %s",
394 strerror(errno));
395 abort();
b57aee66 396 }
ad460058
MD
397 ret = pthread_mutex_unlock(&call_rcu_test_mutex);
398 if (ret) {
399 errno = ret;
400 diag("pthread_mutex_unlock: %s",
401 strerror(errno));
402 abort();
b57aee66
PM
403 }
404 }
8a953620
MD
405 n_updates++;
406 }
b0b31506 407 return NULL;
8a953620
MD
408}
409
410void *rcu_fake_update_stress_test(void *arg)
411{
b57aee66
PM
412 if ((random() & 0xf00) == 0) {
413 struct call_rcu_data *crdp;
414
c1d2c60b 415 crdp = create_call_rcu_data(0, -1);
b57aee66 416 if (crdp != NULL) {
ad460058 417 diag("Using per-thread call_rcu() worker.");
b57aee66
PM
418 set_thread_call_rcu_data(crdp);
419 }
420 }
8a953620 421 while (goflag == GOFLAG_INIT)
775aff2e 422 (void) poll(NULL, 0, 1);
8a953620
MD
423 while (goflag == GOFLAG_RUN) {
424 synchronize_rcu();
775aff2e 425 (void) poll(NULL, 0, 1);
8a953620 426 }
b0b31506 427 return NULL;
8a953620
MD
428}
429
ad460058 430int stresstest(int nreaders)
8a953620
MD
431{
432 int i;
433 int t;
434 long long *p;
435 long long sum;
436
437 init_per_thread(n_reads_pt, 0LL);
438 for_each_thread(t) {
439 p = &per_thread(rcu_stress_count,t)[0];
440 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
441 p[i] = 0LL;
442 }
443 rcu_stress_current = &rcu_stress_array[0];
444 rcu_stress_current->pipe_count = 0;
445 rcu_stress_current->mbtest = 1;
446 for (i = 0; i < nreaders; i++)
447 create_thread(rcu_read_stress_test, NULL);
448 create_thread(rcu_update_stress_test, NULL);
449 for (i = 0; i < 5; i++)
450 create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3 451 cmm_smp_mb();
8a953620 452 goflag = GOFLAG_RUN;
5481ddb3 453 cmm_smp_mb();
8a953620 454 sleep(10);
5481ddb3 455 cmm_smp_mb();
8a953620 456 goflag = GOFLAG_STOP;
5481ddb3 457 cmm_smp_mb();
8a953620
MD
458 wait_all_threads();
459 for_each_thread(t)
460 n_reads += per_thread(n_reads_pt, t);
ad460058 461 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
8a953620 462 n_reads, n_updates, n_mberror);
ad460058
MD
463 rdiag_start();
464 rdiag("rcu_stress_count:");
8a953620
MD
465 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
466 sum = 0LL;
467 for_each_thread(t) {
468 sum += per_thread(rcu_stress_count, t)[i];
469 }
ad460058 470 rdiag(" %lld", sum);
8a953620 471 }
ad460058 472 rdiag_end();
7106ddf8 473 if (get_cpu_call_rcu_data(0)) {
ad460058 474 diag("Deallocating per-CPU call_rcu threads.");
7106ddf8
PM
475 free_all_cpu_call_rcu_data();
476 }
ad460058
MD
477 if (!n_mberror)
478 return 0;
479 else
480 return -1;
8a953620
MD
481}
482
483/*
484 * Mainprogram.
485 */
486
487void usage(int argc, char *argv[])
488{
ad460058 489 diag("Usage: %s [nreaders [ perf | rperf | uperf | stress ] ]\n", argv[0]);
8a953620
MD
490 exit(-1);
491}
492
493int main(int argc, char *argv[])
494{
495 int nreaders = 1;
496 int cpustride = 1;
497
ad460058
MD
498 plan_tests(NR_TESTS);
499
8a953620
MD
500 smp_init();
501 //rcu_init();
b57aee66
PM
502 srandom(time(NULL));
503 if (random() & 0x100) {
ad460058 504 diag("Allocating per-CPU call_rcu threads.");
b57aee66 505 if (create_all_cpu_call_rcu_data(0))
ad460058
MD
506 diag("create_all_cpu_call_rcu_data: %s",
507 strerror(errno));
b57aee66 508 }
8a953620 509
9b171f46
MD
510#ifdef DEBUG_YIELD
511 yield_active |= YIELD_READ;
512 yield_active |= YIELD_WRITE;
513#endif
514
8a953620
MD
515 if (argc > 1) {
516 nreaders = strtoul(argv[1], NULL, 0);
ad460058
MD
517 if (argc == 2) {
518 ok(!perftest(nreaders, cpustride),
519 "perftest readers: %d, stride: %d",
520 nreaders, cpustride);
521 goto end;
522 }
8a953620
MD
523 if (argc > 3)
524 cpustride = strtoul(argv[3], NULL, 0);
525 if (strcmp(argv[2], "perf") == 0)
ad460058
MD
526 ok(!perftest(nreaders, cpustride),
527 "perftest readers: %d, stride: %d",
528 nreaders, cpustride);
8a953620 529 else if (strcmp(argv[2], "rperf") == 0)
ad460058
MD
530 ok(!rperftest(nreaders, cpustride),
531 "rperftest readers: %d, stride: %d",
532 nreaders, cpustride);
8a953620 533 else if (strcmp(argv[2], "uperf") == 0)
ad460058
MD
534 ok(!uperftest(nreaders, cpustride),
535 "uperftest readers: %d, stride: %d",
536 nreaders, cpustride);
8a953620 537 else if (strcmp(argv[2], "stress") == 0)
ad460058
MD
538 ok(!stresstest(nreaders),
539 "stresstest readers: %d, stride: %d",
540 nreaders, cpustride);
541 else
542 usage(argc, argv);
543 } else {
544 ok(!perftest(nreaders, cpustride),
545 "perftest readers: %d, stride: %d",
546 nreaders, cpustride);
8a953620 547 }
ad460058
MD
548end:
549 return exit_status();
8a953620 550}
This page took 0.089808 seconds and 4 git commands to generate.