Maintain list of struct call_rcu_data to keep valgrind happy.
[urcu.git] / tests / rcutorture.h
CommitLineData
8a953620
MD
1/*
2 * rcutorture.h: simple user-level performance/stress test of RCU.
3 *
4 * Usage:
5 * ./rcu <nreaders> rperf [ <cpustride> ]
6 * Run a read-side performance test with the specified
7 * number of readers spaced by <cpustride>.
8 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
9 * CPUs from 0 to 30.
10 * ./rcu <nupdaters> uperf [ <cpustride> ]
11 * Run an update-side performance test with the specified
12 * number of updaters and specified CPU spacing.
13 * ./rcu <nreaders> perf [ <cpustride> ]
14 * Run a combined read/update performance test with the specified
15 * number of readers and one updater and specified CPU spacing.
16 * The readers run on the low-numbered CPUs and the updater
17 * of the highest-numbered CPU.
18 *
19 * The above tests produce output as follows:
20 *
21 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
22 * ns/read: 43.4707 ns/update: 6848.1
23 *
24 * The first line lists the total number of RCU reads and updates executed
25 * during the test, the number of reader threads, the number of updater
26 * threads, and the duration of the test in seconds. The second line
27 * lists the average duration of each type of operation in nanoseconds,
28 * or "nan" if the corresponding type of operation was not performed.
29 *
30 * ./rcu <nreaders> stress
31 * Run a stress test with the specified number of readers and
32 * one updater. None of the threads are affinitied to any
33 * particular CPU.
34 *
35 * This test produces output as follows:
36 *
37 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
38 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
39 *
40 * The first line lists the number of RCU read and update operations
41 * executed, followed by the number of memory-ordering violations
42 * (which will be zero in a correct RCU implementation). The second
43 * line lists the number of readers observing progressively more stale
44 * data. A correct RCU implementation will have all but the first two
45 * numbers non-zero.
46 *
47 * This program is free software; you can redistribute it and/or modify
48 * it under the terms of the GNU General Public License as published by
49 * the Free Software Foundation; either version 2 of the License, or
50 * (at your option) any later version.
51 *
52 * This program is distributed in the hope that it will be useful,
53 * but WITHOUT ANY WARRANTY; without even the implied warranty of
54 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
55 * GNU General Public License for more details.
56 *
57 * You should have received a copy of the GNU General Public License
58 * along with this program; if not, write to the Free Software
59 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
60 *
61 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
62 */
63
64/*
65 * Test variables.
66 */
67
b57aee66
PM
68#include <stdlib.h>
69#include "../urcu-call-rcu.h"
70
8a953620
MD
71DEFINE_PER_THREAD(long long, n_reads_pt);
72DEFINE_PER_THREAD(long long, n_updates_pt);
73
74long long n_reads = 0LL;
75long n_updates = 0L;
6ee91d83 76int nthreadsrunning;
8a953620
MD
77char argsbuf[64];
78
79#define GOFLAG_INIT 0
80#define GOFLAG_RUN 1
81#define GOFLAG_STOP 2
82
06f22bdb 83int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))) = GOFLAG_INIT;
8a953620
MD
84
85#define RCU_READ_RUN 1000
86
87//MD
88#define RCU_READ_NESTABLE
89
90#ifdef RCU_READ_NESTABLE
91#define rcu_read_lock_nest() rcu_read_lock()
92#define rcu_read_unlock_nest() rcu_read_unlock()
93#else /* #ifdef RCU_READ_NESTABLE */
94#define rcu_read_lock_nest()
95#define rcu_read_unlock_nest()
96#endif /* #else #ifdef RCU_READ_NESTABLE */
97
1a43bbd8
MD
98#ifdef TORTURE_QSBR
99#define mark_rcu_quiescent_state rcu_quiescent_state
100#define put_thread_offline rcu_thread_offline
101#define put_thread_online rcu_thread_online
102#endif
103
8a953620
MD
104#ifndef mark_rcu_quiescent_state
105#define mark_rcu_quiescent_state() do ; while (0)
106#endif /* #ifdef mark_rcu_quiescent_state */
107
108#ifndef put_thread_offline
109#define put_thread_offline() do ; while (0)
110#define put_thread_online() do ; while (0)
111#define put_thread_online_delay() do ; while (0)
112#else /* #ifndef put_thread_offline */
113#define put_thread_online_delay() synchronize_rcu()
114#endif /* #else #ifndef put_thread_offline */
115
116/*
117 * Performance test.
118 */
119
120void *rcu_read_perf_test(void *arg)
121{
122 int i;
123 int me = (long)arg;
8a953620
MD
124 long long n_reads_local = 0;
125
121a5d44 126 rcu_register_thread();
8a953620 127 run_on(me);
ec4e58a3 128 uatomic_inc(&nthreadsrunning);
8a953620
MD
129 while (goflag == GOFLAG_INIT)
130 poll(NULL, 0, 1);
131 mark_rcu_quiescent_state();
132 while (goflag == GOFLAG_RUN) {
133 for (i = 0; i < RCU_READ_RUN; i++) {
134 rcu_read_lock();
135 /* rcu_read_lock_nest(); */
136 /* rcu_read_unlock_nest(); */
137 rcu_read_unlock();
138 }
139 n_reads_local += RCU_READ_RUN;
140 mark_rcu_quiescent_state();
141 }
142 __get_thread_var(n_reads_pt) += n_reads_local;
143 put_thread_offline();
121a5d44 144 rcu_unregister_thread();
8a953620
MD
145
146 return (NULL);
147}
148
149void *rcu_update_perf_test(void *arg)
150{
151 long long n_updates_local = 0;
152
b57aee66
PM
153 if ((random() & 0xf00) == 0) {
154 struct call_rcu_data *crdp;
155
156 crdp = create_call_rcu_data(0);
157 if (crdp != NULL) {
158 fprintf(stderr,
159 "Using per-thread call_rcu() worker.\n");
160 set_thread_call_rcu_data(crdp);
161 }
162 }
ec4e58a3 163 uatomic_inc(&nthreadsrunning);
8a953620
MD
164 while (goflag == GOFLAG_INIT)
165 poll(NULL, 0, 1);
166 while (goflag == GOFLAG_RUN) {
167 synchronize_rcu();
168 n_updates_local++;
169 }
170 __get_thread_var(n_updates_pt) += n_updates_local;
b0b31506 171 return NULL;
8a953620
MD
172}
173
174void perftestinit(void)
175{
176 init_per_thread(n_reads_pt, 0LL);
177 init_per_thread(n_updates_pt, 0LL);
ec4e58a3 178 uatomic_set(&nthreadsrunning, 0);
8a953620
MD
179}
180
181void perftestrun(int nthreads, int nreaders, int nupdaters)
182{
183 int t;
184 int duration = 1;
185
5481ddb3 186 cmm_smp_mb();
ec4e58a3 187 while (uatomic_read(&nthreadsrunning) < nthreads)
8a953620
MD
188 poll(NULL, 0, 1);
189 goflag = GOFLAG_RUN;
5481ddb3 190 cmm_smp_mb();
8a953620 191 sleep(duration);
5481ddb3 192 cmm_smp_mb();
8a953620 193 goflag = GOFLAG_STOP;
5481ddb3 194 cmm_smp_mb();
8a953620
MD
195 wait_all_threads();
196 for_each_thread(t) {
197 n_reads += per_thread(n_reads_pt, t);
198 n_updates += per_thread(n_updates_pt, t);
199 }
200 printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n",
201 n_reads, n_updates, nreaders, nupdaters, duration);
202 printf("ns/read: %g ns/update: %g\n",
203 ((duration * 1000*1000*1000.*(double)nreaders) /
204 (double)n_reads),
205 ((duration * 1000*1000*1000.*(double)nupdaters) /
206 (double)n_updates));
207 exit(0);
208}
209
210void perftest(int nreaders, int cpustride)
211{
212 int i;
213 long arg;
214
215 perftestinit();
216 for (i = 0; i < nreaders; i++) {
217 arg = (long)(i * cpustride);
218 create_thread(rcu_read_perf_test, (void *)arg);
219 }
220 arg = (long)(i * cpustride);
221 create_thread(rcu_update_perf_test, (void *)arg);
222 perftestrun(i + 1, nreaders, 1);
223}
224
225void rperftest(int nreaders, int cpustride)
226{
227 int i;
228 long arg;
229
230 perftestinit();
231 init_per_thread(n_reads_pt, 0LL);
232 for (i = 0; i < nreaders; i++) {
233 arg = (long)(i * cpustride);
234 create_thread(rcu_read_perf_test, (void *)arg);
235 }
236 perftestrun(i, nreaders, 0);
237}
238
239void uperftest(int nupdaters, int cpustride)
240{
241 int i;
242 long arg;
243
244 perftestinit();
245 init_per_thread(n_reads_pt, 0LL);
246 for (i = 0; i < nupdaters; i++) {
247 arg = (long)(i * cpustride);
248 create_thread(rcu_update_perf_test, (void *)arg);
249 }
250 perftestrun(i, 0, nupdaters);
251}
252
253/*
254 * Stress test.
255 */
256
257#define RCU_STRESS_PIPE_LEN 10
258
259struct rcu_stress {
260 int pipe_count;
261 int mbtest;
262};
263
b0b31506 264struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
8a953620
MD
265struct rcu_stress *rcu_stress_current;
266int rcu_stress_idx = 0;
267
268int n_mberror = 0;
269DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
270
271int garbage = 0;
272
273void *rcu_read_stress_test(void *arg)
274{
275 int i;
276 int itercnt = 0;
277 struct rcu_stress *p;
278 int pc;
279
121a5d44 280 rcu_register_thread();
8a953620
MD
281 while (goflag == GOFLAG_INIT)
282 poll(NULL, 0, 1);
283 mark_rcu_quiescent_state();
284 while (goflag == GOFLAG_RUN) {
285 rcu_read_lock();
286 p = rcu_dereference(rcu_stress_current);
287 if (p->mbtest == 0)
288 n_mberror++;
289 rcu_read_lock_nest();
290 for (i = 0; i < 100; i++)
291 garbage++;
292 rcu_read_unlock_nest();
293 pc = p->pipe_count;
294 rcu_read_unlock();
295 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
296 pc = RCU_STRESS_PIPE_LEN;
297 __get_thread_var(rcu_stress_count)[pc]++;
298 __get_thread_var(n_reads_pt)++;
299 mark_rcu_quiescent_state();
300 if ((++itercnt % 0x1000) == 0) {
301 put_thread_offline();
302 put_thread_online_delay();
303 put_thread_online();
304 }
305 }
306 put_thread_offline();
121a5d44 307 rcu_unregister_thread();
8a953620
MD
308
309 return (NULL);
310}
311
b57aee66
PM
312static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
313static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
314
315void rcu_update_stress_test_rcu(struct rcu_head *head)
316{
317 if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) {
318 perror("pthread_mutex_lock");
319 exit(-1);
320 }
321 if (pthread_cond_signal(&call_rcu_test_cond) != 0) {
322 perror("pthread_cond_signal");
323 exit(-1);
324 }
325 if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) {
326 perror("pthread_mutex_unlock");
327 exit(-1);
328 }
329}
330
8a953620
MD
331void *rcu_update_stress_test(void *arg)
332{
333 int i;
334 struct rcu_stress *p;
b57aee66 335 struct rcu_head rh;
8a953620
MD
336
337 while (goflag == GOFLAG_INIT)
338 poll(NULL, 0, 1);
339 while (goflag == GOFLAG_RUN) {
340 i = rcu_stress_idx + 1;
341 if (i >= RCU_STRESS_PIPE_LEN)
342 i = 0;
343 p = &rcu_stress_array[i];
344 p->mbtest = 0;
5481ddb3 345 cmm_smp_mb();
8a953620
MD
346 p->pipe_count = 0;
347 p->mbtest = 1;
348 rcu_assign_pointer(rcu_stress_current, p);
349 rcu_stress_idx = i;
350 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
351 if (i != rcu_stress_idx)
352 rcu_stress_array[i].pipe_count++;
b57aee66
PM
353 if (n_updates & 0x1)
354 synchronize_rcu();
355 else {
356 if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) {
357 perror("pthread_mutex_lock");
358 exit(-1);
359 }
360 call_rcu(&rh, rcu_update_stress_test_rcu);
361 if (pthread_cond_wait(&call_rcu_test_cond,
362 &call_rcu_test_mutex) != 0) {
363 perror("pthread_cond_wait");
364 exit(-1);
365 }
366 if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) {
367 perror("pthread_mutex_unlock");
368 exit(-1);
369 }
370 }
8a953620
MD
371 n_updates++;
372 }
b0b31506 373 return NULL;
8a953620
MD
374}
375
376void *rcu_fake_update_stress_test(void *arg)
377{
b57aee66
PM
378 if ((random() & 0xf00) == 0) {
379 struct call_rcu_data *crdp;
380
381 crdp = create_call_rcu_data(0);
382 if (crdp != NULL) {
383 fprintf(stderr,
384 "Using per-thread call_rcu() worker.\n");
385 set_thread_call_rcu_data(crdp);
386 }
387 }
8a953620
MD
388 while (goflag == GOFLAG_INIT)
389 poll(NULL, 0, 1);
390 while (goflag == GOFLAG_RUN) {
391 synchronize_rcu();
392 poll(NULL, 0, 1);
393 }
b0b31506 394 return NULL;
8a953620
MD
395}
396
397void stresstest(int nreaders)
398{
399 int i;
400 int t;
401 long long *p;
402 long long sum;
403
404 init_per_thread(n_reads_pt, 0LL);
405 for_each_thread(t) {
406 p = &per_thread(rcu_stress_count,t)[0];
407 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
408 p[i] = 0LL;
409 }
410 rcu_stress_current = &rcu_stress_array[0];
411 rcu_stress_current->pipe_count = 0;
412 rcu_stress_current->mbtest = 1;
413 for (i = 0; i < nreaders; i++)
414 create_thread(rcu_read_stress_test, NULL);
415 create_thread(rcu_update_stress_test, NULL);
416 for (i = 0; i < 5; i++)
417 create_thread(rcu_fake_update_stress_test, NULL);
5481ddb3 418 cmm_smp_mb();
8a953620 419 goflag = GOFLAG_RUN;
5481ddb3 420 cmm_smp_mb();
8a953620 421 sleep(10);
5481ddb3 422 cmm_smp_mb();
8a953620 423 goflag = GOFLAG_STOP;
5481ddb3 424 cmm_smp_mb();
8a953620
MD
425 wait_all_threads();
426 for_each_thread(t)
427 n_reads += per_thread(n_reads_pt, t);
b0b31506 428 printf("n_reads: %lld n_updates: %ld n_mberror: %d\n",
8a953620
MD
429 n_reads, n_updates, n_mberror);
430 printf("rcu_stress_count:");
431 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
432 sum = 0LL;
433 for_each_thread(t) {
434 sum += per_thread(rcu_stress_count, t)[i];
435 }
436 printf(" %lld", sum);
437 }
438 printf("\n");
439 exit(0);
440}
441
442/*
443 * Mainprogram.
444 */
445
446void usage(int argc, char *argv[])
447{
448 fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]);
449 exit(-1);
450}
451
452int main(int argc, char *argv[])
453{
454 int nreaders = 1;
455 int cpustride = 1;
456
457 smp_init();
458 //rcu_init();
b57aee66
PM
459 srandom(time(NULL));
460 if (random() & 0x100) {
461 fprintf(stderr, "Allocating per-CPU call_rcu threads.\n");
462 if (create_all_cpu_call_rcu_data(0))
463 perror("create_all_cpu_call_rcu_data");
464 }
8a953620 465
9b171f46
MD
466#ifdef DEBUG_YIELD
467 yield_active |= YIELD_READ;
468 yield_active |= YIELD_WRITE;
469#endif
470
8a953620
MD
471 if (argc > 1) {
472 nreaders = strtoul(argv[1], NULL, 0);
473 if (argc == 2)
474 perftest(nreaders, cpustride);
475 if (argc > 3)
476 cpustride = strtoul(argv[3], NULL, 0);
477 if (strcmp(argv[2], "perf") == 0)
478 perftest(nreaders, cpustride);
479 else if (strcmp(argv[2], "rperf") == 0)
480 rperftest(nreaders, cpustride);
481 else if (strcmp(argv[2], "uperf") == 0)
482 uperftest(nreaders, cpustride);
483 else if (strcmp(argv[2], "stress") == 0)
484 stresstest(nreaders);
485 usage(argc, argv);
486 }
487 perftest(nreaders, cpustride);
0578089f 488 return 0;
8a953620 489}
This page took 0.067628 seconds and 4 git commands to generate.