tests/regression/rcutorture.h

   1 /*
   2  * rcutorture.h: simple user-level performance/stress test of RCU.
   3  *
   4  * Usage:
   5  *      ./rcu <nreaders> rperf [ <cpustride> ]
   6  *              Run a read-side performance test with the specified
   7  *              number of readers spaced by <cpustride>.
   8  *              Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
   9  *              CPUs from 0 to 30.
  10  *      ./rcu <nupdaters> uperf [ <cpustride> ]
  11  *              Run an update-side performance test with the specified
  12  *              number of updaters and specified CPU spacing.
  13  *      ./rcu <nreaders> perf [ <cpustride> ]
  14  *              Run a combined read/update performance test with the specified
  15  *              number of readers and one updater and specified CPU spacing.
  16  *              The readers run on the low-numbered CPUs and the updater
  17  *              of the highest-numbered CPU.
  18  *
  19  * The above tests produce output as follows:
  20  *
  21  * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
  22  * ns/read: 43.4707  ns/update: 6848.1
  23  *
  24  * The first line lists the total number of RCU reads and updates executed
  25  * during the test, the number of reader threads, the number of updater
  26  * threads, and the duration of the test in seconds.  The second line
  27  * lists the average duration of each type of operation in nanoseconds,
  28  * or "nan" if the corresponding type of operation was not performed.
  29  *
  30  *      ./rcu <nreaders> stress
  31  *              Run a stress test with the specified number of readers and
  32  *              one updater.  None of the threads are affinitied to any
  33  *              particular CPU.
  34  *
  35  * This test produces output as follows:
  36  *
  37  * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
  38  * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
  39  *
  40  * The first line lists the number of RCU read and update operations
  41  * executed, followed by the number of memory-ordering violations
  42  * (which will be zero in a correct RCU implementation).  The second
  43  * line lists the number of readers observing progressively more stale
  44  * data.  A correct RCU implementation will have all but the first two
  45  * numbers non-zero.
  46  *
  47  * This program is free software; you can redistribute it and/or modify
  48  * it under the terms of the GNU General Public License as published by
  49  * the Free Software Foundation; either version 2 of the License, or
  50  * (at your option) any later version.
  51  *
  52  * This program is distributed in the hope that it will be useful,
  53  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  54  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  55  * GNU General Public License for more details.
  56  *
  57  * You should have received a copy of the GNU General Public License
  58  * along with this program; if not, write to the Free Software
  59  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  60  *
  61  * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
  62  */
  63
  64 /*
  65  * Test variables.
  66  */
  67
  68 #include <stdlib.h>
  69 #include "tap.h"
  70
  71 #define NR_TESTS        1
  72
  73 DEFINE_PER_THREAD(long long, n_reads_pt);
  74 DEFINE_PER_THREAD(long long, n_updates_pt);
  75
  76 enum callrcu_type {
  77         CALLRCU_GLOBAL,
  78         CALLRCU_PERCPU,
  79         CALLRCU_PERTHREAD,
  80 };
  81
  82 static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
  83
  84 long long n_reads = 0LL;
  85 long n_updates = 0L;
  86 int nthreadsrunning;
  87 char argsbuf[64];
  88
  89 #define GOFLAG_INIT 0
  90 #define GOFLAG_RUN  1
  91 #define GOFLAG_STOP 2
  92
  93 volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
  94         = GOFLAG_INIT;
  95
  96 #define RCU_READ_RUN 1000
  97
  98 //MD
  99 #define RCU_READ_NESTABLE
 100
 101 #ifdef RCU_READ_NESTABLE
 102 #define rcu_read_lock_nest() rcu_read_lock()
 103 #define rcu_read_unlock_nest() rcu_read_unlock()
 104 #else /* #ifdef RCU_READ_NESTABLE */
 105 #define rcu_read_lock_nest()
 106 #define rcu_read_unlock_nest()
 107 #endif /* #else #ifdef RCU_READ_NESTABLE */
 108
 109 #ifdef TORTURE_QSBR
 110 #define mark_rcu_quiescent_state        rcu_quiescent_state
 111 #define put_thread_offline              rcu_thread_offline
 112 #define put_thread_online               rcu_thread_online
 113 #endif
 114
 115 #ifndef mark_rcu_quiescent_state
 116 #define mark_rcu_quiescent_state() do ; while (0)
 117 #endif /* #ifdef mark_rcu_quiescent_state */
 118
 119 #ifndef put_thread_offline
 120 #define put_thread_offline()            do ; while (0)
 121 #define put_thread_online()             do ; while (0)
 122 #define put_thread_online_delay()       do ; while (0)
 123 #else /* #ifndef put_thread_offline */
 124 #define put_thread_online_delay()       synchronize_rcu()
 125 #endif /* #else #ifndef put_thread_offline */
 126
 127 /*
 128  * Performance test.
 129  */
 130
 131 void *rcu_read_perf_test(void *arg)
 132 {
 133         int i;
 134         int me = (long)arg;
 135         long long n_reads_local = 0;
 136
 137         rcu_register_thread();
 138         run_on(me);
 139         uatomic_inc(&nthreadsrunning);
 140         put_thread_offline();
 141         while (goflag == GOFLAG_INIT)
 142                 (void) poll(NULL, 0, 1);
 143         put_thread_online();
 144         while (goflag == GOFLAG_RUN) {
 145                 for (i = 0; i < RCU_READ_RUN; i++) {
 146                         rcu_read_lock();
 147                         /* rcu_read_lock_nest(); */
 148                         /* rcu_read_unlock_nest(); */
 149                         rcu_read_unlock();
 150                 }
 151                 n_reads_local += RCU_READ_RUN;
 152                 mark_rcu_quiescent_state();
 153         }
 154         __get_thread_var(n_reads_pt) += n_reads_local;
 155         put_thread_offline();
 156         rcu_unregister_thread();
 157
 158         return (NULL);
 159 }
 160
 161 void *rcu_update_perf_test(void *arg)
 162 {
 163         long long n_updates_local = 0;
 164
 165         if (callrcu_type == CALLRCU_PERTHREAD) {
 166                 struct call_rcu_data *crdp;
 167
 168                 crdp = create_call_rcu_data(0, -1);
 169                 if (crdp != NULL) {
 170                         diag("Successfully using per-thread call_rcu() worker.");
 171                         set_thread_call_rcu_data(crdp);
 172                 }
 173         }
 174         uatomic_inc(&nthreadsrunning);
 175         while (goflag == GOFLAG_INIT)
 176                 (void) poll(NULL, 0, 1);
 177         while (goflag == GOFLAG_RUN) {
 178                 synchronize_rcu();
 179                 n_updates_local++;
 180         }
 181         __get_thread_var(n_updates_pt) += n_updates_local;
 182         if (callrcu_type == CALLRCU_PERTHREAD) {
 183                 struct call_rcu_data *crdp;
 184
 185                 crdp = get_thread_call_rcu_data();
 186                 set_thread_call_rcu_data(NULL);
 187                 call_rcu_data_free(crdp);
 188         }
 189         return NULL;
 190 }
 191
 192 void perftestinit(void)
 193 {
 194         init_per_thread(n_reads_pt, 0LL);
 195         init_per_thread(n_updates_pt, 0LL);
 196         uatomic_set(&nthreadsrunning, 0);
 197 }
 198
 199 int perftestrun(int nthreads, int nreaders, int nupdaters)
 200 {
 201         int t;
 202         int duration = 1;
 203
 204         cmm_smp_mb();
 205         while (uatomic_read(&nthreadsrunning) < nthreads)
 206                 (void) poll(NULL, 0, 1);
 207         goflag = GOFLAG_RUN;
 208         cmm_smp_mb();
 209         sleep(duration);
 210         cmm_smp_mb();
 211         goflag = GOFLAG_STOP;
 212         cmm_smp_mb();
 213         wait_all_threads();
 214         for_each_thread(t) {
 215                 n_reads += per_thread(n_reads_pt, t);
 216                 n_updates += per_thread(n_updates_pt, t);
 217         }
 218         diag("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d",
 219                n_reads, n_updates, nreaders, nupdaters, duration);
 220         diag("ns/read: %g  ns/update: %g",
 221                ((duration * 1000*1000*1000.*(double)nreaders) /
 222                 (double)n_reads),
 223                ((duration * 1000*1000*1000.*(double)nupdaters) /
 224                 (double)n_updates));
 225         if (get_cpu_call_rcu_data(0)) {
 226                 diag("Deallocating per-CPU call_rcu threads.\n");
 227                 free_all_cpu_call_rcu_data();
 228         }
 229         return 0;
 230 }
 231
 232 int perftest(int nreaders, int cpustride)
 233 {
 234         int i;
 235         long arg;
 236
 237         perftestinit();
 238         for (i = 0; i < nreaders; i++) {
 239                 arg = (long)(i * cpustride);
 240                 create_thread(rcu_read_perf_test, (void *)arg);
 241         }
 242         arg = (long)(i * cpustride);
 243         create_thread(rcu_update_perf_test, (void *)arg);
 244         return perftestrun(i + 1, nreaders, 1);
 245 }
 246
 247 int rperftest(int nreaders, int cpustride)
 248 {
 249         int i;
 250         long arg;
 251
 252         perftestinit();
 253         init_per_thread(n_reads_pt, 0LL);
 254         for (i = 0; i < nreaders; i++) {
 255                 arg = (long)(i * cpustride);
 256                 create_thread(rcu_read_perf_test, (void *)arg);
 257         }
 258         return perftestrun(i, nreaders, 0);
 259 }
 260
 261 int uperftest(int nupdaters, int cpustride)
 262 {
 263         int i;
 264         long arg;
 265
 266         perftestinit();
 267         init_per_thread(n_reads_pt, 0LL);
 268         for (i = 0; i < nupdaters; i++) {
 269                 arg = (long)(i * cpustride);
 270                 create_thread(rcu_update_perf_test, (void *)arg);
 271         }
 272         return perftestrun(i, 0, nupdaters);
 273 }
 274
 275 /*
 276  * Stress test.
 277  */
 278
 279 #define RCU_STRESS_PIPE_LEN 10
 280
 281 struct rcu_stress {
 282         int pipe_count;
 283         int mbtest;
 284 };
 285
 286 struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
 287 struct rcu_stress *rcu_stress_current;
 288 int rcu_stress_idx = 0;
 289
 290 int n_mberror = 0;
 291 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
 292
 293 int garbage = 0;
 294
 295 void *rcu_read_stress_test(void *arg)
 296 {
 297         int i;
 298         int itercnt = 0;
 299         struct rcu_stress *p;
 300         int pc;
 301
 302         rcu_register_thread();
 303         put_thread_offline();
 304         while (goflag == GOFLAG_INIT)
 305                 (void) poll(NULL, 0, 1);
 306         put_thread_online();
 307         while (goflag == GOFLAG_RUN) {
 308                 rcu_read_lock();
 309                 p = rcu_dereference(rcu_stress_current);
 310                 if (p->mbtest == 0)
 311                         n_mberror++;
 312                 rcu_read_lock_nest();
 313                 for (i = 0; i < 100; i++)
 314                         garbage++;
 315                 rcu_read_unlock_nest();
 316                 pc = p->pipe_count;
 317                 rcu_read_unlock();
 318                 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
 319                         pc = RCU_STRESS_PIPE_LEN;
 320                 __get_thread_var(rcu_stress_count)[pc]++;
 321                 __get_thread_var(n_reads_pt)++;
 322                 mark_rcu_quiescent_state();
 323                 if ((++itercnt % 0x1000) == 0) {
 324                         put_thread_offline();
 325                         put_thread_online_delay();
 326                         put_thread_online();
 327                 }
 328         }
 329         put_thread_offline();
 330         rcu_unregister_thread();
 331
 332         return (NULL);
 333 }
 334
 335 static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
 336 static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
 337
 338 void rcu_update_stress_test_rcu(struct rcu_head *head)
 339 {
 340         int ret;
 341
 342         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 343         if (ret) {
 344                 errno = ret;
 345                 diag("pthread_mutex_lock: %s",
 346                         strerror(errno));
 347                 abort();
 348         }
 349         ret = pthread_cond_signal(&call_rcu_test_cond);
 350         if (ret) {
 351                 errno = ret;
 352                 diag("pthread_cond_signal: %s",
 353                         strerror(errno));
 354                 abort();
 355         }
 356         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 357         if (ret) {
 358                 errno = ret;
 359                 diag("pthread_mutex_unlock: %s",
 360                         strerror(errno));
 361                 abort();
 362         }
 363 }
 364
 365 void *rcu_update_stress_test(void *arg)
 366 {
 367         int i;
 368         struct rcu_stress *p;
 369         struct rcu_head rh;
 370
 371         while (goflag == GOFLAG_INIT)
 372                 (void) poll(NULL, 0, 1);
 373         while (goflag == GOFLAG_RUN) {
 374                 i = rcu_stress_idx + 1;
 375                 if (i >= RCU_STRESS_PIPE_LEN)
 376                         i = 0;
 377                 p = &rcu_stress_array[i];
 378                 p->mbtest = 0;
 379                 cmm_smp_mb();
 380                 p->pipe_count = 0;
 381                 p->mbtest = 1;
 382                 rcu_assign_pointer(rcu_stress_current, p);
 383                 rcu_stress_idx = i;
 384                 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
 385                         if (i != rcu_stress_idx)
 386                                 rcu_stress_array[i].pipe_count++;
 387                 if (n_updates & 0x1)
 388                         synchronize_rcu();
 389                 else {
 390                         int ret;
 391
 392                         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 393                         if (ret) {
 394                                 errno = ret;
 395                                 diag("pthread_mutex_lock: %s",
 396                                         strerror(errno));
 397                                 abort();
 398                         }
 399                         call_rcu(&rh, rcu_update_stress_test_rcu);
 400                         ret = pthread_cond_wait(&call_rcu_test_cond,
 401                                         &call_rcu_test_mutex);
 402                         if (ret) {
 403                                 errno = ret;
 404                                 diag("pthread_cond_signal: %s",
 405                                         strerror(errno));
 406                                 abort();
 407                         }
 408                         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 409                         if (ret) {
 410                                 errno = ret;
 411                                 diag("pthread_mutex_unlock: %s",
 412                                         strerror(errno));
 413                                 abort();
 414                         }
 415                 }
 416                 n_updates++;
 417         }
 418         return NULL;
 419 }
 420
 421 void *rcu_fake_update_stress_test(void *arg)
 422 {
 423         if (callrcu_type == CALLRCU_PERTHREAD) {
 424                 struct call_rcu_data *crdp;
 425
 426                 crdp = create_call_rcu_data(0, -1);
 427                 if (crdp != NULL) {
 428                         diag("Successfully using per-thread call_rcu() worker.");
 429                         set_thread_call_rcu_data(crdp);
 430                 }
 431         }
 432         while (goflag == GOFLAG_INIT)
 433                 (void) poll(NULL, 0, 1);
 434         while (goflag == GOFLAG_RUN) {
 435                 synchronize_rcu();
 436                 (void) poll(NULL, 0, 1);
 437         }
 438         if (callrcu_type == CALLRCU_PERTHREAD) {
 439                 struct call_rcu_data *crdp;
 440
 441                 crdp = get_thread_call_rcu_data();
 442                 set_thread_call_rcu_data(NULL);
 443                 call_rcu_data_free(crdp);
 444         }
 445         return NULL;
 446 }
 447
 448 int stresstest(int nreaders)
 449 {
 450         int i;
 451         int t;
 452         long long *p;
 453         long long sum;
 454
 455         init_per_thread(n_reads_pt, 0LL);
 456         for_each_thread(t) {
 457                 p = &per_thread(rcu_stress_count,t)[0];
 458                 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
 459                         p[i] = 0LL;
 460         }
 461         rcu_stress_current = &rcu_stress_array[0];
 462         rcu_stress_current->pipe_count = 0;
 463         rcu_stress_current->mbtest = 1;
 464         for (i = 0; i < nreaders; i++)
 465                 create_thread(rcu_read_stress_test, NULL);
 466         create_thread(rcu_update_stress_test, NULL);
 467         for (i = 0; i < 5; i++)
 468                 create_thread(rcu_fake_update_stress_test, NULL);
 469         cmm_smp_mb();
 470         goflag = GOFLAG_RUN;
 471         cmm_smp_mb();
 472         sleep(10);
 473         cmm_smp_mb();
 474         goflag = GOFLAG_STOP;
 475         cmm_smp_mb();
 476         wait_all_threads();
 477         for_each_thread(t)
 478                 n_reads += per_thread(n_reads_pt, t);
 479         diag("n_reads: %lld  n_updates: %ld  n_mberror: %d",
 480                n_reads, n_updates, n_mberror);
 481         rdiag_start();
 482         rdiag("rcu_stress_count:");
 483         for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
 484                 sum = 0LL;
 485                 for_each_thread(t) {
 486                         sum += per_thread(rcu_stress_count, t)[i];
 487                 }
 488                 rdiag(" %lld", sum);
 489         }
 490         rdiag_end();
 491         if (get_cpu_call_rcu_data(0)) {
 492                 diag("Deallocating per-CPU call_rcu threads.");
 493                 free_all_cpu_call_rcu_data();
 494         }
 495         if (!n_mberror)
 496                 return 0;
 497         else
 498                 return -1;
 499 }
 500
 501 /*
 502  * Mainprogram.
 503  */
 504
 505 void usage(int argc, char *argv[])
 506 {
 507         diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
 508         exit(-1);
 509 }
 510
 511 int main(int argc, char *argv[])
 512 {
 513         int nreaders = 1;
 514         int cpustride = 1;
 515
 516         plan_tests(NR_TESTS);
 517
 518         smp_init();
 519         //rcu_init();
 520         if (argc > 4) {
 521                 const char *callrcu_str = argv[4];;
 522
 523                 if (strcmp(callrcu_str, "callrcu_global") == 0) {
 524                         callrcu_type = CALLRCU_GLOBAL;
 525                 } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
 526                         callrcu_type = CALLRCU_PERCPU;
 527                 } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
 528                         callrcu_type = CALLRCU_PERTHREAD;
 529                 } else {
 530                         usage(argc, argv);
 531                         goto end;
 532                 }
 533         }
 534
 535         switch (callrcu_type) {
 536         case CALLRCU_GLOBAL:
 537                 diag("Using global per-process call_rcu thread.");
 538                 break;
 539         case CALLRCU_PERCPU:
 540                 diag("Using per-CPU call_rcu threads.");
 541                 if (create_all_cpu_call_rcu_data(0))
 542                         diag("create_all_cpu_call_rcu_data: %s",
 543                                 strerror(errno));
 544                 break;
 545         case CALLRCU_PERTHREAD:
 546                 diag("Using per-thread call_rcu() worker.");
 547                 break;
 548         default:
 549                 abort();
 550         }
 551
 552 #ifdef DEBUG_YIELD
 553         yield_active |= YIELD_READ;
 554         yield_active |= YIELD_WRITE;
 555 #endif
 556
 557         if (argc > 1) {
 558                 if (strcmp(argv[1], "-h") == 0
 559                                 || strcmp(argv[1], "--help") == 0) {
 560                         usage(argc, argv);
 561                         goto end;
 562                 }
 563                 nreaders = strtoul(argv[1], NULL, 0);
 564                 if (argc == 2) {
 565                         ok(!perftest(nreaders, cpustride),
 566                                 "perftest readers: %d, stride: %d",
 567                                 nreaders, cpustride);
 568                         goto end;
 569                 }
 570                 if (argc > 3)
 571                         cpustride = strtoul(argv[3], NULL, 0);
 572                 if (strcmp(argv[2], "perf") == 0)
 573                         ok(!perftest(nreaders, cpustride),
 574                                 "perftest readers: %d, stride: %d",
 575                                 nreaders, cpustride);
 576                 else if (strcmp(argv[2], "rperf") == 0)
 577                         ok(!rperftest(nreaders, cpustride),
 578                                 "rperftest readers: %d, stride: %d",
 579                                 nreaders, cpustride);
 580                 else if (strcmp(argv[2], "uperf") == 0)
 581                         ok(!uperftest(nreaders, cpustride),
 582                                 "uperftest readers: %d, stride: %d",
 583                                 nreaders, cpustride);
 584                 else if (strcmp(argv[2], "stress") == 0)
 585                         ok(!stresstest(nreaders),
 586                                 "stresstest readers: %d, stride: %d",
 587                                 nreaders, cpustride);
 588                 else
 589                         usage(argc, argv);
 590         } else {
 591                 usage(argc, argv);
 592         }
 593 end:
 594         return exit_status();
 595 }