tests/regression/rcutorture.h

   1 /*
   2  * rcutorture.h: simple user-level performance/stress test of RCU.
   3  *
   4  * Usage:
   5  *      ./rcu <nreaders> rperf [ <cpustride> ]
   6  *              Run a read-side performance test with the specified
   7  *              number of readers spaced by <cpustride>.
   8  *              Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
   9  *              CPUs from 0 to 30.
  10  *      ./rcu <nupdaters> uperf [ <cpustride> ]
  11  *              Run an update-side performance test with the specified
  12  *              number of updaters and specified CPU spacing.
  13  *      ./rcu <nreaders> perf [ <cpustride> ]
  14  *              Run a combined read/update performance test with the specified
  15  *              number of readers and one updater and specified CPU spacing.
  16  *              The readers run on the low-numbered CPUs and the updater
  17  *              of the highest-numbered CPU.
  18  *
  19  * The above tests produce output as follows:
  20  *
  21  * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
  22  * ns/read: 43.4707  ns/update: 6848.1
  23  *
  24  * The first line lists the total number of RCU reads and updates executed
  25  * during the test, the number of reader threads, the number of updater
  26  * threads, and the duration of the test in seconds.  The second line
  27  * lists the average duration of each type of operation in nanoseconds,
  28  * or "nan" if the corresponding type of operation was not performed.
  29  *
  30  *      ./rcu <nreaders> stress
  31  *              Run a stress test with the specified number of readers and
  32  *              one updater.  None of the threads are affinitied to any
  33  *              particular CPU.
  34  *
  35  * This test produces output as follows:
  36  *
  37  * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
  38  * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
  39  *
  40  * The first line lists the number of RCU read and update operations
  41  * executed, followed by the number of memory-ordering violations
  42  * (which will be zero in a correct RCU implementation).  The second
  43  * line lists the number of readers observing progressively more stale
  44  * data.  A correct RCU implementation will have all but the first two
  45  * numbers non-zero.
  46  *
  47  * This program is free software; you can redistribute it and/or modify
  48  * it under the terms of the GNU General Public License as published by
  49  * the Free Software Foundation; either version 2 of the License, or
  50  * (at your option) any later version.
  51  *
  52  * This program is distributed in the hope that it will be useful,
  53  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  54  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  55  * GNU General Public License for more details.
  56  *
  57  * You should have received a copy of the GNU General Public License
  58  * along with this program; if not, write to the Free Software
  59  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  60  *
  61  * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
  62  */
  63
  64 /*
  65  * Test variables.
  66  */
  67
  68 #include <stdlib.h>
  69 #include "tap.h"
  70
  71 #define NR_TESTS        1
  72
  73 DEFINE_PER_THREAD(long long, n_reads_pt);
  74 DEFINE_PER_THREAD(long long, n_updates_pt);
  75
  76 enum callrcu_type {
  77         CALLRCU_GLOBAL,
  78         CALLRCU_PERCPU,
  79         CALLRCU_PERTHREAD,
  80 };
  81
  82 static enum callrcu_type callrcu_type = CALLRCU_GLOBAL;
  83
  84 long long n_reads = 0LL;
  85 long n_updates = 0L;
  86 int nthreadsrunning;
  87 char argsbuf[64];
  88
  89 #define GOFLAG_INIT 0
  90 #define GOFLAG_RUN  1
  91 #define GOFLAG_STOP 2
  92
  93 volatile int goflag __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)))
  94         = GOFLAG_INIT;
  95
  96 #define RCU_READ_RUN 1000
  97
  98 //MD
  99 #define RCU_READ_NESTABLE
 100
 101 #ifdef RCU_READ_NESTABLE
 102 #define rcu_read_lock_nest() rcu_read_lock()
 103 #define rcu_read_unlock_nest() rcu_read_unlock()
 104 #else /* #ifdef RCU_READ_NESTABLE */
 105 #define rcu_read_lock_nest()
 106 #define rcu_read_unlock_nest()
 107 #endif /* #else #ifdef RCU_READ_NESTABLE */
 108
 109 #ifdef TORTURE_QSBR
 110 #define mark_rcu_quiescent_state        rcu_quiescent_state
 111 #define put_thread_offline              rcu_thread_offline
 112 #define put_thread_online               rcu_thread_online
 113 #endif
 114
 115 #ifndef mark_rcu_quiescent_state
 116 #define mark_rcu_quiescent_state() do {} while (0)
 117 #endif /* #ifdef mark_rcu_quiescent_state */
 118
 119 #ifndef put_thread_offline
 120 #define put_thread_offline()            do {} while (0)
 121 #define put_thread_online()             do {} while (0)
 122 #define put_thread_online_delay()       do {} while (0)
 123 #else /* #ifndef put_thread_offline */
 124 #define put_thread_online_delay()       synchronize_rcu()
 125 #endif /* #else #ifndef put_thread_offline */
 126
 127 /*
 128  * Performance test.
 129  */
 130
 131 static
 132 void *rcu_read_perf_test(void *arg)
 133 {
 134         int i;
 135         int me = (long)arg;
 136         long long n_reads_local = 0;
 137
 138         rcu_register_thread();
 139         run_on(me);
 140         uatomic_inc(&nthreadsrunning);
 141         put_thread_offline();
 142         while (goflag == GOFLAG_INIT)
 143                 (void) poll(NULL, 0, 1);
 144         put_thread_online();
 145         while (goflag == GOFLAG_RUN) {
 146                 for (i = 0; i < RCU_READ_RUN; i++) {
 147                         rcu_read_lock();
 148                         /* rcu_read_lock_nest(); */
 149                         /* rcu_read_unlock_nest(); */
 150                         rcu_read_unlock();
 151                 }
 152                 n_reads_local += RCU_READ_RUN;
 153                 mark_rcu_quiescent_state();
 154         }
 155         __get_thread_var(n_reads_pt) += n_reads_local;
 156         put_thread_offline();
 157         rcu_unregister_thread();
 158
 159         return (NULL);
 160 }
 161
 162 static
 163 void *rcu_update_perf_test(void *arg __attribute__((unused)))
 164 {
 165         long long n_updates_local = 0;
 166
 167         if (callrcu_type == CALLRCU_PERTHREAD) {
 168                 struct call_rcu_data *crdp;
 169
 170                 crdp = create_call_rcu_data(0, -1);
 171                 if (crdp != NULL) {
 172                         diag("Successfully using per-thread call_rcu() worker.");
 173                         set_thread_call_rcu_data(crdp);
 174                 }
 175         }
 176         uatomic_inc(&nthreadsrunning);
 177         while (goflag == GOFLAG_INIT)
 178                 (void) poll(NULL, 0, 1);
 179         while (goflag == GOFLAG_RUN) {
 180                 synchronize_rcu();
 181                 n_updates_local++;
 182         }
 183         __get_thread_var(n_updates_pt) += n_updates_local;
 184         if (callrcu_type == CALLRCU_PERTHREAD) {
 185                 struct call_rcu_data *crdp;
 186
 187                 crdp = get_thread_call_rcu_data();
 188                 set_thread_call_rcu_data(NULL);
 189                 call_rcu_data_free(crdp);
 190         }
 191         return NULL;
 192 }
 193
 194 static
 195 void perftestinit(void)
 196 {
 197         init_per_thread(n_reads_pt, 0LL);
 198         init_per_thread(n_updates_pt, 0LL);
 199         uatomic_set(&nthreadsrunning, 0);
 200 }
 201
 202 static
 203 int perftestrun(int nthreads, int nreaders, int nupdaters)
 204 {
 205         int t;
 206         int duration = 1;
 207
 208         cmm_smp_mb();
 209         while (uatomic_read(&nthreadsrunning) < nthreads)
 210                 (void) poll(NULL, 0, 1);
 211         goflag = GOFLAG_RUN;
 212         cmm_smp_mb();
 213         sleep(duration);
 214         cmm_smp_mb();
 215         goflag = GOFLAG_STOP;
 216         cmm_smp_mb();
 217         wait_all_threads();
 218         for_each_thread(t) {
 219                 n_reads += per_thread(n_reads_pt, t);
 220                 n_updates += per_thread(n_updates_pt, t);
 221         }
 222         diag("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d",
 223                n_reads, n_updates, nreaders, nupdaters, duration);
 224         diag("ns/read: %g  ns/update: %g",
 225                ((duration * 1000*1000*1000.*(double)nreaders) /
 226                 (double)n_reads),
 227                ((duration * 1000*1000*1000.*(double)nupdaters) /
 228                 (double)n_updates));
 229         if (get_cpu_call_rcu_data(0)) {
 230                 diag("Deallocating per-CPU call_rcu threads.\n");
 231                 free_all_cpu_call_rcu_data();
 232         }
 233         return 0;
 234 }
 235
 236 static
 237 int perftest(int nreaders, int cpustride)
 238 {
 239         int i;
 240         long arg;
 241
 242         perftestinit();
 243         for (i = 0; i < nreaders; i++) {
 244                 arg = (long)(i * cpustride);
 245                 create_thread(rcu_read_perf_test, (void *)arg);
 246         }
 247         arg = (long)(i * cpustride);
 248         create_thread(rcu_update_perf_test, (void *)arg);
 249         return perftestrun(i + 1, nreaders, 1);
 250 }
 251
 252 static
 253 int rperftest(int nreaders, int cpustride)
 254 {
 255         int i;
 256         long arg;
 257
 258         perftestinit();
 259         init_per_thread(n_reads_pt, 0LL);
 260         for (i = 0; i < nreaders; i++) {
 261                 arg = (long)(i * cpustride);
 262                 create_thread(rcu_read_perf_test, (void *)arg);
 263         }
 264         return perftestrun(i, nreaders, 0);
 265 }
 266
 267 static
 268 int uperftest(int nupdaters, int cpustride)
 269 {
 270         int i;
 271         long arg;
 272
 273         perftestinit();
 274         init_per_thread(n_reads_pt, 0LL);
 275         for (i = 0; i < nupdaters; i++) {
 276                 arg = (long)(i * cpustride);
 277                 create_thread(rcu_update_perf_test, (void *)arg);
 278         }
 279         return perftestrun(i, 0, nupdaters);
 280 }
 281
 282 /*
 283  * Stress test.
 284  */
 285
 286 #define RCU_STRESS_PIPE_LEN 10
 287
 288 struct rcu_stress {
 289         int pipe_count;
 290         int mbtest;
 291 };
 292
 293 struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
 294 struct rcu_stress *rcu_stress_current;
 295 int rcu_stress_idx = 0;
 296
 297 int n_mberror = 0;
 298 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN + 1], rcu_stress_count);
 299
 300 int garbage = 0;
 301
 302 static
 303 void *rcu_read_stress_test(void *arg __attribute__((unused)))
 304 {
 305         int i;
 306         int itercnt = 0;
 307         struct rcu_stress *p;
 308         int pc;
 309
 310         rcu_register_thread();
 311         put_thread_offline();
 312         while (goflag == GOFLAG_INIT)
 313                 (void) poll(NULL, 0, 1);
 314         put_thread_online();
 315         while (goflag == GOFLAG_RUN) {
 316                 rcu_read_lock();
 317                 p = rcu_dereference(rcu_stress_current);
 318                 if (p->mbtest == 0)
 319                         n_mberror++;
 320                 rcu_read_lock_nest();
 321                 for (i = 0; i < 100; i++)
 322                         garbage++;
 323                 rcu_read_unlock_nest();
 324                 pc = p->pipe_count;
 325                 rcu_read_unlock();
 326                 if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0))
 327                         pc = RCU_STRESS_PIPE_LEN;
 328                 __get_thread_var(rcu_stress_count)[pc]++;
 329                 __get_thread_var(n_reads_pt)++;
 330                 mark_rcu_quiescent_state();
 331                 if ((++itercnt % 0x1000) == 0) {
 332                         put_thread_offline();
 333                         put_thread_online_delay();
 334                         put_thread_online();
 335                 }
 336         }
 337         put_thread_offline();
 338         rcu_unregister_thread();
 339
 340         return (NULL);
 341 }
 342
 343 static pthread_mutex_t call_rcu_test_mutex = PTHREAD_MUTEX_INITIALIZER;
 344 static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER;
 345 static bool call_rcu_wait;
 346
 347 static
 348 void rcu_update_stress_test_rcu(struct rcu_head *head __attribute__((unused)))
 349 {
 350         int ret;
 351
 352         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 353         if (ret) {
 354                 errno = ret;
 355                 diag("pthread_mutex_lock: %s",
 356                         strerror(errno));
 357                 abort();
 358         }
 359         ret = pthread_cond_signal(&call_rcu_test_cond);
 360         if (ret) {
 361                 errno = ret;
 362                 diag("pthread_cond_signal: %s",
 363                         strerror(errno));
 364                 abort();
 365         }
 366         call_rcu_wait = false;
 367         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 368         if (ret) {
 369                 errno = ret;
 370                 diag("pthread_mutex_unlock: %s",
 371                         strerror(errno));
 372                 abort();
 373         }
 374 }
 375
 376 static
 377 void *rcu_update_stress_test(void *arg __attribute__((unused)))
 378 {
 379         int i;
 380         struct rcu_stress *p;
 381         struct rcu_head rh;
 382
 383         while (goflag == GOFLAG_INIT)
 384                 (void) poll(NULL, 0, 1);
 385         while (goflag == GOFLAG_RUN) {
 386                 i = rcu_stress_idx + 1;
 387                 if (i >= RCU_STRESS_PIPE_LEN)
 388                         i = 0;
 389                 p = &rcu_stress_array[i];
 390                 p->mbtest = 0;
 391                 cmm_smp_mb();
 392                 p->pipe_count = 0;
 393                 p->mbtest = 1;
 394                 rcu_assign_pointer(rcu_stress_current, p);
 395                 rcu_stress_idx = i;
 396                 for (i = 0; i < RCU_STRESS_PIPE_LEN; i++)
 397                         if (i != rcu_stress_idx)
 398                                 rcu_stress_array[i].pipe_count++;
 399                 if (n_updates & 0x1)
 400                         synchronize_rcu();
 401                 else {
 402                         int ret;
 403
 404                         ret = pthread_mutex_lock(&call_rcu_test_mutex);
 405                         if (ret) {
 406                                 errno = ret;
 407                                 diag("pthread_mutex_lock: %s",
 408                                         strerror(errno));
 409                                 abort();
 410                         }
 411                         rcu_register_thread();
 412                         call_rcu(&rh, rcu_update_stress_test_rcu);
 413                         rcu_unregister_thread();
 414                         /*
 415                          * Our MacOS X test machine with the following
 416                          * config:
 417                          * 15.6.0 Darwin Kernel Version 15.6.0
 418                          * root:xnu-3248.60.10~1/RELEASE_X86_64
 419                          * appears to have issues with liburcu-signal
 420                          * signal being delivered on top of
 421                          * pthread_cond_wait. It seems to make the
 422                          * thread continue, and therefore corrupt the
 423                          * rcu_head. Work around this issue by
 424                          * unregistering the RCU read-side thread
 425                          * immediately after call_rcu (call_rcu needs
 426                          * us to be registered RCU readers).
 427                          */
 428                         call_rcu_wait = true;
 429                         do {
 430                                 ret = pthread_cond_wait(&call_rcu_test_cond,
 431                                                         &call_rcu_test_mutex);
 432                         } while (call_rcu_wait);
 433                         if (ret) {
 434                                 errno = ret;
 435                                 diag("pthread_cond_signal: %s",
 436                                         strerror(errno));
 437                                 abort();
 438                         }
 439                         ret = pthread_mutex_unlock(&call_rcu_test_mutex);
 440                         if (ret) {
 441                                 errno = ret;
 442                                 diag("pthread_mutex_unlock: %s",
 443                                         strerror(errno));
 444                                 abort();
 445                         }
 446                 }
 447                 n_updates++;
 448         }
 449
 450         return NULL;
 451 }
 452
 453 static
 454 void *rcu_fake_update_stress_test(void *arg __attribute__((unused)))
 455 {
 456         if (callrcu_type == CALLRCU_PERTHREAD) {
 457                 struct call_rcu_data *crdp;
 458
 459                 crdp = create_call_rcu_data(0, -1);
 460                 if (crdp != NULL) {
 461                         diag("Successfully using per-thread call_rcu() worker.");
 462                         set_thread_call_rcu_data(crdp);
 463                 }
 464         }
 465         while (goflag == GOFLAG_INIT)
 466                 (void) poll(NULL, 0, 1);
 467         while (goflag == GOFLAG_RUN) {
 468                 synchronize_rcu();
 469                 (void) poll(NULL, 0, 1);
 470         }
 471         if (callrcu_type == CALLRCU_PERTHREAD) {
 472                 struct call_rcu_data *crdp;
 473
 474                 crdp = get_thread_call_rcu_data();
 475                 set_thread_call_rcu_data(NULL);
 476                 call_rcu_data_free(crdp);
 477         }
 478         return NULL;
 479 }
 480
 481 static
 482 int stresstest(int nreaders)
 483 {
 484         int i;
 485         int t;
 486         long long *p;
 487         long long sum;
 488
 489         init_per_thread(n_reads_pt, 0LL);
 490         for_each_thread(t) {
 491                 p = &per_thread(rcu_stress_count,t)[0];
 492                 for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++)
 493                         p[i] = 0LL;
 494         }
 495         rcu_stress_current = &rcu_stress_array[0];
 496         rcu_stress_current->pipe_count = 0;
 497         rcu_stress_current->mbtest = 1;
 498         for (i = 0; i < nreaders; i++)
 499                 create_thread(rcu_read_stress_test, NULL);
 500         create_thread(rcu_update_stress_test, NULL);
 501         for (i = 0; i < 5; i++)
 502                 create_thread(rcu_fake_update_stress_test, NULL);
 503         cmm_smp_mb();
 504         goflag = GOFLAG_RUN;
 505         cmm_smp_mb();
 506         sleep(10);
 507         cmm_smp_mb();
 508         goflag = GOFLAG_STOP;
 509         cmm_smp_mb();
 510         wait_all_threads();
 511         for_each_thread(t)
 512                 n_reads += per_thread(n_reads_pt, t);
 513         diag("n_reads: %lld  n_updates: %ld  n_mberror: %d",
 514                n_reads, n_updates, n_mberror);
 515         rdiag_start();
 516         rdiag("rcu_stress_count:");
 517         for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
 518                 sum = 0LL;
 519                 for_each_thread(t) {
 520                         sum += per_thread(rcu_stress_count, t)[i];
 521                 }
 522                 rdiag(" %lld", sum);
 523         }
 524         rdiag_end();
 525         if (get_cpu_call_rcu_data(0)) {
 526                 diag("Deallocating per-CPU call_rcu threads.");
 527                 free_all_cpu_call_rcu_data();
 528         }
 529         if (!n_mberror)
 530                 return 0;
 531         else
 532                 return -1;
 533 }
 534
 535 /*
 536  * Mainprogram.
 537  */
 538
 539 static
 540 void usage(char *argv[]) __attribute__((__noreturn__));
 541
 542 static
 543 void usage(char *argv[])
 544 {
 545         diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv[0]);
 546         exit(-1);
 547 }
 548
 549 int main(int argc, char *argv[])
 550 {
 551         int nreaders = 1;
 552         int cpustride = 1;
 553
 554         plan_tests(NR_TESTS);
 555
 556         smp_init();
 557         //rcu_init();
 558         if (argc > 4) {
 559                 const char *callrcu_str = argv[4];;
 560
 561                 if (strcmp(callrcu_str, "callrcu_global") == 0) {
 562                         callrcu_type = CALLRCU_GLOBAL;
 563                 } else if (strcmp(callrcu_str, "callrcu_percpu") == 0) {
 564                         callrcu_type = CALLRCU_PERCPU;
 565                 } else if (strcmp(callrcu_str, "callrcu_perthread") == 0) {
 566                         callrcu_type = CALLRCU_PERTHREAD;
 567                 } else {
 568                         usage(argv);
 569                         goto end;
 570                 }
 571         }
 572
 573         switch (callrcu_type) {
 574         case CALLRCU_GLOBAL:
 575                 diag("Using global per-process call_rcu thread.");
 576                 break;
 577         case CALLRCU_PERCPU:
 578                 diag("Using per-CPU call_rcu threads.");
 579                 if (create_all_cpu_call_rcu_data(0))
 580                         diag("create_all_cpu_call_rcu_data: %s",
 581                                 strerror(errno));
 582                 break;
 583         case CALLRCU_PERTHREAD:
 584                 diag("Using per-thread call_rcu() worker.");
 585                 break;
 586         default:
 587                 abort();
 588         }
 589
 590 #ifdef DEBUG_YIELD
 591         yield_active |= YIELD_READ;
 592         yield_active |= YIELD_WRITE;
 593 #endif
 594
 595         if (argc > 1) {
 596                 if (strcmp(argv[1], "-h") == 0
 597                                 || strcmp(argv[1], "--help") == 0) {
 598                         usage(argv);
 599                         goto end;
 600                 }
 601                 nreaders = strtoul(argv[1], NULL, 0);
 602                 if (argc == 2) {
 603                         ok(!perftest(nreaders, cpustride),
 604                                 "perftest readers: %d, stride: %d",
 605                                 nreaders, cpustride);
 606                         goto end;
 607                 }
 608                 if (argc > 3)
 609                         cpustride = strtoul(argv[3], NULL, 0);
 610                 if (strcmp(argv[2], "perf") == 0)
 611                         ok(!perftest(nreaders, cpustride),
 612                                 "perftest readers: %d, stride: %d",
 613                                 nreaders, cpustride);
 614                 else if (strcmp(argv[2], "rperf") == 0)
 615                         ok(!rperftest(nreaders, cpustride),
 616                                 "rperftest readers: %d, stride: %d",
 617                                 nreaders, cpustride);
 618                 else if (strcmp(argv[2], "uperf") == 0)
 619                         ok(!uperftest(nreaders, cpustride),
 620                                 "uperftest readers: %d, stride: %d",
 621                                 nreaders, cpustride);
 622                 else if (strcmp(argv[2], "stress") == 0)
 623                         ok(!stresstest(nreaders),
 624                                 "stresstest readers: %d, stride: %d",
 625                                 nreaders, cpustride);
 626                 else
 627                         usage(argv);
 628         } else {
 629                 usage(argv);
 630         }
 631 end:
 632         return exit_status();
 633 }