tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 RUNAS_BIN="lttng-runas"
  19 RUNAS_MATCH=".*lttng-runas.*"
  20 CONSUMERD_BIN="lttng-consumerd"
  21 CONSUMERD_MATCH=".*lttng-consumerd.*"
  22 RELAYD_BIN="lttng-relayd"
  23 RELAYD_MATCH=".*lttng-relayd.*"
  24 LTTNG_BIN="lttng"
  25 BABELTRACE_BIN="babeltrace"
  26 OUTPUT_DEST=/dev/null
  27 ERROR_OUTPUT_DEST=/dev/null
  28
  29 # Minimal kernel version supported for session daemon tests
  30 KERNEL_MAJOR_VERSION=2
  31 KERNEL_MINOR_VERSION=6
  32 KERNEL_PATCHLEVEL_VERSION=27
  33
  34 # We set the default UST register timeout and network and app socket timeout to
  35 # "wait forever", so that basic tests don't have to worry about hitting
  36 # timeouts on busy systems. Specialized tests should test those corner-cases.
  37 export LTTNG_UST_REGISTER_TIMEOUT=-1
  38 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  39 export LTTNG_APP_SOCKET_TIMEOUT=-1
  40
  41 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  42 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  43 # its own sessiond if none is running. It also ensures that 'lttng create'
  44 # fails when no sessiond is running.
  45 export LTTNG_SESSIOND_PATH="/bin/true"
  46
  47 source $TESTDIR/utils/tap/tap.sh
  48
  49 if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
  50         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  51 fi
  52
  53 function full_cleanup ()
  54 {
  55         # Try to kill daemons gracefully
  56         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # If daemons are still present, forcibly kill them
  60         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  61         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  62         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  63
  64         # Disable trap for SIGTERM since the following kill to the
  65         # pidgroup will be SIGTERM. Otherwise it loops.
  66         # The '-' before the pid number ($$) indicates 'kill' to signal the
  67         # whole process group.
  68         trap - SIGTERM && kill -- -$$
  69 }
  70
  71 function null_pipes ()
  72 {
  73         exec 0>/dev/null
  74         exec 1>/dev/null
  75         exec 2>/dev/null
  76 }
  77
  78 trap full_cleanup SIGINT SIGTERM
  79
  80 # perl prove closes its child pipes before giving it a chance to run its
  81 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
  82 # to allow those trap handlers to proceed.
  83
  84 trap null_pipes SIGPIPE
  85
  86 function print_ok ()
  87 {
  88         # Check if we are a terminal
  89         if [ -t 1 ]; then
  90                 echo -e "\e[1;32mOK\e[0m"
  91         else
  92                 echo -e "OK"
  93         fi
  94 }
  95
  96 function print_fail ()
  97 {
  98         # Check if we are a terminal
  99         if [ -t 1 ]; then
 100                 echo -e "\e[1;31mFAIL\e[0m"
 101         else
 102                 echo -e "FAIL"
 103         fi
 104 }
 105
 106 function print_test_banner ()
 107 {
 108         local desc="$1"
 109         diag "$desc"
 110 }
 111
 112 function validate_kernel_version ()
 113 {
 114         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 115         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 116                 return 0
 117         fi
 118         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 119                 return 0
 120         fi
 121         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 122                 return 0
 123         fi
 124         return 1
 125 }
 126
 127 # Generate a random string
 128 #  $1 = number of characters; defaults to 16
 129 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 130 function randstring()
 131 {
 132         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 133         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 134         echo
 135 }
 136
 137 # Return the number of _configured_ CPUs.
 138 function conf_proc_count()
 139 {
 140         getconf _NPROCESSORS_CONF
 141         if [ $? -ne 0 ]; then
 142                 diag "Failed to get the number of configured CPUs"
 143         fi
 144         echo
 145 }
 146
 147 # Check if base lttng-modules are present.
 148 # Bail out on failure
 149 function validate_lttng_modules_present ()
 150 {
 151         modprobe -n lttng-tracer 2>/dev/null
 152         if [ $? -ne 0  ]; then
 153                 BAIL_OUT "LTTng modules not detected."
 154         fi
 155 }
 156
 157 function enable_kernel_lttng_event
 158 {
 159         local withtap="$1"
 160         local expected_to_fail="$2"
 161         local sess_name="$3"
 162         local event_name="$4"
 163         local channel_name="$5"
 164
 165         if [ -z "$event_name" ]; then
 166                 # Enable all event if no event name specified
 167                 event_name="-a"
 168         fi
 169
 170         if [ -z "$channel_name" ]; then
 171                 # default channel if none specified
 172                 chan=""
 173         else
 174                 chan="-c $channel_name"
 175         fi
 176
 177         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 178         ret=$?
 179         if [[ $expected_to_fail -eq "1" ]]; then
 180                 test $ret -ne "0"
 181                 ret=$?
 182                 if [ $withtap -eq "1" ]; then
 183                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 184                 fi
 185         else
 186                 if [ $withtap -eq "1" ]; then
 187                         ok $ret "Enable kernel event $event_name for session $sess_name"
 188                 fi
 189         fi
 190 }
 191
 192 function enable_kernel_lttng_event_ok ()
 193 {
 194         enable_kernel_lttng_event 1 0 "$@"
 195 }
 196
 197 function enable_kernel_lttng_event_fail ()
 198 {
 199         enable_kernel_lttng_event 1 1 "$@"
 200 }
 201
 202 function enable_kernel_lttng_event_notap ()
 203 {
 204         enable_kernel_lttng_event 0 0 "$@"
 205 }
 206
 207 # Old interface
 208 function lttng_enable_kernel_event
 209 {
 210         enable_kernel_lttng_event_ok "$@"
 211 }
 212
 213 function lttng_enable_kernel_syscall()
 214 {
 215         local expected_to_fail=$1
 216         local sess_name=$2
 217         local syscall_name=$3
 218         local channel_name=$4
 219
 220         if [ -z $syscall_name ]; then
 221                 # Enable all event if no syscall name specified
 222                 syscall_name="-a"
 223         fi
 224
 225         if [ -z $channel_name ]; then
 226                 # default channel if none specified
 227                 chan=""
 228         else
 229                 chan="-c $channel_name"
 230         fi
 231
 232         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 233         ret=$?
 234         if [[ $expected_to_fail -eq "1" ]]; then
 235                 test $ret -ne "0"
 236                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 237         else
 238                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 239         fi
 240 }
 241
 242 function lttng_enable_kernel_syscall_ok()
 243 {
 244         lttng_enable_kernel_syscall 0 "$@"
 245 }
 246
 247 function lttng_enable_kernel_syscall_fail()
 248 {
 249         lttng_enable_kernel_syscall 1 "$@"
 250 }
 251
 252 function lttng_disable_kernel_syscall()
 253 {
 254         local expected_to_fail=$1
 255         local sess_name=$2
 256         local syscall_name=$3
 257         local channel_name=$4
 258
 259         if [ -z $syscall_name ]; then
 260                 # Enable all event if no syscall name specified
 261                 syscall_name="-a"
 262         fi
 263
 264         if [ -z $channel_name ]; then
 265                 # default channel if none specified
 266                 chan=""
 267         else
 268                 chan="-c $channel_name"
 269         fi
 270
 271         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 272
 273         ret=$?
 274         if [[ $expected_to_fail -eq "1" ]]; then
 275                 test $ret -ne "0"
 276                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 277         else
 278                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 279         fi
 280 }
 281
 282 function lttng_disable_kernel_syscall_ok()
 283 {
 284         lttng_disable_kernel_syscall 0 "$@"
 285 }
 286
 287 function lttng_disable_kernel_syscall_fail()
 288 {
 289         lttng_disable_kernel_syscall 1 "$@"
 290 }
 291
 292 function lttng_enable_kernel_channel()
 293 {
 294         local withtap=$1
 295         local expected_to_fail=$2
 296         local sess_name=$3
 297         local channel_name=$4
 298         local opt=$5
 299
 300         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 301         ret=$?
 302         if [[ $expected_to_fail -eq "1" ]]; then
 303                 test "$ret" -ne "0"
 304                 ret=$?
 305                 if [ $withtap -eq "1" ]; then
 306                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 307                 fi
 308         else
 309                 if [ $withtap -eq "1" ]; then
 310                         ok $ret "Enable channel $channel_name for session $sess_name"
 311                 fi
 312         fi
 313 }
 314
 315 function lttng_enable_kernel_channel_ok()
 316 {
 317         lttng_enable_kernel_channel 1 0 "$@"
 318 }
 319
 320 function lttng_enable_kernel_channel_fail()
 321 {
 322         lttng_enable_kernel_channel 1 1 "$@"
 323 }
 324
 325 function lttng_enable_kernel_channel_notap()
 326 {
 327         lttng_enable_kernel_channel 0 0 "$@"
 328 }
 329
 330 function enable_kernel_lttng_channel_ok()
 331 {
 332         lttng_enable_kernel_channel 1 0 "$@"
 333 }
 334
 335 function lttng_disable_kernel_channel()
 336 {
 337         local expected_to_fail=$1
 338         local sess_name=$2
 339         local channel_name=$3
 340
 341         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 342         ret=$?
 343         if [[ $expected_to_fail -eq "1" ]]; then
 344                 test "$ret" -ne "0"
 345                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 346         else
 347                 ok $ret "Disable channel $channel_name for session $sess_name"
 348         fi
 349 }
 350
 351 function lttng_disable_kernel_channel_ok()
 352 {
 353         lttng_disable_kernel_channel 0 "$@"
 354 }
 355
 356 function lttng_disable_kernel_channel_fail()
 357 {
 358         lttng_disable_kernel_channel 1 "$@"
 359 }
 360
 361 function start_lttng_relayd_opt()
 362 {
 363         local withtap=$1
 364         local opt=$2
 365
 366         DIR=$(readlink -f "$TESTDIR")
 367
 368         if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
 369                 # shellcheck disable=SC2086
 370                 "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 371                 #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
 372                 if [ $? -eq 1 ]; then
 373                         if [ "$withtap" -eq "1" ]; then
 374                                 fail "Start lttng-relayd (opt: $opt)"
 375                         fi
 376                         return 1
 377                 else
 378                         if [ "$withtap" -eq "1" ]; then
 379                                 pass "Start lttng-relayd (opt: $opt)"
 380                         fi
 381                 fi
 382         else
 383                 pass "Start lttng-relayd (opt: $opt)"
 384         fi
 385 }
 386
 387 function start_lttng_relayd()
 388 {
 389         start_lttng_relayd_opt 1 "$@"
 390 }
 391
 392 function start_lttng_relayd_notap()
 393 {
 394         start_lttng_relayd_opt 0 "$@"
 395 }
 396
 397 function stop_lttng_relayd_opt()
 398 {
 399         local withtap=$1
 400         local signal=$2
 401
 402         if [ -z "$signal" ]; then
 403                 signal="SIGTERM"
 404         fi
 405
 406         local timeout_s=$3
 407         local dtimeleft_s=
 408
 409         # Multiply time by 2 to simplify integer arithmetic
 410         if [ -n "$timeout_s" ]; then
 411                 dtimeleft_s=$((timeout_s * 2))
 412         fi
 413
 414         local retval=0
 415         local pids=
 416
 417         pids=$(pgrep "$RELAYD_MATCH")
 418         if [ -z "$pids" ]; then
 419                 if [ "$withtap" -eq "1" ]; then
 420                         pass "No relay daemon to kill"
 421                 fi
 422                 return 0
 423         fi
 424
 425         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 426
 427         # shellcheck disable=SC2086
 428         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 429                 retval=1
 430                 if [ "$withtap" -eq "1" ]; then
 431                         fail "Kill relay daemon"
 432                 fi
 433         else
 434                 out=1
 435                 while [ -n "$out" ]; do
 436                         out=$(pgrep "$RELAYD_MATCH")
 437                         if [ -n "$dtimeleft_s" ]; then
 438                                 if [ $dtimeleft_s -lt 0 ]; then
 439                                         out=
 440                                         retval=1
 441                                 fi
 442                                 dtimeleft_s=$((dtimeleft_s - 1))
 443                         fi
 444                         sleep 0.5
 445                 done
 446                 if [ "$withtap" -eq "1" ]; then
 447                         if [ "$retval" -eq "0" ]; then
 448                                 pass "Wait after kill relay daemon"
 449                         else
 450                                 fail "Wait after kill relay daemon"
 451                         fi
 452                 fi
 453         fi
 454         return $retval
 455 }
 456
 457 function stop_lttng_relayd()
 458 {
 459         stop_lttng_relayd_opt 1 "$@"
 460 }
 461
 462 function stop_lttng_relayd_notap()
 463 {
 464         stop_lttng_relayd_opt 0 "$@"
 465 }
 466
 467 #First arg: show tap output
 468 #Second argument: load path for automatic loading
 469 function start_lttng_sessiond_opt()
 470 {
 471         local withtap=$1
 472         local load_path=$2
 473
 474         local env_vars=""
 475         local consumerd=""
 476
 477         local long_bit_value=
 478         long_bit_value=$(getconf LONG_BIT)
 479
 480         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 481                 # Env variable requested no session daemon
 482                 return
 483         fi
 484
 485         DIR=$(readlink -f "$TESTDIR")
 486
 487         # Get long_bit value for 32/64 consumerd
 488         case "$long_bit_value" in
 489                 32)
 490                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 491                         ;;
 492                 64)
 493                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 494                         ;;
 495                 *)
 496                         return
 497                         ;;
 498         esac
 499
 500         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 501         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 502                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 503         fi
 504         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 505
 506         if ! validate_kernel_version; then
 507             fail "Start session daemon"
 508             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 509         fi
 510
 511         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 512         export LTTNG_SESSION_CONFIG_XSD_PATH
 513
 514         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 515                 # Have a load path ?
 516                 if [ -n "$load_path" ]; then
 517                         # shellcheck disable=SC2086
 518                         env $env_vars --load "$load_path" --background "$consumerd"
 519                 else
 520                         # shellcheck disable=SC2086
 521                         env $env_vars --background "$consumerd"
 522                 fi
 523                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 524                 status=$?
 525                 if [ "$withtap" -eq "1" ]; then
 526                         ok $status "Start session daemon"
 527                 fi
 528         fi
 529 }
 530
 531 function start_lttng_sessiond()
 532 {
 533         start_lttng_sessiond_opt 1 "$@"
 534 }
 535
 536 function start_lttng_sessiond_notap()
 537 {
 538         start_lttng_sessiond_opt 0 "$@"
 539 }
 540
 541 function stop_lttng_sessiond_opt()
 542 {
 543         local withtap=$1
 544         local signal=$2
 545
 546         if [ -z "$signal" ]; then
 547                 signal=SIGTERM
 548         fi
 549
 550         local timeout_s=$3
 551         local dtimeleft_s=
 552
 553         # Multiply time by 2 to simplify integer arithmetic
 554         if [ -n "$timeout_s" ]; then
 555                 dtimeleft_s=$((timeout_s * 2))
 556         fi
 557
 558         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 559                 # Env variable requested no session daemon
 560                 return 0
 561         fi
 562
 563         local retval=0
 564
 565         local runas_pids=
 566         runas_pids=$(pgrep "$RUNAS_MATCH")
 567
 568         local pids=
 569         pids=$(pgrep "$SESSIOND_MATCH")
 570
 571         if [ -n "$runas_pids" ]; then
 572                 pids="$pids $runas_pids"
 573         fi
 574
 575         if [ -z "$pids" ]; then
 576                 if [ "$withtap" -eq "1" ]; then
 577                         pass "No session daemon to kill"
 578                 fi
 579                 return 0
 580         fi
 581
 582         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 583
 584         # shellcheck disable=SC2086
 585         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 586                 retval=1
 587                 if [ "$withtap" -eq "1" ]; then
 588                         fail "Kill sessions daemon"
 589                 fi
 590         else
 591                 out=1
 592                 while [ -n "$out" ]; do
 593                         out=$(pgrep "${SESSIOND_MATCH}")
 594                         if [ -n "$dtimeleft_s" ]; then
 595                                 if [ $dtimeleft_s -lt 0 ]; then
 596                                         out=
 597                                         retval=1
 598                                 fi
 599                                 dtimeleft_s=$((dtimeleft_s - 1))
 600                         fi
 601                         sleep 0.5
 602                 done
 603                 out=1
 604                 while [ -n "$out" ]; do
 605                         out=$(pgrep "$CONSUMERD_MATCH")
 606                         if [ -n "$dtimeleft_s" ]; then
 607                                 if [ $dtimeleft_s -lt 0 ]; then
 608                                         out=
 609                                         retval=1
 610                                 fi
 611                                 dtimeleft_s=$((dtimeleft_s - 1))
 612                         fi
 613                         sleep 0.5
 614                 done
 615
 616                 if [ "$withtap" -eq "1" ]; then
 617                         if [ "$retval" -eq "0" ]; then
 618                                 pass "Wait after kill session daemon"
 619                         else
 620                                 fail "Wait after kill session daemon"
 621                         fi
 622                 fi
 623         fi
 624         if [ "$signal" = "SIGKILL" ]; then
 625                 if [ "$(id -u)" -eq "0" ]; then
 626                         local modules=
 627                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 628
 629                         if [ -n "$modules" ]; then
 630                                 diag "Unloading all LTTng modules"
 631                                 modprobe -r "$modules"
 632                         fi
 633                 fi
 634         fi
 635
 636         return $retval
 637 }
 638
 639 function stop_lttng_sessiond()
 640 {
 641         stop_lttng_sessiond_opt 1 "$@"
 642 }
 643
 644 function stop_lttng_sessiond_notap()
 645 {
 646         stop_lttng_sessiond_opt 0 "$@"
 647 }
 648
 649 function sigstop_lttng_sessiond_opt()
 650 {
 651         local withtap=$1
 652         local signal=SIGSTOP
 653
 654         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 655                 # Env variable requested no session daemon
 656                 return
 657         fi
 658
 659         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 660
 661         if [ "$withtap" -eq "1" ]; then
 662                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 663         fi
 664
 665         # shellcheck disable=SC2086
 666         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 667                 if [ "$withtap" -eq "1" ]; then
 668                         fail "Sending SIGSTOP to session daemon"
 669                 fi
 670         else
 671                 out=1
 672                 while [ $out -ne 0 ]; do
 673                         pid="$(pgrep "$SESSIOND_MATCH")"
 674
 675                         # Wait until state becomes stopped for session
 676                         # daemon(s).
 677                         out=0
 678                         for sessiond_pid in $pid; do
 679                                 state="$(ps -p "$sessiond_pid" -o state= )"
 680                                 if [[ -n "$state" && "$state" != "T" ]]; then
 681                                         out=1
 682                                 fi
 683                         done
 684                         sleep 0.5
 685                 done
 686                 if [ "$withtap" -eq "1" ]; then
 687                         pass "Sending SIGSTOP to session daemon"
 688                 fi
 689         fi
 690 }
 691
 692 function sigstop_lttng_sessiond()
 693 {
 694         sigstop_lttng_sessiond_opt 1 "$@"
 695 }
 696
 697 function sigstop_lttng_sessiond_notap()
 698 {
 699         sigstop_lttng_sessiond_opt 0 "$@"
 700 }
 701
 702 function stop_lttng_consumerd_opt()
 703 {
 704         local withtap=$1
 705         local signal=$2
 706
 707         if [ -z "$signal" ]; then
 708                 signal=SIGTERM
 709         fi
 710
 711         local timeout_s=$3
 712         local dtimeleft_s=
 713
 714         # Multiply time by 2 to simplify integer arithmetic
 715         if [ -n "$timeout_s" ]; then
 716                 dtimeleft_s=$((timeout_s * 2))
 717         fi
 718
 719         local retval=0
 720
 721         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 722
 723         if [ -z "$PID_CONSUMERD" ]; then
 724                 if [ "$withtap" -eq "1" ]; then
 725                         pass "No consumer daemon to kill"
 726                 fi
 727                 return 0
 728         fi
 729
 730         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 731
 732         # shellcheck disable=SC2086
 733         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 734                 retval=1
 735                 if [ "$withtap" -eq "1" ]; then
 736                         fail "Kill consumer daemon"
 737                 fi
 738         else
 739                 out=1
 740                 while [ $out -ne 0 ]; do
 741                         pid="$(pgrep "$CONSUMERD_MATCH")"
 742
 743                         # If consumerds are still present check their status.
 744                         # A zombie status qualifies the consumerd as *killed*
 745                         out=0
 746                         for consumer_pid in $pid; do
 747                                 state="$(ps -p "$consumer_pid" -o state= )"
 748                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 749                                         out=1
 750                                 fi
 751                         done
 752                         if [ -n "$dtimeleft_s" ]; then
 753                                 if [ $dtimeleft_s -lt 0 ]; then
 754                                         out=0
 755                                         retval=1
 756                                 fi
 757                                 dtimeleft_s=$((dtimeleft_s - 1))
 758                         fi
 759                         sleep 0.5
 760                 done
 761                 if [ "$withtap" -eq "1" ]; then
 762                         if [ "$retval" -eq "0" ]; then
 763                                 pass "Wait after kill consumer daemon"
 764                         else
 765                                 fail "Wait after kill consumer daemon"
 766                         fi
 767                 fi
 768         fi
 769
 770         return $retval
 771 }
 772
 773 function stop_lttng_consumerd()
 774 {
 775         stop_lttng_consumerd_opt 1 "$@"
 776 }
 777
 778 function stop_lttng_consumerd_notap()
 779 {
 780         stop_lttng_consumerd_opt 0 "$@"
 781 }
 782
 783 function sigstop_lttng_consumerd_opt()
 784 {
 785         local withtap=$1
 786         local signal=SIGSTOP
 787
 788         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 789
 790         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 791
 792         # shellcheck disable=SC2086
 793         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 794         retval=$?
 795
 796         if [ $retval -eq 1 ]; then
 797                 if [ "$withtap" -eq "1" ]; then
 798                         fail "Sending SIGSTOP to consumer daemon"
 799                 fi
 800                 return 1
 801         else
 802                 out=1
 803                 while [ $out -ne 0 ]; do
 804                         pid="$(pgrep "$CONSUMERD_MATCH")"
 805
 806                         # Wait until state becomes stopped for all
 807                         # consumers.
 808                         out=0
 809                         for consumer_pid in $pid; do
 810                                 state="$(ps -p "$consumer_pid" -o state= )"
 811                                 if [[ -n "$state" && "$state" != "T" ]]; then
 812                                         out=1
 813                                 fi
 814                         done
 815                         sleep 0.5
 816                 done
 817                 if [ "$withtap" -eq "1" ]; then
 818                         pass "Sending SIGSTOP to consumer daemon"
 819                 fi
 820         fi
 821         return $retval
 822 }
 823
 824 function sigstop_lttng_consumerd()
 825 {
 826         sigstop_lttng_consumerd_opt 1 "$@"
 827 }
 828
 829 function sigstop_lttng_consumerd_notap()
 830 {
 831         sigstop_lttng_consumerd_opt 0 "$@"
 832 }
 833
 834 function list_lttng_with_opts ()
 835 {
 836         local opts=$1
 837         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 838         ok $? "Lttng-tool list command with option $opts"
 839 }
 840
 841 function create_lttng_session_no_output ()
 842 {
 843         local sess_name=$1
 844
 845         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 846         ok $? "Create session $sess_name in no-output mode"
 847 }
 848
 849 function create_lttng_session ()
 850 {
 851         local withtap=$1
 852         local expected_to_fail=$2
 853         local sess_name=$3
 854         local trace_path=$4
 855         local opt=$5
 856
 857         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 858         ret=$?
 859         if [ $expected_to_fail -eq "1" ]; then
 860                 test "$ret" -ne "0"
 861                 ret=$?
 862                 if [ $withtap -eq "1" ]; then
 863                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 864                 fi
 865         else
 866                 if [ $withtap -eq "1" ]; then
 867                         ok $ret "Create session $sess_name in $trace_path"
 868                 fi
 869         fi
 870         return $ret
 871 }
 872
 873 function create_lttng_session_ok ()
 874 {
 875         create_lttng_session 1 0 "$@"
 876 }
 877
 878 function create_lttng_session_fail ()
 879 {
 880         create_lttng_session 1 1 "$@"
 881 }
 882
 883 function create_lttng_session_notap ()
 884 {
 885         create_lttng_session 0 0 "$@"
 886 }
 887
 888
 889 function enable_ust_lttng_channel ()
 890 {
 891         local withtap=$1
 892         local expected_to_fail=$2
 893         local sess_name=$3
 894         local channel_name=$4
 895         local opt=$5
 896
 897         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 898         ret=$?
 899         if [[ $expected_to_fail -eq "1" ]]; then
 900                 test "$ret" -ne "0"
 901                 ret=$?
 902                 if [ $withtap -eq "1" ]; then
 903                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 904                 fi
 905         else
 906                 if [ $withtap -eq "1" ]; then
 907                         ok $ret "Enable channel $channel_name for session $sess_name"
 908                 fi
 909         fi
 910         return $ret
 911 }
 912
 913 function enable_ust_lttng_channel_ok ()
 914 {
 915         enable_ust_lttng_channel 1 0 "$@"
 916 }
 917
 918 function enable_ust_lttng_channel_fail ()
 919 {
 920         enable_ust_lttng_channel 1 1 "$@"
 921 }
 922
 923 function enable_ust_lttng_channel_notap ()
 924 {
 925         enable_ust_lttng_channel 0 0 "$@"
 926 }
 927
 928 function disable_ust_lttng_channel()
 929 {
 930         local sess_name=$1
 931         local channel_name=$2
 932
 933         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 934         ok $? "Disable channel $channel_name for session $sess_name"
 935 }
 936
 937 function enable_lttng_mmap_overwrite_kernel_channel()
 938 {
 939         local sess_name=$1
 940         local channel_name=$2
 941
 942         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 943         ok $? "Enable channel $channel_name for session $sess_name"
 944 }
 945
 946 function enable_lttng_mmap_discard_small_kernel_channel()
 947 {
 948         local sess_name=$1
 949         local channel_name=$2
 950
 951         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 952         ok $? "Enable small discard channel $channel_name for session $sess_name"
 953 }
 954
 955 function enable_lttng_mmap_overwrite_small_kernel_channel()
 956 {
 957         local sess_name=$1
 958         local channel_name=$2
 959
 960         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 961         ok $? "Enable small discard channel $channel_name for session $sess_name"
 962 }
 963
 964 function enable_lttng_mmap_overwrite_ust_channel()
 965 {
 966         local sess_name=$1
 967         local channel_name=$2
 968
 969         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 970         ok $? "Enable channel $channel_name for session $sess_name"
 971 }
 972
 973 function enable_ust_lttng_event ()
 974 {
 975         local withtap=$1
 976         local expected_to_fail=$2
 977         local sess_name=$3
 978         local event_name="$4"
 979         local channel_name=$5
 980
 981         if [ -z $channel_name ]; then
 982                 # default channel if none specified
 983                 chan=""
 984         else
 985                 chan="-c $channel_name"
 986         fi
 987
 988         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 989         ret=$?
 990         if [[ $expected_to_fail -eq "1" ]]; then
 991                 test $ret -ne "0"
 992                 ret=$?
 993                 if [[ $withtap -eq "1" ]]; then
 994                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
 995                 fi
 996         else
 997                 if [[ $withtap -eq "1" ]]; then
 998                         ok $ret "Enable ust event $event_name for session $sess_name"
 999                 fi
1000         fi
1001         return $ret
1002 }
1003
1004 function enable_ust_lttng_event_ok ()
1005 {
1006         enable_ust_lttng_event 1 0 "$@"
1007 }
1008
1009 function enable_ust_lttng_event_fail ()
1010 {
1011         enable_ust_lttng_event 1 1 "$@"
1012 }
1013
1014 function enable_ust_lttng_event_notap ()
1015 {
1016         enable_ust_lttng_event 0 0 "$@"
1017 }
1018
1019 function enable_jul_lttng_event()
1020 {
1021         sess_name=$1
1022         event_name="$2"
1023         channel_name=$3
1024
1025         if [ -z $channel_name ]; then
1026                 # default channel if none specified
1027                 chan=""
1028         else
1029                 chan="-c $channel_name"
1030         fi
1031
1032         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1033         ok $? "Enable JUL event $event_name for session $sess_name"
1034 }
1035
1036 function enable_jul_lttng_event_loglevel()
1037 {
1038         local sess_name=$1
1039         local event_name="$2"
1040         local loglevel=$3
1041         local channel_name=$4
1042
1043         if [ -z $channel_name ]; then
1044                 # default channel if none specified
1045                 chan=""
1046         else
1047                 chan="-c $channel_name"
1048         fi
1049
1050         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1051         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1052 }
1053
1054 function enable_log4j_lttng_event()
1055 {
1056         sess_name=$1
1057         event_name="$2"
1058         channel_name=$3
1059
1060         if [ -z $channel_name ]; then
1061                 # default channel if none specified
1062                 chan=""
1063         else
1064                 chan="-c $channel_name"
1065         fi
1066
1067         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1068         ok $? "Enable LOG4J event $event_name for session $sess_name"
1069 }
1070
1071 function enable_log4j_lttng_event_loglevel()
1072 {
1073         local sess_name=$1
1074         local event_name="$2"
1075         local loglevel=$3
1076         local channel_name=$4
1077
1078         if [ -z $channel_name ]; then
1079                 # default channel if none specified
1080                 chan=""
1081         else
1082                 chan="-c $channel_name"
1083         fi
1084
1085         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1086         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1087 }
1088
1089 function enable_python_lttng_event()
1090 {
1091         sess_name=$1
1092         event_name="$2"
1093         channel_name=$3
1094
1095         if [ -z $channel_name ]; then
1096                 # default channel if none specified
1097                 chan=""
1098         else
1099                 chan="-c $channel_name"
1100         fi
1101
1102         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1103         ok $? "Enable Python event $event_name for session $sess_name"
1104 }
1105
1106 function enable_python_lttng_event_loglevel()
1107 {
1108         local sess_name=$1
1109         local event_name="$2"
1110         local loglevel=$3
1111         local channel_name=$4
1112
1113         if [ -z $channel_name ]; then
1114                 # default channel if none specified
1115                 chan=""
1116         else
1117                 chan="-c $channel_name"
1118         fi
1119
1120         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1121         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1122 }
1123
1124 function enable_ust_lttng_event_filter()
1125 {
1126         local sess_name="$1"
1127         local event_name="$2"
1128         local filter="$3"
1129
1130         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1131         ok $? "Enable event $event_name with filtering for session $sess_name"
1132 }
1133
1134 function enable_ust_lttng_event_loglevel()
1135 {
1136         local sess_name="$1"
1137         local event_name="$2"
1138         local loglevel="$3"
1139
1140         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1141         ok $? "Enable event $event_name with loglevel $loglevel"
1142 }
1143
1144 function enable_ust_lttng_event_loglevel_only()
1145 {
1146         local sess_name="$1"
1147         local event_name="$2"
1148         local loglevel="$3"
1149
1150         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1151         ok $? "Enable event $event_name with loglevel-only $loglevel"
1152 }
1153
1154 function disable_ust_lttng_event ()
1155 {
1156         local sess_name="$1"
1157         local event_name="$2"
1158         local channel_name="$3"
1159
1160         if [ -z $channel_name ]; then
1161                 # default channel if none specified
1162                 chan=""
1163         else
1164                 chan="-c $channel_name"
1165         fi
1166
1167         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1168         ok $? "Disable event $event_name for session $sess_name"
1169 }
1170
1171 function disable_jul_lttng_event ()
1172 {
1173         local sess_name="$1"
1174         local event_name="$2"
1175
1176         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1177         ok $? "Disable JUL event $event_name for session $sess_name"
1178 }
1179
1180 function disable_log4j_lttng_event ()
1181 {
1182         local sess_name="$1"
1183         local event_name="$2"
1184
1185         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1186         ok $? "Disable LOG4J event $event_name for session $sess_name"
1187 }
1188
1189 function disable_python_lttng_event ()
1190 {
1191         local sess_name="$1"
1192         local event_name="$2"
1193
1194         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1195         ok $? "Disable Python event $event_name for session $sess_name"
1196 }
1197
1198 function start_lttng_tracing ()
1199 {
1200         local expected_to_fail=$1
1201         local sess_name=$2
1202
1203         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1204         ret=$?
1205         if [[ $expected_to_fail -eq "1" ]]; then
1206                 test "$ret" -ne "0"
1207                 ok $? "Start tracing for session $sess_name failed as expected"
1208         else
1209                 ok $ret "Start tracing for session $sess_name"
1210         fi
1211 }
1212
1213 function start_lttng_tracing_ok ()
1214 {
1215         start_lttng_tracing 0 "$@"
1216 }
1217
1218 function start_lttng_tracing_fail ()
1219 {
1220         start_lttng_tracing 1 "$@"
1221 }
1222
1223 function stop_lttng_tracing ()
1224 {
1225         local expected_to_fail=$1
1226         local sess_name=$2
1227
1228         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1229         ret=$?
1230         if [[ $expected_to_fail -eq "1" ]]; then
1231                 test "$ret" -ne "0"
1232                 ok $? "Stop lttng tracing for session $sess_name failed as expected"
1233         else
1234                 ok $ret "Stop lttng tracing for session $sess_name"
1235         fi
1236 }
1237
1238 function stop_lttng_tracing_ok ()
1239 {
1240         stop_lttng_tracing 0 "$@"
1241 }
1242
1243 function stop_lttng_tracing_fail ()
1244 {
1245         stop_lttng_tracing 1 "$@"
1246 }
1247
1248 function destroy_lttng_session ()
1249 {
1250         local withtap=$1
1251         local expected_to_fail=$2
1252         local sess_name=$3
1253
1254         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1255         ret=$?
1256         if [[ $expected_to_fail -eq "1" ]]; then
1257                 test "$ret" -ne "0"
1258                 ret=$?
1259                 if [ $withtap -eq "1" ]; then
1260                         ok $ret "Destroy session $sess_name failed as expected"
1261                 fi
1262         else
1263                 if [ $withtap -eq "1" ]; then
1264                         ok $ret "Destroy session $sess_name"
1265                 fi
1266         fi
1267 }
1268
1269 function destroy_lttng_session_ok ()
1270 {
1271         destroy_lttng_session 1 0 "$@"
1272
1273 }
1274
1275 function destroy_lttng_session_fail ()
1276 {
1277         destroy_lttng_session 1 1 "$@"
1278 }
1279
1280 function destroy_lttng_session_notap ()
1281 {
1282         destroy_lttng_session 0 0 "$@"
1283 }
1284
1285 function destroy_lttng_sessions ()
1286 {
1287         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1288         ok $? "Destroy all lttng sessions"
1289 }
1290
1291 function lttng_snapshot_add_output ()
1292 {
1293         local expected_to_fail=$1
1294         local sess_name=$2
1295         local trace_path=$3
1296
1297         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name file://$trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1298         ret=$?
1299         if [[ $expected_to_fail -eq 1 ]]; then
1300                 test "$ret" -ne "0"
1301                 ok $? "Added snapshot output file://$trace_path failed as expected"
1302         else
1303                 ok $ret "Added snapshot output file://$trace_path"
1304         fi
1305 }
1306
1307 function lttng_snapshot_add_output_ok ()
1308 {
1309         lttng_snapshot_add_output 0 "$@"
1310 }
1311
1312 function lttng_snapshot_add_output_fail ()
1313 {
1314         lttng_snapshot_add_output 1 "$@"
1315 }
1316
1317 function lttng_snapshot_del_output ()
1318 {
1319         local expected_to_fail=$1
1320         local sess_name=$2
1321         local id=$3
1322
1323         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1324         ret=$?
1325         if [[ $expected_to_fail -eq "1" ]]; then
1326                 test "$ret" -ne "0"
1327                 ok $? "Deleted snapshot output id $id failed as expected"
1328         else
1329                 ok $ret "Deleted snapshot output id $id"
1330         fi
1331 }
1332
1333 function lttng_snapshot_del_output_ok ()
1334 {
1335         lttng_snapshot_del_output 0 "$@"
1336 }
1337
1338 function lttng_snapshot_del_output_fail ()
1339 {
1340         lttng_snapshot_del_output 1 "$@"
1341 }
1342
1343 function lttng_snapshot_record ()
1344 {
1345         local sess_name=$1
1346         local trace_path=$2
1347
1348         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1349         ok $? "Snapshot recorded"
1350 }
1351
1352 function lttng_snapshot_list ()
1353 {
1354         local sess_name=$1
1355         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1356         ok $? "Snapshot list"
1357 }
1358
1359 function lttng_save()
1360 {
1361         local sess_name=$1
1362         local opts=$2
1363
1364         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1365         ok $? "Session saved"
1366 }
1367
1368 function lttng_load()
1369 {
1370         local expected_to_fail=$1
1371         local opts=$2
1372
1373         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1374         ret=$?
1375         if [[ $expected_to_fail -eq "1" ]]; then
1376                 test $ret -ne "0"
1377                 ok $? "Load command failed as expected with opts: $opts"
1378         else
1379                 ok $ret "Load command with opts: $opts"
1380         fi
1381 }
1382
1383 function lttng_load_ok()
1384 {
1385         lttng_load 0 "$@"
1386 }
1387
1388 function lttng_load_fail()
1389 {
1390         lttng_load 1 "$@"
1391 }
1392
1393 function lttng_track()
1394 {
1395         local expected_to_fail=$1
1396         local opts=$2
1397         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1398         ret=$?
1399         if [[ $expected_to_fail -eq "1" ]]; then
1400                 test $ret -ne "0"
1401                 ok $? "Track command failed as expected with opts: $opts"
1402         else
1403                 ok $ret "Track command with opts: $opts"
1404         fi
1405 }
1406
1407 function lttng_track_ok()
1408 {
1409         lttng_track 0 "$@"
1410 }
1411
1412 function lttng_track_fail()
1413 {
1414         lttng_track 1 "$@"
1415 }
1416
1417 function lttng_untrack()
1418 {
1419         local expected_to_fail=$1
1420         local opts=$2
1421         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1422         ret=$?
1423         if [[ $expected_to_fail -eq "1" ]]; then
1424                 test $ret -ne "0"
1425                 ok $? "Untrack command failed as expected with opts: $opts"
1426         else
1427                 ok $ret "Untrack command with opts: $opts"
1428         fi
1429 }
1430
1431 function lttng_untrack_ok()
1432 {
1433         lttng_untrack 0 "$@"
1434 }
1435
1436 function lttng_untrack_fail()
1437 {
1438         lttng_untrack 1 "$@"
1439 }
1440
1441 function add_context_lttng()
1442 {
1443         local expected_to_fail="$1"
1444         local domain="$2"
1445         local session_name="$3"
1446         local channel_name="$4"
1447         local type="$5"
1448
1449         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1450         ret=$?
1451         if [[ $expected_to_fail -eq "1" ]]; then
1452                 test $ret -ne "0"
1453                 ok $? "Add context command failed as expected for type: $type"
1454         else
1455                 ok $ret "Add context command for type: $type"
1456         fi
1457 }
1458
1459 function add_context_ust_ok()
1460 {
1461         add_context_lttng 0 -u "$@"
1462 }
1463
1464 function add_context_ust_fail()
1465 {
1466         add_context_lttng 1 -u "$@"
1467 }
1468
1469 function add_context_kernel_ok()
1470 {
1471         add_context_lttng 0 -k "$@"
1472 }
1473
1474 function add_context_kernel_fail()
1475 {
1476         add_context_lttng 1 -k "$@"
1477 }
1478
1479 function trace_matches ()
1480 {
1481         local event_name=$1
1482         local nr_iter=$2
1483         local trace_path=$3
1484
1485         which $BABELTRACE_BIN >/dev/null
1486         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1487
1488         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1489
1490         if [ "$count" -ne "$nr_iter" ]; then
1491                 fail "Trace match"
1492                 diag "$count matching events found in trace"
1493         else
1494                 pass "Trace match"
1495         fi
1496 }
1497
1498 function trace_match_only()
1499 {
1500         local event_name=$1
1501         local nr_iter=$2
1502         local trace_path=$3
1503
1504         which $BABELTRACE_BIN >/dev/null
1505         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1506
1507         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1508         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1509
1510     if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1511         pass "Trace match with $total event $event_name"
1512     else
1513         fail "Trace match"
1514         diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1515     fi
1516 }
1517
1518 function validate_trace
1519 {
1520         local event_name=$1
1521         local trace_path=$2
1522
1523         which $BABELTRACE_BIN >/dev/null
1524         if [ $? -ne 0 ]; then
1525             skip 0 "Babeltrace binary not found. Skipping trace validation"
1526         fi
1527
1528         OLDIFS=$IFS
1529         IFS=","
1530         for i in $event_name; do
1531                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1532                 if [ "$traced" -ne 0 ]; then
1533                         pass "Validate trace for event $i, $traced events"
1534                 else
1535                         fail "Validate trace for event $i"
1536                         diag "Found $traced occurences of $i"
1537                 fi
1538         done
1539         ret=$?
1540         IFS=$OLDIFS
1541         return $ret
1542 }
1543
1544 function validate_trace_count
1545 {
1546         local event_name=$1
1547         local trace_path=$2
1548         local expected_count=$3
1549
1550         which $BABELTRACE_BIN >/dev/null
1551         if [ $? -ne 0 ]; then
1552             skip 0 "Babeltrace binary not found. Skipping trace validation"
1553         fi
1554
1555         cnt=0
1556         OLDIFS=$IFS
1557         IFS=","
1558         for i in $event_name; do
1559                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1560                 if [ "$traced" -ne 0 ]; then
1561                         pass "Validate trace for event $i, $traced events"
1562                 else
1563                         fail "Validate trace for event $i"
1564                         diag "Found $traced occurences of $i"
1565                 fi
1566                 cnt=$(($cnt + $traced))
1567         done
1568         IFS=$OLDIFS
1569         test $cnt -eq $expected_count
1570         ok $? "Read a total of $cnt events, expected $expected_count"
1571 }
1572
1573 function trace_first_line
1574 {
1575         local trace_path=$1
1576
1577         which $BABELTRACE_BIN >/dev/null
1578         if [ $? -ne 0 ]; then
1579             skip 0 "Babeltrace binary not found. Skipping trace validation"
1580         fi
1581
1582         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1583 }
1584
1585 function validate_trace_exp()
1586 {
1587         local event_exp=$1
1588         local trace_path=$2
1589
1590         which $BABELTRACE_BIN >/dev/null
1591         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1592
1593         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep ${event_exp} | wc -l)
1594         if [ "$traced" -ne 0 ]; then
1595                 pass "Validate trace for expression '${event_exp}', $traced events"
1596         else
1597                 fail "Validate trace for expression '${event_exp}'"
1598                 diag "Found $traced occurences of '${event_exp}'"
1599         fi
1600         ret=$?
1601         return $ret
1602 }
1603
1604 function validate_trace_only_exp()
1605 {
1606         local event_exp=$1
1607         local trace_path=$2
1608
1609         which $BABELTRACE_BIN >/dev/null
1610         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1611
1612         local count=$($BABELTRACE_BIN $trace_path | grep ${event_exp} | wc -l)
1613         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1614
1615         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1616                 pass "Trace match with $total for expression '${event_exp}"
1617         else
1618                 fail "Trace match"
1619                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1620         fi
1621         ret=$?
1622         return $ret
1623 }
1624
1625 function validate_trace_empty()
1626 {
1627         local trace_path=$1
1628
1629         which $BABELTRACE_BIN >/dev/null
1630         if [ $? -ne 0 ]; then
1631             skip 0 "Babeltrace binary not found. Skipping trace validation"
1632         fi
1633
1634         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1635         ret=$?
1636         if [ $ret -ne 0 ]; then
1637                 fail "Failed to parse trace"
1638                 return $ret
1639         fi
1640
1641         traced=$(echo -n "$events" | wc -l)
1642         if [ "$traced" -eq 0 ]; then
1643                 pass "Validate empty trace"
1644         else
1645                 fail "Validate empty trace"
1646                 diag "Found $traced events in trace"
1647         fi
1648         ret=$?
1649         return $ret
1650 }
1651
1652 function regenerate_metadata ()
1653 {
1654         local expected_to_fail=$1
1655         local sess_name=$2
1656
1657         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1658         ret=$?
1659         if [[ $expected_to_fail -eq "1" ]]; then
1660                 test "$ret" -ne "0"
1661                 ok $? "Expected fail on regenerate metadata $sess_name"
1662         else
1663                 ok $ret "Metadata regenerate $sess_name"
1664         fi
1665 }
1666
1667 function regenerate_metadata_ok ()
1668 {
1669         regenerate_metadata 0 "$@"
1670 }
1671
1672 function regenerate_metadata_fail ()
1673 {
1674         regenerate_metadata 1 "$@"
1675 }
1676
1677 function regenerate_statedump ()
1678 {
1679         local expected_to_fail=$1
1680         local sess_name=$2
1681
1682         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1683         ret=$?
1684         if [[ $expected_to_fail -eq "1" ]]; then
1685                 test "$ret" -ne "0"
1686                 ok $? "Expected fail on regenerate statedump $sess_name"
1687         else
1688                 ok $ret "Metadata regenerate $sess_name"
1689         fi
1690 }
1691
1692 function regenerate_statedump_ok ()
1693 {
1694         regenerate_statedump 0 "$@"
1695 }
1696
1697 function regenerate_statedump_fail ()
1698 {
1699         regenerate_statedump 1 "$@"
1700 }
1701
1702 function destructive_tests_enabled ()
1703 {
1704         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1705                 return 0
1706         else
1707                 return 1
1708         fi
1709 }