tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 RUNAS_BIN="lttng-runas"
  19 RUNAS_MATCH=".*lttng-runas.*"
  20 CONSUMERD_BIN="lttng-consumerd"
  21 CONSUMERD_MATCH=".*lttng-consumerd.*"
  22 RELAYD_BIN="lttng-relayd"
  23 RELAYD_MATCH=".*lttng-relayd.*"
  24 LTTNG_BIN="lttng"
  25 BABELTRACE_BIN="babeltrace"
  26 OUTPUT_DEST=/dev/null
  27 ERROR_OUTPUT_DEST=/dev/null
  28
  29 # Minimal kernel version supported for session daemon tests
  30 KERNEL_MAJOR_VERSION=2
  31 KERNEL_MINOR_VERSION=6
  32 KERNEL_PATCHLEVEL_VERSION=27
  33
  34 # We set the default UST register timeout and network and app socket timeout to
  35 # "wait forever", so that basic tests don't have to worry about hitting
  36 # timeouts on busy systems. Specialized tests should test those corner-cases.
  37 export LTTNG_UST_REGISTER_TIMEOUT=-1
  38 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  39 export LTTNG_APP_SOCKET_TIMEOUT=-1
  40
  41 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  42 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  43 # its own sessiond if none is running. It also ensures that 'lttng create'
  44 # fails when no sessiond is running.
  45 export LTTNG_SESSIOND_PATH="/bin/true"
  46
  47 source $TESTDIR/utils/tap/tap.sh
  48
  49 if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
  50         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  51 fi
  52
  53 function full_cleanup ()
  54 {
  55         # Try to kill daemons gracefully
  56         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # If daemons are still present, forcibly kill them
  60         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  61         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  62         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  63
  64         # Disable trap for SIGTERM since the following kill to the
  65         # pidgroup will be SIGTERM. Otherwise it loops.
  66         # The '-' before the pid number ($$) indicates 'kill' to signal the
  67         # whole process group.
  68         trap - SIGTERM && kill -- -$$
  69 }
  70
  71
  72 trap full_cleanup SIGINT SIGTERM
  73
  74 function print_ok ()
  75 {
  76         # Check if we are a terminal
  77         if [ -t 1 ]; then
  78                 echo -e "\e[1;32mOK\e[0m"
  79         else
  80                 echo -e "OK"
  81         fi
  82 }
  83
  84 function print_fail ()
  85 {
  86         # Check if we are a terminal
  87         if [ -t 1 ]; then
  88                 echo -e "\e[1;31mFAIL\e[0m"
  89         else
  90                 echo -e "FAIL"
  91         fi
  92 }
  93
  94 function print_test_banner ()
  95 {
  96         local desc="$1"
  97         diag "$desc"
  98 }
  99
 100 function validate_kernel_version ()
 101 {
 102         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 103         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 104                 return 0
 105         fi
 106         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 107                 return 0
 108         fi
 109         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 110                 return 0
 111         fi
 112         return 1
 113 }
 114
 115 # Generate a random string
 116 #  $1 = number of characters; defaults to 16
 117 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 118 function randstring()
 119 {
 120         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 121         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 122         echo
 123 }
 124
 125 # Return the number of _configured_ CPUs.
 126 function conf_proc_count()
 127 {
 128         getconf _NPROCESSORS_CONF
 129         if [ $? -ne 0 ]; then
 130                 diag "Failed to get the number of configured CPUs"
 131         fi
 132         echo
 133 }
 134
 135 # Check if base lttng-modules are present.
 136 # Bail out on failure
 137 function validate_lttng_modules_present ()
 138 {
 139         modprobe -n lttng-tracer 2>/dev/null
 140         if [ $? -ne 0  ]; then
 141                 BAIL_OUT "LTTng modules not detected."
 142         fi
 143 }
 144
 145 function enable_kernel_lttng_event
 146 {
 147         local withtap="$1"
 148         local expected_to_fail="$2"
 149         local sess_name="$3"
 150         local event_name="$4"
 151         local channel_name="$5"
 152
 153         if [ -z "$event_name" ]; then
 154                 # Enable all event if no event name specified
 155                 event_name="-a"
 156         fi
 157
 158         if [ -z "$channel_name" ]; then
 159                 # default channel if none specified
 160                 chan=""
 161         else
 162                 chan="-c $channel_name"
 163         fi
 164
 165         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 166         ret=$?
 167         if [[ $expected_to_fail -eq "1" ]]; then
 168                 test $ret -ne "0"
 169                 ret=$?
 170                 if [ $withtap -eq "1" ]; then
 171                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 172                 fi
 173         else
 174                 if [ $withtap -eq "1" ]; then
 175                         ok $ret "Enable kernel event $event_name for session $sess_name"
 176                 fi
 177         fi
 178 }
 179
 180 function enable_kernel_lttng_event_ok ()
 181 {
 182         enable_kernel_lttng_event 1 0 "$@"
 183 }
 184
 185 function enable_kernel_lttng_event_fail ()
 186 {
 187         enable_kernel_lttng_event 1 1 "$@"
 188 }
 189
 190 function enable_kernel_lttng_event_notap ()
 191 {
 192         enable_kernel_lttng_event 0 0 "$@"
 193 }
 194
 195 # Old interface
 196 function lttng_enable_kernel_event
 197 {
 198         enable_kernel_lttng_event_ok "$@"
 199 }
 200
 201 function lttng_enable_kernel_syscall()
 202 {
 203         local expected_to_fail=$1
 204         local sess_name=$2
 205         local syscall_name=$3
 206         local channel_name=$4
 207
 208         if [ -z $syscall_name ]; then
 209                 # Enable all event if no syscall name specified
 210                 syscall_name="-a"
 211         fi
 212
 213         if [ -z $channel_name ]; then
 214                 # default channel if none specified
 215                 chan=""
 216         else
 217                 chan="-c $channel_name"
 218         fi
 219
 220         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 221         ret=$?
 222         if [[ $expected_to_fail -eq "1" ]]; then
 223                 test $ret -ne "0"
 224                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 225         else
 226                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 227         fi
 228 }
 229
 230 function lttng_enable_kernel_syscall_ok()
 231 {
 232         lttng_enable_kernel_syscall 0 "$@"
 233 }
 234
 235 function lttng_enable_kernel_syscall_fail()
 236 {
 237         lttng_enable_kernel_syscall 1 "$@"
 238 }
 239
 240 function lttng_disable_kernel_syscall()
 241 {
 242         local expected_to_fail=$1
 243         local sess_name=$2
 244         local syscall_name=$3
 245         local channel_name=$4
 246
 247         if [ -z $syscall_name ]; then
 248                 # Enable all event if no syscall name specified
 249                 syscall_name="-a"
 250         fi
 251
 252         if [ -z $channel_name ]; then
 253                 # default channel if none specified
 254                 chan=""
 255         else
 256                 chan="-c $channel_name"
 257         fi
 258
 259         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 260
 261         ret=$?
 262         if [[ $expected_to_fail -eq "1" ]]; then
 263                 test $ret -ne "0"
 264                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 265         else
 266                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 267         fi
 268 }
 269
 270 function lttng_disable_kernel_syscall_ok()
 271 {
 272         lttng_disable_kernel_syscall 0 "$@"
 273 }
 274
 275 function lttng_disable_kernel_syscall_fail()
 276 {
 277         lttng_disable_kernel_syscall 1 "$@"
 278 }
 279
 280 function lttng_enable_kernel_channel()
 281 {
 282         local withtap=$1
 283         local expected_to_fail=$2
 284         local sess_name=$3
 285         local channel_name=$4
 286         local opt=$5
 287
 288         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 289         ret=$?
 290         if [[ $expected_to_fail -eq "1" ]]; then
 291                 test "$ret" -ne "0"
 292                 ret=$?
 293                 if [ $withtap -eq "1" ]; then
 294                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 295                 fi
 296         else
 297                 if [ $withtap -eq "1" ]; then
 298                         ok $ret "Enable channel $channel_name for session $sess_name"
 299                 fi
 300         fi
 301 }
 302
 303 function lttng_enable_kernel_channel_ok()
 304 {
 305         lttng_enable_kernel_channel 1 0 "$@"
 306 }
 307
 308 function lttng_enable_kernel_channel_fail()
 309 {
 310         lttng_enable_kernel_channel 1 1 "$@"
 311 }
 312
 313 function lttng_enable_kernel_channel_notap()
 314 {
 315         lttng_enable_kernel_channel 0 0 "$@"
 316 }
 317
 318 function enable_kernel_lttng_channel_ok()
 319 {
 320         lttng_enable_kernel_channel 1 0 "$@"
 321 }
 322
 323 function lttng_disable_kernel_channel()
 324 {
 325         local expected_to_fail=$1
 326         local sess_name=$2
 327         local channel_name=$3
 328
 329         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 330         ret=$?
 331         if [[ $expected_to_fail -eq "1" ]]; then
 332                 test "$ret" -ne "0"
 333                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 334         else
 335                 ok $ret "Disable channel $channel_name for session $sess_name"
 336         fi
 337 }
 338
 339 function lttng_disable_kernel_channel_ok()
 340 {
 341         lttng_disable_kernel_channel 0 "$@"
 342 }
 343
 344 function lttng_disable_kernel_channel_fail()
 345 {
 346         lttng_disable_kernel_channel 1 "$@"
 347 }
 348
 349 function start_lttng_relayd_opt()
 350 {
 351         local withtap=$1
 352         local opt=$2
 353
 354         DIR=$(readlink -f "$TESTDIR")
 355
 356         if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
 357                 # shellcheck disable=SC2086
 358                 "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 359                 #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
 360                 if [ $? -eq 1 ]; then
 361                         if [ "$withtap" -eq "1" ]; then
 362                                 fail "Start lttng-relayd (opt: $opt)"
 363                         fi
 364                         return 1
 365                 else
 366                         if [ "$withtap" -eq "1" ]; then
 367                                 pass "Start lttng-relayd (opt: $opt)"
 368                         fi
 369                 fi
 370         else
 371                 pass "Start lttng-relayd (opt: $opt)"
 372         fi
 373 }
 374
 375 function start_lttng_relayd()
 376 {
 377         start_lttng_relayd_opt 1 "$@"
 378 }
 379
 380 function start_lttng_relayd_notap()
 381 {
 382         start_lttng_relayd_opt 0 "$@"
 383 }
 384
 385 function stop_lttng_relayd_opt()
 386 {
 387         local withtap=$1
 388         local signal=$2
 389
 390         if [ -z "$signal" ]; then
 391                 signal="SIGTERM"
 392         fi
 393
 394         local timeout_s=$3
 395         local dtimeleft_s=
 396
 397         # Multiply time by 2 to simplify integer arithmetic
 398         if [ -n "$timeout_s" ]; then
 399                 dtimeleft_s=$((timeout_s * 2))
 400         fi
 401
 402         local retval=0
 403         local pids=
 404
 405         pids=$(pgrep "$RELAYD_MATCH")
 406         if [ -z "$pids" ]; then
 407                 if [ "$withtap" -eq "1" ]; then
 408                         pass "No relay daemon to kill"
 409                 fi
 410                 return 0
 411         fi
 412
 413         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 414
 415         # shellcheck disable=SC2086
 416         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 417                 retval=1
 418                 if [ "$withtap" -eq "1" ]; then
 419                         fail "Kill relay daemon"
 420                 fi
 421         else
 422                 out=1
 423                 while [ -n "$out" ]; do
 424                         out=$(pgrep "$RELAYD_MATCH")
 425                         if [ -n "$dtimeleft_s" ]; then
 426                                 if [ $dtimeleft_s -lt 0 ]; then
 427                                         out=
 428                                         retval=1
 429                                 fi
 430                                 dtimeleft_s=$((dtimeleft_s - 1))
 431                         fi
 432                         sleep 0.5
 433                 done
 434                 if [ "$withtap" -eq "1" ]; then
 435                         if [ "$retval" -eq "0" ]; then
 436                                 pass "Wait after kill relay daemon"
 437                         else
 438                                 fail "Wait after kill relay daemon"
 439                         fi
 440                 fi
 441         fi
 442         return $retval
 443 }
 444
 445 function stop_lttng_relayd()
 446 {
 447         stop_lttng_relayd_opt 1 "$@"
 448 }
 449
 450 function stop_lttng_relayd_notap()
 451 {
 452         stop_lttng_relayd_opt 0 "$@"
 453 }
 454
 455 #First arg: show tap output
 456 #Second argument: load path for automatic loading
 457 function start_lttng_sessiond_opt()
 458 {
 459         local withtap=$1
 460         local load_path=$2
 461
 462         local env_vars=""
 463         local consumerd=""
 464
 465         local long_bit_value=
 466         long_bit_value=$(getconf LONG_BIT)
 467
 468         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 469                 # Env variable requested no session daemon
 470                 return
 471         fi
 472
 473         DIR=$(readlink -f "$TESTDIR")
 474
 475         # Get long_bit value for 32/64 consumerd
 476         case "$long_bit_value" in
 477                 32)
 478                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 479                         ;;
 480                 64)
 481                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 482                         ;;
 483                 *)
 484                         return
 485                         ;;
 486         esac
 487
 488         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 489         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 490                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 491         fi
 492         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 493
 494         if ! validate_kernel_version; then
 495             fail "Start session daemon"
 496             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 497         fi
 498
 499         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 500         export LTTNG_SESSION_CONFIG_XSD_PATH
 501
 502         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 503                 # Have a load path ?
 504                 if [ -n "$load_path" ]; then
 505                         # shellcheck disable=SC2086
 506                         env $env_vars --load "$load_path" --background "$consumerd"
 507                 else
 508                         # shellcheck disable=SC2086
 509                         env $env_vars --background "$consumerd"
 510                 fi
 511                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 512                 status=$?
 513                 if [ "$withtap" -eq "1" ]; then
 514                         ok $status "Start session daemon"
 515                 fi
 516         fi
 517 }
 518
 519 function start_lttng_sessiond()
 520 {
 521         start_lttng_sessiond_opt 1 "$@"
 522 }
 523
 524 function start_lttng_sessiond_notap()
 525 {
 526         start_lttng_sessiond_opt 0 "$@"
 527 }
 528
 529 function stop_lttng_sessiond_opt()
 530 {
 531         local withtap=$1
 532         local signal=$2
 533
 534         if [ -z "$signal" ]; then
 535                 signal=SIGTERM
 536         fi
 537
 538         local timeout_s=$3
 539         local dtimeleft_s=
 540
 541         # Multiply time by 2 to simplify integer arithmetic
 542         if [ -n "$timeout_s" ]; then
 543                 dtimeleft_s=$((timeout_s * 2))
 544         fi
 545
 546         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 547                 # Env variable requested no session daemon
 548                 return 0
 549         fi
 550
 551         local retval=0
 552
 553         local runas_pids=
 554         runas_pids=$(pgrep "$RUNAS_MATCH")
 555
 556         local pids=
 557         pids=$(pgrep "$SESSIOND_MATCH")
 558
 559         if [ -n "$runas_pids" ]; then
 560                 pids="$pids $runas_pids"
 561         fi
 562
 563         if [ -z "$pids" ]; then
 564                 if [ "$withtap" -eq "1" ]; then
 565                         pass "No session daemon to kill"
 566                 fi
 567                 return 0
 568         fi
 569
 570         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 571
 572         # shellcheck disable=SC2086
 573         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 574                 retval=1
 575                 if [ "$withtap" -eq "1" ]; then
 576                         fail "Kill sessions daemon"
 577                 fi
 578         else
 579                 out=1
 580                 while [ -n "$out" ]; do
 581                         out=$(pgrep "${SESSIOND_MATCH}")
 582                         if [ -n "$dtimeleft_s" ]; then
 583                                 if [ $dtimeleft_s -lt 0 ]; then
 584                                         out=
 585                                         retval=1
 586                                 fi
 587                                 dtimeleft_s=$((dtimeleft_s - 1))
 588                         fi
 589                         sleep 0.5
 590                 done
 591                 out=1
 592                 while [ -n "$out" ]; do
 593                         out=$(pgrep "$CONSUMERD_MATCH")
 594                         if [ -n "$dtimeleft_s" ]; then
 595                                 if [ $dtimeleft_s -lt 0 ]; then
 596                                         out=
 597                                         retval=1
 598                                 fi
 599                                 dtimeleft_s=$((dtimeleft_s - 1))
 600                         fi
 601                         sleep 0.5
 602                 done
 603
 604                 if [ "$withtap" -eq "1" ]; then
 605                         if [ "$retval" -eq "0" ]; then
 606                                 pass "Wait after kill session daemon"
 607                         else
 608                                 fail "Wait after kill session daemon"
 609                         fi
 610                 fi
 611         fi
 612         if [ "$signal" = "SIGKILL" ]; then
 613                 if [ "$(id -u)" -eq "0" ]; then
 614                         local modules=
 615                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 616
 617                         if [ -n "$modules" ]; then
 618                                 diag "Unloading all LTTng modules"
 619                                 modprobe -r "$modules"
 620                         fi
 621                 fi
 622         fi
 623
 624         return $retval
 625 }
 626
 627 function stop_lttng_sessiond()
 628 {
 629         stop_lttng_sessiond_opt 1 "$@"
 630 }
 631
 632 function stop_lttng_sessiond_notap()
 633 {
 634         stop_lttng_sessiond_opt 0 "$@"
 635 }
 636
 637 function sigstop_lttng_sessiond_opt()
 638 {
 639         local withtap=$1
 640         local signal=SIGSTOP
 641
 642         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 643                 # Env variable requested no session daemon
 644                 return
 645         fi
 646
 647         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 648
 649         if [ "$withtap" -eq "1" ]; then
 650                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 651         fi
 652
 653         # shellcheck disable=SC2086
 654         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 655                 if [ "$withtap" -eq "1" ]; then
 656                         fail "Sending SIGSTOP to session daemon"
 657                 fi
 658         else
 659                 out=1
 660                 while [ $out -ne 0 ]; do
 661                         pid="$(pgrep "$SESSIOND_MATCH")"
 662
 663                         # Wait until state becomes stopped for session
 664                         # daemon(s).
 665                         out=0
 666                         for sessiond_pid in $pid; do
 667                                 state="$(ps -p "$sessiond_pid" -o state= )"
 668                                 if [[ -n "$state" && "$state" != "T" ]]; then
 669                                         out=1
 670                                 fi
 671                         done
 672                         sleep 0.5
 673                 done
 674                 if [ "$withtap" -eq "1" ]; then
 675                         pass "Sending SIGSTOP to session daemon"
 676                 fi
 677         fi
 678 }
 679
 680 function sigstop_lttng_sessiond()
 681 {
 682         sigstop_lttng_sessiond_opt 1 "$@"
 683 }
 684
 685 function sigstop_lttng_sessiond_notap()
 686 {
 687         sigstop_lttng_sessiond_opt 0 "$@"
 688 }
 689
 690 function stop_lttng_consumerd_opt()
 691 {
 692         local withtap=$1
 693         local signal=$2
 694
 695         if [ -z "$signal" ]; then
 696                 signal=SIGTERM
 697         fi
 698
 699         local timeout_s=$3
 700         local dtimeleft_s=
 701
 702         # Multiply time by 2 to simplify integer arithmetic
 703         if [ -n "$timeout_s" ]; then
 704                 dtimeleft_s=$((timeout_s * 2))
 705         fi
 706
 707         local retval=0
 708
 709         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 710
 711         if [ -z "$PID_CONSUMERD" ]; then
 712                 if [ "$withtap" -eq "1" ]; then
 713                         pass "No consumer daemon to kill"
 714                 fi
 715                 return 0
 716         fi
 717
 718         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 719
 720         # shellcheck disable=SC2086
 721         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 722                 retval=1
 723                 if [ "$withtap" -eq "1" ]; then
 724                         fail "Kill consumer daemon"
 725                 fi
 726         else
 727                 out=1
 728                 while [ $out -ne 0 ]; do
 729                         pid="$(pgrep "$CONSUMERD_MATCH")"
 730
 731                         # If consumerds are still present check their status.
 732                         # A zombie status qualifies the consumerd as *killed*
 733                         out=0
 734                         for consumer_pid in $pid; do
 735                                 state="$(ps -p "$consumer_pid" -o state= )"
 736                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 737                                         out=1
 738                                 fi
 739                         done
 740                         if [ -n "$dtimeleft_s" ]; then
 741                                 if [ $dtimeleft_s -lt 0 ]; then
 742                                         out=0
 743                                         retval=1
 744                                 fi
 745                                 dtimeleft_s=$((dtimeleft_s - 1))
 746                         fi
 747                         sleep 0.5
 748                 done
 749                 if [ "$withtap" -eq "1" ]; then
 750                         if [ "$retval" -eq "0" ]; then
 751                                 pass "Wait after kill consumer daemon"
 752                         else
 753                                 fail "Wait after kill consumer daemon"
 754                         fi
 755                 fi
 756         fi
 757
 758         return $retval
 759 }
 760
 761 function stop_lttng_consumerd()
 762 {
 763         stop_lttng_consumerd_opt 1 "$@"
 764 }
 765
 766 function stop_lttng_consumerd_notap()
 767 {
 768         stop_lttng_consumerd_opt 0 "$@"
 769 }
 770
 771 function sigstop_lttng_consumerd_opt()
 772 {
 773         local withtap=$1
 774         local signal=SIGSTOP
 775
 776         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 777
 778         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 779
 780         # shellcheck disable=SC2086
 781         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 782         retval=$?
 783
 784         if [ $retval -eq 1 ]; then
 785                 if [ "$withtap" -eq "1" ]; then
 786                         fail "Sending SIGSTOP to consumer daemon"
 787                 fi
 788                 return 1
 789         else
 790                 out=1
 791                 while [ $out -ne 0 ]; do
 792                         pid="$(pgrep "$CONSUMERD_MATCH")"
 793
 794                         # Wait until state becomes stopped for all
 795                         # consumers.
 796                         out=0
 797                         for consumer_pid in $pid; do
 798                                 state="$(ps -p "$consumer_pid" -o state= )"
 799                                 if [[ -n "$state" && "$state" != "T" ]]; then
 800                                         out=1
 801                                 fi
 802                         done
 803                         sleep 0.5
 804                 done
 805                 if [ "$withtap" -eq "1" ]; then
 806                         pass "Sending SIGSTOP to consumer daemon"
 807                 fi
 808         fi
 809         return $retval
 810 }
 811
 812 function sigstop_lttng_consumerd()
 813 {
 814         sigstop_lttng_consumerd_opt 1 "$@"
 815 }
 816
 817 function sigstop_lttng_consumerd_notap()
 818 {
 819         sigstop_lttng_consumerd_opt 0 "$@"
 820 }
 821
 822 function list_lttng_with_opts ()
 823 {
 824         local opts=$1
 825         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 826         ok $? "Lttng-tool list command with option $opts"
 827 }
 828
 829 function create_lttng_session_no_output ()
 830 {
 831         local sess_name=$1
 832
 833         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 834         ok $? "Create session $sess_name in no-output mode"
 835 }
 836
 837 function create_lttng_session ()
 838 {
 839         local withtap=$1
 840         local expected_to_fail=$2
 841         local sess_name=$3
 842         local trace_path=$4
 843         local opt=$5
 844
 845         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 846         ret=$?
 847         if [ $expected_to_fail -eq "1" ]; then
 848                 test "$ret" -ne "0"
 849                 ret=$?
 850                 if [ $withtap -eq "1" ]; then
 851                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 852                 fi
 853         else
 854                 if [ $withtap -eq "1" ]; then
 855                         ok $ret "Create session $sess_name in $trace_path"
 856                 fi
 857         fi
 858         return $ret
 859 }
 860
 861 function create_lttng_session_ok ()
 862 {
 863         create_lttng_session 1 0 "$@"
 864 }
 865
 866 function create_lttng_session_fail ()
 867 {
 868         create_lttng_session 1 1 "$@"
 869 }
 870
 871 function create_lttng_session_notap ()
 872 {
 873         create_lttng_session 0 0 "$@"
 874 }
 875
 876
 877 function enable_ust_lttng_channel ()
 878 {
 879         local withtap=$1
 880         local expected_to_fail=$2
 881         local sess_name=$3
 882         local channel_name=$4
 883         local opt=$5
 884
 885         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 886         ret=$?
 887         if [[ $expected_to_fail -eq "1" ]]; then
 888                 test "$ret" -ne "0"
 889                 ret=$?
 890                 if [ $withtap -eq "1" ]; then
 891                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 892                 fi
 893         else
 894                 if [ $withtap -eq "1" ]; then
 895                         ok $ret "Enable channel $channel_name for session $sess_name"
 896                 fi
 897         fi
 898         return $ret
 899 }
 900
 901 function enable_ust_lttng_channel_ok ()
 902 {
 903         enable_ust_lttng_channel 1 0 "$@"
 904 }
 905
 906 function enable_ust_lttng_channel_fail ()
 907 {
 908         enable_ust_lttng_channel 1 1 "$@"
 909 }
 910
 911 function enable_ust_lttng_channel_notap ()
 912 {
 913         enable_ust_lttng_channel 0 0 "$@"
 914 }
 915
 916 function disable_ust_lttng_channel()
 917 {
 918         local sess_name=$1
 919         local channel_name=$2
 920
 921         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 922         ok $? "Disable channel $channel_name for session $sess_name"
 923 }
 924
 925 function enable_lttng_mmap_overwrite_kernel_channel()
 926 {
 927         local sess_name=$1
 928         local channel_name=$2
 929
 930         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 931         ok $? "Enable channel $channel_name for session $sess_name"
 932 }
 933
 934 function enable_lttng_mmap_discard_small_kernel_channel()
 935 {
 936         local sess_name=$1
 937         local channel_name=$2
 938
 939         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 940         ok $? "Enable small discard channel $channel_name for session $sess_name"
 941 }
 942
 943 function enable_lttng_mmap_overwrite_small_kernel_channel()
 944 {
 945         local sess_name=$1
 946         local channel_name=$2
 947
 948         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 949         ok $? "Enable small discard channel $channel_name for session $sess_name"
 950 }
 951
 952 function enable_lttng_mmap_overwrite_ust_channel()
 953 {
 954         local sess_name=$1
 955         local channel_name=$2
 956
 957         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 958         ok $? "Enable channel $channel_name for session $sess_name"
 959 }
 960
 961 function enable_ust_lttng_event ()
 962 {
 963         local withtap=$1
 964         local expected_to_fail=$2
 965         local sess_name=$3
 966         local event_name="$4"
 967         local channel_name=$5
 968
 969         if [ -z $channel_name ]; then
 970                 # default channel if none specified
 971                 chan=""
 972         else
 973                 chan="-c $channel_name"
 974         fi
 975
 976         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 977         ret=$?
 978         if [[ $expected_to_fail -eq "1" ]]; then
 979                 test $ret -ne "0"
 980                 ret=$?
 981                 if [[ $withtap -eq "1" ]]; then
 982                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
 983                 fi
 984         else
 985                 if [[ $withtap -eq "1" ]]; then
 986                         ok $ret "Enable ust event $event_name for session $sess_name"
 987                 fi
 988         fi
 989         return $ret
 990 }
 991
 992 function enable_ust_lttng_event_ok ()
 993 {
 994         enable_ust_lttng_event 1 0 "$@"
 995 }
 996
 997 function enable_ust_lttng_event_fail ()
 998 {
 999         enable_ust_lttng_event 1 1 "$@"
1000 }
1001
1002 function enable_ust_lttng_event_notap ()
1003 {
1004         enable_ust_lttng_event 0 0 "$@"
1005 }
1006
1007 function enable_jul_lttng_event()
1008 {
1009         sess_name=$1
1010         event_name="$2"
1011         channel_name=$3
1012
1013         if [ -z $channel_name ]; then
1014                 # default channel if none specified
1015                 chan=""
1016         else
1017                 chan="-c $channel_name"
1018         fi
1019
1020         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1021         ok $? "Enable JUL event $event_name for session $sess_name"
1022 }
1023
1024 function enable_jul_lttng_event_loglevel()
1025 {
1026         local sess_name=$1
1027         local event_name="$2"
1028         local loglevel=$3
1029         local channel_name=$4
1030
1031         if [ -z $channel_name ]; then
1032                 # default channel if none specified
1033                 chan=""
1034         else
1035                 chan="-c $channel_name"
1036         fi
1037
1038         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1039         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1040 }
1041
1042 function enable_log4j_lttng_event()
1043 {
1044         sess_name=$1
1045         event_name="$2"
1046         channel_name=$3
1047
1048         if [ -z $channel_name ]; then
1049                 # default channel if none specified
1050                 chan=""
1051         else
1052                 chan="-c $channel_name"
1053         fi
1054
1055         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1056         ok $? "Enable LOG4J event $event_name for session $sess_name"
1057 }
1058
1059 function enable_log4j_lttng_event_loglevel()
1060 {
1061         local sess_name=$1
1062         local event_name="$2"
1063         local loglevel=$3
1064         local channel_name=$4
1065
1066         if [ -z $channel_name ]; then
1067                 # default channel if none specified
1068                 chan=""
1069         else
1070                 chan="-c $channel_name"
1071         fi
1072
1073         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1074         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1075 }
1076
1077 function enable_python_lttng_event()
1078 {
1079         sess_name=$1
1080         event_name="$2"
1081         channel_name=$3
1082
1083         if [ -z $channel_name ]; then
1084                 # default channel if none specified
1085                 chan=""
1086         else
1087                 chan="-c $channel_name"
1088         fi
1089
1090         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1091         ok $? "Enable Python event $event_name for session $sess_name"
1092 }
1093
1094 function enable_python_lttng_event_loglevel()
1095 {
1096         local sess_name=$1
1097         local event_name="$2"
1098         local loglevel=$3
1099         local channel_name=$4
1100
1101         if [ -z $channel_name ]; then
1102                 # default channel if none specified
1103                 chan=""
1104         else
1105                 chan="-c $channel_name"
1106         fi
1107
1108         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1109         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1110 }
1111
1112 function enable_ust_lttng_event_filter()
1113 {
1114         local sess_name="$1"
1115         local event_name="$2"
1116         local filter="$3"
1117
1118         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1119         ok $? "Enable event $event_name with filtering for session $sess_name"
1120 }
1121
1122 function enable_ust_lttng_event_loglevel()
1123 {
1124         local sess_name="$1"
1125         local event_name="$2"
1126         local loglevel="$3"
1127
1128         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1129         ok $? "Enable event $event_name with loglevel $loglevel"
1130 }
1131
1132 function enable_ust_lttng_event_loglevel_only()
1133 {
1134         local sess_name="$1"
1135         local event_name="$2"
1136         local loglevel="$3"
1137
1138         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1139         ok $? "Enable event $event_name with loglevel-only $loglevel"
1140 }
1141
1142 function disable_ust_lttng_event ()
1143 {
1144         local sess_name="$1"
1145         local event_name="$2"
1146         local channel_name="$3"
1147
1148         if [ -z $channel_name ]; then
1149                 # default channel if none specified
1150                 chan=""
1151         else
1152                 chan="-c $channel_name"
1153         fi
1154
1155         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1156         ok $? "Disable event $event_name for session $sess_name"
1157 }
1158
1159 function disable_jul_lttng_event ()
1160 {
1161         local sess_name="$1"
1162         local event_name="$2"
1163
1164         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1165         ok $? "Disable JUL event $event_name for session $sess_name"
1166 }
1167
1168 function disable_log4j_lttng_event ()
1169 {
1170         local sess_name="$1"
1171         local event_name="$2"
1172
1173         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1174         ok $? "Disable LOG4J event $event_name for session $sess_name"
1175 }
1176
1177 function disable_python_lttng_event ()
1178 {
1179         local sess_name="$1"
1180         local event_name="$2"
1181
1182         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1183         ok $? "Disable Python event $event_name for session $sess_name"
1184 }
1185
1186 function start_lttng_tracing ()
1187 {
1188         local expected_to_fail=$1
1189         local sess_name=$2
1190
1191         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1192         ret=$?
1193         if [[ $expected_to_fail -eq "1" ]]; then
1194                 test "$ret" -ne "0"
1195                 ok $? "Start tracing for session $sess_name failed as expected"
1196         else
1197                 ok $ret "Start tracing for session $sess_name"
1198         fi
1199 }
1200
1201 function start_lttng_tracing_ok ()
1202 {
1203         start_lttng_tracing 0 "$@"
1204 }
1205
1206 function start_lttng_tracing_fail ()
1207 {
1208         start_lttng_tracing 1 "$@"
1209 }
1210
1211 function stop_lttng_tracing ()
1212 {
1213         local expected_to_fail=$1
1214         local sess_name=$2
1215
1216         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1217         ret=$?
1218         if [[ $expected_to_fail -eq "1" ]]; then
1219                 test "$ret" -ne "0"
1220                 ok $? "Stop lttng tracing for session $sess_name failed as expected"
1221         else
1222                 ok $ret "Stop lttng tracing for session $sess_name"
1223         fi
1224 }
1225
1226 function stop_lttng_tracing_ok ()
1227 {
1228         stop_lttng_tracing 0 "$@"
1229 }
1230
1231 function stop_lttng_tracing_fail ()
1232 {
1233         stop_lttng_tracing 1 "$@"
1234 }
1235
1236 function destroy_lttng_session ()
1237 {
1238         local withtap=$1
1239         local expected_to_fail=$2
1240         local sess_name=$3
1241
1242         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1243         ret=$?
1244         if [[ $expected_to_fail -eq "1" ]]; then
1245                 test "$ret" -ne "0"
1246                 ret=$?
1247                 if [ $withtap -eq "1" ]; then
1248                         ok $ret "Destroy session $sess_name failed as expected"
1249                 fi
1250         else
1251                 if [ $withtap -eq "1" ]; then
1252                         ok $ret "Destroy session $sess_name"
1253                 fi
1254         fi
1255 }
1256
1257 function destroy_lttng_session_ok ()
1258 {
1259         destroy_lttng_session 1 0 "$@"
1260
1261 }
1262
1263 function destroy_lttng_session_fail ()
1264 {
1265         destroy_lttng_session 1 1 "$@"
1266 }
1267
1268 function destroy_lttng_session_notap ()
1269 {
1270         destroy_lttng_session 0 0 "$@"
1271 }
1272
1273 function destroy_lttng_sessions ()
1274 {
1275         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1276         ok $? "Destroy all lttng sessions"
1277 }
1278
1279 function lttng_snapshot_add_output ()
1280 {
1281         local expected_to_fail=$1
1282         local sess_name=$2
1283         local trace_path=$3
1284
1285         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name file://$trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1286         ret=$?
1287         if [[ $expected_to_fail -eq 1 ]]; then
1288                 test "$ret" -ne "0"
1289                 ok $? "Added snapshot output file://$trace_path failed as expected"
1290         else
1291                 ok $ret "Added snapshot output file://$trace_path"
1292         fi
1293 }
1294
1295 function lttng_snapshot_add_output_ok ()
1296 {
1297         lttng_snapshot_add_output 0 "$@"
1298 }
1299
1300 function lttng_snapshot_add_output_fail ()
1301 {
1302         lttng_snapshot_add_output 1 "$@"
1303 }
1304
1305 function lttng_snapshot_del_output ()
1306 {
1307         local expected_to_fail=$1
1308         local sess_name=$2
1309         local id=$3
1310
1311         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1312         ret=$?
1313         if [[ $expected_to_fail -eq "1" ]]; then
1314                 test "$ret" -ne "0"
1315                 ok $? "Deleted snapshot output id $id failed as expected"
1316         else
1317                 ok $ret "Deleted snapshot output id $id"
1318         fi
1319 }
1320
1321 function lttng_snapshot_del_output_ok ()
1322 {
1323         lttng_snapshot_del_output 0 "$@"
1324 }
1325
1326 function lttng_snapshot_del_output_fail ()
1327 {
1328         lttng_snapshot_del_output 1 "$@"
1329 }
1330
1331 function lttng_snapshot_record ()
1332 {
1333         local sess_name=$1
1334         local trace_path=$2
1335
1336         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1337         ok $? "Snapshot recorded"
1338 }
1339
1340 function lttng_snapshot_list ()
1341 {
1342         local sess_name=$1
1343         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1344         ok $? "Snapshot list"
1345 }
1346
1347 function lttng_save()
1348 {
1349         local sess_name=$1
1350         local opts=$2
1351
1352         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1353         ok $? "Session saved"
1354 }
1355
1356 function lttng_load()
1357 {
1358         local expected_to_fail=$1
1359         local opts=$2
1360
1361         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1362         ret=$?
1363         if [[ $expected_to_fail -eq "1" ]]; then
1364                 test $ret -ne "0"
1365                 ok $? "Load command failed as expected with opts: $opts"
1366         else
1367                 ok $ret "Load command with opts: $opts"
1368         fi
1369 }
1370
1371 function lttng_load_ok()
1372 {
1373         lttng_load 0 "$@"
1374 }
1375
1376 function lttng_load_fail()
1377 {
1378         lttng_load 1 "$@"
1379 }
1380
1381 function lttng_track()
1382 {
1383         local expected_to_fail=$1
1384         local opts=$2
1385         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1386         ret=$?
1387         if [[ $expected_to_fail -eq "1" ]]; then
1388                 test $ret -ne "0"
1389                 ok $? "Track command failed as expected with opts: $opts"
1390         else
1391                 ok $ret "Track command with opts: $opts"
1392         fi
1393 }
1394
1395 function lttng_track_ok()
1396 {
1397         lttng_track 0 "$@"
1398 }
1399
1400 function lttng_track_fail()
1401 {
1402         lttng_track 1 "$@"
1403 }
1404
1405 function lttng_untrack()
1406 {
1407         local expected_to_fail=$1
1408         local opts=$2
1409         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1410         ret=$?
1411         if [[ $expected_to_fail -eq "1" ]]; then
1412                 test $ret -ne "0"
1413                 ok $? "Untrack command failed as expected with opts: $opts"
1414         else
1415                 ok $ret "Untrack command with opts: $opts"
1416         fi
1417 }
1418
1419 function lttng_untrack_ok()
1420 {
1421         lttng_untrack 0 "$@"
1422 }
1423
1424 function lttng_untrack_fail()
1425 {
1426         lttng_untrack 1 "$@"
1427 }
1428
1429 function add_context_lttng()
1430 {
1431         local expected_to_fail="$1"
1432         local domain="$2"
1433         local session_name="$3"
1434         local channel_name="$4"
1435         local type="$5"
1436
1437         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1438         ret=$?
1439         if [[ $expected_to_fail -eq "1" ]]; then
1440                 test $ret -ne "0"
1441                 ok $? "Add context command failed as expected for type: $type"
1442         else
1443                 ok $ret "Add context command for type: $type"
1444         fi
1445 }
1446
1447 function add_context_ust_ok()
1448 {
1449         add_context_lttng 0 -u "$@"
1450 }
1451
1452 function add_context_ust_fail()
1453 {
1454         add_context_lttng 1 -u "$@"
1455 }
1456
1457 function add_context_kernel_ok()
1458 {
1459         add_context_lttng 0 -k "$@"
1460 }
1461
1462 function add_context_kernel_fail()
1463 {
1464         add_context_lttng 1 -k "$@"
1465 }
1466
1467 function trace_matches ()
1468 {
1469         local event_name=$1
1470         local nr_iter=$2
1471         local trace_path=$3
1472
1473         which $BABELTRACE_BIN >/dev/null
1474         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1475
1476         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1477
1478         if [ "$count" -ne "$nr_iter" ]; then
1479                 fail "Trace match"
1480                 diag "$count matching events found in trace"
1481         else
1482                 pass "Trace match"
1483         fi
1484 }
1485
1486 function trace_match_only()
1487 {
1488         local event_name=$1
1489         local nr_iter=$2
1490         local trace_path=$3
1491
1492         which $BABELTRACE_BIN >/dev/null
1493         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1494
1495         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1496         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1497
1498     if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1499         pass "Trace match with $total event $event_name"
1500     else
1501         fail "Trace match"
1502         diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1503     fi
1504 }
1505
1506 function validate_trace
1507 {
1508         local event_name=$1
1509         local trace_path=$2
1510
1511         which $BABELTRACE_BIN >/dev/null
1512         if [ $? -ne 0 ]; then
1513             skip 0 "Babeltrace binary not found. Skipping trace validation"
1514         fi
1515
1516         OLDIFS=$IFS
1517         IFS=","
1518         for i in $event_name; do
1519                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1520                 if [ "$traced" -ne 0 ]; then
1521                         pass "Validate trace for event $i, $traced events"
1522                 else
1523                         fail "Validate trace for event $i"
1524                         diag "Found $traced occurences of $i"
1525                 fi
1526         done
1527         ret=$?
1528         IFS=$OLDIFS
1529         return $ret
1530 }
1531
1532 function validate_trace_count
1533 {
1534         local event_name=$1
1535         local trace_path=$2
1536         local expected_count=$3
1537
1538         which $BABELTRACE_BIN >/dev/null
1539         if [ $? -ne 0 ]; then
1540             skip 0 "Babeltrace binary not found. Skipping trace validation"
1541         fi
1542
1543         cnt=0
1544         OLDIFS=$IFS
1545         IFS=","
1546         for i in $event_name; do
1547                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1548                 if [ "$traced" -ne 0 ]; then
1549                         pass "Validate trace for event $i, $traced events"
1550                 else
1551                         fail "Validate trace for event $i"
1552                         diag "Found $traced occurences of $i"
1553                 fi
1554                 cnt=$(($cnt + $traced))
1555         done
1556         IFS=$OLDIFS
1557         test $cnt -eq $expected_count
1558         ok $? "Read a total of $cnt events, expected $expected_count"
1559 }
1560
1561 function trace_first_line
1562 {
1563         local trace_path=$1
1564
1565         which $BABELTRACE_BIN >/dev/null
1566         if [ $? -ne 0 ]; then
1567             skip 0 "Babeltrace binary not found. Skipping trace validation"
1568         fi
1569
1570         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1571 }
1572
1573 function validate_trace_exp()
1574 {
1575         local event_exp=$1
1576         local trace_path=$2
1577
1578         which $BABELTRACE_BIN >/dev/null
1579         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1580
1581         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep ${event_exp} | wc -l)
1582         if [ "$traced" -ne 0 ]; then
1583                 pass "Validate trace for expression '${event_exp}', $traced events"
1584         else
1585                 fail "Validate trace for expression '${event_exp}'"
1586                 diag "Found $traced occurences of '${event_exp}'"
1587         fi
1588         ret=$?
1589         return $ret
1590 }
1591
1592 function validate_trace_only_exp()
1593 {
1594         local event_exp=$1
1595         local trace_path=$2
1596
1597         which $BABELTRACE_BIN >/dev/null
1598         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1599
1600         local count=$($BABELTRACE_BIN $trace_path | grep ${event_exp} | wc -l)
1601         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1602
1603         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1604                 pass "Trace match with $total for expression '${event_exp}"
1605         else
1606                 fail "Trace match"
1607                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1608         fi
1609         ret=$?
1610         return $ret
1611 }
1612
1613 function validate_trace_empty()
1614 {
1615         local trace_path=$1
1616
1617         which $BABELTRACE_BIN >/dev/null
1618         if [ $? -ne 0 ]; then
1619             skip 0 "Babeltrace binary not found. Skipping trace validation"
1620         fi
1621
1622         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1623         ret=$?
1624         if [ $ret -ne 0 ]; then
1625                 fail "Failed to parse trace"
1626                 return $ret
1627         fi
1628
1629         traced=$(echo -n "$events" | wc -l)
1630         if [ "$traced" -eq 0 ]; then
1631                 pass "Validate empty trace"
1632         else
1633                 fail "Validate empty trace"
1634                 diag "Found $traced events in trace"
1635         fi
1636         ret=$?
1637         return $ret
1638 }
1639
1640 function regenerate_metadata ()
1641 {
1642         local expected_to_fail=$1
1643         local sess_name=$2
1644
1645         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1646         ret=$?
1647         if [[ $expected_to_fail -eq "1" ]]; then
1648                 test "$ret" -ne "0"
1649                 ok $? "Expected fail on regenerate metadata $sess_name"
1650         else
1651                 ok $ret "Metadata regenerate $sess_name"
1652         fi
1653 }
1654
1655 function regenerate_metadata_ok ()
1656 {
1657         regenerate_metadata 0 "$@"
1658 }
1659
1660 function regenerate_metadata_fail ()
1661 {
1662         regenerate_metadata 1 "$@"
1663 }
1664
1665 function regenerate_statedump ()
1666 {
1667         local expected_to_fail=$1
1668         local sess_name=$2
1669
1670         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1671         ret=$?
1672         if [[ $expected_to_fail -eq "1" ]]; then
1673                 test "$ret" -ne "0"
1674                 ok $? "Expected fail on regenerate statedump $sess_name"
1675         else
1676                 ok $ret "Metadata regenerate $sess_name"
1677         fi
1678 }
1679
1680 function regenerate_statedump_ok ()
1681 {
1682         regenerate_statedump 0 "$@"
1683 }
1684
1685 function regenerate_statedump_fail ()
1686 {
1687         regenerate_statedump 1 "$@"
1688 }
1689
1690 function destructive_tests_enabled ()
1691 {
1692         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1693                 return 0
1694         else
1695                 return 1
1696         fi
1697 }