tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 RUNAS_BIN="lttng-runas"
  19 RUNAS_MATCH=".*lttng-runas.*"
  20 CONSUMERD_BIN="lttng-consumerd"
  21 CONSUMERD_MATCH=".*lttng-consumerd.*"
  22 RELAYD_BIN="lttng-relayd"
  23 RELAYD_MATCH=".*lttng-relayd.*"
  24 LTTNG_BIN="lttng"
  25 BABELTRACE_BIN="babeltrace"
  26 OUTPUT_DEST=/dev/null
  27 ERROR_OUTPUT_DEST=/dev/null
  28
  29 # Minimal kernel version supported for session daemon tests
  30 KERNEL_MAJOR_VERSION=2
  31 KERNEL_MINOR_VERSION=6
  32 KERNEL_PATCHLEVEL_VERSION=27
  33
  34 # We set the default UST register timeout and network and app socket timeout to
  35 # "wait forever", so that basic tests don't have to worry about hitting
  36 # timeouts on busy systems. Specialized tests should test those corner-cases.
  37 export LTTNG_UST_REGISTER_TIMEOUT=-1
  38 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  39 export LTTNG_APP_SOCKET_TIMEOUT=-1
  40
  41 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  42 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  43 # its own sessiond if none is running. It also ensures that 'lttng create'
  44 # fails when no sessiond is running.
  45 export LTTNG_SESSIOND_PATH="/bin/true"
  46
  47 source $TESTDIR/utils/tap/tap.sh
  48
  49 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  50         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  51 fi
  52
  53 function full_cleanup ()
  54 {
  55         # Try to kill daemons gracefully
  56         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # If daemons are still present, forcibly kill them
  60         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  61         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  62         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  63
  64         # Disable trap for SIGTERM since the following kill to the
  65         # pidgroup will be SIGTERM. Otherwise it loops.
  66         # The '-' before the pid number ($$) indicates 'kill' to signal the
  67         # whole process group.
  68         trap - SIGTERM && kill -- -$$
  69         exit 1
  70 }
  71
  72 function null_pipes ()
  73 {
  74         exec 0>/dev/null
  75         exec 1>/dev/null
  76         exec 2>/dev/null
  77 }
  78
  79 trap full_cleanup SIGINT SIGTERM
  80
  81 # perl prove closes its child pipes before giving it a chance to run its
  82 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
  83 # to allow those trap handlers to proceed.
  84
  85 trap null_pipes SIGPIPE
  86
  87 function print_ok ()
  88 {
  89         # Check if we are a terminal
  90         if [ -t 1 ]; then
  91                 echo -e "\e[1;32mOK\e[0m"
  92         else
  93                 echo -e "OK"
  94         fi
  95 }
  96
  97 function print_fail ()
  98 {
  99         # Check if we are a terminal
 100         if [ -t 1 ]; then
 101                 echo -e "\e[1;31mFAIL\e[0m"
 102         else
 103                 echo -e "FAIL"
 104         fi
 105 }
 106
 107 function print_test_banner ()
 108 {
 109         local desc="$1"
 110         diag "$desc"
 111 }
 112
 113 function validate_kernel_version ()
 114 {
 115         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 116         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 117                 return 0
 118         fi
 119         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 120                 return 0
 121         fi
 122         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 123                 return 0
 124         fi
 125         return 1
 126 }
 127
 128 # Generate a random string
 129 #  $1 = number of characters; defaults to 16
 130 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 131 function randstring()
 132 {
 133         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 134         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 135         echo
 136 }
 137
 138 # Return the number of _configured_ CPUs.
 139 function conf_proc_count()
 140 {
 141         getconf _NPROCESSORS_CONF
 142         if [ $? -ne 0 ]; then
 143                 diag "Failed to get the number of configured CPUs"
 144         fi
 145         echo
 146 }
 147
 148 # Check if base lttng-modules are present.
 149 # Bail out on failure
 150 function validate_lttng_modules_present ()
 151 {
 152         modprobe -n lttng-tracer 2>/dev/null
 153         if [ $? -ne 0  ]; then
 154                 BAIL_OUT "LTTng modules not detected."
 155         fi
 156 }
 157
 158 function enable_kernel_lttng_event
 159 {
 160         local withtap="$1"
 161         local expected_to_fail="$2"
 162         local sess_name="$3"
 163         local event_name="$4"
 164         local channel_name="$5"
 165
 166         if [ -z "$event_name" ]; then
 167                 # Enable all event if no event name specified
 168                 event_name="-a"
 169         fi
 170
 171         if [ -z "$channel_name" ]; then
 172                 # default channel if none specified
 173                 chan=""
 174         else
 175                 chan="-c $channel_name"
 176         fi
 177
 178         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 179         ret=$?
 180         if [[ $expected_to_fail -eq "1" ]]; then
 181                 test $ret -ne "0"
 182                 ret=$?
 183                 if [ $withtap -eq "1" ]; then
 184                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 185                 fi
 186         else
 187                 if [ $withtap -eq "1" ]; then
 188                         ok $ret "Enable kernel event $event_name for session $sess_name"
 189                 fi
 190         fi
 191 }
 192
 193 function enable_kernel_lttng_event_ok ()
 194 {
 195         enable_kernel_lttng_event 1 0 "$@"
 196 }
 197
 198 function enable_kernel_lttng_event_fail ()
 199 {
 200         enable_kernel_lttng_event 1 1 "$@"
 201 }
 202
 203 function enable_kernel_lttng_event_notap ()
 204 {
 205         enable_kernel_lttng_event 0 0 "$@"
 206 }
 207
 208 # Old interface
 209 function lttng_enable_kernel_event
 210 {
 211         enable_kernel_lttng_event_ok "$@"
 212 }
 213
 214 function lttng_enable_kernel_syscall()
 215 {
 216         local expected_to_fail=$1
 217         local sess_name=$2
 218         local syscall_name=$3
 219         local channel_name=$4
 220
 221         if [ -z $syscall_name ]; then
 222                 # Enable all event if no syscall name specified
 223                 syscall_name="-a"
 224         fi
 225
 226         if [ -z $channel_name ]; then
 227                 # default channel if none specified
 228                 chan=""
 229         else
 230                 chan="-c $channel_name"
 231         fi
 232
 233         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 234         ret=$?
 235         if [[ $expected_to_fail -eq "1" ]]; then
 236                 test $ret -ne "0"
 237                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 238         else
 239                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 240         fi
 241 }
 242
 243 function lttng_enable_kernel_syscall_ok()
 244 {
 245         lttng_enable_kernel_syscall 0 "$@"
 246 }
 247
 248 function lttng_enable_kernel_syscall_fail()
 249 {
 250         lttng_enable_kernel_syscall 1 "$@"
 251 }
 252
 253 function lttng_disable_kernel_syscall()
 254 {
 255         local expected_to_fail=$1
 256         local sess_name=$2
 257         local syscall_name=$3
 258         local channel_name=$4
 259
 260         if [ -z $syscall_name ]; then
 261                 # Enable all event if no syscall name specified
 262                 syscall_name="-a"
 263         fi
 264
 265         if [ -z $channel_name ]; then
 266                 # default channel if none specified
 267                 chan=""
 268         else
 269                 chan="-c $channel_name"
 270         fi
 271
 272         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 273
 274         ret=$?
 275         if [[ $expected_to_fail -eq "1" ]]; then
 276                 test $ret -ne "0"
 277                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 278         else
 279                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 280         fi
 281 }
 282
 283 function lttng_disable_kernel_syscall_ok()
 284 {
 285         lttng_disable_kernel_syscall 0 "$@"
 286 }
 287
 288 function lttng_disable_kernel_syscall_fail()
 289 {
 290         lttng_disable_kernel_syscall 1 "$@"
 291 }
 292
 293 function lttng_enable_kernel_channel()
 294 {
 295         local withtap=$1
 296         local expected_to_fail=$2
 297         local sess_name=$3
 298         local channel_name=$4
 299         local opt=$5
 300
 301         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 302         ret=$?
 303         if [[ $expected_to_fail -eq "1" ]]; then
 304                 test "$ret" -ne "0"
 305                 ret=$?
 306                 if [ $withtap -eq "1" ]; then
 307                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 308                 fi
 309         else
 310                 if [ $withtap -eq "1" ]; then
 311                         ok $ret "Enable channel $channel_name for session $sess_name"
 312                 fi
 313         fi
 314 }
 315
 316 function lttng_enable_kernel_channel_ok()
 317 {
 318         lttng_enable_kernel_channel 1 0 "$@"
 319 }
 320
 321 function lttng_enable_kernel_channel_fail()
 322 {
 323         lttng_enable_kernel_channel 1 1 "$@"
 324 }
 325
 326 function lttng_enable_kernel_channel_notap()
 327 {
 328         lttng_enable_kernel_channel 0 0 "$@"
 329 }
 330
 331 function enable_kernel_lttng_channel_ok()
 332 {
 333         lttng_enable_kernel_channel 1 0 "$@"
 334 }
 335
 336 function lttng_disable_kernel_channel()
 337 {
 338         local expected_to_fail=$1
 339         local sess_name=$2
 340         local channel_name=$3
 341
 342         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 343         ret=$?
 344         if [[ $expected_to_fail -eq "1" ]]; then
 345                 test "$ret" -ne "0"
 346                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 347         else
 348                 ok $ret "Disable channel $channel_name for session $sess_name"
 349         fi
 350 }
 351
 352 function lttng_disable_kernel_channel_ok()
 353 {
 354         lttng_disable_kernel_channel 0 "$@"
 355 }
 356
 357 function lttng_disable_kernel_channel_fail()
 358 {
 359         lttng_disable_kernel_channel 1 "$@"
 360 }
 361
 362 function start_lttng_relayd_opt()
 363 {
 364         local withtap=$1
 365         local opt=$2
 366
 367         DIR=$(readlink -f "$TESTDIR")
 368
 369         if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
 370                 # shellcheck disable=SC2086
 371                 "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 372                 #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
 373                 if [ $? -eq 1 ]; then
 374                         if [ "$withtap" -eq "1" ]; then
 375                                 fail "Start lttng-relayd (opt: $opt)"
 376                         fi
 377                         return 1
 378                 else
 379                         if [ "$withtap" -eq "1" ]; then
 380                                 pass "Start lttng-relayd (opt: $opt)"
 381                         fi
 382                 fi
 383         else
 384                 pass "Start lttng-relayd (opt: $opt)"
 385         fi
 386 }
 387
 388 function start_lttng_relayd()
 389 {
 390         start_lttng_relayd_opt 1 "$@"
 391 }
 392
 393 function start_lttng_relayd_notap()
 394 {
 395         start_lttng_relayd_opt 0 "$@"
 396 }
 397
 398 function stop_lttng_relayd_opt()
 399 {
 400         local withtap=$1
 401         local signal=$2
 402
 403         if [ -z "$signal" ]; then
 404                 signal="SIGTERM"
 405         fi
 406
 407         local timeout_s=$3
 408         local dtimeleft_s=
 409
 410         # Multiply time by 2 to simplify integer arithmetic
 411         if [ -n "$timeout_s" ]; then
 412                 dtimeleft_s=$((timeout_s * 2))
 413         fi
 414
 415         local retval=0
 416         local pids=
 417
 418         pids=$(pgrep "$RELAYD_MATCH")
 419         if [ -z "$pids" ]; then
 420                 if [ "$withtap" -eq "1" ]; then
 421                         pass "No relay daemon to kill"
 422                 fi
 423                 return 0
 424         fi
 425
 426         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 427
 428         # shellcheck disable=SC2086
 429         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 430                 retval=1
 431                 if [ "$withtap" -eq "1" ]; then
 432                         fail "Kill relay daemon"
 433                 fi
 434         else
 435                 out=1
 436                 while [ -n "$out" ]; do
 437                         out=$(pgrep "$RELAYD_MATCH")
 438                         if [ -n "$dtimeleft_s" ]; then
 439                                 if [ $dtimeleft_s -lt 0 ]; then
 440                                         out=
 441                                         retval=1
 442                                 fi
 443                                 dtimeleft_s=$((dtimeleft_s - 1))
 444                         fi
 445                         sleep 0.5
 446                 done
 447                 if [ "$withtap" -eq "1" ]; then
 448                         if [ "$retval" -eq "0" ]; then
 449                                 pass "Wait after kill relay daemon"
 450                         else
 451                                 fail "Wait after kill relay daemon"
 452                         fi
 453                 fi
 454         fi
 455         return $retval
 456 }
 457
 458 function stop_lttng_relayd()
 459 {
 460         stop_lttng_relayd_opt 1 "$@"
 461 }
 462
 463 function stop_lttng_relayd_notap()
 464 {
 465         stop_lttng_relayd_opt 0 "$@"
 466 }
 467
 468 #First arg: show tap output
 469 #Second argument: load path for automatic loading
 470 function start_lttng_sessiond_opt()
 471 {
 472         local withtap=$1
 473         local load_path=$2
 474
 475         local env_vars=""
 476         local consumerd=""
 477
 478         local long_bit_value=
 479         long_bit_value=$(getconf LONG_BIT)
 480
 481         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 482                 # Env variable requested no session daemon
 483                 return
 484         fi
 485
 486         DIR=$(readlink -f "$TESTDIR")
 487
 488         # Get long_bit value for 32/64 consumerd
 489         case "$long_bit_value" in
 490                 32)
 491                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 492                         ;;
 493                 64)
 494                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 495                         ;;
 496                 *)
 497                         return
 498                         ;;
 499         esac
 500
 501         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 502         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 503                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 504         fi
 505         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 506
 507         if ! validate_kernel_version; then
 508             fail "Start session daemon"
 509             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 510         fi
 511
 512         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 513         export LTTNG_SESSION_CONFIG_XSD_PATH
 514
 515         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 516                 # Have a load path ?
 517                 if [ -n "$load_path" ]; then
 518                         # shellcheck disable=SC2086
 519                         env $env_vars --load "$load_path" --background "$consumerd"
 520                 else
 521                         # shellcheck disable=SC2086
 522                         env $env_vars --background "$consumerd"
 523                 fi
 524                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 525                 status=$?
 526                 if [ "$withtap" -eq "1" ]; then
 527                         ok $status "Start session daemon"
 528                 fi
 529         fi
 530 }
 531
 532 function start_lttng_sessiond()
 533 {
 534         start_lttng_sessiond_opt 1 "$@"
 535 }
 536
 537 function start_lttng_sessiond_notap()
 538 {
 539         start_lttng_sessiond_opt 0 "$@"
 540 }
 541
 542 function stop_lttng_sessiond_opt()
 543 {
 544         local withtap=$1
 545         local signal=$2
 546
 547         if [ -z "$signal" ]; then
 548                 signal=SIGTERM
 549         fi
 550
 551         local timeout_s=$3
 552         local dtimeleft_s=
 553
 554         # Multiply time by 2 to simplify integer arithmetic
 555         if [ -n "$timeout_s" ]; then
 556                 dtimeleft_s=$((timeout_s * 2))
 557         fi
 558
 559         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 560                 # Env variable requested no session daemon
 561                 return 0
 562         fi
 563
 564         local retval=0
 565
 566         local runas_pids=
 567         runas_pids=$(pgrep "$RUNAS_MATCH")
 568
 569         local pids=
 570         pids=$(pgrep "$SESSIOND_MATCH")
 571
 572         if [ -n "$runas_pids" ]; then
 573                 pids="$pids $runas_pids"
 574         fi
 575
 576         if [ -z "$pids" ]; then
 577                 if [ "$withtap" -eq "1" ]; then
 578                         pass "No session daemon to kill"
 579                 fi
 580                 return 0
 581         fi
 582
 583         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 584
 585         # shellcheck disable=SC2086
 586         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 587                 retval=1
 588                 if [ "$withtap" -eq "1" ]; then
 589                         fail "Kill sessions daemon"
 590                 fi
 591         else
 592                 out=1
 593                 while [ -n "$out" ]; do
 594                         out=$(pgrep "${SESSIOND_MATCH}")
 595                         if [ -n "$dtimeleft_s" ]; then
 596                                 if [ $dtimeleft_s -lt 0 ]; then
 597                                         out=
 598                                         retval=1
 599                                 fi
 600                                 dtimeleft_s=$((dtimeleft_s - 1))
 601                         fi
 602                         sleep 0.5
 603                 done
 604                 out=1
 605                 while [ -n "$out" ]; do
 606                         out=$(pgrep "$CONSUMERD_MATCH")
 607                         if [ -n "$dtimeleft_s" ]; then
 608                                 if [ $dtimeleft_s -lt 0 ]; then
 609                                         out=
 610                                         retval=1
 611                                 fi
 612                                 dtimeleft_s=$((dtimeleft_s - 1))
 613                         fi
 614                         sleep 0.5
 615                 done
 616
 617                 if [ "$withtap" -eq "1" ]; then
 618                         if [ "$retval" -eq "0" ]; then
 619                                 pass "Wait after kill session daemon"
 620                         else
 621                                 fail "Wait after kill session daemon"
 622                         fi
 623                 fi
 624         fi
 625         if [ "$signal" = "SIGKILL" ]; then
 626                 if [ "$(id -u)" -eq "0" ]; then
 627                         local modules=
 628                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 629
 630                         if [ -n "$modules" ]; then
 631                                 diag "Unloading all LTTng modules"
 632                                 modprobe -r "$modules"
 633                         fi
 634                 fi
 635         fi
 636
 637         return $retval
 638 }
 639
 640 function stop_lttng_sessiond()
 641 {
 642         stop_lttng_sessiond_opt 1 "$@"
 643 }
 644
 645 function stop_lttng_sessiond_notap()
 646 {
 647         stop_lttng_sessiond_opt 0 "$@"
 648 }
 649
 650 function sigstop_lttng_sessiond_opt()
 651 {
 652         local withtap=$1
 653         local signal=SIGSTOP
 654
 655         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 656                 # Env variable requested no session daemon
 657                 return
 658         fi
 659
 660         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 661
 662         if [ "$withtap" -eq "1" ]; then
 663                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 664         fi
 665
 666         # shellcheck disable=SC2086
 667         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 668                 if [ "$withtap" -eq "1" ]; then
 669                         fail "Sending SIGSTOP to session daemon"
 670                 fi
 671         else
 672                 out=1
 673                 while [ $out -ne 0 ]; do
 674                         pid="$(pgrep "$SESSIOND_MATCH")"
 675
 676                         # Wait until state becomes stopped for session
 677                         # daemon(s).
 678                         out=0
 679                         for sessiond_pid in $pid; do
 680                                 state="$(ps -p "$sessiond_pid" -o state= )"
 681                                 if [[ -n "$state" && "$state" != "T" ]]; then
 682                                         out=1
 683                                 fi
 684                         done
 685                         sleep 0.5
 686                 done
 687                 if [ "$withtap" -eq "1" ]; then
 688                         pass "Sending SIGSTOP to session daemon"
 689                 fi
 690         fi
 691 }
 692
 693 function sigstop_lttng_sessiond()
 694 {
 695         sigstop_lttng_sessiond_opt 1 "$@"
 696 }
 697
 698 function sigstop_lttng_sessiond_notap()
 699 {
 700         sigstop_lttng_sessiond_opt 0 "$@"
 701 }
 702
 703 function stop_lttng_consumerd_opt()
 704 {
 705         local withtap=$1
 706         local signal=$2
 707
 708         if [ -z "$signal" ]; then
 709                 signal=SIGTERM
 710         fi
 711
 712         local timeout_s=$3
 713         local dtimeleft_s=
 714
 715         # Multiply time by 2 to simplify integer arithmetic
 716         if [ -n "$timeout_s" ]; then
 717                 dtimeleft_s=$((timeout_s * 2))
 718         fi
 719
 720         local retval=0
 721
 722         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 723
 724         if [ -z "$PID_CONSUMERD" ]; then
 725                 if [ "$withtap" -eq "1" ]; then
 726                         pass "No consumer daemon to kill"
 727                 fi
 728                 return 0
 729         fi
 730
 731         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 732
 733         # shellcheck disable=SC2086
 734         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 735                 retval=1
 736                 if [ "$withtap" -eq "1" ]; then
 737                         fail "Kill consumer daemon"
 738                 fi
 739         else
 740                 out=1
 741                 while [ $out -ne 0 ]; do
 742                         pid="$(pgrep "$CONSUMERD_MATCH")"
 743
 744                         # If consumerds are still present check their status.
 745                         # A zombie status qualifies the consumerd as *killed*
 746                         out=0
 747                         for consumer_pid in $pid; do
 748                                 state="$(ps -p "$consumer_pid" -o state= )"
 749                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 750                                         out=1
 751                                 fi
 752                         done
 753                         if [ -n "$dtimeleft_s" ]; then
 754                                 if [ $dtimeleft_s -lt 0 ]; then
 755                                         out=0
 756                                         retval=1
 757                                 fi
 758                                 dtimeleft_s=$((dtimeleft_s - 1))
 759                         fi
 760                         sleep 0.5
 761                 done
 762                 if [ "$withtap" -eq "1" ]; then
 763                         if [ "$retval" -eq "0" ]; then
 764                                 pass "Wait after kill consumer daemon"
 765                         else
 766                                 fail "Wait after kill consumer daemon"
 767                         fi
 768                 fi
 769         fi
 770
 771         return $retval
 772 }
 773
 774 function stop_lttng_consumerd()
 775 {
 776         stop_lttng_consumerd_opt 1 "$@"
 777 }
 778
 779 function stop_lttng_consumerd_notap()
 780 {
 781         stop_lttng_consumerd_opt 0 "$@"
 782 }
 783
 784 function sigstop_lttng_consumerd_opt()
 785 {
 786         local withtap=$1
 787         local signal=SIGSTOP
 788
 789         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 790
 791         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 792
 793         # shellcheck disable=SC2086
 794         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 795         retval=$?
 796
 797         if [ $retval -eq 1 ]; then
 798                 if [ "$withtap" -eq "1" ]; then
 799                         fail "Sending SIGSTOP to consumer daemon"
 800                 fi
 801                 return 1
 802         else
 803                 out=1
 804                 while [ $out -ne 0 ]; do
 805                         pid="$(pgrep "$CONSUMERD_MATCH")"
 806
 807                         # Wait until state becomes stopped for all
 808                         # consumers.
 809                         out=0
 810                         for consumer_pid in $pid; do
 811                                 state="$(ps -p "$consumer_pid" -o state= )"
 812                                 if [[ -n "$state" && "$state" != "T" ]]; then
 813                                         out=1
 814                                 fi
 815                         done
 816                         sleep 0.5
 817                 done
 818                 if [ "$withtap" -eq "1" ]; then
 819                         pass "Sending SIGSTOP to consumer daemon"
 820                 fi
 821         fi
 822         return $retval
 823 }
 824
 825 function sigstop_lttng_consumerd()
 826 {
 827         sigstop_lttng_consumerd_opt 1 "$@"
 828 }
 829
 830 function sigstop_lttng_consumerd_notap()
 831 {
 832         sigstop_lttng_consumerd_opt 0 "$@"
 833 }
 834
 835 function list_lttng_with_opts ()
 836 {
 837         local opts=$1
 838         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 839         ok $? "Lttng-tool list command with option $opts"
 840 }
 841
 842 function create_lttng_session_no_output ()
 843 {
 844         local sess_name=$1
 845
 846         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 847         ok $? "Create session $sess_name in no-output mode"
 848 }
 849
 850 function create_lttng_session ()
 851 {
 852         local withtap=$1
 853         local expected_to_fail=$2
 854         local sess_name=$3
 855         local trace_path=$4
 856         local opt=$5
 857
 858         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 859         ret=$?
 860         if [ $expected_to_fail -eq "1" ]; then
 861                 test "$ret" -ne "0"
 862                 ret=$?
 863                 if [ $withtap -eq "1" ]; then
 864                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 865                 fi
 866         else
 867                 if [ $withtap -eq "1" ]; then
 868                         ok $ret "Create session $sess_name in $trace_path"
 869                 fi
 870         fi
 871         return $ret
 872 }
 873
 874 function create_lttng_session_ok ()
 875 {
 876         create_lttng_session 1 0 "$@"
 877 }
 878
 879 function create_lttng_session_fail ()
 880 {
 881         create_lttng_session 1 1 "$@"
 882 }
 883
 884 function create_lttng_session_notap ()
 885 {
 886         create_lttng_session 0 0 "$@"
 887 }
 888
 889
 890 function enable_ust_lttng_channel ()
 891 {
 892         local withtap=$1
 893         local expected_to_fail=$2
 894         local sess_name=$3
 895         local channel_name=$4
 896         local opt=$5
 897
 898         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 899         ret=$?
 900         if [[ $expected_to_fail -eq "1" ]]; then
 901                 test "$ret" -ne "0"
 902                 ret=$?
 903                 if [ $withtap -eq "1" ]; then
 904                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 905                 fi
 906         else
 907                 if [ $withtap -eq "1" ]; then
 908                         ok $ret "Enable channel $channel_name for session $sess_name"
 909                 fi
 910         fi
 911         return $ret
 912 }
 913
 914 function enable_ust_lttng_channel_ok ()
 915 {
 916         enable_ust_lttng_channel 1 0 "$@"
 917 }
 918
 919 function enable_ust_lttng_channel_fail ()
 920 {
 921         enable_ust_lttng_channel 1 1 "$@"
 922 }
 923
 924 function enable_ust_lttng_channel_notap ()
 925 {
 926         enable_ust_lttng_channel 0 0 "$@"
 927 }
 928
 929 function disable_ust_lttng_channel()
 930 {
 931         local sess_name=$1
 932         local channel_name=$2
 933
 934         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 935         ok $? "Disable channel $channel_name for session $sess_name"
 936 }
 937
 938 function enable_lttng_mmap_overwrite_kernel_channel()
 939 {
 940         local sess_name=$1
 941         local channel_name=$2
 942
 943         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 944         ok $? "Enable channel $channel_name for session $sess_name"
 945 }
 946
 947 function enable_lttng_mmap_discard_small_kernel_channel()
 948 {
 949         local sess_name=$1
 950         local channel_name=$2
 951
 952         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 953         ok $? "Enable small discard channel $channel_name for session $sess_name"
 954 }
 955
 956 function enable_lttng_mmap_overwrite_small_kernel_channel()
 957 {
 958         local sess_name=$1
 959         local channel_name=$2
 960
 961         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 962         ok $? "Enable small discard channel $channel_name for session $sess_name"
 963 }
 964
 965 function enable_lttng_mmap_overwrite_ust_channel()
 966 {
 967         local sess_name=$1
 968         local channel_name=$2
 969
 970         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 971         ok $? "Enable channel $channel_name for session $sess_name"
 972 }
 973
 974 function enable_ust_lttng_event ()
 975 {
 976         local withtap=$1
 977         local expected_to_fail=$2
 978         local sess_name=$3
 979         local event_name="$4"
 980         local channel_name=$5
 981
 982         if [ -z $channel_name ]; then
 983                 # default channel if none specified
 984                 chan=""
 985         else
 986                 chan="-c $channel_name"
 987         fi
 988
 989         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 990         ret=$?
 991         if [[ $expected_to_fail -eq "1" ]]; then
 992                 test $ret -ne "0"
 993                 ret=$?
 994                 if [[ $withtap -eq "1" ]]; then
 995                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
 996                 fi
 997         else
 998                 if [[ $withtap -eq "1" ]]; then
 999                         ok $ret "Enable ust event $event_name for session $sess_name"
1000                 fi
1001         fi
1002         return $ret
1003 }
1004
1005 function enable_ust_lttng_event_ok ()
1006 {
1007         enable_ust_lttng_event 1 0 "$@"
1008 }
1009
1010 function enable_ust_lttng_event_fail ()
1011 {
1012         enable_ust_lttng_event 1 1 "$@"
1013 }
1014
1015 function enable_ust_lttng_event_notap ()
1016 {
1017         enable_ust_lttng_event 0 0 "$@"
1018 }
1019
1020 function enable_jul_lttng_event()
1021 {
1022         sess_name=$1
1023         event_name="$2"
1024         channel_name=$3
1025
1026         if [ -z $channel_name ]; then
1027                 # default channel if none specified
1028                 chan=""
1029         else
1030                 chan="-c $channel_name"
1031         fi
1032
1033         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1034         ok $? "Enable JUL event $event_name for session $sess_name"
1035 }
1036
1037 function enable_jul_lttng_event_loglevel()
1038 {
1039         local sess_name=$1
1040         local event_name="$2"
1041         local loglevel=$3
1042         local channel_name=$4
1043
1044         if [ -z $channel_name ]; then
1045                 # default channel if none specified
1046                 chan=""
1047         else
1048                 chan="-c $channel_name"
1049         fi
1050
1051         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1052         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1053 }
1054
1055 function enable_log4j_lttng_event()
1056 {
1057         sess_name=$1
1058         event_name="$2"
1059         channel_name=$3
1060
1061         if [ -z $channel_name ]; then
1062                 # default channel if none specified
1063                 chan=""
1064         else
1065                 chan="-c $channel_name"
1066         fi
1067
1068         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1069         ok $? "Enable LOG4J event $event_name for session $sess_name"
1070 }
1071
1072 function enable_log4j_lttng_event_loglevel()
1073 {
1074         local sess_name=$1
1075         local event_name="$2"
1076         local loglevel=$3
1077         local channel_name=$4
1078
1079         if [ -z $channel_name ]; then
1080                 # default channel if none specified
1081                 chan=""
1082         else
1083                 chan="-c $channel_name"
1084         fi
1085
1086         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1087         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1088 }
1089
1090 function enable_python_lttng_event()
1091 {
1092         sess_name=$1
1093         event_name="$2"
1094         channel_name=$3
1095
1096         if [ -z $channel_name ]; then
1097                 # default channel if none specified
1098                 chan=""
1099         else
1100                 chan="-c $channel_name"
1101         fi
1102
1103         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1104         ok $? "Enable Python event $event_name for session $sess_name"
1105 }
1106
1107 function enable_python_lttng_event_loglevel()
1108 {
1109         local sess_name=$1
1110         local event_name="$2"
1111         local loglevel=$3
1112         local channel_name=$4
1113
1114         if [ -z $channel_name ]; then
1115                 # default channel if none specified
1116                 chan=""
1117         else
1118                 chan="-c $channel_name"
1119         fi
1120
1121         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1122         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1123 }
1124
1125 function enable_ust_lttng_event_filter()
1126 {
1127         local sess_name="$1"
1128         local event_name="$2"
1129         local filter="$3"
1130
1131         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1132         ok $? "Enable event $event_name with filtering for session $sess_name"
1133 }
1134
1135 function enable_ust_lttng_event_loglevel()
1136 {
1137         local sess_name="$1"
1138         local event_name="$2"
1139         local loglevel="$3"
1140
1141         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1142         ok $? "Enable event $event_name with loglevel $loglevel"
1143 }
1144
1145 function enable_ust_lttng_event_loglevel_only()
1146 {
1147         local sess_name="$1"
1148         local event_name="$2"
1149         local loglevel="$3"
1150
1151         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1152         ok $? "Enable event $event_name with loglevel-only $loglevel"
1153 }
1154
1155 function disable_ust_lttng_event ()
1156 {
1157         local sess_name="$1"
1158         local event_name="$2"
1159         local channel_name="$3"
1160
1161         if [ -z $channel_name ]; then
1162                 # default channel if none specified
1163                 chan=""
1164         else
1165                 chan="-c $channel_name"
1166         fi
1167
1168         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1169         ok $? "Disable event $event_name for session $sess_name"
1170 }
1171
1172 function disable_jul_lttng_event ()
1173 {
1174         local sess_name="$1"
1175         local event_name="$2"
1176
1177         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1178         ok $? "Disable JUL event $event_name for session $sess_name"
1179 }
1180
1181 function disable_log4j_lttng_event ()
1182 {
1183         local sess_name="$1"
1184         local event_name="$2"
1185
1186         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1187         ok $? "Disable LOG4J event $event_name for session $sess_name"
1188 }
1189
1190 function disable_python_lttng_event ()
1191 {
1192         local sess_name="$1"
1193         local event_name="$2"
1194
1195         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1196         ok $? "Disable Python event $event_name for session $sess_name"
1197 }
1198
1199 function start_lttng_tracing ()
1200 {
1201         local expected_to_fail=$1
1202         local sess_name=$2
1203
1204         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1205         ret=$?
1206         if [[ $expected_to_fail -eq "1" ]]; then
1207                 test "$ret" -ne "0"
1208                 ok $? "Start tracing for session $sess_name failed as expected"
1209         else
1210                 ok $ret "Start tracing for session $sess_name"
1211         fi
1212 }
1213
1214 function start_lttng_tracing_ok ()
1215 {
1216         start_lttng_tracing 0 "$@"
1217 }
1218
1219 function start_lttng_tracing_fail ()
1220 {
1221         start_lttng_tracing 1 "$@"
1222 }
1223
1224 function stop_lttng_tracing ()
1225 {
1226         local expected_to_fail=$1
1227         local sess_name=$2
1228
1229         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1230         ret=$?
1231         if [[ $expected_to_fail -eq "1" ]]; then
1232                 test "$ret" -ne "0"
1233                 ok $? "Stop lttng tracing for session $sess_name failed as expected"
1234         else
1235                 ok $ret "Stop lttng tracing for session $sess_name"
1236         fi
1237 }
1238
1239 function stop_lttng_tracing_ok ()
1240 {
1241         stop_lttng_tracing 0 "$@"
1242 }
1243
1244 function stop_lttng_tracing_fail ()
1245 {
1246         stop_lttng_tracing 1 "$@"
1247 }
1248
1249 function destroy_lttng_session ()
1250 {
1251         local withtap=$1
1252         local expected_to_fail=$2
1253         local sess_name=$3
1254
1255         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1256         ret=$?
1257         if [[ $expected_to_fail -eq "1" ]]; then
1258                 test "$ret" -ne "0"
1259                 ret=$?
1260                 if [ $withtap -eq "1" ]; then
1261                         ok $ret "Destroy session $sess_name failed as expected"
1262                 fi
1263         else
1264                 if [ $withtap -eq "1" ]; then
1265                         ok $ret "Destroy session $sess_name"
1266                 fi
1267         fi
1268 }
1269
1270 function destroy_lttng_session_ok ()
1271 {
1272         destroy_lttng_session 1 0 "$@"
1273
1274 }
1275
1276 function destroy_lttng_session_fail ()
1277 {
1278         destroy_lttng_session 1 1 "$@"
1279 }
1280
1281 function destroy_lttng_session_notap ()
1282 {
1283         destroy_lttng_session 0 0 "$@"
1284 }
1285
1286 function destroy_lttng_sessions ()
1287 {
1288         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1289         ok $? "Destroy all lttng sessions"
1290 }
1291
1292 function lttng_snapshot_add_output ()
1293 {
1294         local expected_to_fail=$1
1295         local sess_name=$2
1296         local trace_path=$3
1297
1298         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name file://$trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1299         ret=$?
1300         if [[ $expected_to_fail -eq 1 ]]; then
1301                 test "$ret" -ne "0"
1302                 ok $? "Added snapshot output file://$trace_path failed as expected"
1303         else
1304                 ok $ret "Added snapshot output file://$trace_path"
1305         fi
1306 }
1307
1308 function lttng_snapshot_add_output_ok ()
1309 {
1310         lttng_snapshot_add_output 0 "$@"
1311 }
1312
1313 function lttng_snapshot_add_output_fail ()
1314 {
1315         lttng_snapshot_add_output 1 "$@"
1316 }
1317
1318 function lttng_snapshot_del_output ()
1319 {
1320         local expected_to_fail=$1
1321         local sess_name=$2
1322         local id=$3
1323
1324         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1325         ret=$?
1326         if [[ $expected_to_fail -eq "1" ]]; then
1327                 test "$ret" -ne "0"
1328                 ok $? "Deleted snapshot output id $id failed as expected"
1329         else
1330                 ok $ret "Deleted snapshot output id $id"
1331         fi
1332 }
1333
1334 function lttng_snapshot_del_output_ok ()
1335 {
1336         lttng_snapshot_del_output 0 "$@"
1337 }
1338
1339 function lttng_snapshot_del_output_fail ()
1340 {
1341         lttng_snapshot_del_output 1 "$@"
1342 }
1343
1344 function lttng_snapshot_record ()
1345 {
1346         local sess_name=$1
1347         local trace_path=$2
1348
1349         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1350         ok $? "Snapshot recorded"
1351 }
1352
1353 function lttng_snapshot_list ()
1354 {
1355         local sess_name=$1
1356         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1357         ok $? "Snapshot list"
1358 }
1359
1360 function lttng_save()
1361 {
1362         local sess_name=$1
1363         local opts=$2
1364
1365         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1366         ok $? "Session saved"
1367 }
1368
1369 function lttng_load()
1370 {
1371         local expected_to_fail=$1
1372         local opts=$2
1373
1374         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1375         ret=$?
1376         if [[ $expected_to_fail -eq "1" ]]; then
1377                 test $ret -ne "0"
1378                 ok $? "Load command failed as expected with opts: $opts"
1379         else
1380                 ok $ret "Load command with opts: $opts"
1381         fi
1382 }
1383
1384 function lttng_load_ok()
1385 {
1386         lttng_load 0 "$@"
1387 }
1388
1389 function lttng_load_fail()
1390 {
1391         lttng_load 1 "$@"
1392 }
1393
1394 function lttng_track()
1395 {
1396         local expected_to_fail=$1
1397         local opts=$2
1398         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1399         ret=$?
1400         if [[ $expected_to_fail -eq "1" ]]; then
1401                 test $ret -ne "0"
1402                 ok $? "Track command failed as expected with opts: $opts"
1403         else
1404                 ok $ret "Track command with opts: $opts"
1405         fi
1406 }
1407
1408 function lttng_track_ok()
1409 {
1410         lttng_track 0 "$@"
1411 }
1412
1413 function lttng_track_fail()
1414 {
1415         lttng_track 1 "$@"
1416 }
1417
1418 function lttng_untrack()
1419 {
1420         local expected_to_fail=$1
1421         local opts=$2
1422         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1423         ret=$?
1424         if [[ $expected_to_fail -eq "1" ]]; then
1425                 test $ret -ne "0"
1426                 ok $? "Untrack command failed as expected with opts: $opts"
1427         else
1428                 ok $ret "Untrack command with opts: $opts"
1429         fi
1430 }
1431
1432 function lttng_untrack_ok()
1433 {
1434         lttng_untrack 0 "$@"
1435 }
1436
1437 function lttng_untrack_fail()
1438 {
1439         lttng_untrack 1 "$@"
1440 }
1441
1442 function add_context_lttng()
1443 {
1444         local expected_to_fail="$1"
1445         local domain="$2"
1446         local session_name="$3"
1447         local channel_name="$4"
1448         local type="$5"
1449
1450         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1451         ret=$?
1452         if [[ $expected_to_fail -eq "1" ]]; then
1453                 test $ret -ne "0"
1454                 ok $? "Add context command failed as expected for type: $type"
1455         else
1456                 ok $ret "Add context command for type: $type"
1457         fi
1458 }
1459
1460 function add_context_ust_ok()
1461 {
1462         add_context_lttng 0 -u "$@"
1463 }
1464
1465 function add_context_ust_fail()
1466 {
1467         add_context_lttng 1 -u "$@"
1468 }
1469
1470 function add_context_kernel_ok()
1471 {
1472         add_context_lttng 0 -k "$@"
1473 }
1474
1475 function add_context_kernel_fail()
1476 {
1477         add_context_lttng 1 -k "$@"
1478 }
1479
1480 function trace_matches ()
1481 {
1482         local event_name=$1
1483         local nr_iter=$2
1484         local trace_path=$3
1485
1486         which $BABELTRACE_BIN >/dev/null
1487         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1488
1489         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1490
1491         if [ "$count" -ne "$nr_iter" ]; then
1492                 fail "Trace match"
1493                 diag "$count matching events found in trace"
1494         else
1495                 pass "Trace match"
1496         fi
1497 }
1498
1499 function trace_match_only()
1500 {
1501         local event_name=$1
1502         local nr_iter=$2
1503         local trace_path=$3
1504
1505         which $BABELTRACE_BIN >/dev/null
1506         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1507
1508         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1509         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1510
1511     if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1512         pass "Trace match with $total event $event_name"
1513     else
1514         fail "Trace match"
1515         diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1516     fi
1517 }
1518
1519 function validate_trace
1520 {
1521         local event_name=$1
1522         local trace_path=$2
1523
1524         which $BABELTRACE_BIN >/dev/null
1525         if [ $? -ne 0 ]; then
1526             skip 0 "Babeltrace binary not found. Skipping trace validation"
1527         fi
1528
1529         OLDIFS=$IFS
1530         IFS=","
1531         for i in $event_name; do
1532                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1533                 if [ "$traced" -ne 0 ]; then
1534                         pass "Validate trace for event $i, $traced events"
1535                 else
1536                         fail "Validate trace for event $i"
1537                         diag "Found $traced occurences of $i"
1538                 fi
1539         done
1540         ret=$?
1541         IFS=$OLDIFS
1542         return $ret
1543 }
1544
1545 function validate_trace_count
1546 {
1547         local event_name=$1
1548         local trace_path=$2
1549         local expected_count=$3
1550
1551         which $BABELTRACE_BIN >/dev/null
1552         if [ $? -ne 0 ]; then
1553             skip 0 "Babeltrace binary not found. Skipping trace validation"
1554         fi
1555
1556         cnt=0
1557         OLDIFS=$IFS
1558         IFS=","
1559         for i in $event_name; do
1560                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1561                 if [ "$traced" -ne 0 ]; then
1562                         pass "Validate trace for event $i, $traced events"
1563                 else
1564                         fail "Validate trace for event $i"
1565                         diag "Found $traced occurences of $i"
1566                 fi
1567                 cnt=$(($cnt + $traced))
1568         done
1569         IFS=$OLDIFS
1570         test $cnt -eq $expected_count
1571         ok $? "Read a total of $cnt events, expected $expected_count"
1572 }
1573
1574 function trace_first_line
1575 {
1576         local trace_path=$1
1577
1578         which $BABELTRACE_BIN >/dev/null
1579         if [ $? -ne 0 ]; then
1580             skip 0 "Babeltrace binary not found. Skipping trace validation"
1581         fi
1582
1583         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1584 }
1585
1586 function validate_trace_exp()
1587 {
1588         local event_exp=$1
1589         local trace_path=$2
1590
1591         which $BABELTRACE_BIN >/dev/null
1592         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1593
1594         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep ${event_exp} | wc -l)
1595         if [ "$traced" -ne 0 ]; then
1596                 pass "Validate trace for expression '${event_exp}', $traced events"
1597         else
1598                 fail "Validate trace for expression '${event_exp}'"
1599                 diag "Found $traced occurences of '${event_exp}'"
1600         fi
1601         ret=$?
1602         return $ret
1603 }
1604
1605 function validate_trace_only_exp()
1606 {
1607         local event_exp=$1
1608         local trace_path=$2
1609
1610         which $BABELTRACE_BIN >/dev/null
1611         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1612
1613         local count=$($BABELTRACE_BIN $trace_path | grep ${event_exp} | wc -l)
1614         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1615
1616         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1617                 pass "Trace match with $total for expression '${event_exp}"
1618         else
1619                 fail "Trace match"
1620                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1621         fi
1622         ret=$?
1623         return $ret
1624 }
1625
1626 function validate_trace_empty()
1627 {
1628         local trace_path=$1
1629
1630         which $BABELTRACE_BIN >/dev/null
1631         if [ $? -ne 0 ]; then
1632             skip 0 "Babeltrace binary not found. Skipping trace validation"
1633         fi
1634
1635         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1636         ret=$?
1637         if [ $ret -ne 0 ]; then
1638                 fail "Failed to parse trace"
1639                 return $ret
1640         fi
1641
1642         traced=$(echo -n "$events" | wc -l)
1643         if [ "$traced" -eq 0 ]; then
1644                 pass "Validate empty trace"
1645         else
1646                 fail "Validate empty trace"
1647                 diag "Found $traced events in trace"
1648         fi
1649         ret=$?
1650         return $ret
1651 }
1652
1653 function regenerate_metadata ()
1654 {
1655         local expected_to_fail=$1
1656         local sess_name=$2
1657
1658         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1659         ret=$?
1660         if [[ $expected_to_fail -eq "1" ]]; then
1661                 test "$ret" -ne "0"
1662                 ok $? "Expected fail on regenerate metadata $sess_name"
1663         else
1664                 ok $ret "Metadata regenerate $sess_name"
1665         fi
1666 }
1667
1668 function regenerate_metadata_ok ()
1669 {
1670         regenerate_metadata 0 "$@"
1671 }
1672
1673 function regenerate_metadata_fail ()
1674 {
1675         regenerate_metadata 1 "$@"
1676 }
1677
1678 function regenerate_statedump ()
1679 {
1680         local expected_to_fail=$1
1681         local sess_name=$2
1682
1683         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1684         ret=$?
1685         if [[ $expected_to_fail -eq "1" ]]; then
1686                 test "$ret" -ne "0"
1687                 ok $? "Expected fail on regenerate statedump $sess_name"
1688         else
1689                 ok $ret "Statedump regenerate $sess_name"
1690         fi
1691 }
1692
1693 function regenerate_statedump_ok ()
1694 {
1695         regenerate_statedump 0 "$@"
1696 }
1697
1698 function regenerate_statedump_fail ()
1699 {
1700         regenerate_statedump 1 "$@"
1701 }
1702
1703 function destructive_tests_enabled ()
1704 {
1705         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1706                 return 0
1707         else
1708                 return 1
1709         fi
1710 }