lava: Use custom timeout function for make check

[lttng-ci.git] / scripts / system-tests / run-test-suites.sh

diff --git a/scripts/system-tests/run-test-suites.sh b/scripts/system-tests/run-test-suites.sh
index 86de721ff59e4b9260c1904f7d8935e552c7b0c3..62e956db6372dca20cbe66a0c5d905a9d0b8572c 100755 (executable)
--- a/scripts/system-tests/run-test-suites.sh
+++ b/scripts/system-tests/run-test-suites.sh
@@ -68,31 +68,93 @@ verne() {
     [ "$res" -ne "0" ]
 }
 
+# shellcheck disable=SC2317
+function cleanup
+{
+    timedatectl set-ntp true
+    # The false dates used in the tests are far in the past
+    # and it may take some time for the ntp update to actually
+    # happen.
+    # If the date is still in the past, it is possible that
+    # subsequent steps will fail (eg. TLS certificates cannot
+    # be validated).
+    while [[ "$(date +%Y)" -lt "2024" ]] ; do
+        sleep 1
+    done
+}
+
+function test_timeout
+{
+    local TIMEOUT=0
+    local TIMEOUT_MINUTES="${1:-90}"
+    shift 1
+    PID=''
+    "${@}" &
+    PID="${!}"
+    while true; do
+        sleep 1m
+        if ! ps -q "${PID}" > /dev/null ; then
+            # The process ID doesn't exist anymore
+            break
+        fi
+        TIMEOUT=$((TIMEOUT+1))
+        if [[ "${TIMEOUT}" -ge "${TIMEOUT_MINUTES}" ]]; then
+            echo "Command '${@}' timed out (${TIMEOUT} minutes) " \
+                 "attempting to get backtraces for lttng/babeltrace binaries"
+            apt-get install -y --force-yes gdb
+            # Abort all lttng-sessiond, lttng, lttng-relayd, lttng-consumerd,
+            # and babeltrace process so there are coredumps available.
+            PIDS=$(pgrep 'babeltrace*|[l]ttng*')
+            for P in ${PIDS}; do
+                OUTFILE=$(mktemp -t "backtrace-${P}.XXXXXX")
+                ps -f "${P}" | tee -a "${OUTFILE}"
+                gdb -p "${P}" --batch -ex 'thread apply all bt' 2>&1 | tee -a "${OUTFILE}"
+                mv "${OUTFILE}" /tmp/coredump/
+            done
+
+            # Send sigterm to make
+            kill "${PID}"
+
+            # Cleanup, to hopefully not interfere with future tests
+            apt-get purge -y gdb
+            apt-get autoremove -y
+        fi
+    done
+    wait "${PID}"
+    return "${?}"
+}
+
+trap cleanup EXIT SIGINT SIGTERM
+
 lttng_version="$1"
 failed_tests=0
 
-if [[ "$lttng_version" == "master" ]]; then
-       make --keep-going check || failed_tests=1
-       # TODO: remove when root regression tests are merged with make check or
-       # in another make command.
-       if [ -f "./tests/root_regression" ]; then
-               cd "./tests" || exit 1
-               prove --nocolor --verbose --merge --exec '' - < root_regression || failed_tests=1
-               cd ..
-       fi
-elif vergte "$lttng_version" "2.13"; then
-       # All root regression are now part of the make check
-       # *destructive* tests are now part of the `make`-based test suites.
-       export LTTNG_ENABLE_DESTRUCTIVE_TESTS="will-break-my-system"
-       make --keep-going check || failed_tests=1
-else
-       make --keep-going check || failed_tests=1
-       cd "./tests" || exit 1
-       prove --nocolor --verbose --merge --exec '' - < root_regression || failed_tests=1
-       cd ..
-fi
+export LTTNG_ENABLE_DESTRUCTIVE_TESTS="will-break-my-system"
+timedatectl set-ntp false
 
-exit $failed_tests
+# When make check is interrupted, the default test driver
+# (`config/test-driver`) will still delete the log and trs
+# files for the currently running test.
+test_timeout 90 make --keep-going check || failed_tests=1
+
+if [ -f "./tests/root_regression" ]; then
+    cd "./tests" || exit 1
+    prove --nocolor --verbose --merge --exec '' - < root_regression || failed_tests=2
+    cd ..
+fi
 
+# This script doesn't exist in master anymore, but compatibility with old branches
+# should be retained until lttng-tools 2.13 is no longer supported
+if [ -f "./tests/root_destructive_tests" ]; then
+    cd "./tests" || exit 1
+    prove --nocolor --verbose --merge --exec '' - < root_destructive_tests || failed_tests=3
+    cd ..
+else
+    echo 'root_destructive_tests not found'
+fi
 
+if [[ "${failed_tests}" != "0" ]] ; then
+    find tests/ -iname '*.trs' -print0 -or -iname '*.log' -print0 | tar czf /tmp/coredump/logs.tgz --null -T -
+fi
 
+exit $failed_tests