tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 UST_EVENT_NAME="tp:tptest"
  19 KERNEL_EVENT_NAME="sched_switch"
  20 CHANNEL_NAME="testchan"
  21 HEALTH_CHECK_BIN="health_check"
  22 NUM_TESTS=90
  23 SLEEP_TIME=30
  24
  25 source $TESTDIR/utils/utils.sh
  26
  27 function lttng_create_session_uri
  28 {
  29         # Create session with default path
  30         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $SESSION_NAME \
  31                 -U net://localhost >/dev/null 2>&1
  32         ok $? "Create session with default path"
  33 }
  34
  35 function report_errors
  36 {
  37         test_thread_error_string="$1"
  38         test_relayd="$2"
  39         err_no_relayd_match="Error querying relayd health"
  40
  41         # Check for health errors
  42         # Include inability to contact relayd health as an expected
  43         # error, since this can happen whenever the relayd shutdown due
  44         # to an error in any thread.
  45         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  46         if [ $test_relayd -ne 0 ]; then
  47                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  48         else
  49                 outerr=0
  50         fi
  51         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  52                 fail "Validation failure"
  53                 diag "Health returned:"
  54                 diag "stdout:"
  55                 file=${STDOUT_PATH}
  56                 while read line ; do
  57                     diag "$line"
  58                 done < ${file}
  59
  60                 diag "stderr:"
  61                 file=${STDERR_PATH}
  62                 while read line ; do
  63                     diag "$line"
  64                 done < ${file}
  65         else
  66                 pass "Validation OK"
  67         fi
  68 }
  69
  70 function test_health
  71 {
  72         test_suffix="$1"
  73         test_thread_name="$2"
  74         test_thread_error_string="$3"
  75         test_needs_root="$4"
  76         test_consumerd="$5"
  77         test_relayd="$6"
  78
  79         diag "Test health problem detection with ${test_thread_name}"
  80
  81         # Set the socket timeout to 5 so the health check detection
  82         # happens within 25 s
  83         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  84         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  85
  86         # Activate testpoints
  87         export LTTNG_TESTPOINT_ENABLE=1
  88
  89         # Activate specific thread test
  90         export ${test_thread_name}_${test_suffix}=1
  91
  92         # Spawn sessiond with preload healthexit lib
  93         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  94
  95         diag "Start session daemon"
  96         start_lttng_sessiond
  97
  98         if [ ${test_consumerd} -eq 1 ]; then
  99                 create_lttng_session_no_output $SESSION_NAME
 100
 101                 diag "With UST consumer daemons"
 102                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
 103
 104                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
 105                 {
 106                         diag "With kernel consumer daemon"
 107                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 108                 }
 109                 start_lttng_tracing_ok $SESSION_NAME
 110         fi
 111
 112         if [ ${test_relayd} -eq 1 ]; then
 113                 diag "With relay daemon"
 114                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 115
 116                 start_lttng_relayd "-o $TRACE_PATH"
 117         else
 118                 RELAYD_ARGS=
 119         fi
 120
 121         # Check health status, not caring about result
 122         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 123                 > /dev/null
 124
 125         # Wait
 126         diag "Check after running for ${SLEEP_TIME} seconds"
 127         sleep ${SLEEP_TIME}
 128
 129         # Check health status
 130         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 131                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 132
 133
 134         if [ ${test_needs_root} -eq 1 ]; then
 135                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 136                 {
 137                         report_errors "${test_thread_error_string}" "${test_relayd}"
 138                 }
 139         else
 140                 report_errors "${test_thread_error_string}" "${test_relayd}"
 141         fi
 142
 143         if [ ${test_relayd} -eq 1 ]; then
 144                 # We may fail to stop relayd here, and this is OK, since
 145                 # it may have been killed volountarily by testpoint.
 146                 stop_lttng_relayd_notap
 147         fi
 148         stop_lttng_sessiond
 149
 150         unset LTTNG_TESTPOINT_ENABLE
 151         unset ${test_thread_name}_${test_suffix}
 152         unset LD_PRELOAD
 153         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 154         unset LTTNG_RELAYD_HEALTH
 155 }
 156
 157 plan_tests $NUM_TESTS
 158
 159 print_test_banner "$TEST_DESC"
 160
 161 if [ -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
 162         foundobj=1
 163 else
 164         foundobj=0
 165 fi
 166
 167 skip $foundobj "No shared object generated. Skipping all tests." $NUM_TESTS && exit 0
 168
 169 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 170         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 171         "LTTNG_SESSIOND_THREAD_REG_APPS"
 172         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 173         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 174         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 175         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 176
 177         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 178         "LTTNG_CONSUMERD_THREAD_METADATA"
 179         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 180
 181         "LTTNG_RELAYD_THREAD_DISPATCHER"
 182         "LTTNG_RELAYD_THREAD_WORKER"
 183         "LTTNG_RELAYD_THREAD_LISTENER"
 184         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 185         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 186         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 187 )
 188
 189 ERROR_STRING=(
 190         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 191         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 192         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 193         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 194         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 195         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 196         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 197
 198         "Thread \"Consumer daemon channel\" is not responding"
 199         "Thread \"Consumer daemon metadata\" is not responding"
 200         "Thread \"Consumer daemon metadata timer\" is not responding"
 201
 202         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 203         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 204         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 205         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 206         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 207         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 208 )
 209
 210 # TODO
 211 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 212 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 213
 214 # TODO: test kernel consumerd specifically in addition to UST consumerd
 215
 216 # TODO: need refactoring of consumerd teardown
 217 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 218 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 219
 220 # TODO: this thread is responsible for close a file descriptor that
 221 # triggers teardown of metadata thread. We should revisit teardown of
 222 # consumerd.
 223 # "LTTNG_CONSUMERD_THREAD_DATA"
 224 # "Thread \"Consumer daemon data\" is not responding"
 225
 226 NEEDS_ROOT=(
 227         0
 228         0
 229         0
 230         0
 231         0
 232         0
 233         1
 234
 235         0
 236         0
 237         0
 238
 239         0
 240         0
 241         0
 242         0
 243         0
 244         0
 245 )
 246
 247 TEST_CONSUMERD=(
 248         0
 249         0
 250         0
 251         0
 252         0
 253         0
 254         0
 255
 256         1
 257         1
 258         1
 259
 260         1
 261         1
 262         1
 263         1
 264         1
 265         1
 266 )
 267
 268 TEST_RELAYD=(
 269         0
 270         0
 271         0
 272         0
 273         0
 274         0
 275         0
 276
 277         0
 278         0
 279         0
 280
 281         1
 282         1
 283         1
 284         1
 285         1
 286         1
 287 )
 288
 289 STDOUT_PATH=$(mktemp)
 290 STDERR_PATH=$(mktemp)
 291 TRACE_PATH=$(mktemp -d)
 292 HEALTH_PATH=$(mktemp -d)
 293
 294 if [ "$(id -u)" == "0" ]; then
 295         isroot=1
 296 else
 297         isroot=0
 298 fi
 299
 300 THREAD_COUNT=${#THREAD[@]}
 301 i=0
 302 while [ "$i" -lt "$THREAD_COUNT" ]; do
 303         test_health "${TEST_SUFFIX}" \
 304                 "${THREAD[$i]}" \
 305                 "${ERROR_STRING[$i]}" \
 306                 "${NEEDS_ROOT[$i]}" \
 307                 "${TEST_CONSUMERD[$i]}" \
 308                 "${TEST_RELAYD[$i]}"
 309         let "i++"
 310 done
 311
 312 rm -rf ${HEALTH_PATH}
 313 rm -rf ${TRACE_PATH}
 314 rm -f ${STDOUT_PATH}
 315 rm -f ${STDERR_PATH}