tests/utils/lttngtest/environment.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (C) 2022 Jérémie Galarneau <jeremie.galarneau@efficios.com>
   4 #
   5 # SPDX-License-Identifier: GPL-2.0-only
   6 #
   7
   8 from types import FrameType
   9 from typing import Callable, Iterator, Optional, Tuple, List
  10 import sys
  11 import pathlib
  12 import signal
  13 import subprocess
  14 import shlex
  15 import shutil
  16 import os
  17 import queue
  18 import tempfile
  19 from . import logger
  20 import time
  21 import threading
  22 import contextlib
  23
  24
  25 class TemporaryDirectory:
  26     def __init__(self, prefix):
  27         # type: (str) -> None
  28         self._directory_path = tempfile.mkdtemp(prefix=prefix)
  29
  30     def __del__(self):
  31         shutil.rmtree(self._directory_path, ignore_errors=True)
  32
  33     @property
  34     def path(self):
  35         # type: () -> pathlib.Path
  36         return pathlib.Path(self._directory_path)
  37
  38
  39 class _SignalWaitQueue:
  40     """
  41     Utility class useful to wait for a signal before proceeding.
  42
  43     Simply register the `signal` method as the handler for the signal you are
  44     interested in and call `wait_for_signal` to wait for its reception.
  45
  46     Registering a signal:
  47         signal.signal(signal.SIGWHATEVER, queue.signal)
  48
  49     Waiting for the signal:
  50         queue.wait_for_signal()
  51     """
  52
  53     def __init__(self):
  54         self._queue = queue.Queue()  # type: queue.Queue
  55
  56     def signal(
  57         self,
  58         signal_number,
  59         frame,  # type: Optional[FrameType]
  60     ):
  61         self._queue.put_nowait(signal_number)
  62
  63     def wait_for_signal(self):
  64         self._queue.get(block=True)
  65
  66
  67 class WaitTraceTestApplication:
  68     """
  69     Create an application that waits before tracing. This allows a test to
  70     launch an application, get its PID, and get it to start tracing when it
  71     has completed its setup.
  72     """
  73
  74     def __init__(
  75         self,
  76         binary_path,  # type: pathlib.Path
  77         event_count,  # type: int
  78         environment,  # type: Environment
  79         wait_time_between_events_us=0,  # type: int
  80     ):
  81         self._environment = environment  # type: Environment
  82         if event_count % 5:
  83             # The test application currently produces 5 different events per iteration.
  84             raise ValueError("event count must be a multiple of 5")
  85         self._iteration_count = int(event_count / 5)  # type: int
  86         # File that the application will wait to see before tracing its events.
  87         self._app_start_tracing_file_path = pathlib.Path(
  88             tempfile.mktemp(
  89                 prefix="app_",
  90                 suffix="_start_tracing",
  91                 dir=self._compat_open_path(environment.lttng_home_location),
  92             )
  93         )
  94         self._has_returned = False
  95
  96         test_app_env = os.environ.copy()
  97         test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location)
  98         # Make sure the app is blocked until it is properly registered to
  99         # the session daemon.
 100         test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
 101
 102         # File that the application will create to indicate it has completed its initialization.
 103         app_ready_file_path = tempfile.mktemp(
 104             prefix="app_",
 105             suffix="_ready",
 106             dir=self._compat_open_path(environment.lttng_home_location),
 107         )  # type: str
 108
 109         test_app_args = [str(binary_path)]
 110         test_app_args.extend(
 111             shlex.split(
 112                 "--iter {iteration_count} --create-in-main {app_ready_file_path} --wait-before-first-event {app_start_tracing_file_path} --wait {wait_time_between_events_us}".format(
 113                     iteration_count=self._iteration_count,
 114                     app_ready_file_path=app_ready_file_path,
 115                     app_start_tracing_file_path=self._app_start_tracing_file_path,
 116                     wait_time_between_events_us=wait_time_between_events_us,
 117                 )
 118             )
 119         )
 120
 121         self._process = subprocess.Popen(
 122             test_app_args,
 123             env=test_app_env,
 124         )  # type: subprocess.Popen
 125
 126         # Wait for the application to create the file indicating it has fully
 127         # initialized. Make sure the app hasn't crashed in order to not wait
 128         # forever.
 129         while True:
 130             if os.path.exists(app_ready_file_path):
 131                 break
 132
 133             if self._process.poll() is not None:
 134                 # Application has unexepectedly returned.
 135                 raise RuntimeError(
 136                     "Test application has unexepectedly returned during its initialization with return code `{return_code}`".format(
 137                         return_code=self._process.returncode
 138                     )
 139                 )
 140
 141             time.sleep(0.1)
 142
 143     def trace(self):
 144         # type: () -> None
 145         if self._process.poll() is not None:
 146             # Application has unexepectedly returned.
 147             raise RuntimeError(
 148                 "Test application has unexepectedly before tracing with return code `{return_code}`".format(
 149                     return_code=self._process.returncode
 150                 )
 151             )
 152         open(self._compat_open_path(self._app_start_tracing_file_path), mode="x")
 153
 154     def wait_for_exit(self):
 155         # type: () -> None
 156         if self._process.wait() != 0:
 157             raise RuntimeError(
 158                 "Test application has exit with return code `{return_code}`".format(
 159                     return_code=self._process.returncode
 160                 )
 161             )
 162         self._has_returned = True
 163
 164     @property
 165     def vpid(self):
 166         # type: () -> int
 167         return self._process.pid
 168
 169     @staticmethod
 170     def _compat_open_path(path):
 171         # type: (pathlib.Path) -> pathlib.Path | str
 172         """
 173         The builtin open() in python >= 3.6 expects a path-like object while
 174         prior versions expect a string or bytes object. Return the correct type
 175         based on the presence of the "__fspath__" attribute specified in PEP-519.
 176         """
 177         if hasattr(path, "__fspath__"):
 178             return path
 179         else:
 180             return str(path)
 181
 182     def __del__(self):
 183         if not self._has_returned:
 184             # This is potentially racy if the pid has been recycled. However,
 185             # we can't use pidfd_open since it is only available in python >= 3.9.
 186             self._process.kill()
 187             self._process.wait()
 188
 189
 190 class TraceTestApplication:
 191     """
 192     Create an application to trace.
 193     """
 194
 195     def __init__(self, binary_path, environment):
 196         # type: (pathlib.Path, Environment)
 197         self._environment = environment  # type: Environment
 198         self._has_returned = False
 199
 200         test_app_env = os.environ.copy()
 201         test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location)
 202         # Make sure the app is blocked until it is properly registered to
 203         # the session daemon.
 204         test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
 205
 206         test_app_args = [str(binary_path)]
 207
 208         self._process: subprocess.Popen = subprocess.Popen(
 209             test_app_args, env=test_app_env
 210         )
 211
 212     def wait_for_exit(self):
 213         # type: () -> None
 214         if self._process.wait() != 0:
 215             raise RuntimeError(
 216                 "Test application has exit with return code `{return_code}`".format(
 217                     return_code=self._process.returncode
 218                 )
 219             )
 220         self._has_returned = True
 221
 222     def __del__(self):
 223         if not self._has_returned:
 224             # This is potentially racy if the pid has been recycled. However,
 225             # we can't use pidfd_open since it is only available in python >= 3.9.
 226             self._process.kill()
 227             self._process.wait()
 228
 229
 230 class ProcessOutputConsumer(threading.Thread, logger._Logger):
 231     def __init__(
 232         self,
 233         process,  # type: subprocess.Popen
 234         name,  # type: str
 235         log,  # type: Callable[[str], None]
 236     ):
 237         threading.Thread.__init__(self)
 238         self._prefix = name
 239         logger._Logger.__init__(self, log)
 240         self._process = process
 241
 242     def run(self):
 243         # type: () -> None
 244         while self._process.poll() is None:
 245             assert self._process.stdout
 246             line = self._process.stdout.readline().decode("utf-8").replace("\n", "")
 247             if len(line) != 0:
 248                 self._log("{prefix}: {line}".format(prefix=self._prefix, line=line))
 249
 250
 251 # Generate a temporary environment in which to execute a test.
 252 class _Environment(logger._Logger):
 253     def __init__(
 254         self,
 255         with_sessiond,  # type: bool
 256         log=None,  # type: Optional[Callable[[str], None]]
 257     ):
 258         super().__init__(log)
 259         signal.signal(signal.SIGTERM, self._handle_termination_signal)
 260         signal.signal(signal.SIGINT, self._handle_termination_signal)
 261
 262         # Assumes the project's hierarchy to this file is:
 263         # tests/utils/python/this_file
 264         self._project_root = (
 265             pathlib.Path(__file__).absolute().parents[3]
 266         )  # type: pathlib.Path
 267         self._lttng_home = TemporaryDirectory(
 268             "lttng_test_env_home"
 269         )  # type: Optional[TemporaryDirectory]
 270
 271         self._sessiond = (
 272             self._launch_lttng_sessiond() if with_sessiond else None
 273         )  # type: Optional[subprocess.Popen[bytes]]
 274
 275     @property
 276     def lttng_home_location(self):
 277         # type: () -> pathlib.Path
 278         if self._lttng_home is None:
 279             raise RuntimeError("Attempt to access LTTng home after clean-up")
 280         return self._lttng_home.path
 281
 282     @property
 283     def lttng_client_path(self):
 284         # type: () -> pathlib.Path
 285         return self._project_root / "src" / "bin" / "lttng" / "lttng"
 286
 287     def create_temporary_directory(self, prefix=None):
 288         # type: (Optional[str]) -> pathlib.Path
 289         # Simply return a path that is contained within LTTNG_HOME; it will
 290         # be destroyed when the temporary home goes out of scope.
 291         assert self._lttng_home
 292         return pathlib.Path(
 293             tempfile.mkdtemp(
 294                 prefix="tmp" if prefix is None else prefix,
 295                 dir=str(self._lttng_home.path),
 296             )
 297         )
 298
 299     # Unpack a list of environment variables from a string
 300     # such as "HELLO=is_it ME='/you/are/looking/for'"
 301     @staticmethod
 302     def _unpack_env_vars(env_vars_string):
 303         # type: (str) -> List[Tuple[str, str]]
 304         unpacked_vars = []
 305         for var in shlex.split(env_vars_string):
 306             equal_position = var.find("=")
 307             # Must have an equal sign and not end with an equal sign
 308             if equal_position == -1 or equal_position == len(var) - 1:
 309                 raise ValueError(
 310                     "Invalid sessiond environment variable: `{}`".format(var)
 311                 )
 312
 313             var_name = var[0:equal_position]
 314             var_value = var[equal_position + 1 :]
 315             # Unquote any paths
 316             var_value = var_value.replace("'", "")
 317             var_value = var_value.replace('"', "")
 318             unpacked_vars.append((var_name, var_value))
 319
 320         return unpacked_vars
 321
 322     def _launch_lttng_sessiond(self):
 323         # type: () -> Optional[subprocess.Popen]
 324         is_64bits_host = sys.maxsize > 2**32
 325
 326         sessiond_path = (
 327             self._project_root / "src" / "bin" / "lttng-sessiond" / "lttng-sessiond"
 328         )
 329         consumerd_path_option_name = "--consumerd{bitness}-path".format(
 330             bitness="64" if is_64bits_host else "32"
 331         )
 332         consumerd_path = (
 333             self._project_root / "src" / "bin" / "lttng-consumerd" / "lttng-consumerd"
 334         )
 335
 336         no_sessiond_var = os.environ.get("TEST_NO_SESSIOND")
 337         if no_sessiond_var and no_sessiond_var == "1":
 338             # Run test without a session daemon; the user probably
 339             # intends to run one under gdb for example.
 340             return None
 341
 342         # Setup the session daemon's environment
 343         sessiond_env_vars = os.environ.get("LTTNG_SESSIOND_ENV_VARS")
 344         sessiond_env = os.environ.copy()
 345         if sessiond_env_vars:
 346             self._log("Additional lttng-sessiond environment variables:")
 347             additional_vars = self._unpack_env_vars(sessiond_env_vars)
 348             for var_name, var_value in additional_vars:
 349                 self._log("  {name}={value}".format(name=var_name, value=var_value))
 350                 sessiond_env[var_name] = var_value
 351
 352         sessiond_env["LTTNG_SESSION_CONFIG_XSD_PATH"] = str(
 353             self._project_root / "src" / "common"
 354         )
 355
 356         assert self._lttng_home is not None
 357         sessiond_env["LTTNG_HOME"] = str(self._lttng_home.path)
 358
 359         wait_queue = _SignalWaitQueue()
 360         signal.signal(signal.SIGUSR1, wait_queue.signal)
 361
 362         self._log(
 363             "Launching session daemon with LTTNG_HOME=`{home_dir}`".format(
 364                 home_dir=str(self._lttng_home.path)
 365             )
 366         )
 367         process = subprocess.Popen(
 368             [
 369                 str(sessiond_path),
 370                 consumerd_path_option_name,
 371                 str(consumerd_path),
 372                 "--sig-parent",
 373             ],
 374             stdout=subprocess.PIPE,
 375             stderr=subprocess.STDOUT,
 376             env=sessiond_env,
 377         )
 378
 379         if self._logging_function:
 380             self._sessiond_output_consumer = ProcessOutputConsumer(
 381                 process, "lttng-sessiond", self._logging_function
 382             )  # type: Optional[ProcessOutputConsumer]
 383             self._sessiond_output_consumer.daemon = True
 384             self._sessiond_output_consumer.start()
 385
 386         # Wait for SIGUSR1, indicating the sessiond is ready to proceed
 387         wait_queue.wait_for_signal()
 388         signal.signal(signal.SIGUSR1, wait_queue.signal)
 389
 390         return process
 391
 392     def _handle_termination_signal(self, signal_number, frame):
 393         # type: (int, Optional[FrameType]) -> None
 394         self._log(
 395             "Killed by {signal_name} signal, cleaning-up".format(
 396                 signal_name=signal.strsignal(signal_number)
 397             )
 398         )
 399         self._cleanup()
 400
 401     def launch_wait_trace_test_application(self, event_count):
 402         # type: (int) -> WaitTraceTestApplication
 403         """
 404         Launch an application that will wait before tracing `event_count` events.
 405         """
 406         return WaitTraceTestApplication(
 407             self._project_root
 408             / "tests"
 409             / "utils"
 410             / "testapp"
 411             / "gen-ust-nevents"
 412             / "gen-ust-nevents",
 413             event_count,
 414             self,
 415         )
 416
 417     def launch_trace_test_constructor_application(self):
 418         # type () -> TraceTestApplication
 419         """
 420         Launch an application that will trace from within constructors.
 421         """
 422         return TraceTestApplication(
 423             self._project_root
 424             / "tests"
 425             / "utils"
 426             / "testapp"
 427             / "gen-ust-events-constructor"
 428             / "gen-ust-events-constructor",
 429             self,
 430         )
 431
 432     # Clean-up managed processes
 433     def _cleanup(self):
 434         # type: () -> None
 435         if self._sessiond and self._sessiond.poll() is None:
 436             # The session daemon is alive; kill it.
 437             self._log(
 438                 "Killing session daemon (pid = {sessiond_pid})".format(
 439                     sessiond_pid=self._sessiond.pid
 440                 )
 441             )
 442
 443             self._sessiond.terminate()
 444             self._sessiond.wait()
 445             if self._sessiond_output_consumer:
 446                 self._sessiond_output_consumer.join()
 447                 self._sessiond_output_consumer = None
 448
 449             self._log("Session daemon killed")
 450             self._sessiond = None
 451
 452         self._lttng_home = None
 453
 454     def __del__(self):
 455         self._cleanup()
 456
 457
 458 @contextlib.contextmanager
 459 def test_environment(with_sessiond, log=None):
 460     # type: (bool, Optional[Callable[[str], None]]) -> Iterator[_Environment]
 461     env = _Environment(with_sessiond, log)
 462     try:
 463         yield env
 464     finally:
 465         env._cleanup()