tests/utils/lttngtest/environment.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (C) 2022 Jérémie Galarneau <jeremie.galarneau@efficios.com>
   4 #
   5 # SPDX-License-Identifier: GPL-2.0-only
   6 #
   7
   8 from types import FrameType
   9 from typing import Callable, Iterator, Optional, Tuple, List
  10 import sys
  11 import pathlib
  12 import signal
  13 import subprocess
  14 import shlex
  15 import shutil
  16 import os
  17 import queue
  18 import tempfile
  19 from . import logger
  20 import time
  21 import threading
  22 import contextlib
  23
  24
  25 class TemporaryDirectory:
  26     def __init__(self, prefix):
  27         # type: (str) -> None
  28         self._directory_path = tempfile.mkdtemp(prefix=prefix)
  29
  30     def __del__(self):
  31         shutil.rmtree(self._directory_path, ignore_errors=True)
  32
  33     @property
  34     def path(self):
  35         # type: () -> pathlib.Path
  36         return pathlib.Path(self._directory_path)
  37
  38
  39 class _SignalWaitQueue:
  40     """
  41     Utility class useful to wait for a signal before proceeding.
  42
  43     Simply register the `signal` method as the handler for the signal you are
  44     interested in and call `wait_for_signal` to wait for its reception.
  45
  46     Registering a signal:
  47         signal.signal(signal.SIGWHATEVER, queue.signal)
  48
  49     Waiting for the signal:
  50         queue.wait_for_signal()
  51     """
  52
  53     def __init__(self):
  54         self._queue = queue.Queue()  # type: queue.Queue
  55
  56     def signal(
  57         self,
  58         signal_number,
  59         frame,  # type: Optional[FrameType]
  60     ):
  61         self._queue.put_nowait(signal_number)
  62
  63     def wait_for_signal(self):
  64         self._queue.get(block=True)
  65
  66
  67 class WaitTraceTestApplication:
  68     """
  69     Create an application that waits before tracing. This allows a test to
  70     launch an application, get its PID, and get it to start tracing when it
  71     has completed its setup.
  72     """
  73
  74     def __init__(
  75         self,
  76         binary_path,  # type: pathlib.Path
  77         event_count,  # type: int
  78         environment,  # type: Environment
  79         wait_time_between_events_us=0,  # type: int
  80     ):
  81         self._environment = environment  # type: Environment
  82         if event_count % 5:
  83             # The test application currently produces 5 different events per iteration.
  84             raise ValueError("event count must be a multiple of 5")
  85         self._iteration_count = int(event_count / 5)  # type: int
  86         # File that the application will wait to see before tracing its events.
  87         self._app_start_tracing_file_path = pathlib.Path(
  88             tempfile.mktemp(
  89                 prefix="app_",
  90                 suffix="_start_tracing",
  91                 dir=self._compat_open_path(environment.lttng_home_location),
  92             )
  93         )
  94         self._has_returned = False
  95
  96         test_app_env = os.environ.copy()
  97         test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location)
  98         # Make sure the app is blocked until it is properly registered to
  99         # the session daemon.
 100         test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
 101
 102         # File that the application will create to indicate it has completed its initialization.
 103         app_ready_file_path = tempfile.mktemp(
 104             prefix="app_",
 105             suffix="_ready",
 106             dir=self._compat_open_path(environment.lttng_home_location),
 107         )  # type: str
 108
 109         test_app_args = [str(binary_path)]
 110         test_app_args.extend(
 111             shlex.split(
 112                 "--iter {iteration_count} --create-in-main {app_ready_file_path} --wait-before-first-event {app_start_tracing_file_path} --wait {wait_time_between_events_us}".format(
 113                     iteration_count=self._iteration_count,
 114                     app_ready_file_path=app_ready_file_path,
 115                     app_start_tracing_file_path=self._app_start_tracing_file_path,
 116                     wait_time_between_events_us=wait_time_between_events_us,
 117                 )
 118             )
 119         )
 120
 121         self._process = subprocess.Popen(
 122             test_app_args,
 123             env=test_app_env,
 124         )  # type: subprocess.Popen
 125
 126         # Wait for the application to create the file indicating it has fully
 127         # initialized. Make sure the app hasn't crashed in order to not wait
 128         # forever.
 129         while True:
 130             if os.path.exists(app_ready_file_path):
 131                 break
 132
 133             if self._process.poll() is not None:
 134                 # Application has unexepectedly returned.
 135                 raise RuntimeError(
 136                     "Test application has unexepectedly returned during its initialization with return code `{return_code}`".format(
 137                         return_code=self._process.returncode
 138                     )
 139                 )
 140
 141             time.sleep(0.1)
 142
 143     def trace(self):
 144         # type: () -> None
 145         if self._process.poll() is not None:
 146             # Application has unexepectedly returned.
 147             raise RuntimeError(
 148                 "Test application has unexepectedly before tracing with return code `{return_code}`".format(
 149                     return_code=self._process.returncode
 150                 )
 151             )
 152         open(self._compat_open_path(self._app_start_tracing_file_path), mode="x")
 153
 154     def wait_for_exit(self):
 155         # type: () -> None
 156         if self._process.wait() != 0:
 157             raise RuntimeError(
 158                 "Test application has exit with return code `{return_code}`".format(
 159                     return_code=self._process.returncode
 160                 )
 161             )
 162         self._has_returned = True
 163
 164     @property
 165     def vpid(self):
 166         # type: () -> int
 167         return self._process.pid
 168
 169     @staticmethod
 170     def _compat_open_path(path):
 171         # type: (pathlib.Path) -> pathlib.Path | str
 172         """
 173         The builtin open() in python >= 3.6 expects a path-like object while
 174         prior versions expect a string or bytes object. Return the correct type
 175         based on the presence of the "__fspath__" attribute specified in PEP-519.
 176         """
 177         if hasattr(path, "__fspath__"):
 178             return path
 179         else:
 180             return str(path)
 181
 182     def __del__(self):
 183         if not self._has_returned:
 184             # This is potentially racy if the pid has been recycled. However,
 185             # we can't use pidfd_open since it is only available in python >= 3.9.
 186             self._process.kill()
 187             self._process.wait()
 188
 189
 190 class TraceTestApplication:
 191     """
 192     Create an application to trace.
 193     """
 194
 195     def __init__(self, binary_path: pathlib.Path, environment: "Environment"):
 196         self._environment: Environment = environment
 197         self._has_returned = False
 198
 199         test_app_env = os.environ.copy()
 200         test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location)
 201         # Make sure the app is blocked until it is properly registered to
 202         # the session daemon.
 203         test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
 204
 205         test_app_args = [str(binary_path)]
 206
 207         self._process: subprocess.Popen = subprocess.Popen(
 208             test_app_args, env=test_app_env
 209         )
 210
 211     def wait_for_exit(self) -> None:
 212         if self._process.wait() != 0:
 213             raise RuntimeError(
 214                 "Test application has exit with return code `{return_code}`".format(
 215                     return_code=self._process.returncode
 216                 )
 217             )
 218         self._has_returned = True
 219
 220     def __del__(self):
 221         if not self._has_returned:
 222             # This is potentially racy if the pid has been recycled. However,
 223             # we can't use pidfd_open since it is only available in python >= 3.9.
 224             self._process.kill()
 225             self._process.wait()
 226
 227
 228 class ProcessOutputConsumer(threading.Thread, logger._Logger):
 229     def __init__(
 230         self,
 231         process,  # type: subprocess.Popen
 232         name,  # type: str
 233         log,  # type: Callable[[str], None]
 234     ):
 235         threading.Thread.__init__(self)
 236         self._prefix = name
 237         logger._Logger.__init__(self, log)
 238         self._process = process
 239
 240     def run(self):
 241         # type: () -> None
 242         while self._process.poll() is None:
 243             assert self._process.stdout
 244             line = self._process.stdout.readline().decode("utf-8").replace("\n", "")
 245             if len(line) != 0:
 246                 self._log("{prefix}: {line}".format(prefix=self._prefix, line=line))
 247
 248
 249 # Generate a temporary environment in which to execute a test.
 250 class _Environment(logger._Logger):
 251     def __init__(
 252         self,
 253         with_sessiond,  # type: bool
 254         log=None,  # type: Optional[Callable[[str], None]]
 255     ):
 256         super().__init__(log)
 257         signal.signal(signal.SIGTERM, self._handle_termination_signal)
 258         signal.signal(signal.SIGINT, self._handle_termination_signal)
 259
 260         # Assumes the project's hierarchy to this file is:
 261         # tests/utils/python/this_file
 262         self._project_root = (
 263             pathlib.Path(__file__).absolute().parents[3]
 264         )  # type: pathlib.Path
 265         self._lttng_home = TemporaryDirectory(
 266             "lttng_test_env_home"
 267         )  # type: Optional[TemporaryDirectory]
 268
 269         self._sessiond = (
 270             self._launch_lttng_sessiond() if with_sessiond else None
 271         )  # type: Optional[subprocess.Popen[bytes]]
 272
 273     @property
 274     def lttng_home_location(self):
 275         # type: () -> pathlib.Path
 276         if self._lttng_home is None:
 277             raise RuntimeError("Attempt to access LTTng home after clean-up")
 278         return self._lttng_home.path
 279
 280     @property
 281     def lttng_client_path(self):
 282         # type: () -> pathlib.Path
 283         return self._project_root / "src" / "bin" / "lttng" / "lttng"
 284
 285     def create_temporary_directory(self, prefix=None):
 286         # type: (Optional[str]) -> pathlib.Path
 287         # Simply return a path that is contained within LTTNG_HOME; it will
 288         # be destroyed when the temporary home goes out of scope.
 289         assert self._lttng_home
 290         return pathlib.Path(
 291             tempfile.mkdtemp(
 292                 prefix="tmp" if prefix is None else prefix,
 293                 dir=str(self._lttng_home.path),
 294             )
 295         )
 296
 297     # Unpack a list of environment variables from a string
 298     # such as "HELLO=is_it ME='/you/are/looking/for'"
 299     @staticmethod
 300     def _unpack_env_vars(env_vars_string):
 301         # type: (str) -> List[Tuple[str, str]]
 302         unpacked_vars = []
 303         for var in shlex.split(env_vars_string):
 304             equal_position = var.find("=")
 305             # Must have an equal sign and not end with an equal sign
 306             if equal_position == -1 or equal_position == len(var) - 1:
 307                 raise ValueError(
 308                     "Invalid sessiond environment variable: `{}`".format(var)
 309                 )
 310
 311             var_name = var[0:equal_position]
 312             var_value = var[equal_position + 1 :]
 313             # Unquote any paths
 314             var_value = var_value.replace("'", "")
 315             var_value = var_value.replace('"', "")
 316             unpacked_vars.append((var_name, var_value))
 317
 318         return unpacked_vars
 319
 320     def _launch_lttng_sessiond(self):
 321         # type: () -> Optional[subprocess.Popen]
 322         is_64bits_host = sys.maxsize > 2**32
 323
 324         sessiond_path = (
 325             self._project_root / "src" / "bin" / "lttng-sessiond" / "lttng-sessiond"
 326         )
 327         consumerd_path_option_name = "--consumerd{bitness}-path".format(
 328             bitness="64" if is_64bits_host else "32"
 329         )
 330         consumerd_path = (
 331             self._project_root / "src" / "bin" / "lttng-consumerd" / "lttng-consumerd"
 332         )
 333
 334         no_sessiond_var = os.environ.get("TEST_NO_SESSIOND")
 335         if no_sessiond_var and no_sessiond_var == "1":
 336             # Run test without a session daemon; the user probably
 337             # intends to run one under gdb for example.
 338             return None
 339
 340         # Setup the session daemon's environment
 341         sessiond_env_vars = os.environ.get("LTTNG_SESSIOND_ENV_VARS")
 342         sessiond_env = os.environ.copy()
 343         if sessiond_env_vars:
 344             self._log("Additional lttng-sessiond environment variables:")
 345             additional_vars = self._unpack_env_vars(sessiond_env_vars)
 346             for var_name, var_value in additional_vars:
 347                 self._log("  {name}={value}".format(name=var_name, value=var_value))
 348                 sessiond_env[var_name] = var_value
 349
 350         sessiond_env["LTTNG_SESSION_CONFIG_XSD_PATH"] = str(
 351             self._project_root / "src" / "common"
 352         )
 353
 354         assert self._lttng_home is not None
 355         sessiond_env["LTTNG_HOME"] = str(self._lttng_home.path)
 356
 357         wait_queue = _SignalWaitQueue()
 358         signal.signal(signal.SIGUSR1, wait_queue.signal)
 359
 360         self._log(
 361             "Launching session daemon with LTTNG_HOME=`{home_dir}`".format(
 362                 home_dir=str(self._lttng_home.path)
 363             )
 364         )
 365         process = subprocess.Popen(
 366             [
 367                 str(sessiond_path),
 368                 consumerd_path_option_name,
 369                 str(consumerd_path),
 370                 "--sig-parent",
 371             ],
 372             stdout=subprocess.PIPE,
 373             stderr=subprocess.STDOUT,
 374             env=sessiond_env,
 375         )
 376
 377         if self._logging_function:
 378             self._sessiond_output_consumer = ProcessOutputConsumer(
 379                 process, "lttng-sessiond", self._logging_function
 380             )  # type: Optional[ProcessOutputConsumer]
 381             self._sessiond_output_consumer.daemon = True
 382             self._sessiond_output_consumer.start()
 383
 384         # Wait for SIGUSR1, indicating the sessiond is ready to proceed
 385         wait_queue.wait_for_signal()
 386         signal.signal(signal.SIGUSR1, wait_queue.signal)
 387
 388         return process
 389
 390     def _handle_termination_signal(self, signal_number, frame):
 391         # type: (int, Optional[FrameType]) -> None
 392         self._log(
 393             "Killed by {signal_name} signal, cleaning-up".format(
 394                 signal_name=signal.strsignal(signal_number)
 395             )
 396         )
 397         self._cleanup()
 398
 399     def launch_wait_trace_test_application(self, event_count):
 400         # type: (int) -> WaitTraceTestApplication
 401         """
 402         Launch an application that will wait before tracing `event_count` events.
 403         """
 404         return WaitTraceTestApplication(
 405             self._project_root
 406             / "tests"
 407             / "utils"
 408             / "testapp"
 409             / "gen-ust-nevents"
 410             / "gen-ust-nevents",
 411             event_count,
 412             self,
 413         )
 414
 415     def launch_trace_test_constructor_application(
 416         self
 417     ) -> TraceTestApplication:
 418         """
 419         Launch an application that will trace from within constructors.
 420         """
 421         return TraceTestApplication(
 422             self._project_root
 423             / "tests"
 424             / "utils"
 425             / "testapp"
 426             / "gen-ust-events-constructor"
 427             / "gen-ust-events-constructor",
 428             self,
 429         )
 430
 431     # Clean-up managed processes
 432     def _cleanup(self):
 433         # type: () -> None
 434         if self._sessiond and self._sessiond.poll() is None:
 435             # The session daemon is alive; kill it.
 436             self._log(
 437                 "Killing session daemon (pid = {sessiond_pid})".format(
 438                     sessiond_pid=self._sessiond.pid
 439                 )
 440             )
 441
 442             self._sessiond.terminate()
 443             self._sessiond.wait()
 444             if self._sessiond_output_consumer:
 445                 self._sessiond_output_consumer.join()
 446                 self._sessiond_output_consumer = None
 447
 448             self._log("Session daemon killed")
 449             self._sessiond = None
 450
 451         self._lttng_home = None
 452
 453     def __del__(self):
 454         self._cleanup()
 455
 456
 457 @contextlib.contextmanager
 458 def test_environment(with_sessiond, log=None):
 459     # type: (bool, Optional[Callable[[str], None]]) -> Iterator[_Environment]
 460     env = _Environment(with_sessiond, log)
 461     try:
 462         yield env
 463     finally:
 464         env._cleanup()