tests/utils/lttngtest/environment.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright (C) 2022 Jérémie Galarneau <jeremie.galarneau@efficios.com>
   4 #
   5 # SPDX-License-Identifier: GPL-2.0-only
   6 #
   7
   8 from types import FrameType
   9 from typing import Callable, Iterator, Optional, Tuple, List
  10 import sys
  11 import pathlib
  12 import signal
  13 import subprocess
  14 import shlex
  15 import shutil
  16 import os
  17 import queue
  18 import tempfile
  19 from . import logger
  20 import time
  21 import threading
  22 import contextlib
  23
  24
  25 class TemporaryDirectory:
  26     def __init__(self, prefix):
  27         # type: (str) -> None
  28         self._directory_path = tempfile.mkdtemp(prefix=prefix)
  29
  30     def __del__(self):
  31         shutil.rmtree(self._directory_path, ignore_errors=True)
  32
  33     @property
  34     def path(self):
  35         # type: () -> pathlib.Path
  36         return pathlib.Path(self._directory_path)
  37
  38
  39 class _SignalWaitQueue:
  40     """
  41     Utility class useful to wait for a signal before proceeding.
  42
  43     Simply register the `signal` method as the handler for the signal you are
  44     interested in and call `wait_for_signal` to wait for its reception.
  45
  46     Registering a signal:
  47         signal.signal(signal.SIGWHATEVER, queue.signal)
  48
  49     Waiting for the signal:
  50         queue.wait_for_signal()
  51     """
  52
  53     def __init__(self):
  54         self._queue = queue.Queue()  # type: queue.Queue
  55
  56     def signal(
  57         self,
  58         signal_number,
  59         frame,  # type: Optional[FrameType]
  60     ):
  61         self._queue.put_nowait(signal_number)
  62
  63     def wait_for_signal(self):
  64         self._queue.get(block=True)
  65
  66
  67 class WaitTraceTestApplication:
  68     """
  69     Create an application that waits before tracing. This allows a test to
  70     launch an application, get its PID, and get it to start tracing when it
  71     has completed its setup.
  72     """
  73
  74     def __init__(
  75         self,
  76         binary_path,  # type: pathlib.Path
  77         event_count,  # type: int
  78         environment,  # type: Environment
  79         wait_time_between_events_us=0,  # type: int
  80     ):
  81         self._environment = environment  # type: Environment
  82         if event_count % 5:
  83             # The test application currently produces 5 different events per iteration.
  84             raise ValueError("event count must be a multiple of 5")
  85         self._iteration_count = int(event_count / 5)  # type: int
  86         # File that the application will wait to see before tracing its events.
  87         self._app_start_tracing_file_path = pathlib.Path(
  88             tempfile.mktemp(
  89                 prefix="app_",
  90                 suffix="_start_tracing",
  91                 dir=self._compat_open_path(environment.lttng_home_location),
  92             )
  93         )
  94         self._has_returned = False
  95
  96         test_app_env = os.environ.copy()
  97         test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location)
  98         # Make sure the app is blocked until it is properly registered to
  99         # the session daemon.
 100         test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
 101
 102         # File that the application will create to indicate it has completed its initialization.
 103         app_ready_file_path = tempfile.mktemp(
 104             prefix="app_",
 105             suffix="_ready",
 106             dir=self._compat_open_path(environment.lttng_home_location),
 107         )  # type: str
 108
 109         test_app_args = [str(binary_path)]
 110         test_app_args.extend(
 111             shlex.split(
 112                 "--iter {iteration_count} --create-in-main {app_ready_file_path} --wait-before-first-event {app_start_tracing_file_path} --wait {wait_time_between_events_us}".format(
 113                     iteration_count=self._iteration_count,
 114                     app_ready_file_path=app_ready_file_path,
 115                     app_start_tracing_file_path=self._app_start_tracing_file_path,
 116                     wait_time_between_events_us=wait_time_between_events_us,
 117                 )
 118             )
 119         )
 120
 121         self._process = subprocess.Popen(
 122             test_app_args,
 123             env=test_app_env,
 124         )  # type: subprocess.Popen
 125
 126         # Wait for the application to create the file indicating it has fully
 127         # initialized. Make sure the app hasn't crashed in order to not wait
 128         # forever.
 129         while True:
 130             if os.path.exists(app_ready_file_path):
 131                 break
 132
 133             if self._process.poll() is not None:
 134                 # Application has unexepectedly returned.
 135                 raise RuntimeError(
 136                     "Test application has unexepectedly returned during its initialization with return code `{return_code}`".format(
 137                         return_code=self._process.returncode
 138                     )
 139                 )
 140
 141             time.sleep(0.1)
 142
 143     def trace(self):
 144         # type: () -> None
 145         if self._process.poll() is not None:
 146             # Application has unexepectedly returned.
 147             raise RuntimeError(
 148                 "Test application has unexepectedly before tracing with return code `{return_code}`".format(
 149                     return_code=self._process.returncode
 150                 )
 151             )
 152         open(self._compat_open_path(self._app_start_tracing_file_path), mode="x")
 153
 154     def wait_for_exit(self):
 155         # type: () -> None
 156         if self._process.wait() != 0:
 157             raise RuntimeError(
 158                 "Test application has exit with return code `{return_code}`".format(
 159                     return_code=self._process.returncode
 160                 )
 161             )
 162         self._has_returned = True
 163
 164     @property
 165     def vpid(self):
 166         # type: () -> int
 167         return self._process.pid
 168
 169     @staticmethod
 170     def _compat_open_path(path):
 171         # type: (pathlib.Path) -> pathlib.Path | str
 172         """
 173         The builtin open() in python >= 3.6 expects a path-like object while
 174         prior versions expect a string or bytes object. Return the correct type
 175         based on the presence of the "__fspath__" attribute specified in PEP-519.
 176         """
 177         if hasattr(path, "__fspath__"):
 178             return path
 179         else:
 180             return str(path)
 181
 182     def __del__(self):
 183         if not self._has_returned:
 184             # This is potentially racy if the pid has been recycled. However,
 185             # we can't use pidfd_open since it is only available in python >= 3.9.
 186             self._process.kill()
 187             self._process.wait()
 188
 189
 190 class TraceTestApplication:
 191     """
 192     Create an application that emits events as soon as it is launched. In most
 193     scenarios, it is preferable to use a WaitTraceTestApplication.
 194     """
 195
 196     def __init__(self, binary_path, environment):
 197         # type: (pathlib.Path, Environment)
 198         self._environment = environment  # type: Environment
 199         self._has_returned = False
 200
 201         test_app_env = os.environ.copy()
 202         test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location)
 203         # Make sure the app is blocked until it is properly registered to
 204         # the session daemon.
 205         test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
 206
 207         test_app_args = [str(binary_path)]
 208
 209         self._process: subprocess.Popen = subprocess.Popen(
 210             test_app_args, env=test_app_env
 211         )
 212
 213     def wait_for_exit(self):
 214         # type: () -> None
 215         if self._process.wait() != 0:
 216             raise RuntimeError(
 217                 "Test application has exit with return code `{return_code}`".format(
 218                     return_code=self._process.returncode
 219                 )
 220             )
 221         self._has_returned = True
 222
 223     def __del__(self):
 224         if not self._has_returned:
 225             # This is potentially racy if the pid has been recycled. However,
 226             # we can't use pidfd_open since it is only available in python >= 3.9.
 227             self._process.kill()
 228             self._process.wait()
 229
 230
 231 class ProcessOutputConsumer(threading.Thread, logger._Logger):
 232     def __init__(
 233         self,
 234         process,  # type: subprocess.Popen
 235         name,  # type: str
 236         log,  # type: Callable[[str], None]
 237     ):
 238         threading.Thread.__init__(self)
 239         self._prefix = name
 240         logger._Logger.__init__(self, log)
 241         self._process = process
 242
 243     def run(self):
 244         # type: () -> None
 245         while self._process.poll() is None:
 246             assert self._process.stdout
 247             line = self._process.stdout.readline().decode("utf-8").replace("\n", "")
 248             if len(line) != 0:
 249                 self._log("{prefix}: {line}".format(prefix=self._prefix, line=line))
 250
 251
 252 # Generate a temporary environment in which to execute a test.
 253 class _Environment(logger._Logger):
 254     def __init__(
 255         self,
 256         with_sessiond,  # type: bool
 257         log=None,  # type: Optional[Callable[[str], None]]
 258     ):
 259         super().__init__(log)
 260         signal.signal(signal.SIGTERM, self._handle_termination_signal)
 261         signal.signal(signal.SIGINT, self._handle_termination_signal)
 262
 263         # Assumes the project's hierarchy to this file is:
 264         # tests/utils/python/this_file
 265         self._project_root = (
 266             pathlib.Path(__file__).absolute().parents[3]
 267         )  # type: pathlib.Path
 268         self._lttng_home = TemporaryDirectory(
 269             "lttng_test_env_home"
 270         )  # type: Optional[TemporaryDirectory]
 271
 272         self._sessiond = (
 273             self._launch_lttng_sessiond() if with_sessiond else None
 274         )  # type: Optional[subprocess.Popen[bytes]]
 275
 276     @property
 277     def lttng_home_location(self):
 278         # type: () -> pathlib.Path
 279         if self._lttng_home is None:
 280             raise RuntimeError("Attempt to access LTTng home after clean-up")
 281         return self._lttng_home.path
 282
 283     @property
 284     def lttng_client_path(self):
 285         # type: () -> pathlib.Path
 286         return self._project_root / "src" / "bin" / "lttng" / "lttng"
 287
 288     def create_temporary_directory(self, prefix=None):
 289         # type: (Optional[str]) -> pathlib.Path
 290         # Simply return a path that is contained within LTTNG_HOME; it will
 291         # be destroyed when the temporary home goes out of scope.
 292         assert self._lttng_home
 293         return pathlib.Path(
 294             tempfile.mkdtemp(
 295                 prefix="tmp" if prefix is None else prefix,
 296                 dir=str(self._lttng_home.path),
 297             )
 298         )
 299
 300     # Unpack a list of environment variables from a string
 301     # such as "HELLO=is_it ME='/you/are/looking/for'"
 302     @staticmethod
 303     def _unpack_env_vars(env_vars_string):
 304         # type: (str) -> List[Tuple[str, str]]
 305         unpacked_vars = []
 306         for var in shlex.split(env_vars_string):
 307             equal_position = var.find("=")
 308             # Must have an equal sign and not end with an equal sign
 309             if equal_position == -1 or equal_position == len(var) - 1:
 310                 raise ValueError(
 311                     "Invalid sessiond environment variable: `{}`".format(var)
 312                 )
 313
 314             var_name = var[0:equal_position]
 315             var_value = var[equal_position + 1 :]
 316             # Unquote any paths
 317             var_value = var_value.replace("'", "")
 318             var_value = var_value.replace('"', "")
 319             unpacked_vars.append((var_name, var_value))
 320
 321         return unpacked_vars
 322
 323     def _launch_lttng_sessiond(self):
 324         # type: () -> Optional[subprocess.Popen]
 325         is_64bits_host = sys.maxsize > 2**32
 326
 327         sessiond_path = (
 328             self._project_root / "src" / "bin" / "lttng-sessiond" / "lttng-sessiond"
 329         )
 330         consumerd_path_option_name = "--consumerd{bitness}-path".format(
 331             bitness="64" if is_64bits_host else "32"
 332         )
 333         consumerd_path = (
 334             self._project_root / "src" / "bin" / "lttng-consumerd" / "lttng-consumerd"
 335         )
 336
 337         no_sessiond_var = os.environ.get("TEST_NO_SESSIOND")
 338         if no_sessiond_var and no_sessiond_var == "1":
 339             # Run test without a session daemon; the user probably
 340             # intends to run one under gdb for example.
 341             return None
 342
 343         # Setup the session daemon's environment
 344         sessiond_env_vars = os.environ.get("LTTNG_SESSIOND_ENV_VARS")
 345         sessiond_env = os.environ.copy()
 346         if sessiond_env_vars:
 347             self._log("Additional lttng-sessiond environment variables:")
 348             additional_vars = self._unpack_env_vars(sessiond_env_vars)
 349             for var_name, var_value in additional_vars:
 350                 self._log("  {name}={value}".format(name=var_name, value=var_value))
 351                 sessiond_env[var_name] = var_value
 352
 353         sessiond_env["LTTNG_SESSION_CONFIG_XSD_PATH"] = str(
 354             self._project_root / "src" / "common"
 355         )
 356
 357         assert self._lttng_home is not None
 358         sessiond_env["LTTNG_HOME"] = str(self._lttng_home.path)
 359
 360         wait_queue = _SignalWaitQueue()
 361         signal.signal(signal.SIGUSR1, wait_queue.signal)
 362
 363         self._log(
 364             "Launching session daemon with LTTNG_HOME=`{home_dir}`".format(
 365                 home_dir=str(self._lttng_home.path)
 366             )
 367         )
 368         process = subprocess.Popen(
 369             [
 370                 str(sessiond_path),
 371                 consumerd_path_option_name,
 372                 str(consumerd_path),
 373                 "--sig-parent",
 374             ],
 375             stdout=subprocess.PIPE,
 376             stderr=subprocess.STDOUT,
 377             env=sessiond_env,
 378         )
 379
 380         if self._logging_function:
 381             self._sessiond_output_consumer = ProcessOutputConsumer(
 382                 process, "lttng-sessiond", self._logging_function
 383             )  # type: Optional[ProcessOutputConsumer]
 384             self._sessiond_output_consumer.daemon = True
 385             self._sessiond_output_consumer.start()
 386
 387         # Wait for SIGUSR1, indicating the sessiond is ready to proceed
 388         wait_queue.wait_for_signal()
 389         signal.signal(signal.SIGUSR1, wait_queue.signal)
 390
 391         return process
 392
 393     def _handle_termination_signal(self, signal_number, frame):
 394         # type: (int, Optional[FrameType]) -> None
 395         self._log(
 396             "Killed by {signal_name} signal, cleaning-up".format(
 397                 signal_name=signal.strsignal(signal_number)
 398             )
 399         )
 400         self._cleanup()
 401
 402     def launch_wait_trace_test_application(self, event_count):
 403         # type: (int) -> WaitTraceTestApplication
 404         """
 405         Launch an application that will wait before tracing `event_count` events.
 406         """
 407         return WaitTraceTestApplication(
 408             self._project_root
 409             / "tests"
 410             / "utils"
 411             / "testapp"
 412             / "gen-ust-nevents"
 413             / "gen-ust-nevents",
 414             event_count,
 415             self,
 416         )
 417
 418     def launch_trace_test_constructor_application(self):
 419         # type () -> TraceTestApplication
 420         """
 421         Launch an application that will trace from within constructors.
 422         """
 423         return TraceTestApplication(
 424             self._project_root
 425             / "tests"
 426             / "utils"
 427             / "testapp"
 428             / "gen-ust-events-constructor"
 429             / "gen-ust-events-constructor",
 430             self,
 431         )
 432
 433     # Clean-up managed processes
 434     def _cleanup(self):
 435         # type: () -> None
 436         if self._sessiond and self._sessiond.poll() is None:
 437             # The session daemon is alive; kill it.
 438             self._log(
 439                 "Killing session daemon (pid = {sessiond_pid})".format(
 440                     sessiond_pid=self._sessiond.pid
 441                 )
 442             )
 443
 444             self._sessiond.terminate()
 445             self._sessiond.wait()
 446             if self._sessiond_output_consumer:
 447                 self._sessiond_output_consumer.join()
 448                 self._sessiond_output_consumer = None
 449
 450             self._log("Session daemon killed")
 451             self._sessiond = None
 452
 453         self._lttng_home = None
 454
 455     def __del__(self):
 456         self._cleanup()
 457
 458
 459 @contextlib.contextmanager
 460 def test_environment(with_sessiond, log=None):
 461     # type: (bool, Optional[Callable[[str], None]]) -> Iterator[_Environment]
 462     env = _Environment(with_sessiond, log)
 463     try:
 464         yield env
 465     finally:
 466         env._cleanup()