Commit | Line | Data |
---|---|---|
ef945e4d JG |
1 | #!/usr/bin/env python3 |
2 | # | |
3 | # Copyright (C) 2022 Jérémie Galarneau <jeremie.galarneau@efficios.com> | |
4 | # | |
5 | # SPDX-License-Identifier: GPL-2.0-only | |
6 | # | |
7 | ||
8 | from types import FrameType | |
0ac0f70e | 9 | from typing import Callable, Iterator, Optional, Tuple, List, Generator |
ef945e4d JG |
10 | import sys |
11 | import pathlib | |
03775d44 KS |
12 | import pwd |
13 | import random | |
ef945e4d | 14 | import signal |
91118dcc | 15 | import socket |
ef945e4d JG |
16 | import subprocess |
17 | import shlex | |
18 | import shutil | |
03775d44 KS |
19 | import stat |
20 | import string | |
ef945e4d JG |
21 | import os |
22 | import queue | |
23 | import tempfile | |
24 | from . import logger | |
25 | import time | |
26 | import threading | |
27 | import contextlib | |
28 | ||
91118dcc KS |
29 | import bt2 |
30 | ||
ef945e4d JG |
31 | |
32 | class TemporaryDirectory: | |
ce8470c9 MJ |
33 | def __init__(self, prefix): |
34 | # type: (str) -> None | |
ef945e4d JG |
35 | self._directory_path = tempfile.mkdtemp(prefix=prefix) |
36 | ||
37 | def __del__(self): | |
03775d44 KS |
38 | if os.getenv("LTTNG_TEST_PRESERVE_TEST_ENV", "0") != "1": |
39 | shutil.rmtree(self._directory_path, ignore_errors=True) | |
ef945e4d JG |
40 | |
41 | @property | |
ce8470c9 MJ |
42 | def path(self): |
43 | # type: () -> pathlib.Path | |
ef945e4d JG |
44 | return pathlib.Path(self._directory_path) |
45 | ||
46 | ||
47 | class _SignalWaitQueue: | |
48 | """ | |
49 | Utility class useful to wait for a signal before proceeding. | |
50 | ||
51 | Simply register the `signal` method as the handler for the signal you are | |
52 | interested in and call `wait_for_signal` to wait for its reception. | |
53 | ||
54 | Registering a signal: | |
55 | signal.signal(signal.SIGWHATEVER, queue.signal) | |
56 | ||
57 | Waiting for the signal: | |
58 | queue.wait_for_signal() | |
59 | """ | |
60 | ||
61 | def __init__(self): | |
ce8470c9 | 62 | self._queue = queue.Queue() # type: queue.Queue |
ef945e4d | 63 | |
ce8470c9 MJ |
64 | def signal( |
65 | self, | |
66 | signal_number, | |
67 | frame, # type: Optional[FrameType] | |
68 | ): | |
ef945e4d JG |
69 | self._queue.put_nowait(signal_number) |
70 | ||
71 | def wait_for_signal(self): | |
72 | self._queue.get(block=True) | |
73 | ||
0ac0f70e JG |
74 | @contextlib.contextmanager |
75 | def intercept_signal(self, signal_number): | |
76 | # type: (int) -> Generator[None, None, None] | |
77 | original_handler = signal.getsignal(signal_number) | |
78 | signal.signal(signal_number, self.signal) | |
79 | try: | |
80 | yield | |
81 | except: | |
82 | # Restore the original signal handler and forward the exception. | |
83 | raise | |
84 | finally: | |
85 | signal.signal(signal_number, original_handler) | |
86 | ||
ef945e4d | 87 | |
91118dcc KS |
88 | class _LiveViewer: |
89 | """ | |
90 | Create a babeltrace2 live viewer. | |
91 | """ | |
92 | ||
93 | def __init__( | |
94 | self, | |
95 | environment, # type: Environment | |
96 | session, # type: str | |
97 | hostname=None, # type: Optional[str] | |
98 | ): | |
99 | self._environment = environment | |
100 | self._session = session | |
101 | self._hostname = hostname | |
102 | if self._hostname is None: | |
103 | self._hostname = socket.gethostname() | |
104 | self._events = [] | |
105 | ||
106 | ctf_live_cc = bt2.find_plugin("ctf").source_component_classes["lttng-live"] | |
107 | self._live_iterator = bt2.TraceCollectionMessageIterator( | |
108 | bt2.ComponentSpec( | |
109 | ctf_live_cc, | |
110 | { | |
111 | "inputs": [ | |
112 | "net://localhost:{}/host/{}/{}".format( | |
113 | environment.lttng_relayd_live_port, | |
114 | self._hostname, | |
115 | session, | |
116 | ) | |
117 | ], | |
118 | "session-not-found-action": "end", | |
119 | }, | |
120 | ) | |
121 | ) | |
122 | ||
123 | try: | |
124 | # Cause the connection to be initiated since tests | |
125 | # tend to wait for a viewer to be connected before proceeding. | |
126 | msg = next(self._live_iterator) | |
127 | self._events.append(msg) | |
128 | except bt2.TryAgain: | |
129 | pass | |
130 | ||
131 | @property | |
132 | def output(self): | |
133 | return self._events | |
134 | ||
135 | @property | |
136 | def messages(self): | |
137 | return [x for x in self._events if type(x) is bt2._EventMessageConst] | |
138 | ||
139 | def _drain(self, retry=False): | |
140 | while True: | |
141 | try: | |
142 | for msg in self._live_iterator: | |
143 | self._events.append(msg) | |
144 | break | |
145 | except bt2.TryAgain as e: | |
146 | if retry: | |
147 | time.sleep(0.01) | |
148 | continue | |
149 | else: | |
150 | break | |
151 | ||
152 | def wait_until_connected(self, timeout=0): | |
153 | ctf_live_cc = bt2.find_plugin("ctf").source_component_classes["lttng-live"] | |
154 | self._environment._log( | |
155 | "Checking for connected clients at 'net://localhost:{}'".format( | |
156 | self._environment.lttng_relayd_live_port | |
157 | ) | |
158 | ) | |
159 | query_executor = bt2.QueryExecutor( | |
160 | ctf_live_cc, | |
161 | "sessions", | |
162 | params={ | |
163 | "url": "net://localhost:{}".format( | |
164 | self._environment.lttng_relayd_live_port | |
165 | ) | |
166 | }, | |
167 | ) | |
168 | connected = False | |
169 | started = time.time() | |
170 | while not connected: | |
171 | try: | |
172 | if timeout != 0 and (time.time() - started) > timeout: | |
173 | raise RuntimeError( | |
174 | "Timed out waiting for connected clients on session '{}' after {}s".format( | |
175 | self._session, time.time() - started | |
176 | ) | |
177 | ) | |
178 | query_result = query_executor.query() | |
179 | except bt2._Error: | |
180 | time.sleep(0.01) | |
181 | continue | |
182 | for live_session in query_result: | |
183 | if ( | |
184 | live_session["session-name"] == self._session | |
185 | and live_session["client-count"] >= 1 | |
186 | ): | |
187 | connected = True | |
188 | self._environment._log( | |
189 | "Session '{}' has {} connected clients".format( | |
190 | live_session["session-name"], live_session["client-count"] | |
191 | ) | |
192 | ) | |
193 | break | |
194 | return connected | |
195 | ||
196 | def wait(self): | |
197 | if self._live_iterator: | |
198 | self._drain(retry=True) | |
199 | del self._live_iterator | |
200 | self._live_iterator = None | |
201 | ||
202 | def __del__(self): | |
203 | pass | |
204 | ||
205 | ||
c661f2f4 | 206 | class _WaitTraceTestApplication: |
ef945e4d JG |
207 | """ |
208 | Create an application that waits before tracing. This allows a test to | |
209 | launch an application, get its PID, and get it to start tracing when it | |
210 | has completed its setup. | |
211 | """ | |
212 | ||
213 | def __init__( | |
214 | self, | |
ce8470c9 MJ |
215 | binary_path, # type: pathlib.Path |
216 | event_count, # type: int | |
217 | environment, # type: Environment | |
218 | wait_time_between_events_us=0, # type: int | |
c661f2f4 JG |
219 | wait_before_exit=False, # type: bool |
220 | wait_before_exit_file_path=None, # type: Optional[pathlib.Path] | |
03775d44 | 221 | run_as=None, # type: Optional[str] |
ef945e4d | 222 | ): |
cebde614 | 223 | self._process = None |
ce8470c9 | 224 | self._environment = environment # type: Environment |
ef07b7ae | 225 | self._iteration_count = event_count |
ef945e4d | 226 | # File that the application will wait to see before tracing its events. |
03775d44 KS |
227 | dir = self._compat_pathlike(environment.lttng_home_location) |
228 | if run_as is not None: | |
229 | dir = os.path.join(dir, run_as) | |
ce8470c9 | 230 | self._app_start_tracing_file_path = pathlib.Path( |
ef945e4d JG |
231 | tempfile.mktemp( |
232 | prefix="app_", | |
233 | suffix="_start_tracing", | |
03775d44 | 234 | dir=dir, |
ef945e4d JG |
235 | ) |
236 | ) | |
03775d44 | 237 | |
c661f2f4 JG |
238 | # File that the application will create when all events have been emitted. |
239 | self._app_tracing_done_file_path = pathlib.Path( | |
240 | tempfile.mktemp( | |
241 | prefix="app_", | |
242 | suffix="_done_tracing", | |
03775d44 | 243 | dir=dir, |
c661f2f4 JG |
244 | ) |
245 | ) | |
246 | ||
247 | if wait_before_exit and wait_before_exit_file_path is None: | |
248 | wait_before_exit_file_path = pathlib.Path( | |
249 | tempfile.mktemp( | |
250 | prefix="app_", | |
251 | suffix="_exit", | |
c0aaf21b | 252 | dir=dir, |
c661f2f4 JG |
253 | ) |
254 | ) | |
c0aaf21b | 255 | self._wait_before_exit_file_path = wait_before_exit_file_path |
ef945e4d | 256 | self._has_returned = False |
c0aaf21b | 257 | self._tracing_started = False |
ef945e4d JG |
258 | |
259 | test_app_env = os.environ.copy() | |
260 | test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location) | |
261 | # Make sure the app is blocked until it is properly registered to | |
262 | # the session daemon. | |
263 | test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1" | |
264 | ||
265 | # File that the application will create to indicate it has completed its initialization. | |
8466f071 | 266 | app_ready_file_path = tempfile.mktemp( |
2d2198ca MJ |
267 | prefix="app_", |
268 | suffix="_ready", | |
03775d44 | 269 | dir=dir, |
ce8470c9 | 270 | ) # type: str |
ef945e4d JG |
271 | |
272 | test_app_args = [str(binary_path)] | |
c661f2f4 | 273 | test_app_args.extend(["--iter", str(event_count)]) |
ef945e4d | 274 | test_app_args.extend( |
c661f2f4 JG |
275 | ["--sync-application-in-main-touch", str(app_ready_file_path)] |
276 | ) | |
277 | test_app_args.extend( | |
278 | ["--sync-before-first-event", str(self._app_start_tracing_file_path)] | |
279 | ) | |
280 | test_app_args.extend( | |
281 | ["--sync-before-exit-touch", str(self._app_tracing_done_file_path)] | |
ef945e4d | 282 | ) |
c0aaf21b KS |
283 | if wait_before_exit: |
284 | test_app_args.extend( | |
285 | ["--sync-before-exit", str(self._wait_before_exit_file_path)] | |
286 | ) | |
c661f2f4 JG |
287 | if wait_time_between_events_us != 0: |
288 | test_app_args.extend(["--wait", str(wait_time_between_events_us)]) | |
ef945e4d | 289 | |
03775d44 KS |
290 | if run_as is not None: |
291 | # When running as root and reducing the permissions to run as another | |
292 | # user, the test binary needs to be readable and executable by the | |
293 | # world; however, the file may be in a deep path or on systems where | |
294 | # we don't want to modify the filesystem state (eg. for a person who | |
295 | # has downloaded and ran the tests manually). | |
296 | # Therefore, the binary_path is copied to a temporary file in the | |
297 | # `run_as` user's home directory | |
298 | new_binary_path = os.path.join( | |
299 | str(environment.lttng_home_location), | |
300 | run_as, | |
301 | os.path.basename(str(binary_path)), | |
302 | ) | |
303 | ||
304 | if not os.path.exists(new_binary_path): | |
305 | shutil.copy(str(binary_path), new_binary_path) | |
306 | ||
307 | test_app_args[0] = new_binary_path | |
308 | ||
309 | lib_dir = environment.lttng_home_location / run_as / "lib" | |
310 | if not os.path.isdir(str(lib_dir)): | |
311 | os.mkdir(str(lib_dir)) | |
312 | # When running dropping privileges, the libraries built in the | |
313 | # root-owned directories may not be reachable and readable by | |
314 | # the loader running as an unprivileged user. These should also be | |
315 | # copied. | |
316 | _ldd = subprocess.Popen( | |
317 | ["ldd", new_binary_path], | |
318 | stdout=subprocess.PIPE, | |
319 | stderr=subprocess.PIPE, | |
320 | ) | |
321 | if _ldd.wait() != 0: | |
322 | raise RuntimeError( | |
323 | "Error while using `ldd` to determine test application dependencies: `{}`".format( | |
324 | stderr.read().decode("utf-8") | |
325 | ) | |
326 | ) | |
327 | libs = [ | |
328 | x.decode("utf-8").split(sep="=>") for x in _ldd.stdout.readlines() | |
329 | ] | |
330 | libs = [ | |
331 | x[1].split(sep=" ")[1] | |
332 | for x in libs | |
333 | if len(x) >= 2 and x[1].find("lttng") != -1 | |
334 | ] | |
335 | for lib in libs: | |
336 | shutil.copy(lib, lib_dir) | |
337 | ||
338 | test_app_env["LD_LIBRARY_PATH"] = "{}:{}".format( | |
339 | test_app_env["LD_LIBRARY_PATH"], | |
340 | str(lib_dir), | |
341 | ) | |
342 | ||
343 | # As of python 3.9, subprocess.Popen supports a user parameter which | |
344 | # runs `setreuid()` before executing the proces and will be preferable | |
345 | # when support for older python versions is no longer required. | |
346 | test_app_args = [ | |
347 | "runuser", | |
348 | "-u", | |
349 | run_as, | |
350 | "--", | |
351 | ] + test_app_args | |
352 | ||
353 | self._environment._log( | |
354 | "Launching test application: '{}'".format( | |
355 | self._compat_shlex_join(test_app_args) | |
356 | ) | |
357 | ) | |
ce8470c9 | 358 | self._process = subprocess.Popen( |
ef945e4d JG |
359 | test_app_args, |
360 | env=test_app_env, | |
c661f2f4 | 361 | stdout=subprocess.PIPE, |
91118dcc | 362 | stderr=subprocess.PIPE, |
ce8470c9 | 363 | ) # type: subprocess.Popen |
ef945e4d JG |
364 | |
365 | # Wait for the application to create the file indicating it has fully | |
366 | # initialized. Make sure the app hasn't crashed in order to not wait | |
367 | # forever. | |
c661f2f4 JG |
368 | self._wait_for_file_to_be_created(pathlib.Path(app_ready_file_path)) |
369 | ||
370 | def _wait_for_file_to_be_created(self, sync_file_path): | |
371 | # type: (pathlib.Path) -> None | |
ef945e4d | 372 | while True: |
8a5e3824 | 373 | if os.path.exists(self._compat_pathlike(sync_file_path)): |
ef945e4d JG |
374 | break |
375 | ||
376 | if self._process.poll() is not None: | |
377 | # Application has unexepectedly returned. | |
378 | raise RuntimeError( | |
03775d44 KS |
379 | "Test application has unexepectedly returned while waiting for synchronization file to be created: sync_file=`{sync_file}`, return_code=`{return_code}`, output=`{output}`".format( |
380 | sync_file=sync_file_path, | |
381 | return_code=self._process.returncode, | |
382 | output=self._process.stderr.read().decode("utf-8"), | |
ef945e4d JG |
383 | ) |
384 | ) | |
385 | ||
c661f2f4 | 386 | time.sleep(0.001) |
ef945e4d | 387 | |
c0aaf21b KS |
388 | def touch_exit_file(self): |
389 | open(self._compat_pathlike(self._wait_before_exit_file_path), mode="x") | |
390 | ||
ce8470c9 MJ |
391 | def trace(self): |
392 | # type: () -> None | |
ef945e4d JG |
393 | if self._process.poll() is not None: |
394 | # Application has unexepectedly returned. | |
395 | raise RuntimeError( | |
396 | "Test application has unexepectedly before tracing with return code `{return_code}`".format( | |
397 | return_code=self._process.returncode | |
398 | ) | |
399 | ) | |
8a5e3824 | 400 | open(self._compat_pathlike(self._app_start_tracing_file_path), mode="x") |
c0aaf21b KS |
401 | self._environment._log("[{}] Tracing started".format(self.vpid)) |
402 | self._tracing_started = True | |
ef945e4d | 403 | |
c661f2f4 JG |
404 | def wait_for_tracing_done(self): |
405 | # type: () -> None | |
c0aaf21b KS |
406 | if not self._tracing_started: |
407 | raise RuntimeError("Tracing hasn't been started") | |
c661f2f4 | 408 | self._wait_for_file_to_be_created(self._app_tracing_done_file_path) |
c0aaf21b | 409 | self._environment._log("[{}] Tracing done".format(self.vpid)) |
c661f2f4 | 410 | |
ce8470c9 MJ |
411 | def wait_for_exit(self): |
412 | # type: () -> None | |
ef945e4d JG |
413 | if self._process.wait() != 0: |
414 | raise RuntimeError( | |
03775d44 KS |
415 | "Test application [{pid}] has exit with return code `{return_code}`, output=`{output}`".format( |
416 | pid=self.vpid, | |
417 | return_code=self._process.returncode, | |
418 | output=self._process.stderr.read().decode("utf-8"), | |
ef945e4d JG |
419 | ) |
420 | ) | |
421 | self._has_returned = True | |
422 | ||
423 | @property | |
ce8470c9 MJ |
424 | def vpid(self): |
425 | # type: () -> int | |
ef945e4d JG |
426 | return self._process.pid |
427 | ||
2d2198ca | 428 | @staticmethod |
8a5e3824 | 429 | def _compat_pathlike(path): |
ce8470c9 | 430 | # type: (pathlib.Path) -> pathlib.Path | str |
2d2198ca | 431 | """ |
8a5e3824 MJ |
432 | The builtin open() and many methods of the 'os' library in Python >= 3.6 |
433 | expect a path-like object while prior versions expect a string or | |
434 | bytes object. Return the correct type based on the presence of the | |
435 | "__fspath__" attribute specified in PEP-519. | |
2d2198ca MJ |
436 | """ |
437 | if hasattr(path, "__fspath__"): | |
438 | return path | |
439 | else: | |
440 | return str(path) | |
441 | ||
03775d44 KS |
442 | @staticmethod |
443 | def _compat_shlex_join(args): | |
444 | # type: list[str] -> str | |
445 | # shlex.join was added in python 3.8 | |
446 | return " ".join([shlex.quote(x) for x in args]) | |
447 | ||
ef945e4d | 448 | def __del__(self): |
cebde614 | 449 | if self._process is not None and not self._has_returned: |
ef945e4d JG |
450 | # This is potentially racy if the pid has been recycled. However, |
451 | # we can't use pidfd_open since it is only available in python >= 3.9. | |
452 | self._process.kill() | |
453 | self._process.wait() | |
454 | ||
455 | ||
c661f2f4 JG |
456 | class WaitTraceTestApplicationGroup: |
457 | def __init__( | |
458 | self, | |
459 | environment, # type: Environment | |
460 | application_count, # type: int | |
461 | event_count, # type: int | |
462 | wait_time_between_events_us=0, # type: int | |
463 | wait_before_exit=False, # type: bool | |
464 | ): | |
465 | self._wait_before_exit_file_path = ( | |
466 | pathlib.Path( | |
467 | tempfile.mktemp( | |
468 | prefix="app_group_", | |
469 | suffix="_exit", | |
8a5e3824 | 470 | dir=_WaitTraceTestApplication._compat_pathlike( |
c661f2f4 JG |
471 | environment.lttng_home_location |
472 | ), | |
473 | ) | |
474 | ) | |
475 | if wait_before_exit | |
476 | else None | |
477 | ) | |
478 | ||
479 | self._apps = [] | |
480 | self._consumers = [] | |
481 | for i in range(application_count): | |
482 | new_app = environment.launch_wait_trace_test_application( | |
483 | event_count, | |
484 | wait_time_between_events_us, | |
485 | wait_before_exit, | |
486 | self._wait_before_exit_file_path, | |
487 | ) | |
488 | ||
489 | # Attach an output consumer to log the application's error output (if any). | |
490 | if environment._logging_function: | |
491 | app_output_consumer = ProcessOutputConsumer( | |
492 | new_app._process, | |
493 | "app-{}".format(str(new_app.vpid)), | |
494 | environment._logging_function, | |
495 | ) # type: Optional[ProcessOutputConsumer] | |
496 | app_output_consumer.daemon = True | |
497 | app_output_consumer.start() | |
498 | self._consumers.append(app_output_consumer) | |
499 | ||
500 | self._apps.append(new_app) | |
501 | ||
502 | def trace(self): | |
503 | # type: () -> None | |
504 | for app in self._apps: | |
505 | app.trace() | |
506 | ||
507 | def exit( | |
508 | self, wait_for_apps=False # type: bool | |
509 | ): | |
510 | if self._wait_before_exit_file_path is None: | |
511 | raise RuntimeError( | |
512 | "Can't call exit on an application group created with `wait_before_exit=False`" | |
513 | ) | |
514 | ||
515 | # Wait for apps to have produced all of their events so that we can | |
516 | # cause the death of all apps to happen within a short time span. | |
517 | for app in self._apps: | |
518 | app.wait_for_tracing_done() | |
519 | ||
c0aaf21b KS |
520 | self._apps[0].touch_exit_file() |
521 | ||
c661f2f4 JG |
522 | # Performed in two passes to allow tests to stress the unregistration of many applications. |
523 | # Waiting for each app to exit turn-by-turn would defeat the purpose here. | |
524 | if wait_for_apps: | |
525 | for app in self._apps: | |
526 | app.wait_for_exit() | |
527 | ||
528 | ||
529 | class _TraceTestApplication: | |
da1e97c9 | 530 | """ |
e88109fc JG |
531 | Create an application that emits events as soon as it is launched. In most |
532 | scenarios, it is preferable to use a WaitTraceTestApplication. | |
da1e97c9 MD |
533 | """ |
534 | ||
873d3601 MJ |
535 | def __init__(self, binary_path, environment): |
536 | # type: (pathlib.Path, Environment) | |
cebde614 | 537 | self._process = None |
873d3601 | 538 | self._environment = environment # type: Environment |
da1e97c9 MD |
539 | self._has_returned = False |
540 | ||
541 | test_app_env = os.environ.copy() | |
542 | test_app_env["LTTNG_HOME"] = str(environment.lttng_home_location) | |
543 | # Make sure the app is blocked until it is properly registered to | |
544 | # the session daemon. | |
545 | test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1" | |
546 | ||
547 | test_app_args = [str(binary_path)] | |
548 | ||
47ddc6e5 | 549 | self._process = subprocess.Popen( |
da1e97c9 | 550 | test_app_args, env=test_app_env |
47ddc6e5 | 551 | ) # type: subprocess.Popen |
da1e97c9 | 552 | |
873d3601 MJ |
553 | def wait_for_exit(self): |
554 | # type: () -> None | |
da1e97c9 MD |
555 | if self._process.wait() != 0: |
556 | raise RuntimeError( | |
557 | "Test application has exit with return code `{return_code}`".format( | |
558 | return_code=self._process.returncode | |
559 | ) | |
560 | ) | |
561 | self._has_returned = True | |
562 | ||
563 | def __del__(self): | |
cebde614 | 564 | if self._process is not None and not self._has_returned: |
da1e97c9 MD |
565 | # This is potentially racy if the pid has been recycled. However, |
566 | # we can't use pidfd_open since it is only available in python >= 3.9. | |
567 | self._process.kill() | |
568 | self._process.wait() | |
569 | ||
570 | ||
ef945e4d JG |
571 | class ProcessOutputConsumer(threading.Thread, logger._Logger): |
572 | def __init__( | |
ce8470c9 MJ |
573 | self, |
574 | process, # type: subprocess.Popen | |
575 | name, # type: str | |
576 | log, # type: Callable[[str], None] | |
ef945e4d JG |
577 | ): |
578 | threading.Thread.__init__(self) | |
579 | self._prefix = name | |
580 | logger._Logger.__init__(self, log) | |
581 | self._process = process | |
582 | ||
ce8470c9 MJ |
583 | def run(self): |
584 | # type: () -> None | |
ef945e4d JG |
585 | while self._process.poll() is None: |
586 | assert self._process.stdout | |
587 | line = self._process.stdout.readline().decode("utf-8").replace("\n", "") | |
588 | if len(line) != 0: | |
589 | self._log("{prefix}: {line}".format(prefix=self._prefix, line=line)) | |
590 | ||
591 | ||
91118dcc KS |
592 | class SavingProcessOutputConsumer(ProcessOutputConsumer): |
593 | def __init__(self, process, name, log): | |
594 | self._lines = [] | |
595 | super().__init__(process=process, name=name, log=log) | |
596 | ||
597 | def run(self): | |
598 | # type: () -> None | |
599 | while self._process.poll() is None: | |
600 | assert self._process.stdout | |
601 | line = self._process.stdout.readline().decode("utf-8").replace("\n", "") | |
602 | if len(line) != 0: | |
603 | self._lines.append(line) | |
604 | self._log("{prefix}: {line}".format(prefix=self._prefix, line=line)) | |
605 | ||
606 | @property | |
607 | def output(self): | |
608 | return self._lines | |
609 | ||
610 | ||
ef945e4d JG |
611 | # Generate a temporary environment in which to execute a test. |
612 | class _Environment(logger._Logger): | |
613 | def __init__( | |
ce8470c9 MJ |
614 | self, |
615 | with_sessiond, # type: bool | |
616 | log=None, # type: Optional[Callable[[str], None]] | |
45ce5eed | 617 | with_relayd=False, # type: bool |
ef945e4d JG |
618 | ): |
619 | super().__init__(log) | |
620 | signal.signal(signal.SIGTERM, self._handle_termination_signal) | |
621 | signal.signal(signal.SIGINT, self._handle_termination_signal) | |
622 | ||
55aec41f KS |
623 | if os.getenv("LTTNG_TEST_VERBOSE_BABELTRACE", "0") == "1": |
624 | # @TODO: Is there a way to feed the logging output to | |
625 | # the logger._Logger instead of directly to stderr? | |
626 | bt2.set_global_logging_level(bt2.LoggingLevel.TRACE) | |
627 | ||
ef945e4d JG |
628 | # Assumes the project's hierarchy to this file is: |
629 | # tests/utils/python/this_file | |
ce8470c9 MJ |
630 | self._project_root = ( |
631 | pathlib.Path(__file__).absolute().parents[3] | |
632 | ) # type: pathlib.Path | |
03775d44 | 633 | |
ce8470c9 | 634 | self._lttng_home = TemporaryDirectory( |
ef945e4d | 635 | "lttng_test_env_home" |
03775d44 KS |
636 | ) # type: Optional[str] |
637 | os.chmod( | |
638 | str(self._lttng_home.path), | |
639 | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IROTH | stat.S_IXOTH, | |
640 | ) | |
ef945e4d | 641 | |
45ce5eed KS |
642 | self._relayd = ( |
643 | self._launch_lttng_relayd() if with_relayd else None | |
644 | ) # type: Optional[subprocess.Popen[bytes]] | |
645 | self._relayd_output_consumer = None | |
646 | ||
ce8470c9 | 647 | self._sessiond = ( |
ef945e4d | 648 | self._launch_lttng_sessiond() if with_sessiond else None |
ce8470c9 | 649 | ) # type: Optional[subprocess.Popen[bytes]] |
ef945e4d | 650 | |
03775d44 KS |
651 | self._dummy_users = {} # type: Dictionary[int, string] |
652 | self._preserve_test_env = os.getenv("LTTNG_TEST_PRESERVE_TEST_ENV", "0") != "1" | |
653 | ||
ef945e4d | 654 | @property |
ce8470c9 MJ |
655 | def lttng_home_location(self): |
656 | # type: () -> pathlib.Path | |
ef945e4d JG |
657 | if self._lttng_home is None: |
658 | raise RuntimeError("Attempt to access LTTng home after clean-up") | |
659 | return self._lttng_home.path | |
660 | ||
661 | @property | |
ce8470c9 MJ |
662 | def lttng_client_path(self): |
663 | # type: () -> pathlib.Path | |
ef945e4d JG |
664 | return self._project_root / "src" / "bin" / "lttng" / "lttng" |
665 | ||
45ce5eed KS |
666 | @property |
667 | def lttng_relayd_control_port(self): | |
668 | # type: () -> int | |
669 | return 5400 | |
670 | ||
671 | @property | |
672 | def lttng_relayd_data_port(self): | |
673 | # type: () -> int | |
674 | return 5401 | |
675 | ||
676 | @property | |
677 | def lttng_relayd_live_port(self): | |
678 | # type: () -> int | |
679 | return 5402 | |
680 | ||
03775d44 KS |
681 | @property |
682 | def preserve_test_env(self): | |
683 | # type: () -> bool | |
684 | return self._preserve_test_env | |
685 | ||
686 | @staticmethod | |
687 | def allows_destructive(): | |
688 | # type: () -> bool | |
689 | return os.getenv("LTTNG_ENABLE_DESTRUCTIVE_TESTS", "") == "will-break-my-system" | |
690 | ||
691 | def create_dummy_user(self): | |
692 | # type: () -> (int, str) | |
693 | # Create a dummy user. The uid and username will be eturned in a tuple. | |
694 | # If the name already exists, an exception will be thrown. | |
695 | # The users will be removed when the environment is cleaned up. | |
696 | name = "".join([random.choice(string.ascii_lowercase) for x in range(10)]) | |
697 | ||
698 | try: | |
699 | entry = pwd.getpwnam(name) | |
700 | raise Exception("User '{}' already exists".format(name)) | |
701 | except KeyError: | |
702 | pass | |
703 | ||
704 | # Create user | |
705 | proc = subprocess.Popen( | |
706 | [ | |
707 | "useradd", | |
708 | "--base-dir", | |
709 | str(self._lttng_home.path), | |
710 | "--create-home", | |
711 | "--no-user-group", | |
712 | "--shell", | |
713 | "/bin/sh", | |
714 | name, | |
715 | ] | |
716 | ) | |
717 | proc.wait() | |
718 | if proc.returncode != 0: | |
719 | raise Exception( | |
720 | "Failed to create user '{}', useradd returned {}".format( | |
721 | name, proc.returncode | |
722 | ) | |
723 | ) | |
724 | ||
725 | entry = pwd.getpwnam(name) | |
726 | self._dummy_users[entry[2]] = name | |
727 | return (entry[2], name) | |
728 | ||
ce8470c9 MJ |
729 | def create_temporary_directory(self, prefix=None): |
730 | # type: (Optional[str]) -> pathlib.Path | |
ef945e4d JG |
731 | # Simply return a path that is contained within LTTNG_HOME; it will |
732 | # be destroyed when the temporary home goes out of scope. | |
733 | assert self._lttng_home | |
734 | return pathlib.Path( | |
735 | tempfile.mkdtemp( | |
736 | prefix="tmp" if prefix is None else prefix, | |
737 | dir=str(self._lttng_home.path), | |
738 | ) | |
739 | ) | |
740 | ||
741 | # Unpack a list of environment variables from a string | |
742 | # such as "HELLO=is_it ME='/you/are/looking/for'" | |
743 | @staticmethod | |
ce8470c9 MJ |
744 | def _unpack_env_vars(env_vars_string): |
745 | # type: (str) -> List[Tuple[str, str]] | |
ef945e4d JG |
746 | unpacked_vars = [] |
747 | for var in shlex.split(env_vars_string): | |
748 | equal_position = var.find("=") | |
749 | # Must have an equal sign and not end with an equal sign | |
750 | if equal_position == -1 or equal_position == len(var) - 1: | |
751 | raise ValueError( | |
752 | "Invalid sessiond environment variable: `{}`".format(var) | |
753 | ) | |
754 | ||
755 | var_name = var[0:equal_position] | |
756 | var_value = var[equal_position + 1 :] | |
757 | # Unquote any paths | |
758 | var_value = var_value.replace("'", "") | |
759 | var_value = var_value.replace('"', "") | |
760 | unpacked_vars.append((var_name, var_value)) | |
761 | ||
762 | return unpacked_vars | |
763 | ||
45ce5eed KS |
764 | def _launch_lttng_relayd(self): |
765 | # type: () -> Optional[subprocess.Popen] | |
766 | relayd_path = ( | |
767 | self._project_root / "src" / "bin" / "lttng-relayd" / "lttng-relayd" | |
768 | ) | |
769 | if os.environ.get("LTTNG_TEST_NO_RELAYD", "0") == "1": | |
770 | # Run without a relay daemon; the user may be running one | |
771 | # under gdb, for example. | |
772 | return None | |
773 | ||
774 | relayd_env_vars = os.environ.get("LTTNG_RELAYD_ENV_VARS") | |
775 | relayd_env = os.environ.copy() | |
776 | if relayd_env_vars: | |
777 | self._log("Additional lttng-relayd environment variables:") | |
778 | for name, value in self._unpack_env_vars(relayd_env_vars): | |
779 | self._log("{}={}".format(name, value)) | |
780 | relayd_env[name] = value | |
781 | ||
782 | assert self._lttng_home is not None | |
783 | relayd_env["LTTNG_HOME"] = str(self._lttng_home.path) | |
784 | self._log( | |
785 | "Launching relayd with LTTNG_HOME='${}'".format(str(self._lttng_home.path)) | |
786 | ) | |
55aec41f KS |
787 | verbose = [] |
788 | if os.environ.get("LTTNG_TEST_VERBOSE_RELAYD") is not None: | |
789 | verbose = ["-vvv"] | |
45ce5eed KS |
790 | process = subprocess.Popen( |
791 | [ | |
792 | str(relayd_path), | |
793 | "-C", | |
794 | "tcp://0.0.0.0:{}".format(self.lttng_relayd_control_port), | |
795 | "-D", | |
796 | "tcp://0.0.0.0:{}".format(self.lttng_relayd_data_port), | |
797 | "-L", | |
798 | "tcp://localhost:{}".format(self.lttng_relayd_live_port), | |
55aec41f KS |
799 | ] |
800 | + verbose, | |
45ce5eed KS |
801 | stdout=subprocess.PIPE, |
802 | stderr=subprocess.STDOUT, | |
803 | env=relayd_env, | |
804 | ) | |
805 | ||
806 | if self._logging_function: | |
807 | self._relayd_output_consumer = ProcessOutputConsumer( | |
808 | process, "lttng-relayd", self._logging_function | |
809 | ) | |
810 | self._relayd_output_consumer.daemon = True | |
811 | self._relayd_output_consumer.start() | |
812 | ||
813 | return process | |
814 | ||
ce8470c9 MJ |
815 | def _launch_lttng_sessiond(self): |
816 | # type: () -> Optional[subprocess.Popen] | |
ef945e4d JG |
817 | is_64bits_host = sys.maxsize > 2**32 |
818 | ||
819 | sessiond_path = ( | |
820 | self._project_root / "src" / "bin" / "lttng-sessiond" / "lttng-sessiond" | |
821 | ) | |
822 | consumerd_path_option_name = "--consumerd{bitness}-path".format( | |
823 | bitness="64" if is_64bits_host else "32" | |
824 | ) | |
825 | consumerd_path = ( | |
826 | self._project_root / "src" / "bin" / "lttng-consumerd" / "lttng-consumerd" | |
827 | ) | |
828 | ||
829 | no_sessiond_var = os.environ.get("TEST_NO_SESSIOND") | |
830 | if no_sessiond_var and no_sessiond_var == "1": | |
831 | # Run test without a session daemon; the user probably | |
832 | # intends to run one under gdb for example. | |
833 | return None | |
834 | ||
835 | # Setup the session daemon's environment | |
836 | sessiond_env_vars = os.environ.get("LTTNG_SESSIOND_ENV_VARS") | |
837 | sessiond_env = os.environ.copy() | |
838 | if sessiond_env_vars: | |
839 | self._log("Additional lttng-sessiond environment variables:") | |
840 | additional_vars = self._unpack_env_vars(sessiond_env_vars) | |
841 | for var_name, var_value in additional_vars: | |
842 | self._log(" {name}={value}".format(name=var_name, value=var_value)) | |
843 | sessiond_env[var_name] = var_value | |
844 | ||
845 | sessiond_env["LTTNG_SESSION_CONFIG_XSD_PATH"] = str( | |
846 | self._project_root / "src" / "common" | |
847 | ) | |
848 | ||
849 | assert self._lttng_home is not None | |
850 | sessiond_env["LTTNG_HOME"] = str(self._lttng_home.path) | |
851 | ||
852 | wait_queue = _SignalWaitQueue() | |
0ac0f70e JG |
853 | with wait_queue.intercept_signal(signal.SIGUSR1): |
854 | self._log( | |
855 | "Launching session daemon with LTTNG_HOME=`{home_dir}`".format( | |
856 | home_dir=str(self._lttng_home.path) | |
857 | ) | |
858 | ) | |
55aec41f KS |
859 | verbose = [] |
860 | if os.environ.get("LTTNG_TEST_VERBOSE_SESSIOND") is not None: | |
861 | verbose = ["-vvv", "--verbose-consumer"] | |
0ac0f70e JG |
862 | process = subprocess.Popen( |
863 | [ | |
864 | str(sessiond_path), | |
865 | consumerd_path_option_name, | |
866 | str(consumerd_path), | |
867 | "--sig-parent", | |
55aec41f KS |
868 | ] |
869 | + verbose, | |
0ac0f70e JG |
870 | stdout=subprocess.PIPE, |
871 | stderr=subprocess.STDOUT, | |
872 | env=sessiond_env, | |
ef945e4d | 873 | ) |
ef945e4d | 874 | |
0ac0f70e JG |
875 | if self._logging_function: |
876 | self._sessiond_output_consumer = ProcessOutputConsumer( | |
877 | process, "lttng-sessiond", self._logging_function | |
878 | ) # type: Optional[ProcessOutputConsumer] | |
879 | self._sessiond_output_consumer.daemon = True | |
880 | self._sessiond_output_consumer.start() | |
ef945e4d | 881 | |
0ac0f70e JG |
882 | # Wait for SIGUSR1, indicating the sessiond is ready to proceed |
883 | wait_queue.wait_for_signal() | |
ef945e4d JG |
884 | |
885 | return process | |
886 | ||
ce8470c9 MJ |
887 | def _handle_termination_signal(self, signal_number, frame): |
888 | # type: (int, Optional[FrameType]) -> None | |
ef945e4d JG |
889 | self._log( |
890 | "Killed by {signal_name} signal, cleaning-up".format( | |
891 | signal_name=signal.strsignal(signal_number) | |
892 | ) | |
893 | ) | |
894 | self._cleanup() | |
895 | ||
91118dcc KS |
896 | def launch_live_viewer(self, session, hostname=None): |
897 | # Make sure the relayd is ready | |
898 | ready = False | |
899 | ctf_live_cc = bt2.find_plugin("ctf").source_component_classes["lttng-live"] | |
900 | query_executor = bt2.QueryExecutor( | |
901 | ctf_live_cc, | |
902 | "sessions", | |
903 | params={"url": "net://localhost:{}".format(self.lttng_relayd_live_port)}, | |
904 | ) | |
905 | while not ready: | |
906 | try: | |
907 | query_result = query_executor.query() | |
908 | except bt2._Error: | |
909 | time.sleep(0.1) | |
910 | continue | |
911 | for live_session in query_result: | |
912 | if live_session["session-name"] == session: | |
913 | ready = True | |
914 | self._log( | |
915 | "Session '{}' is available at net://localhost:{}".format( | |
916 | session, self.lttng_relayd_live_port | |
917 | ) | |
918 | ) | |
919 | break | |
920 | return _LiveViewer(self, session, hostname) | |
921 | ||
c661f2f4 JG |
922 | def launch_wait_trace_test_application( |
923 | self, | |
924 | event_count, # type: int | |
925 | wait_time_between_events_us=0, | |
926 | wait_before_exit=False, | |
927 | wait_before_exit_file_path=None, | |
03775d44 | 928 | run_as=None, |
c661f2f4 | 929 | ): |
03775d44 | 930 | # type: (int, int, bool, Optional[pathlib.Path], Optional[str]) -> _WaitTraceTestApplication |
ef945e4d JG |
931 | """ |
932 | Launch an application that will wait before tracing `event_count` events. | |
933 | """ | |
c661f2f4 | 934 | return _WaitTraceTestApplication( |
ef945e4d JG |
935 | self._project_root |
936 | / "tests" | |
937 | / "utils" | |
938 | / "testapp" | |
ef07b7ae JG |
939 | / "gen-ust-events" |
940 | / "gen-ust-events", | |
ef945e4d JG |
941 | event_count, |
942 | self, | |
c661f2f4 JG |
943 | wait_time_between_events_us, |
944 | wait_before_exit, | |
945 | wait_before_exit_file_path, | |
03775d44 | 946 | run_as, |
ef945e4d JG |
947 | ) |
948 | ||
09a872ef | 949 | def launch_test_application(self, subpath): |
873d3601 | 950 | # type () -> TraceTestApplication |
da1e97c9 MD |
951 | """ |
952 | Launch an application that will trace from within constructors. | |
953 | """ | |
c661f2f4 | 954 | return _TraceTestApplication( |
9a28bc04 | 955 | self._project_root / "tests" / "utils" / "testapp" / subpath, |
da1e97c9 MD |
956 | self, |
957 | ) | |
958 | ||
9a28bc04 KS |
959 | def _terminate_relayd(self): |
960 | if self._relayd and self._relayd.poll() is None: | |
961 | self._relayd.terminate() | |
962 | self._relayd.wait() | |
963 | if self._relayd_output_consumer: | |
964 | self._relayd_output_consumer.join() | |
965 | self._relayd_output_consumer = None | |
966 | self._log("Relayd killed") | |
967 | self._relayd = None | |
968 | ||
ef945e4d | 969 | # Clean-up managed processes |
ce8470c9 MJ |
970 | def _cleanup(self): |
971 | # type: () -> None | |
ef945e4d JG |
972 | if self._sessiond and self._sessiond.poll() is None: |
973 | # The session daemon is alive; kill it. | |
974 | self._log( | |
975 | "Killing session daemon (pid = {sessiond_pid})".format( | |
976 | sessiond_pid=self._sessiond.pid | |
977 | ) | |
978 | ) | |
979 | ||
980 | self._sessiond.terminate() | |
981 | self._sessiond.wait() | |
982 | if self._sessiond_output_consumer: | |
983 | self._sessiond_output_consumer.join() | |
984 | self._sessiond_output_consumer = None | |
985 | ||
986 | self._log("Session daemon killed") | |
987 | self._sessiond = None | |
988 | ||
9a28bc04 | 989 | self._terminate_relayd() |
45ce5eed | 990 | |
03775d44 KS |
991 | # The user accounts will always be deleted, but the home directories will |
992 | # be retained unless the user has opted to preserve the test environment. | |
993 | userdel = ["userdel"] | |
994 | if not self.preserve_test_env: | |
995 | userdel += ["--remove"] | |
996 | for uid, name in self._dummy_users.items(): | |
997 | # When subprocess is run during the interpreter teardown, ImportError | |
998 | # may be raised; however, the commands seem to execute correctly. | |
999 | # Eg. | |
1000 | # | |
1001 | # Exception ignored in: <function _Environment.__del__ at 0x7f2d62e3b9c0> | |
1002 | # Traceback (most recent call last): | |
1003 | # File "tests/utils/lttngtest/environment.py", line 1024, in __del__ | |
1004 | # File "tests/utils/lttngtest/environment.py", line 1016, in _cleanup | |
1005 | # File "/usr/lib/python3.11/subprocess.py", line 1026, in __init__ | |
1006 | # File "/usr/lib/python3.11/subprocess.py", line 1880, in _execute_child | |
1007 | # File "<frozen os>", line 629, in get_exec_path | |
1008 | # ImportError: sys.meta_path is None, Python is likely shutting down | |
1009 | # | |
1010 | try: | |
1011 | _proc = subprocess.Popen( | |
1012 | ["pkill", "--uid", str(uid)], stderr=subprocess.PIPE | |
1013 | ) | |
1014 | _proc.wait() | |
1015 | except ImportError: | |
1016 | pass | |
1017 | try: | |
1018 | _proc = subprocess.Popen(userdel + [name], stderr=subprocess.PIPE) | |
1019 | _proc.wait() | |
1020 | except ImportError: | |
1021 | pass | |
1022 | ||
ef945e4d JG |
1023 | self._lttng_home = None |
1024 | ||
1025 | def __del__(self): | |
1026 | self._cleanup() | |
1027 | ||
1028 | ||
1029 | @contextlib.contextmanager | |
45ce5eed KS |
1030 | def test_environment(with_sessiond, log=None, with_relayd=False): |
1031 | # type: (bool, Optional[Callable[[str], None]], bool) -> Iterator[_Environment] | |
1032 | env = _Environment(with_sessiond, log, with_relayd) | |
ef945e4d JG |
1033 | try: |
1034 | yield env | |
1035 | finally: | |
1036 | env._cleanup() |