#!/usr/bin/env python3
#
# SPDX-FileCopyrightText: 2022 Jérémie Galarneau <jeremie.galarneau@efficios.com>
#
# SPDX-License-Identifier: GPL-2.0-only
#

import enum
import importlib.util
import json
import multiprocessing
from types import FrameType
from typing import (
    Callable,
    Dict,
    Iterator,
    Optional,
    Tuple,
    Union,
    List,
    Generator,
)
import sys
import pathlib
import platform
import grp
import pwd
import random
import signal
import socket
import subprocess
import shlex
import shutil
import stat
import string
import os
import queue
import re
import tempfile
from . import logger
import time
import threading
import contextlib

import bt2

from . import lttngctl
from . import utils


class ConsumerType(enum.Enum):
    KERNEL = 0
    UST64 = 1
    UST32 = 2


class TemporaryDirectory:
    def __init__(self, prefix):
        # type: (str) -> None
        self._cleanup_callbacks = []
        self._directory_path = tempfile.mkdtemp(prefix=prefix)

    def __del__(self):
        try:
            for func, args, kwargs in self._cleanup_callbacks:
                func(*args, **kwargs)
        finally:
            preserve_env = os.getenv("LTTNG_TEST_PRESERVE_TEST_ENV", "0") != "0" or (
                os.getenv("LTTNG_TEST_PRESERVE_TEST_ENV_ON_FAILURE", "0") != "0"
                and sys.exc_info()[1] is not None
            )
            if preserve_env:
                destination = os.getenv("LTTNG_TEST_PRESERVE_TEST_ENV_DIR", None)
                if destination is not None and pathlib.Path(destination).is_dir():
                    shutil.copytree(
                        self._directory_path,
                        os.path.join(
                            destination, os.path.basename(self._directory_path)
                        ),
                        ignore=utils.shutil_ignore_sockets,
                    )
                    shutil.rmtree(self._directory_path, ignore_errors=True)
            else:
                shutil.rmtree(self._directory_path, ignore_errors=True)

    def add_cleanup_callback(self, func, *args, **kwargs):
        self._cleanup_callbacks.append((func, args, kwargs))

    @property
    def path(self):
        # type: () -> pathlib.Path
        return pathlib.Path(self._directory_path)


class _SignalWaitQueue:
    """
    Utility class useful to wait for a signal before proceeding.

    Simply register the `signal` method as the handler for the signal you are
    interested in and call `wait_for_signal` to wait for its reception.

    Registering a signal:
        signal.signal(signal.SIGWHATEVER, queue.signal)

    Waiting for the signal:
        queue.wait_for_signal()
    """

    def __init__(self):
        self._queue = queue.Queue()  # type: queue.Queue

    def signal(
        self,
        signal_number,
        frame,  # type: Optional[FrameType]
    ):
        self._queue.put_nowait(signal_number)

    def wait_for_signal(self):
        # This uses a timeout + retry loop to avoid
        # issues with hanging on yocto kirkstone powerpc
        wait = True
        while wait:
            try:
                self._queue.get(block=True, timeout=1)
                wait = False
            except queue.Empty:
                continue
            except Exception as e:
                raise e

    @contextlib.contextmanager
    def intercept_signal(self, signal_number):
        # type: (int) -> Generator[None, None, None]
        original_handler = signal.getsignal(signal_number)
        signal.signal(signal_number, self.signal)
        try:
            yield
        except:
            # Restore the original signal handler and forward the exception.
            raise
        finally:
            signal.signal(signal_number, original_handler)


class _LiveViewer:
    """
    Create a babeltrace2 live viewer.
    """

    def __init__(
        self,
        environment,  # type: Environment
        session,  # type: str
        hostname=None,  # type: Optional[str]
    ):
        self._environment = environment
        self._session = session
        self._hostname = hostname
        if self._hostname is None:
            self._hostname = socket.gethostname()
        self._events = []

        ctf_live_cc = bt2.find_plugin("ctf").source_component_classes["lttng-live"]
        self._live_iterator = bt2.TraceCollectionMessageIterator(
            bt2.ComponentSpec(
                ctf_live_cc,
                {
                    "inputs": [
                        "net://localhost:{}/host/{}/{}".format(
                            environment.lttng_relayd_live_port,
                            self._hostname,
                            session,
                        )
                    ],
                    "session-not-found-action": "end",
                },
            )
        )

        try:
            # Cause the connection to be initiated since tests
            # tend to wait for a viewer to be connected before proceeding.
            msg = next(self._live_iterator)
            self._events.append(msg)
        except bt2.TryAgain:
            pass

    @property
    def output(self):
        return self._events

    @property
    def messages(self):
        return [x for x in self._events if type(x) is bt2._EventMessageConst]

    def _drain(self, retry=False, timeout=0):
        start = time.time()
        while True:
            try:
                for msg in self._live_iterator:
                    if type(msg) is bt2._MessageIteratorInactivityMessageConst:
                        break
                    self._events.append(msg)
                break
            except bt2.TryAgain as e:
                if retry:
                    if timeout > 0 and (time.time() - start) > timeout:
                        break
                    time.sleep(0.01)
                    continue
                else:
                    break

    def is_connected(self):
        ctf_live_cc = bt2.find_plugin("ctf").source_component_classes["lttng-live"]
        self._environment._log(
            "Checking for connected clients at 'net://localhost:{}'".format(
                self._environment.lttng_relayd_live_port
            )
        )
        query_executor = bt2.QueryExecutor(
            ctf_live_cc,
            "sessions",
            params={
                "url": "net://localhost:{}".format(
                    self._environment.lttng_relayd_live_port
                )
            },
        )

        for live_session in query_executor.query():
            if (
                live_session["session-name"] == self._session
                and live_session["client-count"] >= 1
            ):
                self._environment._log(
                    "Session '{}' has {} connected clients".format(
                        live_session["session-name"], live_session["client-count"]
                    )
                )
                return True
        return False

    def _wait_until(self, desired_state: bool, timeout=0):
        connected_state = not desired_state
        started = time.time()
        while connected_state != desired_state:
            try:
                if timeout != 0 and (time.time() - started) > timeout:
                    raise RuntimeError(
                        "Timed out waiting for connected clients on session '{}' after {}s".format(
                            self._session, time.time() - started
                        )
                    )

                connected_state = self.is_connected()
            except bt2._Error:
                time.sleep(0.01)
                continue
        return connected_state

    def wait_until_disconnected(self, timeout=0):
        return self._wait_until(False, timeout)

    def wait_until_connected(self, timeout=0):
        return self._wait_until(True, timeout)

    def wait(self, timeout=0, close_iterator=True):
        if self._live_iterator:
            self._drain(retry=True, timeout=timeout)
            if close_iterator:
                del self._live_iterator
                self._live_iterator = None

    def __del__(self):
        pass


def _prepend_library_dirs(
    env: Dict[str, str], library_dirs: Optional[List[pathlib.Path]]
) -> None:
    # Prepend `library_dirs` to the LD_LIBRARY_PATH of `env`, in place.
    #
    # A component of a non-default build profile links against that build's
    # liblttng-ust and liburcu, which typically live outside the loader's
    # default search path (for instance under a 32-bit prefix on a 64-bit
    # host) and are not guaranteed to be reachable through the binary's RPATH.
    # Its declared library directories must therefore be made available
    # explicitly; the in-tree build passes no `library_dirs` and keeps relying
    # on the loader's defaults.
    if not library_dirs:
        return

    existing_library_path = env.get("LD_LIBRARY_PATH", "")
    prepended = ":".join(str(directory) for directory in library_dirs)
    env["LD_LIBRARY_PATH"] = prepended + (
        (":" + existing_library_path) if existing_library_path else ""
    )


class _WaitTraceTestApplication:
    """
    Create an application that waits before tracing. This allows a test to
    launch an application, get its PID, and get it to start tracing when it
    has completed its setup.
    """

    def __init__(
        self,
        binary_path,  # type: pathlib.Path
        event_count,  # type: int
        environment,  # type: Environment
        wait_time_between_events_us=0,  # type: int
        wait_before_exit=False,  # type: bool
        wait_before_exit_file_path=None,  # type: Optional[pathlib.Path]
        run_as=None,  # type: Optional[str]
        text_size=None,  # type: Optional[int]
        fill_text=False,  # type: bool
        wait_before_last_event=False,  # type: bool
        wait_before_last_event_file_path=None,  # type: Optional[pathlib.Path]
        extra_env_vars=dict(),
        emit_event_with_empty_field_name=False,
        emit_blob_events=False,
        register_timeout_s=-1,
        library_dirs: Optional[List[pathlib.Path]] = None,
    ):
        self._process = None
        self._environment = environment  # type: Environment
        self._iteration_count = event_count
        # File that the application will wait to see before tracing its events.
        dir = (
            self._compat_pathlike(environment.lttng_home_location)
            if environment.lttng_home_location
            else None
        )
        if run_as is not None:
            dir = os.path.join(dir, run_as)
        self._app_start_tracing_file_path = pathlib.Path(
            tempfile.mktemp(
                prefix="app_",
                suffix="_start_tracing",
                dir=dir,
            )
        )

        # File that the application will create when all the last event has been emitted
        self._app_before_last_event_file_path = pathlib.Path(
            tempfile.mktemp(
                prefix="app_",
                suffix="_before_last_event",
                dir=dir,
            )
        )

        # File that the application will create when all events have been emitted.
        self._app_tracing_done_file_path = pathlib.Path(
            tempfile.mktemp(
                prefix="app_",
                suffix="_done_tracing",
                dir=dir,
            )
        )

        if wait_before_exit and wait_before_exit_file_path is None:
            wait_before_exit_file_path = pathlib.Path(
                tempfile.mktemp(
                    prefix="app_",
                    suffix="_exit",
                    dir=dir,
                )
            )

        self._wait_before_exit_file_path = wait_before_exit_file_path
        self._has_returned = False
        self._tracing_started = False
        if wait_before_last_event and wait_before_last_event_file_path is None:
            wait_before_last_event_file_path = pathlib.Path(
                tempfile.mktemp(prefix="app_", suffix="_last_event", dir=dir)
            )

        self._wait_before_last_event_file_path = wait_before_last_event_file_path
        test_app_env = environment.get_ust_test_app_env(extra_env_vars)
        # Make sure the app is blocked until it is properly registered to the session daemon.
        if "LTTNG_UST_REGISTER_TIMEOUT" not in extra_env_vars:
            test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = str(register_timeout_s)

        # When the application is built for a non-native ABI (e.g. 32-bit on a
        # 64-bit host), its libraries (e.g. liblttng-ust) live outside the
        # loader's default search path; prepend the directories reported by that
        # build's profile.
        _prepend_library_dirs(test_app_env, library_dirs)

        # File that the application will create to indicate it has completed its initialization.
        app_ready_file_path = tempfile.mktemp(
            prefix="app_",
            suffix="_ready",
            dir=dir,
        )  # type: str

        test_app_args = [str(binary_path)]
        test_app_args.extend(["--iter", str(event_count)])
        test_app_args.extend(
            ["--sync-application-in-main-touch", str(app_ready_file_path)]
        )
        test_app_args.extend(
            ["--sync-before-first-event", str(self._app_start_tracing_file_path)]
        )
        test_app_args.extend(
            [
                "--sync-before-last-event-touch",
                str(self._app_before_last_event_file_path),
            ]
        )
        test_app_args.extend(
            ["--sync-before-exit-touch", str(self._app_tracing_done_file_path)]
        )

        if wait_before_last_event:
            test_app_args.extend(
                [
                    "--sync-before-last-event",
                    str(self._wait_before_last_event_file_path),
                ]
            )

        if wait_before_exit:
            test_app_args.extend(
                ["--sync-before-exit", str(self._wait_before_exit_file_path)]
            )

        if wait_time_between_events_us != 0:
            test_app_args.extend(["--wait", str(wait_time_between_events_us)])

        if text_size is not None:
            test_app_args.extend(["--text-size", str(text_size)])

        if fill_text:
            test_app_args.extend(["--fill-text"])

        if emit_event_with_empty_field_name:
            test_app_args.extend(["--emit-event-with-empty-field-name"])

        if emit_blob_events:
            test_app_args.append("--emit-blob-events")

        if run_as is not None:
            # When running as root and reducing the permissions to run as another
            # user, the test binary needs to be readable and executable by the
            # world; however, the file may be in a deep path or on systems where
            # we don't want to modify the filesystem state (eg. for a person who
            # has downloaded and ran the tests manually).
            # Therefore, the binary_path is copied to a temporary file in the
            # `run_as` user's home directory
            new_binary_path = os.path.join(
                str(environment.lttng_home_location),
                run_as,
                os.path.basename(str(binary_path)),
            )

            if not os.path.exists(new_binary_path):
                shutil.copy(str(binary_path), new_binary_path)

            test_app_args[0] = new_binary_path

            lib_dir = environment.lttng_home_location / run_as / "lib"
            if not os.path.isdir(str(lib_dir)):
                os.mkdir(str(lib_dir))
                # When running dropping privileges, the libraries built in the
                # root-owned directories may not be reachable and readable by
                # the loader running as an unprivileged user. These should also be
                # copied.
                _ldd = subprocess.Popen(
                    ["ldd", new_binary_path],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                )
                if _ldd.wait() != 0:
                    raise RuntimeError(
                        "Error while using `ldd` to determine test application dependencies: `{}`".format(
                            stderr.read().decode("utf-8")
                        )
                    )
                libs = [
                    x.decode("utf-8").split(sep="=>") for x in _ldd.stdout.readlines()
                ]
                libs = [
                    x[1].split(sep=" ")[1]
                    for x in libs
                    if len(x) >= 2 and x[1].find("lttng") != -1
                ]
                for lib in libs:
                    shutil.copy(lib, lib_dir)

            test_app_env["LD_LIBRARY_PATH"] = "{}:{}".format(
                test_app_env.get("LD_LIBRARY_PATH", ""),
                str(lib_dir),
            )

            # As of python 3.9, subprocess.Popen supports a user parameter which
            # runs `setreuid()` before executing the proces and will be preferable
            # when support for older python versions is no longer required.
            test_app_args = [
                "runuser",
                "-u",
                run_as,
                "--",
            ] + test_app_args

        self._environment._log(
            "Launching test application: '{}'".format(
                self._compat_shlex_join(test_app_args)
            )
        )
        self._process = subprocess.Popen(
            test_app_args,
            env=test_app_env,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )  # type: subprocess.Popen

        # Create and start a thread to consume and log the process output
        if self._environment._logging_function:
            self._output_consumer = ProcessOutputConsumer(
                self._process,
                "test-app-{}".format(os.path.basename(str(test_app_args[0]))),
                self._environment._logging_function,
            )
            self._output_consumer.daemon = True
            self._output_consumer.start()
        else:
            self._output_consumer = None

        # Wait for the application to create the file indicating it has fully
        # initialized. Make sure the app hasn't crashed in order to not wait
        # forever.
        self._wait_for_file_to_be_created(pathlib.Path(app_ready_file_path))

    def _wait_for_file_to_be_created(self, sync_file_path):
        # type: (pathlib.Path) -> None
        while True:
            if os.path.exists(self._compat_pathlike(sync_file_path)):
                break

            if self._process.poll() is not None:
                # Application has exited. Re-check for the file since it could
                # have been created between the existence check and the poll.
                if os.path.exists(self._compat_pathlike(sync_file_path)):
                    break

                # Application has unexepectedly returned.
                try:
                    output = self._process.stderr.read().decode("utf-8")
                except AttributeError as e:
                    # e.g. AttributeError: 'NoneType' object has no attribute 'read'
                    # When a process segfaults very early in startup
                    output = (
                        "<process has no stderr attribute>"
                        if self._process.stderr is None
                        else str(e)
                    )
                raise RuntimeError(
                    "Test application has unexpectedly returned while waiting for synchronization file to be created: sync_file=`{sync_file}`, return_code=`{return_code}`, output=`{output}`".format(
                        sync_file=sync_file_path,
                        return_code=self._process.returncode,
                        output=output,
                    )
                )

            time.sleep(0.001)

    def taskset(self, cpu):
        """
        Use taskset to set the CPU affinity for the traced application.
        """
        if not self.vpid:
            raise RuntimeError("No process ID")

        if self._has_returned:
            raise RuntimeError("Application is already marked as returned")

        p = subprocess.Popen(["taskset", "-p", "-c", str(cpu), str(self.vpid)])
        if p.wait() or p.returncode != 0:
            raise RuntimeError("taskset failed, return code: {}".format(p.returncode))

    def taskset_anycpu(self):
        """
        Pin the application to the first available online CPU and return
        its number.
        """
        last_error = None
        for cpu in sorted(online_cpus()):
            try:
                self.taskset(cpu)
                return cpu
            except RuntimeError as taskset_error:
                last_error = taskset_error

        raise RuntimeError("No online CPU could be pinned: {}".format(last_error))

    def touch_last_event_file(self):
        if self._wait_before_last_event_file_path is None:
            raise RuntimeError("wait_before_laswt_event_file_path not set")

        open(self._compat_pathlike(self._wait_before_last_event_file_path), mode="x")

    def touch_exit_file(self):
        open(self._compat_pathlike(self._wait_before_exit_file_path), mode="x")

    @property
    def start_tracing_path(self):
        return self._compat_pathlike(self._app_start_tracing_file_path)

    def trace(self):
        # type: () -> None
        if self._process.poll() is not None:
            # Application has unexpectedly returned.
            raise RuntimeError(
                "Test application has unexpectedly returned before tracing with return code `{return_code}`".format(
                    return_code=self._process.returncode
                )
            )
        open(self._compat_pathlike(self._app_start_tracing_file_path), mode="x")
        self._environment._log("[{}] Tracing started".format(self.vpid))
        self._tracing_started = True

    def wait_for_before_last_event(self):
        # type: () -> None
        if not self._tracing_started:
            raise RuntimeError("Tracing hasn't been started")
        self._wait_for_file_to_be_created(self._app_before_last_event_file_path)
        self._environment._log("[{}] Before last event done".format(self.vpid))

    def wait_for_tracing_done(self):
        # type: () -> None
        if not self._tracing_started:
            raise RuntimeError("Tracing hasn't been started")
        self._wait_for_file_to_be_created(self._app_tracing_done_file_path)
        self._environment._log("[{}] Tracing done".format(self.vpid))

    def wait_for_exit(self):
        # type: () -> None
        if self._process.wait() != 0:
            raise RuntimeError(
                "Test application [{pid}] has exit with return code `{return_code}`".format(
                    pid=self.vpid,
                    return_code=self._process.returncode,
                )
            )
        self._has_returned = True

    @property
    def status(self):
        return self._process.poll()

    @property
    def vpid(self):
        # type: () -> int
        return self._process.pid

    @staticmethod
    def _compat_pathlike(path):
        # type: (pathlib.Path) -> pathlib.Path | str
        """
        The builtin open() and many methods of the 'os' library in Python >= 3.6
        expect a path-like object while prior versions expect a string or
        bytes object. Return the correct type based on the presence of the
        "__fspath__" attribute specified in PEP-519.
        """
        if hasattr(path, "__fspath__"):
            return path
        else:
            return str(path)

    @staticmethod
    def _compat_shlex_join(args):
        # type: list[str] -> str
        # shlex.join was added in python 3.8
        return " ".join([shlex.quote(x) for x in args])

    def __del__(self):
        if self._process is not None and not self._has_returned:
            # This is potentially racy if the pid has been recycled. However,
            # we can't use pidfd_open since it is only available in python >= 3.9.
            self._process.kill()
            self._process.wait()


class WaitTraceTestApplicationGroup:
    def __init__(
        self,
        environment,  # type: Environment
        application_count,  # type: int
        event_count,  # type: int
        wait_time_between_events_us=0,  # type: int
        wait_before_exit=False,  # type: bool
    ):
        self._wait_before_exit_file_path = (
            pathlib.Path(
                tempfile.mktemp(
                    prefix="app_group_",
                    suffix="_exit",
                    dir=_WaitTraceTestApplication._compat_pathlike(
                        environment.lttng_home_location
                    ),
                )
            )
            if wait_before_exit
            else None
        )

        self._apps = []
        self._consumers = []
        for i in range(application_count):
            new_app = environment.launch_wait_trace_test_application(
                event_count,
                wait_time_between_events_us,
                wait_before_exit,
                self._wait_before_exit_file_path,
            )

            # Attach an output consumer to log the application's error output (if any).
            if environment._logging_function:
                app_output_consumer = ProcessOutputConsumer(
                    new_app._process,
                    "app-{}".format(str(new_app.vpid)),
                    environment._logging_function,
                )  # type: Optional[ProcessOutputConsumer]
                app_output_consumer.daemon = True
                app_output_consumer.start()
                self._consumers.append(app_output_consumer)

            self._apps.append(new_app)

    def trace(self):
        # type: () -> None
        for app in self._apps:
            app.trace()

    def wait_for_tracing_done(self):
        # type: () -> None
        for app in self._apps:
            app.wait_for_tracing_done()

    def exit(
        self, wait_for_apps=False  # type: bool
    ):
        if self._wait_before_exit_file_path is None:
            raise RuntimeError(
                "Can't call exit on an application group created with `wait_before_exit=False`"
            )

        # Wait for apps to have produced all of their events so that we can
        # cause the death of all apps to happen within a short time span.
        self.wait_for_tracing_done()

        self._apps[0].touch_exit_file()

        # Performed in two passes to allow tests to stress the unregistration of many applications.
        # Waiting for each app to exit turn-by-turn would defeat the purpose here.
        if wait_for_apps:
            for app in self._apps:
                app.wait_for_exit()


class _TraceTestApplication:
    """
    Create an application that emits events as soon as it is launched. In most
    scenarios, it is preferable to use a WaitTraceTestApplication.
    """

    def __init__(
        self,
        binary_path: pathlib.Path,
        environment: "_Environment",
        extra_env_vars: dict = dict(),
        library_dirs: Optional[List[pathlib.Path]] = None,
        **kwargs,
    ):
        self._process = None
        self._environment = environment  # type: Environment
        self._has_returned = False

        test_app_env = environment.get_ust_test_app_env(extra_env_vars)
        # Make sure the app is blocked until it is properly registered to the session daemon.
        if "LTTNG_UST_REGISTER_TIMEOUT" not in extra_env_vars:
            test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"

        # See the equivalent handling in `_WaitTraceTestApplication`.
        _prepend_library_dirs(test_app_env, library_dirs)

        if type(binary_path) is str:
            test_app_args = [str(binary_path)]
        else:
            # Assume it is a list
            test_app_args = binary_path

        self._process = subprocess.Popen(
            test_app_args, env=test_app_env, **kwargs
        )  # type: subprocess.Popen
        self._environment._log(
            "Launched tested application '{}' with pid '{}'".format(
                test_app_args, self._process.pid
            )
        )

    def wait_for_exit(self):
        # type: () -> None
        if self._process.wait() != 0:
            raise RuntimeError(
                "Test application has exit with return code `{return_code}`".format(
                    return_code=self._process.returncode
                )
            )
        self._has_returned = True

    def __del__(self):
        if self._process is not None and not self._has_returned:
            # This is potentially racy if the pid has been recycled. However,
            # we can't use pidfd_open since it is only available in python >= 3.9.
            self._environment._log(
                "Terminating test app pid '{}'".format(self._process.pid)
            )
            self._process.terminate()
            try:
                self._process.wait(timeout=self._environment.teardown_timeout)
            except TimeoutException:
                self._environment._log(
                    "Killing test app pid '{}'".format(self._process.pid)
                )
                self._process.kill()


# Description of one of the agent domain test applications of the test suite
# (see `tests/regression/ust/java-*/` and
# `tests/utils/testapp/gen-py-events.py`).
#
# Every one of those applications emits, in order:
#
# * `event_count` log statements of `info_log_level` on `logger_name`,
#   optionally interleaved with as many statements of `debug_log_level` on the
#   same logger.
#
# * Optionally, a single `info_log_level` statement on `second_logger_name`.
#
# The agents of every domain trace all the log statements of their domain
# through a single user space event class, `ust_event_name`: the logger name
# and the log level are payload fields, which an event rule matches with a
# filter.
class AgentTestApplicationDescriptor:
    def __init__(
        self,
        domain: lttngctl.TracingDomain,
        ust_event_name: str,
        logger_name: str,
        second_logger_name: str,
        info_log_level: lttngctl.LogLevel,
        debug_log_level: lttngctl.LogLevel,
    ) -> None:
        self._domain = domain
        self._ust_event_name = ust_event_name
        self._logger_name = logger_name
        self._second_logger_name = second_logger_name
        self._info_log_level = info_log_level
        self._debug_log_level = debug_log_level

    @property
    def domain(self) -> lttngctl.TracingDomain:
        return self._domain

    @property
    def ust_event_name(self) -> str:
        return self._ust_event_name

    @property
    def logger_name(self) -> str:
        return self._logger_name

    @property
    def second_logger_name(self) -> str:
        return self._second_logger_name

    @property
    def info_log_level(self) -> lttngctl.LogLevel:
        return self._info_log_level

    @property
    def debug_log_level(self) -> lttngctl.LogLevel:
        return self._debug_log_level


# Where the test application of an agent domain lives in a build and how to
# start it. `launch_agent_test_application()` builds its command line from
# this, while `AgentTestApplicationDescriptor` describes what the application
# logs.
#
# `directory` is relative to the `tests` directory of a build, `java_class` is
# the class to launch (`None` for the Python application), and
# `java_properties` names the extra Java system properties to set, their value
# being a file name relative to `directory`.
class _AgentTestApplicationLaunchInfo:
    def __init__(
        self,
        directory: pathlib.Path,
        java_class: Optional[str],
        java_properties: Dict[str, str],
    ) -> None:
        self._directory = directory
        self._java_class = java_class
        self._java_properties = java_properties

    @property
    def directory(self) -> pathlib.Path:
        return self._directory

    @property
    def java_class(self) -> Optional[str]:
        return self._java_class

    @property
    def java_properties(self) -> Dict[str, str]:
        return self._java_properties


_AGENT_TEST_APPLICATIONS = {
    lttngctl.TracingDomain.JUL: AgentTestApplicationDescriptor(
        domain=lttngctl.TracingDomain.JUL,
        ust_event_name="lttng_jul:event",
        logger_name="JTestLTTng",
        second_logger_name="JTestLTTng2",
        info_log_level=lttngctl.JULLogLevel.INFO,
        debug_log_level=lttngctl.JULLogLevel.FINEST,
    ),
    lttngctl.TracingDomain.Log4j: AgentTestApplicationDescriptor(
        domain=lttngctl.TracingDomain.Log4j,
        ust_event_name="lttng_log4j:event",
        logger_name="log4j-event",
        second_logger_name="log4j-event-2",
        info_log_level=lttngctl.Log4jLogLevel.INFO,
        debug_log_level=lttngctl.Log4jLogLevel.DEBUG,
    ),
    lttngctl.TracingDomain.Log4j2: AgentTestApplicationDescriptor(
        domain=lttngctl.TracingDomain.Log4j2,
        ust_event_name="lttng_log4j2:event",
        logger_name="log4j2-event-1",
        second_logger_name="log4j2-event-2",
        info_log_level=lttngctl.Log4j2LogLevel.INFO,
        debug_log_level=lttngctl.Log4j2LogLevel.DEBUG,
    ),
    lttngctl.TracingDomain.Python: AgentTestApplicationDescriptor(
        domain=lttngctl.TracingDomain.Python,
        ust_event_name="lttng_python:event",
        logger_name="python-ev-test1",
        second_logger_name="python-ev-test2",
        info_log_level=lttngctl.PythonLogLevel.INFO,
        debug_log_level=lttngctl.PythonLogLevel.DEBUG,
    ),
}

_AGENT_TEST_APPLICATION_LAUNCH_INFO = {
    lttngctl.TracingDomain.JUL: _AgentTestApplicationLaunchInfo(
        directory=pathlib.Path("regression") / "ust" / "java-jul",
        java_class="JTestLTTng",
        java_properties={},
    ),
    lttngctl.TracingDomain.Log4j: _AgentTestApplicationLaunchInfo(
        directory=pathlib.Path("regression") / "ust" / "java-log4j",
        java_class="JTestLTTngLog4j",
        java_properties={},
    ),
    lttngctl.TracingDomain.Log4j2: _AgentTestApplicationLaunchInfo(
        directory=pathlib.Path("regression") / "ust" / "java-log4j2",
        java_class="JTestLTTngLog4j2",
        # Selects the Log4j 2.x appender bound to the `log4j2` domain; the same
        # application logs to the legacy `log4j` domain when configured with
        # `domain-log4j.xml`.
        java_properties={"log4j2.configurationFile": "domain-log4j2.xml"},
    ),
    lttngctl.TracingDomain.Python: _AgentTestApplicationLaunchInfo(
        directory=pathlib.Path("utils") / "testapp",
        java_class=None,
        java_properties={},
    ),
}


def agent_test_application_descriptor(
    domain: lttngctl.TracingDomain,
) -> AgentTestApplicationDescriptor:
    """
    Return the descriptor of the agent test application of `domain`.
    """
    try:
        return _AGENT_TEST_APPLICATIONS[domain]
    except KeyError:
        raise ValueError("`{}` is not an agent domain".format(domain))


def _agent_test_application_launch_info(
    domain: lttngctl.TracingDomain,
) -> _AgentTestApplicationLaunchInfo:
    try:
        return _AGENT_TEST_APPLICATION_LAUNCH_INFO[domain]
    except KeyError:
        raise ValueError("`{}` is not an agent domain".format(domain))


class ProcessOutputConsumer(threading.Thread, logger._Logger):
    def __init__(
        self,
        process,  # type: subprocess.Popen
        name,  # type: str
        log,  # type: Callable[[str], None]
    ):
        threading.Thread.__init__(self)
        self._prefix = name
        logger._Logger.__init__(self, log)
        self._process = process

    def run(self):
        # type: () -> None
        while self._process.poll() is None:
            if not self._process.stdout:
                break
            line = self._process.stdout.readline().decode("utf-8").replace("\n", "")
            if len(line) != 0:
                self._log("{prefix}: {line}".format(prefix=self._prefix, line=line))


class SavingProcessOutputConsumer(ProcessOutputConsumer):
    def __init__(self, process, name, log):
        self._lines = []
        super().__init__(process=process, name=name, log=log)

    def run(self):
        # type: () -> None
        while self._process.poll() is None:
            if not self._process.stdout:
                break
            line = self._process.stdout.readline().decode("utf-8").replace("\n", "")
            if len(line) != 0:
                self._lines.append(line)
                self._log("{prefix}: {line}".format(prefix=self._prefix, line=line))

    @property
    def output(self):
        return self._lines


class _BuildProfile:
    """
    Describes the artifacts of a single build.

    A profile is loaded from a file generated at build time (see
    `tests/utils/lttng-build-profile.json.in`) or hand-authored to point the
    test framework at the artifacts of another build. Beyond a name and a
    description, it records the build's ABI word size and the paths of its
    daemons, client and test applications, so that a test can run a given
    component from a build other than the in-tree one.

    Paths in the profile are resolved relative to the profile file's own
    directory, unless they are absolute.
    """

    def __init__(self, profile_path: pathlib.Path) -> None:
        profile_path = pathlib.Path(profile_path)
        self._base_dir = profile_path.parent
        with open(str(profile_path), "r") as profile_file:
            contents = json.load(profile_file)

        abi = contents["abi"]
        self._name = abi["name"]
        self._description = abi["description"]
        self._word_size_bits = int(abi["word_size_bits"])
        self._binaries = contents["binaries"]
        self._testapp_dir = self._resolve(contents["testapp_dir"])
        # Optional: only the test applications which live outside of
        # `testapp_dir` need those (see `tests_path()` and `java`).
        tests_dir = contents.get("tests_dir")
        self._tests_dir = self._resolve(tests_dir) if tests_dir else None
        self._java = contents.get("java", {})
        # An empty entry is possible when a dependency's library directory is
        # unknown (e.g. a build configured without LTTng-UST).
        self._library_dirs = [
            self._resolve(directory)
            for directory in contents["library_dirs"]
            if directory
        ]

    def __repr__(self) -> str:
        return "BuildProfile(name={!r}, word_size_bits={})".format(
            self._name, self._word_size_bits
        )

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, _BuildProfile):
            return NotImplemented
        return self._name == other._name

    def __hash__(self) -> int:
        return hash(self._name)

    def _resolve(self, path: str) -> pathlib.Path:
        path = pathlib.Path(path)
        if not path.is_absolute():
            path = self._base_dir / path
        # Collapse any `..` segments so logged paths stay readable.
        return pathlib.Path(os.path.normpath(str(path)))

    @property
    def name(self) -> str:
        return self._name

    @property
    def description(self) -> str:
        return self._description

    @property
    def word_size_bits(self) -> int:
        return self._word_size_bits

    @property
    def sessiond_path(self) -> pathlib.Path:
        return self._resolve(self._binaries["sessiond"])

    @property
    def relayd_path(self) -> pathlib.Path:
        return self._resolve(self._binaries["relayd"])

    @property
    def consumerd_path(self) -> pathlib.Path:
        return self._resolve(self._binaries["consumerd"])

    @property
    def lttng_client_path(self) -> pathlib.Path:
        return self._resolve(self._binaries["lttng_client"])

    @property
    def library_dirs(self) -> List[pathlib.Path]:
        return self._library_dirs

    def testapp_path(self, *components: str) -> pathlib.Path:
        return self._testapp_dir.joinpath(*components)

    def tests_path(self, *components: Union[str, pathlib.Path]) -> pathlib.Path:
        """
        A path under the `tests` directory of the build, which holds the
        artifacts of the test applications that are not part of `testapp_dir`
        (the Java and Python agent applications, for instance).
        """
        if self._tests_dir is None:
            raise RuntimeError(
                "Build profile `{}` declares no `tests_dir`".format(self._name)
            )

        return self._tests_dir.joinpath(*components)

    @property
    def java_binary(self) -> Optional[str]:
        """
        The `java` command that compiled the Java test applications of the
        build, which may not be the one of the `PATH`, or `None` when the build
        is configured without the Java agent tests.
        """
        return self._java.get("binary") or None

    @property
    def java_classpath(self) -> Optional[str]:
        """
        The class path holding the LTTng-UST Java agent jars that the build was
        configured with, or `None` when it is unknown.
        """
        return self._java.get("classpath") or None


# Generate a temporary environment in which to execute a test.
class _Environment(logger._Logger):
    def __init__(
        self,
        with_sessiond,  # type: bool
        log=None,  # type: Optional[Callable[[str], None]]
        with_relayd=False,  # type: bool
        extra_env_vars=dict(),  # type: dict
        skip_temporary_lttng_home=False,  # type: bool
        enable_kernel_domain=False,  # type: bool
        skip_temporary_lttng_rundir=False,  # type: bool
        sessiond_extra_args=None,  # type: Optional[List[str]]
        sessiond_profile=None,  # type: Optional[_BuildProfile]
        relayd_profile=None,  # type: Optional[_BuildProfile]
        client_profile=None,  # type: Optional[_BuildProfile]
        consumerd_profiles=None,  # type: Optional[List[_BuildProfile]]
    ):
        super().__init__(log)
        self._processes = []

        signal.signal(signal.SIGTERM, self._handle_termination_signal)
        signal.signal(signal.SIGINT, self._handle_termination_signal)
        if os.getenv("LTTNG_TEST_VERBOSE_BABELTRACE", "0") != "0":
            # @TODO: Is there a way to feed the logging output to
            # the logger._Logger instead of directly to stderr?
            bt2.set_global_logging_level(bt2.LoggingLevel.TRACE)

        # Assumes the project's hierarchy to this file is:
        # tests/utils/python/this_file
        self._project_root = (
            pathlib.Path(__file__).absolute().parents[3]
        )  # type: pathlib.Path

        # Build profiles describe the artifacts (daemons, client, test
        # applications) of a given build; see `_BuildProfile` and `profiles()`.
        # Any component left without an explicit profile override is taken from
        # the in-tree build's own profile, so the default behaviour matches a
        # plain build tree.
        self._default_profile = type(self).default_profile()
        self._sessiond_profile = sessiond_profile or self._default_profile
        self._relayd_profile = relayd_profile or self._default_profile
        self._client_profile = client_profile or self._default_profile

        if consumerd_profiles is None:
            consumerd_profiles = [self._default_profile]
        self._consumerd_profiles = self._validate_consumerd_profiles(consumerd_profiles)

        self._extra_env_vars = extra_env_vars

        # There are times when we need to exercise default configurations
        # that don't set LTTNG_HOME. When doing this, it makes it impossible
        # to safely run parallel tests.
        self._lttng_home = None
        if not skip_temporary_lttng_home:
            self._lttng_home = TemporaryDirectory(
                "lttng_test_env_home"
            )  # type: Optional[TemporaryDirectory]
            os.chmod(
                str(self._lttng_home.path),
                stat.S_IRUSR
                | stat.S_IWUSR
                | stat.S_IXUSR
                | stat.S_IROTH
                | stat.S_IXOTH,
            )

        # When starting a sessiond daemon with the root user, the rundir
        # is global by default. Use LTTNG_RUNDIR to set it to a temporary
        # directory to avoid trampling on parallel tests.
        self._lttng_rundir = None
        if os.getuid() == 0 and not skip_temporary_lttng_rundir:
            self._lttng_rundir = TemporaryDirectory("lttng_test_env_rundir")
            self._extra_env_vars["LTTNG_RUNDIR"] = str(self._lttng_rundir.path)

        self._preserve_test_env = os.getenv("LTTNG_TEST_PRESERVE_TEST_ENV", "0") != "1"
        self._relayd_control_port = 0
        self._relayd_data_port = 0
        self._relayd_live_port = 0
        self._relayd_env_vars = {}
        self._relayd = None
        self._relayd_output_consumer = None
        self._dummy_users = {}  # type: Dictionary[int, string]
        self._dummy_groups = {}  # type: Dictionary[int, string]
        self.teardown_timeout = os.getenv("LTTNG_TEST_TEARDOWN_TIMEOUT", "60")
        self._sessiond = None
        self._sessiond_extra_args = sessiond_extra_args if sessiond_extra_args else []

        # Start daemons as required
        self._relayd = (
            self._launch_lttng_relayd() if with_relayd else None
        )  # type: Optional[subprocess.Popen[bytes]]
        self._sessiond = (
            self._launch_lttng_sessiond(enable_kernel_domain) if with_sessiond else None
        )  # type: Optional[subprocess.Popen[bytes]]

        # This is a bit of particularity for the testing infrastructure.
        # When the sessiond is started, it will change the permissions of the
        # set LTTNG_RUNDIR. When running tests that exercise a traced application
        # run as a non-root user, it is convenient that the permissions are set
        # so that the shm wait file in LTTNG_UST_APP_PATH can be created.
        #
        # The behaviour of a traced application is to disable the LTTNG_UST_APP_PATH
        # connection if the shm wait file cannot be created, and subsequent retries
        # only attempt connections to the global socket in /var/run/lttng/.
        if self._lttng_rundir is not None:
            os.chmod(
                str(self._lttng_rundir.path),
                stat.S_IRUSR
                | stat.S_IWUSR
                | stat.S_IXUSR
                | stat.S_IROTH
                | stat.S_IWOTH
                | stat.S_IXOTH,
            )

    def log(self, *args, **kwargs):
        if self._log:
            self._log(*args, **kwargs)

    @property
    def teardown_timeout(self):
        return self._teardown_timeout

    @teardown_timeout.setter
    def teardown_timeout(self, value):
        try:
            self._teardown_timeout = int(value)
        except TypeError:
            self._teardown_timeout = None
        except ValueError:
            self._teardown_timeout = None

    @property
    def lttng_home_location(self):
        # type: () -> pathlib.Path
        if self._lttng_home is not None:
            return self._lttng_home.path
        return None

    @property
    def lttng_log_dir(self):
        path = os.getenv("LTTNG_TEST_LOG_DIR", "")
        if path == "" or path is None or path == "-":
            return None
        return path

    @property
    def lttng_rundir(self):
        # type: () -> pathlib.Path
        if self._lttng_rundir is not None:
            return self._lttng_rundir.path
        return None

    @property
    def lttng_client_path(self):
        # type: () -> pathlib.Path
        return self._client_profile.lttng_client_path

    _profiles_cache: Optional[List[_BuildProfile]] = None

    @classmethod
    def _default_profile_path(cls) -> pathlib.Path:
        # The in-tree build's own profile is generated next to this module at
        # build time.
        return pathlib.Path(__file__).absolute().parents[1] / "lttng-build-profile.json"

    @classmethod
    def profiles(cls) -> "List[_BuildProfile]":
        """
        All build profiles known to the test framework.

        The in-tree build's own profile comes first; any profile found (by
        `*.json` extension) in the directory named by
        `LTTNG_TOOLS_BUILD_PROFILE_DIR` follows. Tests that exercise multi-ABI
        scenarios filter this list (see `profiles_with_word_size_bits()`) and size
        their TAP plan from the result.
        """
        if cls._profiles_cache is not None:
            return cls._profiles_cache

        profiles = [_BuildProfile(cls._default_profile_path())]
        names_seen = {profiles[0].name}

        profile_dir = os.environ.get("LTTNG_TOOLS_BUILD_PROFILE_DIR")
        if profile_dir:
            for profile_file in sorted(pathlib.Path(profile_dir).glob("*.json")):
                profile = _BuildProfile(profile_file)
                if profile.name in names_seen:
                    raise RuntimeError(
                        "Duplicate build profile name `{}` (from `{}`)".format(
                            profile.name, profile_file
                        )
                    )
                names_seen.add(profile.name)
                profiles.append(profile)

        cls._profiles_cache = profiles
        return profiles

    @classmethod
    def profiles_with_word_size_bits(cls, word_size_bits: int) -> "List[_BuildProfile]":
        """All known build profiles whose ABI has the given word size (32 or 64)."""
        return [
            profile
            for profile in cls.profiles()
            if profile.word_size_bits == word_size_bits
        ]

    @classmethod
    def default_profile(cls) -> "_BuildProfile":
        """The in-tree build's own profile, used by default for every component."""
        return cls.profiles()[0]

    @staticmethod
    def _validate_consumerd_profiles(
        consumerd_profiles: "List[_BuildProfile]",
    ) -> "List[_BuildProfile]":
        # A session daemon has a single consumer daemon slot per word size
        # (--consumerd32-path and --consumerd64-path), so at most two profiles
        # may be registered and they must have distinct word sizes.
        if len(consumerd_profiles) > 2:
            raise RuntimeError(
                "At most two consumer daemon profiles may be registered, got {}".format(
                    len(consumerd_profiles)
                )
            )
        word_sizes_bits = [profile.word_size_bits for profile in consumerd_profiles]
        if len(set(word_sizes_bits)) != len(word_sizes_bits):
            raise RuntimeError(
                "Consumer daemon profiles must have distinct word sizes, got {}".format(
                    word_sizes_bits
                )
            )
        return list(consumerd_profiles)

    def _library_dirs_for(
        self, profile: "_BuildProfile"
    ) -> Optional[List[pathlib.Path]]:
        # The in-tree build's libraries are found through the loader's usual
        # mechanisms; only components from another build need their library
        # directories injected explicitly.
        if profile == self._default_profile:
            return None
        return profile.library_dirs

    def _prepend_profile_library_dirs(
        self, env: Dict[str, str], profile: "_BuildProfile"
    ) -> None:
        # Make a daemon's, client's or application's own library directories
        # available through LD_LIBRARY_PATH when it comes from a non-default
        # profile (see `_library_dirs_for` and `_prepend_library_dirs`).
        _prepend_library_dirs(env, self._library_dirs_for(profile))

    def _assert_consumerd_registered_for(self, profile: "_BuildProfile") -> None:
        # An application registers to the consumer daemon of its own word size;
        # fail early with a clear message if none was registered for it.
        if not any(
            registered.word_size_bits == profile.word_size_bits
            for registered in self._consumerd_profiles
        ):
            raise RuntimeError(
                "No {}-bit consumer daemon is registered for application profile "
                "`{}`; pass it to `test_environment(consumerd_profiles=...)`".format(
                    profile.word_size_bits, profile.name
                )
            )

    def get_lttng_client_env(
        self, extra_env_vars: Dict[str, str] = dict()
    ) -> Dict[str, str]:
        client_env = os.environ.copy()
        if self.lttng_home_location is not None:
            client_env["LTTNG_HOME"] = str(self.lttng_home_location)

        if self.lttng_rundir is not None:
            client_env["LTTNG_RUNDIR"] = str(self.lttng_rundir)

        client_env.update(self._extra_env_vars)
        client_env.update(extra_env_vars)
        self._prepend_profile_library_dirs(client_env, self._client_profile)
        return client_env

    def get_ust_test_app_env(
        self, extra_env_vars: Dict[str, str] = dict()
    ) -> Dict[str, str]:
        test_app_env = os.environ.copy()
        if self.lttng_home_location is not None:
            test_app_env["LTTNG_HOME"] = str(self.lttng_home_location)

        if self.lttng_rundir is not None:
            test_app_env["LTTNG_UST_APP_PATH"] = str(self.lttng_rundir)

        test_app_env["LTTNG_UST_REGISTER_TIMEOUT"] = "-1"
        test_app_env.update(self._extra_env_vars)
        test_app_env.update(extra_env_vars)
        return test_app_env

    def _relayd_port(self, port_name):
        port_attr = "_relayd_{}_port".format(port_name)
        if getattr(self, port_attr) == 0:
            try:
                self._update_relayd_port(port_name)
            except Exception:
                pass
        return getattr(self, port_attr)

    def _update_relayd_port(self, port_name):
        port_attr = "_relayd_{}_port".format(port_name)
        port_file = self.lttng_relayd_control_directory / "{}.port".format(port_name)
        try:
            with open(str(port_file), "r") as f:
                setattr(self, port_attr, int(f.read()))
        except Exception as e:
            self._log("Failed to open port file '{}': {}".format(port_file, str(e)))
            raise e

    @property
    def lttng_consumerd_control_directory(self):
        if self.lttng_sessiond_env_vars.get("LTTNG_RUNDIR"):
            return pathlib.Path(self.lttng_sessiond_env_vars.get("LTTNG_RUNDIR"))
        else:
            home_dir = (
                self.lttng_sessiond_env_vars["LTTNG_HOME"]
                if ("LTTNG_HOME" in self.lttng_sessiond_env_vars)
                else os.environ.get("HOME", os.path.expanduser("~"))
            )
            return (
                pathlib.Path("/var/run/lttng")
                if os.getuid() == 0
                else pathlib.Path(home_dir) / ".lttng"
            )

    @property
    def lttng_consumerd_kernel_pid(self):
        pid_file = str(self.lttng_consumerd_control_directory / "kconsumerd" / "pid")
        if os.path.exists(pid_file):
            with open(pid_file, "r") as f:
                return f.read().strip()

        return None

    @property
    def lttng_consumerd_ust32_pid(self):
        pid_file = str(
            self.lttng_consumerd_control_directory / "ustconsumerd32" / "pid"
        )
        if os.path.exists(pid_file):
            with open(pid_file, "r") as f:
                return f.read().strip()

        return None

    @property
    def lttng_consumerd_ust64_pid(self):
        pid_file = str(
            self.lttng_consumerd_control_directory / "ustconsumerd64" / "pid"
        )
        if os.path.exists(pid_file):
            with open(pid_file, "r") as f:
                return f.read().strip()

        return None

    @property
    def lttng_relayd_control_port(self):
        # type: () -> int
        return self._relayd_port("control")

    @property
    def lttng_relayd_data_port(self):
        # type: () -> int
        return self._relayd_port("data")

    @property
    def lttng_relayd_live_port(self):
        # type: () -> int
        return self._relayd_port("live")

    @property
    def lttng_relayd_control_directory(self):
        if self.lttng_relayd_env_vars.get("LTTNG_RUNDIR"):
            return (
                pathlib.Path(self.lttng_relayd_env_vars.get("LTTNG_RUNDIR")) / "relayd"
            )
        else:
            home_dir = (
                self.lttng_relayd_env_vars["LTTNG_HOME"]
                if ("LTTNG_HOME" in self.lttng_relayd_env_vars)
                else os.environ.get("HOME", os.path.expanduser("~"))
            )
            return (
                pathlib.Path("/var/run/lttng")
                if os.getuid() == 0
                else pathlib.Path(home_dir) / ".lttng"
            ) / "relayd"

    @property
    def lttng_relayd_env_vars(self):
        return self._relayd_env_vars

    @property
    def lttng_relayd_output_path(self):
        return os.path.join(
            (
                str(self.lttng_home_location)
                if self.lttng_home_location
                else os.getenv("HOME")
            ),
            "lttng-traces",
            platform.node(),
        )

    @property
    def preserve_test_env(self):
        # type: () -> bool
        return self._preserve_test_env

    @staticmethod
    def allows_destructive():
        # type: () -> bool
        return os.getenv("LTTNG_ENABLE_DESTRUCTIVE_TESTS", "") == "will-break-my-system"

    def lttng_consumerd_get_pid(self, consumerd_type: ConsumerType) -> int:
        pid = None
        if consumerd_type == ConsumerType.KERNEL:
            pid = self.lttng_consumerd_kernel_pid
        elif consumerd_type == ConsumerType.UST64:
            pid = self.lttng_consumerd_ust64_pid
        elif consumerd_type == ConsumerType.UST32:
            pid = self.lttng_consumerd_ust32_pid
        else:
            raise ValueError("Unsupport consumerd_type: {}".format(consumerd_type))

        if pid is None:
            raise Exception(
                "No PID available for consumerd type `{}`".format(consumerd_type)
            )

        return pid

    def lttng_consumerd_is_paused(self, consumerd_type: ConsumerType) -> bool:
        pid = self.lttng_consumerd_get_pid(consumerd_type)
        pattern = re.compile(r"^\$1 = (?P<paused>\d+)$")
        script_commands = [
            "attach {}".format(pid),
            "print data_consumption_paused",
            "detach",
        ]
        gdb, script = utils.gdb_script(
            script_commands, {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT}
        )
        output, _ = gdb.communicate()
        if gdb.returncode != 0:
            raise Exception(
                "GDB returned non-zero exit code: {}".format(gdb.returncode)
            )

        paused = None
        for line in output.decode("utf-8").splitlines():
            m = pattern.match(line)
            if m:
                paused = False if m.group("paused") == "0" else True
                break

        if paused is None:
            raise Exception(
                "Couldn't find data_consumption_paused variable in GDB output: {}".format(
                    output.decode("utf-8")
                )
            )

        return paused

    def _set_lttng_consumerd_paused(
        self, consumerd_type: ConsumerType, paused: bool
    ) -> None:
        pid = self.lttng_consumerd_get_pid(consumerd_type)
        script_commands = [
            "attach {}".format(pid),
            "set data_consumption_paused = {}".format("1" if paused else "0"),
        ]
        gdb, script = utils.gdb_script(script_commands)
        gdb.wait()
        if gdb.returncode != 0:
            raise Exception(
                "GDB returned non-zero exit code: {}".format(gdb.returncode)
            )

    def lttng_consumerd_pause(self, consumerd_type: ConsumerType) -> None:
        self._set_lttng_consumerd_paused(consumerd_type, True)

    def lttng_consumerd_resume(self, consumerd_type: ConsumerType) -> None:
        self._set_lttng_consumerd_paused(consumerd_type, False)

    def lttng_sessiond_is_notification_paused(self) -> bool:
        pid = self._sessiond.pid
        pattern = re.compile(r"^\$1 = (?P<paused>\d+)$")
        script_commands = [
            "attach {}".format(pid),
            "print notifier_consumption_paused",
            "detach",
        ]
        gdb, script = utils.gdb_script(
            script_commands, {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT}
        )
        output, _ = gdb.communicate()
        if gdb.returncode != 0:
            raise Exception(
                "GDB returned non-zero exit code: {}".format(gdb.returncode)
            )

        paused = None
        for line in output.decode("utf-8").splitlines():
            m = pattern.match(line)
            if m:
                paused = False if m.group("paused") == "0" else True
                break

        if paused is None:
            raise Exception(
                "Couldn't find notifier_consumption_paused variable in GDB output: {}".format(
                    output.decode("utf-8")
                )
            )

        return paused

    def lttng_sessiond_pause_notifications(self, paused=True) -> None:
        pid = self._sessiond.pid
        script_commands = [
            "attach {}".format(pid),
            "set notifier_consumption_paused = {}".format("1" if paused else "0"),
        ]
        gdb, script = utils.gdb_script(script_commands)
        gdb.wait()
        if gdb.returncode != 0:
            raise Exception(
                "GDB returned non-zero exit code: {}".format(gdb.returncode)
            )

    def create_dummy_user(self):
        # type: () -> (int, str)
        # Create a dummy user. The uid and username will be returned in a tuple.
        # If the name already exists, an exception will be thrown.
        # The users will be removed when the environment is cleaned up.
        name = "".join([random.choice(string.ascii_lowercase) for x in range(10)])

        try:
            entry = pwd.getpwnam(name)
            raise Exception("User '{}' already exists".format(name))
        except KeyError:
            pass

        # Create user
        proc = subprocess.Popen(
            [
                "useradd",
                "--base-dir",
                str(self._lttng_home.path),
                "--create-home",
                "--no-user-group",
                "--shell",
                "/bin/sh",
                name,
            ]
        )
        proc.wait()
        if proc.returncode != 0:
            raise Exception(
                "Failed to create user '{}', useradd returned {}".format(
                    name, proc.returncode
                )
            )

        entry = pwd.getpwnam(name)
        self._dummy_users[entry[2]] = name
        return (entry[2], name)

    def create_dummy_group(self):
        # type: () -> (int, str)
        # Create a dummy group. The gid and group name will be returned in a tuple.
        # If the name already exists, an exception will be thrown.
        # The groups will be removed when the environment is cleaned up.
        name = "".join([random.choice(string.ascii_lowercase) for x in range(10)])

        try:
            entry = grp.getgrnam(name)
            raise Exception("Group '{}' already exists".format(name))
        except KeyError:
            pass

        # Create group
        proc = subprocess.Popen(["groupadd", name])
        proc.wait()
        if proc.returncode != 0:
            raise Exception(
                "Failed to create group '{}', groupadd returned {}".format(
                    name, proc.returncode
                )
            )

        entry = grp.getgrnam(name)
        self._dummy_groups[entry[2]] = name
        return (entry[2], name)

    def create_temporary_directory(self, prefix=None, dir=None):
        # type: (Optional[str], Optional[str]) -> pathlib.Path
        # Simply return a path that is contained within LTTNG_HOME; it will
        # be destroyed when the temporary home goes out of scope.
        if dir is None:
            dir = str(self.lttng_home_location) if self.lttng_home_location else None

        return pathlib.Path(
            tempfile.mkdtemp(prefix="tmp" if prefix is None else prefix, dir=dir)
        )

    @staticmethod
    def run_kernel_tests():
        # type: () -> bool
        return (
            os.getenv("LTTNG_TOOLS_DISABLE_KERNEL_TESTS", "0") != "1"
            and os.getuid() == 0
        )

    @staticmethod
    def run_long_regression_tests():
        # type: () -> bool
        value = os.getenv("LTTNG_TOOLS_RUN_TESTS_LONG_REGRESSION")
        return bool(value and value != "0")

    @staticmethod
    def _java_class_path(profile: "_BuildProfile") -> Optional[str]:
        # The agent jars come from the `CLASSPATH` of the environment, like the
        # shell agent tests use it, and from the build's own configuration when
        # it is unset.
        return os.environ.get("CLASSPATH") or profile.java_classpath

    @classmethod
    def run_agent_domain_tests(
        cls,
        domain: lttngctl.TracingDomain,
        profile: "Optional[_BuildProfile]" = None,
    ) -> bool:
        """
        Whether the test application of the agent domain `domain` of the build
        `profile` (the in-tree build by default) can run.

        The Java applications are only compiled when the build is configured
        with the Java agent tests of their domain (`--enable-test-java-agent-*`),
        which also validates that the agent classes are on the `CLASSPATH`; that
        class path must still be known when they run. The Python application
        needs the `lttngust` package of the interpreter that runs the test.

        Pass the same `profile` to `launch_agent_test_application()`.
        """
        launch_info = _agent_test_application_launch_info(domain)
        profile = profile or cls.default_profile()

        if launch_info.java_class is None:
            # Locating the package is enough: importing it would start an agent
            # in this process, which then attempts to register with any
            # reachable session daemon.
            try:
                return importlib.util.find_spec("lttngust") is not None
            except (ImportError, ValueError):
                return False

        if profile.java_binary is None and shutil.which("java") is None:
            return False

        if cls._java_class_path(profile) is None:
            return False

        return profile.tests_path(
            launch_info.directory, launch_info.java_class + ".class"
        ).exists()

    def launch_agent_test_application(
        self,
        domain: lttngctl.TracingDomain,
        event_count: int,
        wait_time_between_events_ms: int = 0,
        fire_debug_events: bool = False,
        fire_second_logger_event: bool = False,
        extra_env_vars: dict = dict(),
        profile: "Optional[_BuildProfile]" = None,
    ) -> "_TraceTestApplication":
        """
        Launch the test application of the agent domain `domain` of the build
        `profile` (the in-tree build by default), which emits its log statements
        as soon as its agent registers with the session daemon.

        Register the triggers and event rules which must count those log
        statements before launching it: the session daemon sends an agent its
        configuration before confirming its registration, which the application
        waits for, but nothing holds back an application which is already
        running.

        Only launch it when `run_agent_domain_tests()` returns `True` for
        `domain`; the application is otherwise missing its agent.
        """
        launch_info = _agent_test_application_launch_info(domain)
        profile = profile or self._default_profile
        directory = profile.tests_path(launch_info.directory)

        if launch_info.java_class is None:
            # The Python application expresses its inter-event delay in
            # seconds, unlike the Java ones which use milliseconds.
            args = [
                sys.executable,
                str(directory / "gen-py-events.py"),
                "--nr-iter",
                str(event_count),
                "--wait",
                str(wait_time_between_events_ms / 1000),
            ]

            if fire_debug_events:
                args.append("--fire-debug-event")

            if fire_second_logger_event:
                args.append("--fire-second-event")
        else:
            # The application's own class is in its build directory, the agent
            # classes are on the class path of the build or of the environment.
            agent_class_path = self._java_class_path(profile)
            class_path = os.pathsep.join(
                [str(directory)] + ([agent_class_path] if agent_class_path else [])
            )
            # The JNI libraries of the agents live in the library directories of
            # the build, which are not necessarily those of the loader nor of
            # this process's `LD_LIBRARY_PATH`.
            library_path = os.pathsep.join(
                [str(library_dir) for library_dir in profile.library_dirs]
                + (
                    [os.environ["LD_LIBRARY_PATH"]]
                    if os.environ.get("LD_LIBRARY_PATH")
                    else []
                )
                + ["/usr/local/lib", "/usr/lib"]
            )
            args = [
                profile.java_binary or "java",
                "-cp",
                class_path,
                "-Djava.library.path=" + library_path,
            ]

            for name, file_name in launch_info.java_properties.items():
                args.append("-D{}={}".format(name, directory / file_name))

            args += [
                launch_info.java_class,
                str(event_count),
                str(wait_time_between_events_ms),
                "1" if fire_debug_events else "0",
                "1" if fire_second_logger_event else "0",
            ]

        return self.launch_test_application(
            args,
            extra_env_vars=extra_env_vars,
            profile=profile,
            # The Log4j 1.x and Log4j 2.x applications log to the standard
            # output, which is the TAP stream of the test.
            stdout=subprocess.DEVNULL,
        )

    # Unpack a list of environment variables from a string
    # such as "HELLO=is_it ME='/you/are/looking/for'"
    @staticmethod
    def _unpack_env_vars(env_vars_string):
        # type: (str) -> List[Tuple[str, str]]
        unpacked_vars = []
        for var in shlex.split(env_vars_string):
            equal_position = var.find("=")
            # Must have an equal sign and not end with an equal sign
            if equal_position == -1 or equal_position == len(var) - 1:
                raise ValueError(
                    "Invalid sessiond environment variable: `{}`".format(var)
                )

            var_name = var[0:equal_position]
            var_value = var[equal_position + 1 :]
            # Unquote any paths
            var_value = var_value.replace("'", "")
            var_value = var_value.replace('"', "")
            unpacked_vars.append((var_name, var_value))

        return unpacked_vars

    def _launch_lttng_relayd(self):
        # type: () -> Optional[subprocess.Popen]
        relayd_path = self._relayd_profile.relayd_path
        if os.environ.get("LTTNG_TEST_NO_RELAYD", "0") == "1":
            # Run without a relay daemon; the user may be running one
            # under gdb, for example.
            return None

        relayd_env_vars = os.environ.get("LTTNG_RELAYD_ENV_VARS")
        relayd_env = os.environ.copy()
        relayd_env.update(self._extra_env_vars)
        if relayd_env_vars:
            self._log("Additional lttng-relayd environment variables:")
            for name, value in self._unpack_env_vars(relayd_env_vars):
                self._log("{}={}".format(name, value))
                relayd_env[name] = value

        if self.lttng_home_location is not None:
            relayd_env["LTTNG_HOME"] = str(self.lttng_home_location)

        self._prepend_profile_library_dirs(relayd_env, self._relayd_profile)

        self._relayd_env_vars = relayd_env
        self._log(
            "Launching relayd with LTTNG_HOME='{}'".format(
                str(self.lttng_home_location)
            )
        )

        verbose = []
        if os.getenv("LTTNG_TEST_VERBOSE_RELAYD", "0") != "0":
            verbose = ["-vvv"]
        self._relayd_log_file = None
        if self.lttng_log_dir:
            self._relayd_log_file = tempfile.NamedTemporaryFile(
                prefix="relayd_",
                dir=self.lttng_log_dir,
                delete=False,
            )
        process = subprocess.Popen(
            [
                str(relayd_path),
                "--dynamic-port-allocation",
                "-C",
                "tcp://0.0.0.0:{}".format(self.lttng_relayd_control_port),
                "-D",
                "tcp://0.0.0.0:{}".format(self.lttng_relayd_data_port),
                "-L",
                "tcp://localhost:{}".format(self.lttng_relayd_live_port),
            ]
            + verbose,
            stdout=(
                self._relayd_log_file.file if self._relayd_log_file else subprocess.PIPE
            ),
            stderr=subprocess.STDOUT,
            env=relayd_env,
        )

        if self._logging_function:
            self._relayd_output_consumer = ProcessOutputConsumer(
                process, "lttng-relayd", self._logging_function
            )
            self._relayd_output_consumer.daemon = True
            self._relayd_output_consumer.start()

        if os.environ.get("LTTNG_TEST_GDBSERVER_RELAYD") is not None:
            subprocess.Popen(
                [
                    "gdbserver",
                    "--attach",
                    "localhost:{}".format(
                        os.environ.get("LTTNG_TEST_GDBSERVER_RELAYD_PORT", "1025")
                    ),
                    str(process.pid),
                ]
            )

            if os.environ.get("LTTNG_TEST_GDBSERVER_RELAYD_WAIT", ""):
                input("Waiting for user input. Press `Enter` to continue")
            else:
                subprocess.Popen(
                    [
                        "gdb",
                        "--batch-silent",
                        "-ex",
                        "target remote localhost:{}".format(
                            os.environ.get("LTTNG_TEST_GDBSERVER_RELAYD_PORT", "1025")
                        ),
                        "-ex",
                        "continue",
                        "-ex",
                        "disconnect",
                    ]
                )

        # Wait until the ports are written to the run directory
        self._log(
            "Checking for relayd port files in '{}'".format(
                self.lttng_relayd_control_directory
            )
        )
        while (
            self.lttng_relayd_control_port == 0
            or self.lttng_relayd_live_port == 0
            or self.lttng_relayd_data_port == 0
        ):
            self._log(
                "Waiting for lttng-relayd control ({}), data ({}), and live ({}) ports to be non-zero".format(
                    self.lttng_relayd_control_port,
                    self.lttng_relayd_data_port,
                    self.lttng_relayd_live_port,
                )
            )
            time.sleep(0.1)
        self._log(
            "lttng-relayd using the following ports: control {}, data {}, and live {}".format(
                self.lttng_relayd_control_port,
                self.lttng_relayd_data_port,
                self.lttng_relayd_live_port,
            )
        )

        return process

    def _launch_lttng_sessiond(self, enable_kernel_domain=False):
        # type: () -> Optional[subprocess.Popen]
        sessiond_path = self._sessiond_profile.sessiond_path

        # Provide the session daemon with a consumer daemon for each registered
        # build profile (at most one per word size). A session daemon eagerly
        # spawns the consumer daemon of its own word size regardless of which
        # applications register, so add the session daemon's own profile when no
        # consumer daemon of its word size was registered. Without an explicit
        # path the session daemon looks for that consumer daemon at its
        # compiled-in install location (`<prefix>/lib/lttng/libexec`), which
        # only exists once the build has been installed; pointing it at the
        # profile's build-tree consumer daemon keeps the test self-contained.
        launch_consumerd_profiles = list(self._consumerd_profiles)
        if not any(
            profile.word_size_bits == self._sessiond_profile.word_size_bits
            for profile in launch_consumerd_profiles
        ):
            launch_consumerd_profiles.append(self._sessiond_profile)

        consumerd_options = []
        for profile in launch_consumerd_profiles:
            bitness = "64" if profile.word_size_bits == 64 else "32"
            consumerd_options += [
                "--consumerd{}-path".format(bitness),
                str(profile.consumerd_path),
            ]
            # A consumer daemon from another build must be told where its
            # libraries are so that it can be spawned with the right
            # LD_LIBRARY_PATH; the in-tree one relies on the loader's defaults.
            if profile != self._default_profile and profile.library_dirs:
                consumerd_options += [
                    "--consumerd{}-libdir".format(bitness),
                    ":".join(str(directory) for directory in profile.library_dirs),
                ]

        no_sessiond_var = os.environ.get("TEST_NO_SESSIOND")
        if no_sessiond_var and no_sessiond_var == "1":
            # Run test without a session daemon; the user probably
            # intends to run one under gdb for example.
            return None

        # Setup the session daemon's environment
        sessiond_env_vars = os.environ.get("LTTNG_SESSIOND_ENV_VARS")
        sessiond_env = os.environ.copy()
        sessiond_env.update(self._extra_env_vars)
        if sessiond_env_vars:
            self._log("Additional lttng-sessiond environment variables:")
            additional_vars = self._unpack_env_vars(sessiond_env_vars)
            for var_name, var_value in additional_vars:
                self._log("  {name}={value}".format(name=var_name, value=var_value))
                sessiond_env[var_name] = var_value

        sessiond_env["LTTNG_SESSION_CONFIG_XSD_PATH"] = str(
            self._project_root / "src" / "common"
        )

        if self.lttng_home_location is not None:
            sessiond_env["LTTNG_HOME"] = str(self.lttng_home_location)

        self._prepend_profile_library_dirs(sessiond_env, self._sessiond_profile)

        self.lttng_sessiond_env_vars = sessiond_env
        wait_queue = _SignalWaitQueue()
        with wait_queue.intercept_signal(signal.SIGUSR1):
            self._log(
                "Launching session daemon with LTTNG_HOME=`{home_dir}`".format(
                    home_dir=str(self.lttng_home_location)
                )
            )
            verbose = []
            sessiond_command = (
                [str(sessiond_path)] + consumerd_options + ["--sig-parent"]
            )
            if os.getenv("LTTNG_TEST_VERBOSE_SESSIOND", "0") != "0":
                sessiond_command.extend(["-vvv", "--verbose-consumer"])

            if not enable_kernel_domain:
                sessiond_command.extend(["--no-kernel"])

            if self._sessiond_extra_args:
                sessiond_command.extend(self._sessiond_extra_args)

            if (
                os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST32") is not None
                or os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST64") is not None
            ):
                sessiond_command.extend(["--spawn-consumers"])

            self._lttng_sessiond_log_file = None
            if self.lttng_log_dir:
                self._lttng_sessiond_log_file = tempfile.NamedTemporaryFile(
                    prefix="sessiond_",
                    dir=self.lttng_log_dir,
                    delete=False,
                )

            self._log(" ".join(sessiond_command))
            process = subprocess.Popen(
                sessiond_command,
                stdout=(
                    self._lttng_sessiond_log_file.file
                    if self._lttng_sessiond_log_file
                    else subprocess.PIPE
                ),
                stderr=subprocess.STDOUT,
                env=sessiond_env,
            )

            if self._logging_function:
                self._sessiond_output_consumer = ProcessOutputConsumer(
                    process, "lttng-sessiond", self._logging_function
                )  # type: Optional[ProcessOutputConsumer]
                self._sessiond_output_consumer.daemon = True
                self._sessiond_output_consumer.start()

            # Wait for SIGUSR1, indicating the sessiond is ready to proceed
            wait_queue.wait_for_signal()

        if os.environ.get("LTTNG_TEST_GDBSERVER_SESSIOND") is not None:
            self.attach_gdbserver(
                "localhost:{}".format(
                    os.environ.get("LTTNG_TEST_GDBSERVER_SESSIOND_PORT", "1024")
                ),
                process.pid,
            )

            if os.environ.get("LTTNG_TEST_GDBSERVER_SESSIOND_WAIT", ""):
                input("Waiting for user input. Press `Enter` to continue")
            else:
                self.detach_gdbserver(
                    "localhost:{}".format(
                        os.environ.get("LTTNG_TEST_GDBSERVER_SESSIOND_PORT", "1024")
                    ),
                    True,
                )

        if (
            os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_KERNEL") is not None
            and enable_kernel_domain
            and os.getuid() == 0
        ):
            while self.lttng_consumerd_kernel_pid is None:
                self.log("Waiting for lttng-consumerd kernel pid")
                time.sleep(1)

            self.attach_gdbserver(
                "localhost:{}".format(
                    os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_KERNEL_PORT", "1026")
                ),
                self.lttng_consumerd_kernel_pid,
            )

            if os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_KERNEL_WAIT", ""):
                input(
                    "Waiting for user input (post gdbserver attach to consumer kernel). Press `Enter` to continue"
                )
            else:
                self.detach_gdbserver(
                    "localhost:{}".format(
                        os.environ.get(
                            "LTTNG_TEST_GDBSERVER_CONSUMERD_KERNEL_PORT", "1026"
                        )
                    )
                )
        if os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST32") is not None:
            while self.lttng_consumerd_ust32_pid is None:
                self.log("Waiting for lttng-consumerd ust32 pid")
                time.sleep(1)

            self.attach_gdbserver(
                "localhost:{}".format(
                    os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST32_PORT", "1027")
                ),
                self.lttng_consumerd_ust32_pid,
            )

            if os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST32_WAIT", ""):
                input(
                    "Waiting for user input (post gdbserver attach to consumer ust32). Press `Enter` to continue"
                )
            else:
                self.detach_gdbserver(
                    "localhost:{}".format(
                        os.environ.get(
                            "LTTNG_TEST_GDBSERVER_CONSUMERD_UST32_PORT", "1027"
                        )
                    )
                )

        if os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST64") is not None:
            while self.lttng_consumerd_ust64_pid is None:
                self.log("Waiting for lttng-consumerd ust64 pid")
                time.sleep(1)

            self.attach_gdbserver(
                "localhost:{}".format(
                    os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST64_PORT", "1028")
                ),
                self.lttng_consumerd_ust64_pid,
            )

            if os.environ.get("LTTNG_TEST_GDBSERVER_CONSUMERD_UST64_WAIT", ""):
                input(
                    "Waiting for user input (post gdbserver attach to consumer ust64). Press `Enter` to continue"
                )
            else:
                self.detach_gdbserver(
                    "localhost:{}".format(
                        os.environ.get(
                            "LTTNG_TEST_GDBSERVER_CONSUMERD_UST64_PORT", "1028"
                        )
                    )
                )

        return process

    def _handle_termination_signal(self, signal_number, frame):
        # type: (int, Optional[FrameType]) -> None
        if sys.version_info[0] == 3 and sys.version_info[1] >= 8:
            # signal.strsignal is introduced in python 3.8
            self._log(
                "Killed by {signal_name} signal, cleaning-up".format(
                    signal_name=signal.strsignal(signal_number)  # novermin
                )
            )
        else:
            self._log("Killed by signal {}, cleaning-up".format(str(signal_number)))
        self._cleanup()

    def launch_live_viewer(self, session, hostname=None):
        # Make sure the relayd is ready
        ready = False
        ctf_live_cc = bt2.find_plugin("ctf").source_component_classes["lttng-live"]
        query_executor = bt2.QueryExecutor(
            ctf_live_cc,
            "sessions",
            params={"url": "net://localhost:{}".format(self.lttng_relayd_live_port)},
        )
        while not ready:
            try:
                query_result = query_executor.query()
            except bt2._Error:
                time.sleep(0.1)
                continue
            for live_session in query_result:
                if live_session["session-name"] == session:
                    ready = True
                    self._log(
                        "Session '{}' is available at net://localhost:{}".format(
                            session, self.lttng_relayd_live_port
                        )
                    )
                    break
        return _LiveViewer(self, session, hostname)

    def launch_wait_trace_test_application(
        self,
        event_count: int,
        wait_time_between_events_us: int = 0,
        wait_before_exit: bool = False,
        wait_before_exit_file_path: Optional[pathlib.Path] = None,
        run_as: Optional[str] = None,
        text_size: Optional[int] = None,
        fill_text: bool = False,
        wait_before_last_event: bool = False,
        wait_before_last_event_file_path: Optional[pathlib.Path] = None,
        extra_env_vars: dict = dict(),
        emit_event_with_empty_field_name: bool = False,
        emit_blob_events: bool = False,
        profile: "Optional[_BuildProfile]" = None,
        **kwargs,
    ) -> "_WaitTraceTestApplication":
        """
        Launch an application that will wait before tracing `event_count` events.

        `profile` selects the build whose test application is launched; it
        defaults to the in-tree build. A consumer daemon matching the profile's
        word size must be registered (see `test_environment`'s
        `consumerd_profiles`).
        """
        profile = profile or self._default_profile
        self._assert_consumerd_registered_for(profile)
        return _WaitTraceTestApplication(
            profile.testapp_path("gen-ust-events"),
            event_count,
            self,
            wait_time_between_events_us,
            wait_before_exit,
            wait_before_exit_file_path,
            run_as,
            text_size,
            fill_text,
            wait_before_last_event,
            wait_before_last_event_file_path,
            extra_env_vars,
            emit_event_with_empty_field_name,
            emit_blob_events,
            library_dirs=self._library_dirs_for(profile),
            **kwargs,
        )

    def launch_multi_event_wait_trace_test_application(
        self,
        event_count: int,
        wait_time_between_events_us: int = 0,
        wait_before_exit: bool = False,
        wait_before_exit_file_path: Optional[pathlib.Path] = None,
        run_as: Optional[str] = None,
        wait_before_last_event: bool = False,
        wait_before_last_event_file_path: Optional[pathlib.Path] = None,
        profile: "Optional[_BuildProfile]" = None,
    ) -> "_WaitTraceTestApplication":
        """
        Launch the gen-many-event-classes application that will wait before tracing `event_count` iterations.
        Each iteration emits events from multiple event classes.
        Note: text_size and fill_text parameters are not supported by this application.

        `profile` selects the build whose test application is launched; it
        defaults to the in-tree build. A consumer daemon matching the profile's
        word size must be registered (see `test_environment`'s
        `consumerd_profiles`).
        """
        profile = profile or self._default_profile
        self._assert_consumerd_registered_for(profile)
        return _WaitTraceTestApplication(
            profile.testapp_path("gen-many-event-classes", "gen-many-event-classes"),
            event_count,
            self,
            wait_time_between_events_us,
            wait_before_exit,
            wait_before_exit_file_path,
            run_as,
            text_size=None,
            fill_text=False,
            wait_before_last_event=wait_before_last_event,
            wait_before_last_event_file_path=wait_before_last_event_file_path,
            library_dirs=self._library_dirs_for(profile),
        )

    def launch_test_application(
        self,
        path: pathlib.Path,
        extra_env_vars: dict = dict(),
        profile: "Optional[_BuildProfile]" = None,
        **kwargs,
    ) -> "_TraceTestApplication":
        """
        Launch an application with it's environment set for being traced
        by this test environment's session daemon.

        `profile`, when set, selects the build (see `profiles()`) whose library
        directories are made available to the application; `path` remains
        caller-provided.
        """
        profile = profile or self._default_profile
        self._assert_consumerd_registered_for(profile)
        return _TraceTestApplication(
            path,
            self,
            extra_env_vars,
            library_dirs=self._library_dirs_for(profile),
            **kwargs,
        )

    def launch_notification_client(
        self,
        trigger_name,  # type: str
        sync=True,  # type: bool
        timeout_s=None,  # type: Optional[float]
        count=None,  # type: Optional[int]
        end_trigger_name=None,  # type: Optional[str]
    ):
        # type: (...) -> subprocess.Popen
        """
        Launch the notification-client helper and optionally wait for it
        to register with the session daemon.

        If sync is True (the default), this method blocks until the client
        has registered for notifications. If timeout_s is None (the default),
        it waits indefinitely.

        If count is specified, the client will exit after receiving that many
        notifications for the trigger (default is 1).

        If end_trigger_name is specified, the client will exit only when a
        notification matching that trigger is received, instead of exiting
        after count notifications.
        """
        notification_client_path = (
            self._project_root
            / "tests"
            / "regression"
            / "tools"
            / "trigger"
            / "utils"
            / "notification-client"
        )

        args = [
            str(notification_client_path),
            "--trigger",
            trigger_name,
        ]

        if count is not None:
            args.extend(["--count", str(count)])

        if end_trigger_name is not None:
            args.extend(["--end-trigger", end_trigger_name])

        env = os.environ.copy()
        if self.lttng_home_location is not None:
            env["LTTNG_HOME"] = str(self.lttng_home_location)
        if self.lttng_rundir is not None:
            env["LTTNG_RUNDIR"] = str(self.lttng_rundir)

        sync_file = None
        if sync:
            sync_dir = self.create_temporary_directory("notif-client-sync-")
            sync_file = sync_dir / "sync"
            args.extend(["--sync-after-notif-register", str(sync_file)])

        self._log("Launching notification client: '{}'".format(" ".join(args)))

        process = subprocess.Popen(
            args,
            env=env,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
        )

        if sync:
            deadline = None if timeout_s is None else time.time() + timeout_s
            while not os.path.exists(sync_file):
                if deadline is not None and time.time() > deadline:
                    process.kill()
                    raise RuntimeError(
                        "Timed out waiting for notification client to register"
                    )
                if process.poll() is not None:
                    raise RuntimeError(
                        "Notification client exited unexpectedly with code {}".format(
                            process.returncode
                        )
                    )
                time.sleep(0.01)
            self._log(
                "Notification client registered for trigger '{}'".format(trigger_name)
            )

        return process

    def _terminate_relayd(self):
        if self._relayd and self._relayd.poll() is None:
            try:
                self._relayd.terminate()
                self._log(
                    "Termination signal sent to relayd (pid = {})".format(
                        self._relayd.pid
                    )
                )
                self._relayd.wait(timeout=self.teardown_timeout)
            except subprocess.TimeoutExpired:
                self._relayd.kill()
                self._log(
                    "Kill signal sent to relayd (pid = {}) after waiting {}s".format(
                        self._relayd.pid, self.teardown_timeout
                    )
                )

            if self._relayd_output_consumer:
                self._relayd_output_consumer.join()
                self._relayd_output_consumer = None
            self._log("Relayd killed")
            self._relayd = None

    # Clean-up managed processes
    def _cleanup(self):
        # type: () -> None
        for process in self._processes:
            if process.poll() is None:
                try:
                    process.terminate()
                    self.log(
                        "Termination signal sent to process (pid = {})".format(
                            process.pid
                        )
                    )
                    process.wait(timeout=self.teardown_timeout)
                except subprocess.TimeoutExpired:
                    process.kill()
                    self.log(
                        "Kill signal sent to process (pid = {}) after waiting {}s".format(
                            process.pid, self.teardown_timeout
                        )
                    )

        if self._sessiond and self._sessiond.poll() is None:
            # The session daemon is alive; kill it.
            try:
                self._sessiond.terminate()
                self._log(
                    "Termination signal sent to session daemon (pid = {})".format(
                        self._sessiond.pid
                    )
                )
                self._sessiond.wait(timeout=self.teardown_timeout)
            except subprocess.TimeoutExpired:
                self._sessiond.kill()
                self._log(
                    "Kill signal sent to lttng-sessiond (pid = {}) after waiting {}s".format(
                        self._sessiond.pid, self.teardown_timeout
                    )
                )
            if self._sessiond_output_consumer:
                self._sessiond_output_consumer.join()
                self._sessiond_output_consumer = None

            self._log("Session daemon killed")
            self._sessiond = None

        self._terminate_relayd()

        # The user accounts will always be deleted, but the home directories will
        # be retained unless the user has opted to preserve the test environment.
        userdel = ["userdel"]
        if not self.preserve_test_env:
            userdel += ["--remove"]
        for uid, name in self._dummy_users.items():
            # When subprocess is run during the interpreter teardown, ImportError
            # may be raised; however, the commands seem to execute correctly.
            # Eg.
            #
            # Exception ignored in: <function _Environment.__del__ at 0x7f2d62e3b9c0>
            # Traceback (most recent call last):
            #   File "tests/utils/lttngtest/environment.py", line 1024, in __del__
            #   File "tests/utils/lttngtest/environment.py", line 1016, in _cleanup
            #   File "/usr/lib/python3.11/subprocess.py", line 1026, in __init__
            #   File "/usr/lib/python3.11/subprocess.py", line 1880, in _execute_child
            #   File "<frozen os>", line 629, in get_exec_path
            # ImportError: sys.meta_path is None, Python is likely shutting down
            #
            try:
                _proc = subprocess.Popen(
                    ["pkill", "--uid", str(uid)], stderr=subprocess.PIPE
                )
                _proc.wait()
            except ImportError:
                pass
            try:
                _proc = subprocess.Popen(userdel + [name], stderr=subprocess.PIPE)
                _proc.wait()
            except ImportError:
                pass

        for gid, name in self._dummy_groups.items():
            try:
                _proc = subprocess.Popen(["groupdel", name], stderr=subprocess.PIPE)
                _proc.wait()
            except ImportError:
                pass

        self._lttng_home = None

    def __del__(self):
        self._cleanup()

    def attach_gdbserver(self, target, pid):
        process = subprocess.Popen(["gdbserver", "--attach", target, str(pid)])
        self._processes.append(process)

    def detach_gdbserver(self, target, continue_and_dont_wait=False):
        command = [
            "gdb",
            "--nx",
            "--batch",
            "-ex",
            "target remote {}".format(target),
        ]
        if continue_and_dont_wait:
            command.extend(["-ex", "continue"])

        command.extend(["-ex", "disconnect"])
        process = subprocess.Popen(command)
        if not continue_and_dont_wait:
            process.wait()
        else:
            self._processes.append(process)


def getconf(name):
    p = subprocess.Popen(["getconf", str(name)], stdout=subprocess.PIPE)
    if p.wait() != 0:
        raise subprocess.CalledProcessError(
            returncode=p.returncode, cmd=["getconf", str(name)]
        )
    return p.stdout.read().decode("utf-8").strip()


def get_machine():
    """
    Returns a machine identifier
    """
    return "{}-{}-{}".format(
        platform.machine(), platform.architecture()[0], platform.system().lower()
    )


def count_events(trace_path, ignore_exceptions=False):
    """
    Returns a tuple of (received events, discarded events) from the trace path.

    trace_path may be an iterable (e.g., array, map from pathlib.Path.glob()).
    """
    received = 0
    discarded = 0
    dirs = []
    if isinstance(trace_path, str) or isinstance(trace_path, pathlib.Path):
        dirs.append(str(trace_path))
    else:
        dirs = trace_path

    for dir in dirs:
        try:
            for msg in bt2.TraceCollectionMessageIterator(str(dir)):
                if type(msg) is bt2._EventMessageConst:
                    received += 1
                if type(msg) is bt2._DiscardedEventsMessageConst:
                    # msg.count may be None when the value is indeterminate
                    discarded += msg.count if msg.count is not None else 1
        except Exception as e:
            if not ignore_exceptions:
                raise e

    return (received, discarded)


def count_events_worker(args):
    trace_path = args
    received = 0
    discarded = 0
    for msg in bt2.TraceCollectionMessageIterator(str(trace_path)):
        if type(msg) is bt2._EventMessageConst:
            received += 1
            continue

        if type(msg) is bt2._DiscardedEventsMessageConst:
            discarded += msg.count if msg.count is not None else 1

    return (received, discarded)


def parallel_count_events(trace_path, test_env):
    """
    Count the number of events in a trace using parallel processing.
    This is a workaround for the performance issues with bt2 bindings.

    trace_path may be an iterable (e.g., array, map from pathlib.Path.glob()).

    When multiple trace_paths are given, each one is handled iteratively, but
    the processing of each stream is farmed out to a worker.
    """
    dirs = []
    received = 0
    discarded = 0

    if isinstance(trace_path, str) or isinstance(trace_path, pathlib.Path):
        dirs.append(str(trace_path))
    else:
        dirs = trace_path

    for dir in dirs:
        streams_path = None
        for root, dirs, files in os.walk(str(dir)):
            if "metadata" in files:
                streams_path = pathlib.Path(root)
                break

        if streams_path is None:
            raise RuntimeError("No metadata found in trace path: {}".format(trace_path))

        metadata_path = streams_path / "metadata"
        stream_dirs = []

        for file in streams_path.iterdir():
            if file.name == "metadata":
                continue

            temp_dir_path = test_env.create_temporary_directory()

            # Symlink to the stream file
            stream_link = temp_dir_path / file.name
            stream_link.symlink_to(file)
            # Symlink to the metadata file
            metadata_link = temp_dir_path / "metadata"
            metadata_link.symlink_to(metadata_path)

            stream_dirs.append(temp_dir_path)

        with multiprocessing.Pool() as pool:
            results = pool.map(count_events_worker, stream_dirs)

        for this_received, this_discarded in results:
            received += this_received
            discarded += this_discarded

    return received, discarded


def validate_trace(trace_location):
    """Validate that the trace at location `trace_location` is a valid trace.

    This is done by running the following bt2 graph on the trace:

    trace_location -> [source:ctf:fs] -> [filter:utils:muxer] ->
    [sink:utils:dummy]

    """

    inputs = []
    for dirpath, dirs, files in os.walk(trace_location):
        if "metadata" in files:
            inputs.append(dirpath)

    ctf = bt2.find_plugin("ctf")
    assert ctf

    fs = ctf.source_component_classes["fs"]
    assert fs

    utils = bt2.find_plugin("utils")
    assert utils

    dummy = utils.sink_component_classes["dummy"]
    assert dummy

    muxer = utils.filter_component_classes["muxer"]
    assert muxer

    graph = bt2.Graph(mip_version=1)

    src = graph.add_component(fs, "source", params={"inputs": inputs})
    mux = graph.add_component(muxer, "filter")
    sink = graph.add_component(dummy, "sink")

    for src_name in tuple(src.output_ports):
        available_input_ports = [
            x for x in mux.input_ports.values() if not x.is_connected
        ]
        graph.connect_ports(src.output_ports[src_name], available_input_ports[0])

    graph.connect_ports(mux.output_ports["out"], sink.input_ports["in"])

    graph.run()


@contextlib.contextmanager
def kernel_module(module_name):
    """
    Context manager to load a kernel module and unload it when it goes out of scope.
    """
    try:
        subprocess.run(["modprobe", module_name], check=True)
        yield module_name
    finally:
        subprocess.run(["modprobe", "-r", module_name], check=True)


def _cpu_mask_to_set(mask):
    "Return the set of CPU ids described by a sysfs CPU mask string."
    cpus = set()

    for cpu_range in mask.split(","):
        if cpu_range.find("-") != -1:
            start, end = cpu_range.split("-")
            for cpu in range(int(start), int(end) + 1):
                cpus.add(cpu)
        else:
            cpus.add(int(cpu_range.strip()))
    return cpus


def online_cpus():
    "Return a set of CPU that are currently online."
    with open("/sys/devices/system/cpu/online") as online:
        return _cpu_mask_to_set(online.readline())


def possible_cpus_array_len():
    """
    Return the length of an array indexed by possible CPU id (the highest
    possible CPU id plus one), mirroring how the tracers size per-CPU
    resources such as the streams of a per-CPU channel.

    In a container, the online mask can be virtualized to a sparse subset of
    the possible range while the possible mask still reflects the host.
    """
    with open("/sys/devices/system/cpu/possible") as possible:
        return max(_cpu_mask_to_set(possible.readline())) + 1


@contextlib.contextmanager
def test_environment(
    with_sessiond,
    log=None,
    with_relayd=False,
    extra_env_vars=dict(),
    skip_temporary_lttng_home=False,
    enable_kernel_domain=False,
    skip_temporary_lttng_rundir=False,
    sessiond_extra_args=None,
    sessiond_profile=None,
    relayd_profile=None,
    client_profile=None,
    consumerd_profiles=None,
):
    # type: (bool, Optional[Callable[[str], None]], bool, dict, bool, bool, bool, Optional[List[str]], Optional[_BuildProfile], Optional[_BuildProfile], Optional[_BuildProfile], Optional[List[_BuildProfile]]) -> Iterator[_Environment]
    env = _Environment(
        with_sessiond,
        log,
        with_relayd,
        extra_env_vars,
        skip_temporary_lttng_home,
        enable_kernel_domain,
        skip_temporary_lttng_rundir,
        sessiond_extra_args,
        sessiond_profile,
        relayd_profile,
        client_profile,
        consumerd_profiles,
    )
    try:
        yield env
    finally:
        env._cleanup()
