crosvm/run_tests

#!/usr/bin/env python3
# Copyright 2021 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Runs tests for crosvm.
#
# This script gives us more flexibility in running tests than using
# `cargo test --workspace`:
# - We can also test crates that are not part of the workspace.
# - We can pick out tests that need to be run single-threaded.
# - We can filter out tests that cannot be built or run due to missing build-
#   dependencies or missing runtime requirements.

from typing import List, Dict, Set
import argparse
import enum
import os
import platform
import subprocess
import sys

# Print debug info. Overriden by --verbose.
VERBOSE = False

# Runs tests using the exec_file wrapper, which will run the test inside the
# builders built-in VM.
VM_TEST_RUNNER = "/workspace/vm/exec_file --no-sync"

# Runs tests using QEMU user-space emulation.
QEMU_TEST_RUNNER = (
    "qemu-aarch64-static -E LD_LIBRARY_PATH=/workspace/scratch/lib"
)


class Requirements(enum.Enum):
    # Test can only be built for aarch64.
    AARCH64 = "aarch64"

    # Test can only be built for x86_64.
    X86_64 = "x86_64"

    # Requires ChromeOS build environment.
    CROS_BUILD = "cros_build"

    # Test is disabled explicitly.
    DISABLED = "disabled"

    # Test needs to be executed with expanded privileges for device access.
    PRIVILEGED = "privileged"

    # Test needs to run single-threaded
    SINGLE_THREADED = "single_threaded"


BUILD_TIME_REQUIREMENTS = [
    Requirements.AARCH64,
    Requirements.X86_64,
    Requirements.CROS_BUILD,
    Requirements.DISABLED,
]

# A list of all crates and their requirements
CRATE_REQUIREMENTS: Dict[str, List[Requirements]] = {
    "aarch64": [Requirements.AARCH64],
    "crosvm": [Requirements.DISABLED],
    "aarch64": [Requirements.AARCH64],
    "acpi_tables": [],
    "arch": [],
    "assertions": [],
    "base": [],
    "bit_field": [],
    "bit_field_derive": [],
    "cros_async": [Requirements.DISABLED],
    "crosvm_plugin": [Requirements.X86_64],
    "data_model": [],
    "devices": [
        Requirements.SINGLE_THREADED,
        Requirements.PRIVILEGED,
        Requirements.X86_64,
    ],
    "disk": [Requirements.DISABLED],
    "enumn": [],
    "fuse": [],
    "fuzz": [Requirements.DISABLED],
    "gpu_display": [],
    "hypervisor": [Requirements.PRIVILEGED, Requirements.X86_64],
    "io_uring": [Requirements.DISABLED],
    "kernel_cmdline": [],
    "kernel_loader": [Requirements.PRIVILEGED],
    "kvm_sys": [Requirements.PRIVILEGED],
    "kvm": [Requirements.PRIVILEGED, Requirements.X86_64],
    "linux_input_sys": [],
    "msg_socket": [Requirements.PRIVILEGED],
    "msg_on_socket_derive": [],
    "net_sys": [],
    "net_util": [Requirements.PRIVILEGED],
    "power_monitor": [],
    "protos": [],
    "qcow_utils": [],
    "rand_ish": [],
    "resources": [],
    "rutabaga_gfx": [Requirements.CROS_BUILD],
    "sync": [],
    "sys_util": [Requirements.SINGLE_THREADED, Requirements.PRIVILEGED],
    "poll_token_derive": [],
    "syscall_defines": [],
    "tempfile": [],
    "tpm2-sys": [],
    "tpm2": [],
    "usb_sys": [],
    "usb_util": [],
    "vfio_sys": [],
    "vhost": [Requirements.PRIVILEGED],
    "virtio_sys": [],
    "vm_control": [],
    "vm_memory": [Requirements.DISABLED],
    "x86_64": [Requirements.X86_64, Requirements.PRIVILEGED],
}

# Just like for crates, lists requirements for each cargo feature flag.
FEATURE_REQUIREMENTS: Dict[str, List[Requirements]] = {
    "chromeos": [],
    "audio": [],
    "gpu": [Requirements.CROS_BUILD],
    "plugin": [Requirements.PRIVILEGED, Requirements.X86_64],
    "power-monitor-powerd": [],
    "tpm": [Requirements.CROS_BUILD],
    "video-decoder": [Requirements.DISABLED],
    "video-encoder": [Requirements.DISABLED],
    "wl-dmabuf": [Requirements.DISABLED],
    "x": [],
    "virgl_renderer_next": [Requirements.CROS_BUILD],
    "composite-disk": [],
    "virgl_renderer": [Requirements.CROS_BUILD],
    "gfxstream": [Requirements.DISABLED],
    "gdb": [],
}


class CrateInfo(object):
    """Informaton about whether a crate can be built or run on this host."""

    def __init__(
        self,
        name: str,
        requirements: Set[Requirements],
        capabilities: Set[Requirements],
    ):
        self.name = name
        self.requirements = requirements
        self.single_threaded = Requirements.SINGLE_THREADED in requirements
        self.needs_privilege = Requirements.PRIVILEGED in requirements

        build_reqs = requirements.intersection(BUILD_TIME_REQUIREMENTS)
        self.can_build = all(req in capabilities for req in build_reqs)

        self.can_run = self.can_build and (
            not self.needs_privilege or Requirements.PRIVILEGED in capabilities
        )

    def __repr__(self):
        return f"{self.name} {self.requirements}"


def target_arch():
    """Returns architecture cargo is set up to build for."""
    if "CARGO_BUILD_TARGET" in os.environ:
        target = os.environ["CARGO_BUILD_TARGET"]
        return target.split("-")[0]
    else:
        return platform.machine()


def get_test_runner_env(use_vm: bool):
    """Sets the target.*.runner cargo setting to use the correct test runner."""
    env = os.environ.copy()
    key = f"CARGO_TARGET_{target_arch().upper()}_UNKNOWN_LINUX_GNU_RUNNER"
    if use_vm:
        env[key] = VM_TEST_RUNNER
    else:
        if target_arch() == "aarch64":
            env[key] = QEMU_TEST_RUNNER
        else:
            if key in env:
                del env[key]
    return env


def cargo_test(
    crates: List[CrateInfo],
    features: Set[str],
    run: bool = True,
    single_threaded: bool = False,
    use_vm: bool = False,
):
    """Executes the list of crates via `cargo test`."""
    if not crates:
        return True

    cmd = ["cargo", "test", "-q"]
    if not run:
        cmd += ["--no-run"]
    if features:
        cmd += ["--no-default-features", "--features", ",".join(features)]
    for crate in sorted(crate.name for crate in crates):
        cmd += ["-p", crate]
    if single_threaded:
        cmd += ["--", "--test-threads=1"]
    env = get_test_runner_env(use_vm)

    msg = ["Running" if run else "Building"]
    if use_vm:
        msg.append("in vm")
    if single_threaded:
        msg.append("(single-threaded)")

    print()
    print(f"{' '.join(msg)}: {', '.join(crate.name for crate in crates)}")
    if VERBOSE:
        print("ENV", env)
        print("CMD", " ".join(cmd))
    print()

    process = subprocess.run(cmd, env=env)
    return process.returncode == 0


def execute_batched_by_parallelism(
    crates: List[CrateInfo], features: Set[str], use_vm: bool
):
    """Batches tests by single-threaded and parallel, then executes them."""
    passed = True
    run_single = [crate for crate in crates if crate.single_threaded]
    if not cargo_test(
        run_single, features, single_threaded=True, use_vm=use_vm
    ):
        passed = False

    run_parallel = [crate for crate in crates if not crate.single_threaded]
    if not cargo_test(run_parallel, features, use_vm=use_vm):
        passed = False
    return passed


def execute_batched_by_privilege(
    crates: List[CrateInfo], features: Set[str], use_vm: bool
):
    """
    Batches tests by whether or not a test needs privileged access to run.

    Non-privileged tests are run first. Privileged tests are executed in
    a VM if use_vm is set.
    """
    passed = True

    build_crates = [crate for crate in crates if crate.can_build]
    if not cargo_test(build_crates, features, run=False):
        return False
    simple_crates = [
        crate for crate in crates if crate.can_run and not crate.needs_privilege
    ]
    execute_batched_by_parallelism(simple_crates, features, use_vm=False)

    privileged_crates = [
        crate for crate in crates if crate.can_run and crate.needs_privilege
    ]
    if privileged_crates:
        if use_vm:
            subprocess.run("/workspace/vm/sync_so", check=True)
            execute_batched_by_parallelism(
                privileged_crates, features, use_vm=True
            )
        else:
            execute_batched_by_parallelism(
                privileged_crates, features, use_vm=False
            )
    return passed


def main(capabilities: Set[Requirements], use_vm: bool):
    print("Capabilities:", ", ".join(cap.value for cap in capabilities))

    # Select all features where capabilities meet the requirements
    features = set(
        feature
        for (feature, requirements) in FEATURE_REQUIREMENTS.items()
        if all(r in capabilities for r in requirements)
    )

    # Disable sandboxing for tests until our builders are set up to run with
    # sandboxing.
    features.add("default-no-sandbox")
    print("Features:", ", ".join(features))

    crates = [
        CrateInfo(crate, set(requirements), capabilities)
        for (crate, requirements) in CRATE_REQUIREMENTS.items()
    ]
    passed = execute_batched_by_privilege(crates, features, use_vm)

    # TODO: We should parse test output and summarize the results
    # Unfortunately machine readable output for `cargo test` is still a nightly
    # rust feature.

    print()
    crates_not_built = [crate.name for crate in crates if not crate.can_build]
    print(f"Tests not built: {', '.join(crates_not_built)}")

    crates_not_run = [
        crate.name for crate in crates if crate.can_build and not crate.can_run
    ]
    print(f"Tests not run: {', '.join(crates_not_run)}")

    disabled_features = set(FEATURE_REQUIREMENTS.keys()).difference(features)
    print(f"Disabled features: {', '.join(disabled_features)}")

    print()
    if not passed:
        print("Some tests failed.", file=sys.stderr)
        exit(-1)
    else:
        print("All tests passed.")


DESCRIPTION = """\
Selects a subset of tests from crosvm to run depending on the capabilities of
the local host.
"""

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=DESCRIPTION)
    parser.add_argument(
        "--verbose",
        "-v",
        action="store_true",
        default=False,
        help="Print commands executed.",
    )
    parser.add_argument(
        "--run-privileged",
        action="store_true",
        default=False,
        help="Enable tests that requires privileged access to the system.",
    )
    parser.add_argument(
        "--cros-build",
        action="store_true",
        default=False,
        help=(
            "Enables tests that require a ChromeOS build environment. "
            "Can also be set by CROSVM_CROS_BUILD"
        ),
    )
    parser.add_argument(
        "--use-vm",
        action="store_true",
        default=False,
        help=(
            "Enables privileged tests to run in a VM. "
            "Can also be set by CROSVM_USE_VM"
        ),
    )
    parser.add_argument(
        "--require-all",
        action="store_true",
        default=False,
        help="Requires all tests to run, fail if tests would be disabled.",
    )

    args = parser.parse_args()
    VERBOSE = args.verbose  # type: ignore
    use_vm = os.environ.get("CROSVM_USE_VM") != None or args.use_vm
    cros_build = os.environ.get("CROSVM_CROS_BUILD") != None or args.cros_build

    capabilities = set()
    if target_arch() == "aarch64":
        capabilities.add(Requirements.AARCH64)
    elif target_arch() == "x86_64":
        capabilities.add(Requirements.X86_64)

    if cros_build:
        capabilities.add(Requirements.CROS_BUILD)

    if use_vm:
        if not os.path.exists("/workspace/vm"):
            print("--use-vm can only be used within the ./ci/builder's.")
            exit(1)
        capabilities.add(Requirements.PRIVILEGED)

    if args.run_privileged:
        capabilities.add(Requirements.PRIVILEGED)

    if args.require_all and not Requirements.PRIVILEGED in capabilities:
        print("--require-all needs to be run with --use-vm or --run-privileged")
        exit(1)

    main(capabilities, use_vm)