diff --git a/tools/impl/dev_container/version b/tools/impl/dev_container/version index 8d1c6eebe4..ea1d9ddebb 100644 --- a/tools/impl/dev_container/version +++ b/tools/impl/dev_container/version @@ -1 +1 @@ -r0015 +r0016 diff --git a/tools/impl/test_runner.py b/tools/impl/test_runner.py index 5f03bce5be..d30d58fed8 100644 --- a/tools/impl/test_runner.py +++ b/tools/impl/test_runner.py @@ -13,7 +13,7 @@ import subprocess import sys from multiprocessing import Pool from pathlib import Path -from typing import Dict, Iterable, List, NamedTuple +from typing import Dict, Iterable, List, NamedTuple, Optional from . import test_target, testvm from .test_target import TestTarget, Triple @@ -65,12 +65,20 @@ class ExecutableResults(object): """Container for results of a test executable.""" def __init__( - self, name: str, success: bool, test_log: str, previous_attempts: List["ExecutableResults"] + self, + name: str, + binary_file: Path, + success: bool, + test_log: str, + previous_attempts: List["ExecutableResults"], + profile_file: Optional[Path], ): self.name = name + self.binary_file = binary_file self.success = success self.test_log = test_log self.previous_attempts = previous_attempts + self.profile_file = profile_file class Executable(NamedTuple): @@ -232,10 +240,12 @@ def build_common_crate(build_env: Dict[str, str], crate: Crate): return list(cargo_build_executables([], env=build_env, cwd=crate.path)) -def build_all_binaries(target: TestTarget, crosvm_direct: bool): +def build_all_binaries(target: TestTarget, crosvm_direct: bool, instrument_coverage: bool): """Discover all crates and build them.""" build_env = os.environ.copy() build_env.update(test_target.get_cargo_env(target)) + if instrument_coverage: + build_env["RUSTFLAGS"] = "-C instrument-coverage" print("Building crosvm workspace") features = BUILD_FEATURES[str(target.build_triple)] @@ -276,7 +286,7 @@ def get_test_timeout(target: TestTarget, executable: Executable): return timeout * EMULATION_TIMEOUT_MULTIPLIER -def execute_test(target: TestTarget, attempts: int, executable: Executable): +def execute_test(target: TestTarget, attempts: int, collect_coverage: bool, executable: Executable): """ Executes a single test on the given test targed @@ -301,29 +311,43 @@ def execute_test(target: TestTarget, attempts: int, executable: Executable): print(f"Running test {executable.name} on {target}... (attempt {i}/{attempts})") try: + profile_file = None + if collect_coverage: + profile_file = binary_path.with_suffix(".profraw") + # Pipe stdout/err to be printed in the main process if needed. test_process = test_target.exec_file_on_target( target, binary_path, args=args, timeout=get_test_timeout(target, executable), + profile_file=profile_file, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) + if profile_file and not profile_file.exists(): + print() + print(f"Warning: Running {binary_path} did not produce profile file.") + profile_file = None + result = ExecutableResults( executable.name, + binary_path, test_process.returncode == 0, test_process.stdout, previous_attempts, + profile_file, ) except subprocess.TimeoutExpired as e: # Append a note about the timeout to the stdout of the process. msg = f"\n\nProcess timed out after {e.timeout}s\n" result = ExecutableResults( executable.name, + binary_path, False, e.stdout.decode("utf-8") + msg, previous_attempts, + None, ) if result.success: break @@ -358,6 +382,7 @@ def execute_all( executables: List[Executable], target: test_target.TestTarget, attempts: int, + collect_coverage: bool, ): """Executes all tests in the `executables` list in parallel.""" @@ -369,7 +394,7 @@ def execute_all( sys.stdout.flush() with Pool(PARALLELISM) as pool: for result in pool.imap( - functools.partial(execute_test, target, attempts), pool_executables + functools.partial(execute_test, target, attempts, collect_coverage), pool_executables ): print_test_progress(result) yield result @@ -379,7 +404,7 @@ def execute_all( sys.stdout.write(f"Running {len(exclusive_executables)} test binaries on {target}") sys.stdout.flush() for executable in exclusive_executables: - result = execute_test(target, attempts, executable) + result = execute_test(target, attempts, collect_coverage, executable) print_test_progress(result) yield result print() @@ -392,6 +417,27 @@ def find_crosvm_binary(executables: List[Executable]): raise Exception("Cannot find crosvm executable") +def generate_lcov(results: List[ExecutableResults], lcov_file: str): + print("Merging profiles") + merged_file = testvm.cargo_target_dir() / "merged.profraw" + profiles = [str(r.profile_file) for r in results if r.profile_file] + subprocess.check_call(["rust-profdata", "merge", "-sparse", *profiles, "-o", str(merged_file)]) + + print("Exporting report") + lcov_data = subprocess.check_output( + [ + "rust-cov", + "export", + "--format=lcov", + "--ignore-filename-regex='/.cargo/registry'", + f"--instr-profile={merged_file}", + *(f"--object={r.binary_file}" for r in results), + ], + text=True, + ) + open(lcov_file, "w").write(lcov_data) + + def main(): parser = argparse.ArgumentParser(usage=USAGE) parser.add_argument( @@ -424,6 +470,10 @@ def main(): "--build-only", action="store_true", ) + parser.add_argument( + "--generate-lcov", + help="Generate an lcov code coverage profile", + ) parser.add_argument( "--crosvm-direct", action="store_true", @@ -465,6 +515,7 @@ def main(): print() build_target = Triple.from_shorthand(args.arch) + collect_coverage = args.generate_lcov emulator_cmd = args.emulator.split(" ") if args.emulator else None build_target = Triple.from_shorthand(args.build_target) if args.build_target else None target = test_target.TestTarget(args.target, build_target, emulator_cmd) @@ -475,7 +526,7 @@ def main(): testvm.build_if_needed(target.vm) testvm.up(target.vm) - executables = list(build_all_binaries(target, args.crosvm_direct)) + executables = list(build_all_binaries(target, args.crosvm_direct, collect_coverage)) if args.build_only: print("Not running tests as requested.") @@ -501,9 +552,12 @@ def main(): if args.repeat > 1: print() print(f"Round {i+1}/{args.repeat}:") - all_results.extend(execute_all(test_executables, target, args.retry + 1)) + all_results.extend(execute_all(test_executables, target, args.retry + 1, collect_coverage)) random.shuffle(test_executables) + if args.generate_lcov: + generate_lcov(all_results, args.generate_lcov) + flakes = [r for r in all_results if r.previous_attempts] if flakes: print() diff --git a/tools/impl/test_target.py b/tools/impl/test_target.py index a90b95a59e..184fb660df 100755 --- a/tools/impl/test_target.py +++ b/tools/impl/test_target.py @@ -344,6 +344,7 @@ def exec_file_on_target( timeout: int, args: List[str] = [], extra_files: List[Path] = [], + profile_file: Optional[Path] = None, **kwargs: Any, ): """Executes a file on the test target. @@ -370,6 +371,8 @@ def exec_file_on_target( env["PATH"] += ";" + str(find_rust_lib_dir()) else: raise Exception(f"Unsupported build target: {os.name}") + if profile_file: + env["LLVM_PROFILE_FILE"] = str(profile_file) cmd_line = [*prefix, str(filepath), *args] return subprocess.run( @@ -383,6 +386,8 @@ def exec_file_on_target( filename = Path(filepath).name target.ssh.upload_files([filepath] + extra_files, quiet=True) cmd_line = [*prefix, f"./{filename}", *args] + if profile_file: + raise Exception("Coverage collection on remote hosts is not supported.") try: result = target.ssh.run( f"chmod +x {filename} && sudo LD_LIBRARY_PATH=. {' '.join(cmd_line)}", diff --git a/tools/install-deps b/tools/install-deps index 77a8d4ae7c..f087211781 100755 --- a/tools/install-deps +++ b/tools/install-deps @@ -54,10 +54,16 @@ pip3 install \ rustup component add clippy rustup component add rustfmt +# LLVM tools are used to generate and process coverage files +rustup component add llvm-tools-preview + rustup target add x86_64-pc-windows-gnu # The bindgen tool is required to build a crosvm dependency. cargo install bindgen +# binutils are wrappers to call the rustup bundled versions of llvm tools. +cargo install cargo-binutils + # Install dependencies used to generate mdbook documentation. $(dirname "$0")/install-docs-deps