Add Kokoro for ci/builders

Adds the crosvm-side infrastructure to build and test
in kokoro.
There is a build script for testing on x86, aarch64
and a separte script for analysis (clippy, fmt).
These will run in parallel on Kokoro. To test the
scripts locally, a simulate script is provided.

Runtime on my workstation:
- aarch64: 10m
- x86: 2:30m
- analysis: 1:40m

BUG=b:177951955
TEST=./ci/kokoro/simulate_all

Change-Id: I2f666ec768e6c3391a258dc7f0cbd999ad9b2fb1
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2654413
Tested-by: Dennis Kempin <denniskempin@google.com>
Commit-Queue: Dennis Kempin <denniskempin@google.com>
Reviewed-by: Stephen Barber <smbarber@chromium.org>

Cargo.lock

@@ -558,6 +558,7 @@ dependencies = [
  "libc",
  "log",
  "protobuf",
+ "thiserror",
 ]
 
 [[package]]

bin/clippy

@@ -10,14 +10,14 @@
 set -eo pipefail
 
 USE_CACHE=false
-CLIPPY_ARGS=( "$@" )
+CLIPPY_ARGS=("$@")
 
 # TODO: When we add more options, use a fancier parsing mechanism such as
 # getopts. Also use the Rust convention of -- separating the arguments for this
 # script itself from the ones for clippy.
-if (( "$#" > 0 )) && [[ "$1" == "--use-cache" ]]; then
+if (("$#" > 0)) && [[ "$1" == "--use-cache" ]]; then
     USE_CACHE=true
-    CLIPPY_ARGS=( "${CLIPPY_ARGS[@]:1}" )
+    CLIPPY_ARGS=("${CLIPPY_ARGS[@]:1}")
 fi
 
 # Change into directory of script, which is crosvm/bin.
@@ -77,6 +77,7 @@ SUPPRESS=(
     unreadable_literal
     useless_let_if_seq
     useless_transmute
+    new-ret-no-self
 )
 
 # Needed or else clippy won't re-run on code that has already compiled.
@@ -91,4 +92,4 @@ RUSTFLAGS="${RUSTFLAGS:-}"
 export RUSTFLAGS="$RUSTFLAGS --sysroot=$RUST_SYSROOT"
 
 cargo clippy --all-features --all-targets -- ${SUPPRESS[@]/#/-Aclippy::} \
-      "${CLIPPY_ARGS[@]}" -D warnings
+    "${CLIPPY_ARGS[@]}" -D warnings

ci/Makefile

@@ -20,9 +20,11 @@ DOCKER ?= docker
 all: crosvm_builder crosvm_aarch64_builder
 
 upload: all
-	$(DOCKER) push $(TAG_BASE)/crosvm_base
-	$(DOCKER) push $(TAG_BASE)/crosvm_builder
-	$(DOCKER) push $(TAG_BASE)/crosvm_aarch64_builder
+	$(DOCKER) push $(TAG_BASE)/crosvm_base:$(TAG_VERSION)
+	$(DOCKER) push $(TAG_BASE)/crosvm_builder:$(TAG_VERSION)
+	$(DOCKER) push $(TAG_BASE)/crosvm_aarch64_builder:$(TAG_VERSION)
+	$(DOCKER) push $(TAG_BASE)/crosvm_test_vm_amd64:$(TAG_VERSION)
+	$(DOCKER) push $(TAG_BASE)/crosvm_test_vm_arm64:$(TAG_VERSION)
 
 crosvm_base:
 	cd $@ && $(DOCKER) build -t $(TAG_BASE)/$@:$(TAG_VERSION) .

ci/kokoro/build-aarch64.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+source "$(dirname $0)/common.sh"
+
+cd "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"
+./ci/aarch64_builder --vm ./run_tests --require-all

ci/kokoro/build-x86_64.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+source "$(dirname $0)/common.sh"
+
+./ci/builder --vm ./run_tests --require-all && \
+    ./ci/builder "bin/clippy" && \
+    ./ci/builder "bin/fmt --check"

ci/kokoro/common.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+crosvm_root="${KOKORO_ARTIFACTS_DIR}"/git/crosvm
+
+setup_source() {
+  if [ -z "${KOKORO_ARTIFACTS_DIR}" ]; then
+    echo "This script must be run in kokoro"
+    exit 1
+  fi
+
+  cd "${KOKORO_ARTIFACTS_DIR}"
+
+  echo ""
+  echo "Downloading crosvm dependencies to $(pwd)/cros..."
+  mkdir cros
+  cd cros
+
+  # repo gets confused by pyenv, make sure we select 3.6.1 as our default
+  # version.
+  if command -v pyenv; then
+    echo "Selecting Python 3.6.1"
+    pyenv global 3.6.1
+  fi
+  curl -s https://storage.googleapis.com/git-repo-downloads/repo >repo
+  chmod +x repo
+  ./repo init --depth 1 -u https://chromium.googlesource.com/chromiumos/manifest.git \
+    --repo-url https://chromium.googlesource.com/external/repo.git -g crosvm || return 1
+  ./repo sync -j8 -c -m "${crosvm_root}/ci/kokoro/manifest.xml" || return 1
+
+  # Bind mount source into cros checkout.
+  echo ""
+  echo "Mounting crosvm source to $(pwd)/src/platform/crosvm..."
+  rm -rf src/platform/crosvm && mkdir -p src/platform/crosvm
+  if command -v bindfs; then
+    bindfs "${crosvm_root}" src/platform/crosvm || return 1
+  else
+    sudo mount --bind "${crosvm_root}" src/platform/crosvm || return 1
+  fi
+}
+
+cleanup() {
+  if command -v bindfs; then
+    fusermount -uz "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"
+  else
+    sudo umount --lazy "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"
+  fi
+}
+
+# Setup source when the script is loaded. Clean up on exit.
+trap cleanup EXIT
+setup_source || {
+  echo "Failed to setup_source"
+  exit 1
+}
+cd "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"

ci/kokoro/continuous-aarch64.cfg

@@ -0,0 +1,3 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "crosvm/ci/kokoro/build-aarch64.sh"

ci/kokoro/continuous-x86_64.cfg

@@ -0,0 +1,3 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "crosvm/ci/kokoro/build-x86_64.sh"

ci/kokoro/manifest.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<manifest>
+  <remote fetch="https://android.googlesource.com" name="aosp" review="https://android-review.googlesource.com"/>
+  <remote alias="cros" fetch="https://chromium.googlesource.com/" name="chromium"/>
+  <remote fetch="https://chromium.googlesource.com" name="cros" review="https://chromium-review.googlesource.com"/>
+  
+  <default remote="cros" revision="refs/heads/main" sync-j="8"/>
+  
+  <project dest-branch="refs/heads/master" groups="minilayout,firmware,buildtools,chromeos-admin,labtools,sysmon,devserver,crosvm" name="chromiumos/chromite" path="chromite" revision="c5b4aa0e96738d18694890a1a1b9da42db5b2155" upstream="refs/heads/master">
+    <copyfile dest="AUTHORS" src="AUTHORS"/>
+    <copyfile dest="LICENSE" src="LICENSE"/>
+  </project>
+  <project dest-branch="refs/heads/main" groups="crosvm" name="chromiumos/docs" path="docs" revision="2caeceae3a6497efc4a9563e1520f7ceada4f228" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/main" groups="crosvm" name="chromiumos/platform/crosvm" path="src/platform/crosvm" revision="4a86ce877a632ad2c6e88553657c870835d864ae" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/main" groups="crosvm" name="chromiumos/platform/minigbm" path="src/platform/minigbm" revision="bfd38d3e80aef198aa525e819d54dd28a1d1b49c" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/main" groups="crosvm" name="chromiumos/platform2" path="src/platform2" revision="c8760e7df3f8923199a58e279b0e0c3c86dd64f7" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/main" groups="minilayout,firmware,buildtools,labtools,crosvm" name="chromiumos/repohooks" path="src/repohooks" revision="c1bd878f8a2c8f03c81ad519866ef4d387d20ea9" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/main" groups="crosvm" name="chromiumos/third_party/adhd" path="src/third_party/adhd" revision="1e85d90675628e788b4e7950089739ade3ca1dfd" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/debian" groups="notdefault,crostini,crosvm" name="chromiumos/third_party/libdrm" path="src/third_party/libdrm-debian" revision="b32233490540efbb6dfc202a972f3b71ac0d80c9" upstream="refs/heads/debian"/>
+  <project dest-branch="refs/heads/main" groups="firmware,crosvm" name="chromiumos/third_party/tpm2" path="src/third_party/tpm2" revision="069cb5acb1d4a8a001404dfecdfab465de639ad1" upstream="refs/heads/main"/>
+  <project dest-branch="refs/heads/master" groups="crosvm" name="chromiumos/third_party/virglrenderer" path="src/third_party/virglrenderer" revision="b9dc30355a2021c6a1cdd8bfbf23f7ffc342d89b" upstream="refs/heads/master"/>
+  <project dest-branch="refs/heads/master" groups="crosvm" name="platform/external/minijail" path="src/aosp/external/minijail" remote="aosp" revision="b49291144e126662b8c7d1551fd1dc572e1d55da" upstream="refs/heads/master"/>
+  
+  <repo-hooks enabled-list="pre-upload" in-project="chromiumos/repohooks"/>
+</manifest>

ci/kokoro/presubmit-aarch64.cfg

@@ -0,0 +1,3 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "crosvm/ci/kokoro/build-aarch64.sh"

ci/kokoro/presubmit-x86_64.cfg

@@ -0,0 +1,3 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+build_file: "crosvm/ci/kokoro/build-x86_64.sh"

ci/kokoro/simulate

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Simulates a Kokoro run executing one of the build-*.sh scripts.
+# e.g. ./ci/kokoro/simulate build-aarch64.sh
+
+kokoro_root=$(mktemp -d)
+kokoro_src="${kokoro_root}/git/crosvm"
+local_src=$(realpath $(dirname $0)/../../)
+
+cleanup() {
+  echo "Unmounting ${kokoro_src}"
+  umount --lazy "${kokoro_src}" || true
+  rm -rf "${kokoro_root}"
+}
+
+main() {
+  # Bind mount source read-only to kokoro_src.
+  echo "Bind-mounting ${local_src} to ${kokoro_src}"
+  mkdir -p "${kokoro_src}"
+  bindfs "${local_src}" "${kokoro_src}" -p 0000,u=rxD
+
+  # Run user-provided kokoro build script.
+  export KOKORO_ARTIFACTS_DIR="${kokoro_root}"
+  echo "Running $1 in ${kokoro_root}"
+  bash $(dirname $0)/$1
+  echo "$1 returned $?"
+}
+
+trap cleanup EXIT
+main "$@"

ci/kokoro/simulate_all

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Simulates all kokoro jobs (In parallel if tmux is installed)
+
+cd $(dirname "$0")
+
+if command -v tmux; then
+  TMUX= tmux \
+    new-session './simulate build-x86_64.sh;
+            echo "press enter to close"; read' \; \
+    split-window './simulate build-aarch64.sh;
+            echo "press enter to close"; read' \;
+else
+  ./simulate build-x86.sh &&
+    ./simulate build-analysis.sh &&
+    ./simulate build-aarch64.sh
+fi

ci/run_container.sh

@@ -4,17 +4,25 @@
 # found in the LICENSE file.
 #
 # Runs a crosvm builder. Will use podman if available, falls back to docker.
+# Usage:
+# run_container.sh builder_name [--clean] entry point args...
 crosvm_root=$(realpath "$(dirname $0)/..")
 cros_root=$(realpath "${crosvm_root}/../../..")
-target=$(
-    cargo metadata --no-deps --format-version 1 | jq -r ".target_directory"
-)
+tmpdir="${TMPDIR:-/tmp}"
 
 if [ ! -d "${cros_root}/.repo" ]; then
     echo "The CI builder must be run from a cros checkout. See ci/README.md"
     exit 1
 fi
 
+# Parse parameters
+builder="$1"
+shift
+if [ "$1" = "--clean" ]; then
+    shift
+    clean=1
+fi
+
 # User podman if available for root-less execution. Fall-back to docker.
 if which podman >/dev/null; then
     run() {
@@ -34,18 +42,32 @@ fi
 
 version=$(cat $(dirname $0)/image_tag)
 src="${cros_root}/src"
-scratch="${target}/ci/$1"
-mkdir -p "${scratch}"
+scratch="${tmpdir}/crosvm-ci/${builder}"
 
-echo "Using builder version: ${version}"
+echo "Using builder: ${builder}:${version}"
 echo "Using source directory: ${src}"
 echo "Using scratch directory: ${scratch}"
-echo ""
 
-run --rm -it \
+if [[ -n "${clean}" ]]; then
+    rm -rf "${scratch}"
+    echo "Cleaned scratch directory."
+fi
+mkdir -p "${scratch}"
+
+docker_args=(
+    --rm
     --device /dev/kvm \
     --volume /dev/log:/dev/log \
     --volume "${src}":/workspace/src:rw \
     --volume "${scratch}":/workspace/scratch:rw \
-    "gcr.io/crosvm-packages/$1:${version}" \
-    "${@:2}"
+)
+
+# Enable interactive mode when running in an interactive terminal.
+if [ -t 1 ]; then
+    docker_args+=( -it )
+fi
+
+echo ""
+run ${docker_args[@]} \
+    "gcr.io/crosvm-packages/${builder}:${version}" \
+    "$@"

devices/src/pci/ac97.rs

@@ -14,10 +14,6 @@ use libcras::{CrasClient, CrasClientType, CrasSocketType};
 use resources::{Alloc, MmioType, SystemAllocator};
 use vm_memory::GuestMemory;
 
-#[cfg(target_os = "linux")]
-use crate::virtio::snd::vios_backend::VioSShmStreamSource;
-#[cfg(not(target_os = "linux"))]
-use crate::virtio::snd::vios_backend::Error as VioSError;
 use crate::pci::ac97_bus_master::Ac97BusMaster;
 use crate::pci::ac97_mixer::Ac97Mixer;
 use crate::pci::ac97_regs::*;
@@ -26,6 +22,10 @@ use crate::pci::pci_configuration::{
 };
 use crate::pci::pci_device::{self, PciDevice, Result};
 use crate::pci::{PciAddress, PciInterruptPin};
+#[cfg(not(target_os = "linux"))]
+use crate::virtio::snd::vios_backend::Error as VioSError;
+#[cfg(target_os = "linux")]
+use crate::virtio::snd::vios_backend::VioSShmStreamSource;
 
 // Use 82801AA because it's what qemu does.
 const PCI_DEVICE_ID_INTEL_82801AA_5: u16 = 0x2415;
@@ -179,7 +179,9 @@ impl Ac97Dev {
             return Ok(vios_audio);
         }
         #[cfg(not(target_os = "linux"))]
-        Err(pci_device::Error::CreateViosClientFailed(VioSError::PlatformNotSupported))
+        Err(pci_device::Error::CreateViosClientFailed(
+            VioSError::PlatformNotSupported,
+        ))
     }
 
     fn create_null_audio_device(mem: GuestMemory) -> Result<Self> {

devices/src/pci/pci_device.rs

@@ -10,9 +10,9 @@ use resources::{Error as SystemAllocatorFaliure, SystemAllocator};
 
 use crate::pci::pci_configuration;
 use crate::pci::{PciAddress, PciInterruptPin};
-use crate::{BusAccessInfo, BusDevice};
 #[cfg(feature = "audio")]
 use crate::virtio::snd::vios_backend::Error as VioSError;
+use crate::{BusAccessInfo, BusDevice};
 
 #[derive(Debug)]
 pub enum Error {