dev_container: Preserve container between calls

This vastly improves iterative build times and enables more flexible
usage of the container.

BUG=None
TEST=./tools/dev_container cargo build
First run will build everything. Second run will finish right away.

Change-Id: I9b4eeee0689f0e9d07f0a32f846d21ab42f689f7
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3292100
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Dennis Kempin <denniskempin@google.com>
Reviewed-by: Junichi Uekawa <uekawa@chromium.org>

ci/kokoro/build-aarch64.sh

@@ -5,4 +5,4 @@
 
 source "$(dirname $0)/common.sh"
 
-./tools/dev_container ./tools/run_tests --target=vm:aarch64
+./tools/dev_container --hermetic ./tools/run_tests --target=vm:aarch64 -v

ci/kokoro/build-x86_64.sh

@@ -4,8 +4,8 @@
 # found in the LICENSE file.
 source "$(dirname $0)/common.sh"
 
-./tools/dev_container bash -c "\
-    ./tools/run_tests --target=host \
+./tools/dev_container --hermetic bash -c "\
+    ./tools/run_tests --target=host -v \
     && ./tools/clippy \
     && ./tools/fmt --check \
     && mdbook build ./docs/book"

ci/kokoro/common.sh

@@ -13,7 +13,7 @@ if [[ ! -z "${DEBUG_SSH_KEY}" ]]; then
   echo "${DEBUG_SSH_KEY}" >>~/.ssh/authorized_keys
   external_ip=$(
     curl -s -H "Metadata-Flavor: Google" \
-    http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip
+      http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip
   )
   echo "SSH Debug enabled. Connect to: kbuilder@${external_ip}"
 fi
@@ -66,7 +66,3 @@ setup_source || {
 # Set logs directory so we can copy them to sponge
 export CROSVM_BUILDER_LOGS_DIR="${KOKORO_ARTIFACTS_DIR}/logs"
 cd "${KOKORO_ARTIFACTS_DIR}/git/crosvm"
-
-# Log how long it takes to pull the docker container
-echo "Downloading dev container image"
-time ./tools/dev_container echo "done"

docs/book/src/building_crosvm/linux.md

@@ -76,6 +76,16 @@ The container image is big and may take a while to download when first used.
 Once started, you can follow all instructions in this document within the
 container shell.
 
+Instead of using the interactive shell, commands to execute can be provided
+directly:
+
+```sh
+$ ./tools/dev_container cargo build
+```
+
+Note: The container and build artifacts are preserved between calls to
+`./tools/dev_container`. If you wish to start fresh, use the `--reset` flag.
+
 ## Building a binary
 
 If you simply want to try crosvm, run `cargo build`. Then the binary is
@@ -89,8 +99,8 @@ the `--features` flag. (e.g. `cargo build --features=gdb`)
 
 ### Iterative development
 
-You can use cargo as usual for crosvm development to `cargo build` and `cargo
-test` single crates that you are working on.
+You can use cargo as usual for crosvm development to `cargo build` and
+`cargo test` single crates that you are working on.
 
 If you are working on aarch64 specific code, you can use the `set_test_target`
 tool to instruct cargo to build for aarch64 and run tests on a VM:
@@ -129,14 +139,6 @@ you can use the dev container to build and run the tests.
 $ ./tools/dev_container ./tools/run_tests --target=vm:aarch64
 ```
 
-Note however, that using an interactive shell in the container is preferred, as
-the build artifacts are not preserved between calls:
-
-```sh
-$ ./tools/dev_container
-crosvm_dev$ ./tools/run_tests --target=vm:aarch64
-```
-
 It is also possible to run tests on a remote machine via ssh. The target
 architecture is automatically detected:
 
@@ -177,14 +179,15 @@ skip some slower checks, like building for other platforms.
     also stop minijail from killing processes that violate the seccomp rule,
     making the sandboxing much less aggressive.
 -   Seccomp policy files have hardcoded absolute paths. You can either fix up
-    the paths locally, or set up an awesome hacky symlink: `sudo mkdir
-    /usr/share/policy && sudo ln -s /path/to/crosvm/seccomp/x86_64
-    /usr/share/policy/crosvm`. We'll eventually build the precompiled policies
+    the paths locally, or set up an awesome hacky symlink:
+    `sudo mkdir /usr/share/policy && sudo ln -s /path/to/crosvm/seccomp/x86_64 /usr/share/policy/crosvm`.
+    We'll eventually build the precompiled policies
     [into the crosvm binary](http://crbug.com/1052126).
--   Devices can't be jailed if `/var/empty` doesn't exist. `sudo mkdir -p
-    /var/empty` to work around this for now.
+-   Devices can't be jailed if `/var/empty` doesn't exist.
+    `sudo mkdir -p /var/empty` to work around this for now.
 -   You need read/write permissions for `/dev/kvm` to run tests or other crosvm
-    instances. Usually it's owned by the `kvm` group, so `sudo usermod -a -G kvm
-    $USER` and then log out and back in again to fix this.
+    instances. Usually it's owned by the `kvm` group, so
+    `sudo usermod -a -G kvm $USER` and then log out and back in again to fix
+    this.
 -   Some other features (networking) require `CAP_NET_ADMIN` so those usually
     need to be run as root.

tools/dev_container

@@ -2,30 +2,96 @@
 # Copyright 2021 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+#
+# Usage:
+#
+# To get an interactive shell for development:
+#   ./tools/dev_container
+#
+# To run a command in the container, e.g. to run presubmits:
+#   ./tools/dev_container ./tools/presubmit
+#
+# The state of the container (including build artifacts) are preserved between
+# calls. To stop the container call:
+#   ./tools/dev_container --stop
+#
+# The dev container can also be called with a fresh container for each call that
+# is cleaned up afterwards (e.g. when run by Kokoro):
+#
+#   ./tools/dev_container --hermetic CMD
 
 set -e
-cd "$(dirname $0)/.."
-
-cli_args=(
-    --rm
-    --volume $(pwd):/workspace:rw
-    --device /dev/net/tun
-    --device /dev/kvm
-    --volume /dev/log:/dev/log
-    --privileged
-)
-
-# Enable interactive mode when running in an interactive terminal.
-if [ -t 1 ]; then
-    cli_args+=(-it)
-fi
+cd "$(dirname "$0")/.."
 
 # Allow to override the container CLI tool, similar to the Makefile. Try
 # "docker" first and fall back to "podman".
 DOCKER=${DOCKER:-$(which docker || which podman)}
 
-"${DOCKER}" run \
-    ${cli_args[@]} \
-    gcr.io/crosvm-packages/crosvm_dev:$(cat tools/impl/dev_container/version) \
-    "$@"
+# Name to identify the running crosvm dev container
+CONTAINER_NAME=crosvm_dev
 
+IMAGE_VERSION=$(cat tools/impl/dev_container/version)
+
+# Enable interactive mode when running in an interactive terminal.
+TTY_ARGS=()
+if [ -t 1 ]; then
+    TTY_ARGS+=(
+        --interactive
+        --tty
+    )
+fi
+
+DOCKER_ARGS=(
+    "${TTY_ARGS[@]}"
+    --volume "$(pwd):/workspace:rw"
+    --device "/dev/net/tun"
+    --device "/dev/kvm"
+    --volume "/dev/log:/dev/log"
+    --privileged
+    "gcr.io/crosvm-packages/crosvm_dev:$IMAGE_VERSION"
+)
+
+docker_run_detached() {
+    "${DOCKER}" run \
+        --detach --name "${CONTAINER_NAME}" \
+        "${DOCKER_ARGS[@]}" >/dev/null
+}
+
+docker_run() {
+    "${DOCKER}" run --rm "${DOCKER_ARGS[@]}" "$@"
+}
+
+get_container_id() {
+    docker ps -q -f name="${CONTAINER_NAME}"
+}
+
+docker_exec() {
+    if [[ $# -gt 0 ]]; then
+        cmd=("$@")
+    else
+        cmd=(/bin/bash)
+    fi
+    "${DOCKER}" exec "${TTY_ARGS[@]}" "$(get_container_id)" "${cmd[@]}"
+}
+
+main() {
+    if [[ "$1" == "--stop" ]]; then
+        if [ -n "$(get_container_id)" ]; then
+            docker rm -f "$(get_container_id)" >/dev/null
+        fi
+        exit
+    fi
+
+    if [[ "$1" == "--hermetic" ]]; then
+        shift
+        docker_run "$@"
+    else
+        if [ -z "$(get_container_id)" ]; then
+            docker_run_detached
+        fi
+        docker_exec "$@"
+    fi
+
+}
+
+main "$@"