Switch to submodules based workflow

Instead of requiring crosvm to be checked out via the chromeos manifest
to access dependencies in the ChromeOS monorepo, this change adds
git submodules to third_party/.

The CI scripts and Cargo.toml are updated to use the new paths.

BUG=b:194336213
TEST=git clone --recursive https://chromium.googlesource.com/chromiumos/platform/crosvm
     cd crosvm
     cargo test
     ./test_all
     ./ci/kokoro/simulate_all

Change-Id: I9859d18176e21909ac3a140976fbd67cc14129bb
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3049003
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Dennis Kempin <denniskempin@google.com>
Reviewed-by: Chirantan Ekbote <chirantan@chromium.org>

.gitmodules

@@ -1,3 +1,15 @@
 [submodule "tpm2-sys/libtpm2"]
 	path = tpm2-sys/libtpm2
 	url = https://chromium.googlesource.com/chromiumos/third_party/tpm2
+[submodule "third_party/minigbm"]
+	path = third_party/minigbm
+	url = https://chromium.googlesource.com/chromiumos/platform/minigbm
+[submodule "third_party/virglrenderer"]
+	path = third_party/virglrenderer
+	url = https://chromium.googlesource.com/chromiumos/third_party/virglrenderer
+[submodule "third_party/minijail"]
+	path = third_party/minijail
+	url = https://android.googlesource.com/platform/external/minijail
+[submodule "third_party/vmm_vhost"]
+	path = third_party/vmm_vhost
+	url = https://chromium.googlesource.com/chromiumos/third_party/rust-vmm/vhost/

Cargo.toml

@@ -120,10 +120,10 @@ base = { path = "base" }
 cros_fuzz = { path = "common/cros-fuzz" } # ignored by ebuild
 data_model = { path = "data_model" }
 libcras = { path = "libcras_stub" } # ignored by ebuild
-minijail = { path = "../../aosp/external/minijail/rust/minijail" } # ignored by ebuild
 p9 = { path = "common/p9" } # ignored by ebuild
 sync = { path = "sync" }
 sys_util = { path = "sys_util" }
 tempfile = { path = "tempfile" }
-vmm_vhost = { path = "../../third_party/rust-vmm/vhost", features = ["vhost-user-master", "vhost-user-slave"] } # ignored by ebuild
 wire_format_derive = { path = "common/p9/wire_format_derive" } # ignored by ebuild
+minijail = { path = "third_party/minijail/rust/minijail" } # ignored by ebuild
+vmm_vhost = { path = "third_party/vmm_vhost", features = ["vhost-user-master", "vhost-user-slave"] } # ignored by ebuild

README.md

@@ -32,15 +32,17 @@ to build and test crosvm in enviroments outside of the Chrome OS chroot.
 
 >**NOTE:** Building for Linux natively is new and not fully supported.
 
-First, [set up depot_tools] and use `repo` to sync down the crosvm source
-tree. This is a subset of the entire Chromium OS manifest with just enough repos
-to build crosvm.
+Crosvm uses submodules to manage external dependencies. Initialize them via:
 
 ```sh
-mkdir crosvm
-cd crosvm
-repo init -g crosvm -u https://chromium.googlesource.com/chromiumos/manifest.git --repo-url=https://chromium.googlesource.com/external/repo.git
-repo sync
+git submodule update --init
+```
+
+It is recommended to enable automatic recursive operations to keep the
+submodules in sync with the main repository:
+
+```sh
+git config --global submodule.recurse true
 ```
 
 A basic crosvm build links against `libcap`. On a Debian-based system,
@@ -77,8 +79,6 @@ Known issues:
 
 And that's it! You should be able to `cargo build/run/test`.
 
-[set up depot_tools]: https://commondatastorage.googleapis.com/chrome-infra-docs/flat/depot_tools/docs/html/depot_tools_tutorial.html#_setting_up
-
 ## Usage
 
 To see the usage information for your version of crosvm, run `crosvm` or `crosvm

bin/clippy

@@ -71,7 +71,6 @@ FEATURES=(
     audio
     gpu
     plugin
-    power-monitor-powerd
     tpm
     usb
     video-decoder

ci/README.md

@@ -10,19 +10,10 @@ testing.
 
 ### Setting up the source
 
-Since crosvm is part of chromiumos, and uses a couple of it's projects as
-dependencies, you need a standard chromiumos checkout as described by the
-[ChromiumOS Developer Guide](https://chromium.googlesource.com/chromiumos/docs/+/master/developer_guide.md#Get-the-Source).
-
-To reduce the number of repositories to download, you can use the `-g crosvm`
-argument on `repo init`. This will be significantly faster:
-
-In summary:
+Crosvm requires a bunch of dependencies that are checked out via submodules:
 
 ```
-$ repo init -u https://chromium.googlesource.com/chromiumos/manifest.git --repo-url https://chromium.googlesource.com/external/repo.git -g crosvm
-$ repo sync -j4
-$ cd src/platform/crosvm
+$ git submodule update --init
 ```
 
 ### Installing Podman (or Docker)

ci/build_environment/Makefile

@@ -2,12 +2,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 #
-# This makefile is run by the ./ci/crosvm_* containers to build ChromiumOS
-# dependencies required by crosvm.
+# This makefile is run by the ./ci/crosvm_* containers to build native third
+# party dependencies required by crosvm.
 #
 # Setting TARGET_ARCH=aarch64 enables cross-compilation for aarch64.
 
-SRC ?= /workspace/src
+SRC ?= /workspace/src/crosvm
 BUILD ?= /workspace/scratch/build
 LIB ?= /workspace/scratch/lib
 TARGET_ARCH ?=
@@ -36,7 +36,7 @@ clean:
 
 tpm2:
 	mkdir -p $(BUILD)/tpm2
-	$(MAKE) -C $(SRC)/third_party/tpm2 \
+	$(MAKE) -C $(SRC)/tpm2-sys/libtpm2 \
 		obj=$(BUILD)/tpm2 \
 		AR=$(CROSS_COMPILE)ar \
 		CC=$(CROSS_COMPILE)gcc \
@@ -44,13 +44,13 @@ tpm2:
 
 minijail:
 	mkdir -p $(BUILD)/minijail
-	$(MAKE) -C $(SRC)/aosp/external/minijail \
+	$(MAKE) -C $(SRC)/third_party/minijail \
 		OUT=$(BUILD)/minijail \
 		CROSS_COMPILE=$(CROSS_COMPILE)
 
 minigbm:
 	mkdir -p $(BUILD)/minigbm
-	$(MAKE) -C $(SRC)/platform/minigbm \
+	$(MAKE) -C $(SRC)/third_party/minigbm \
 		OUT=$(BUILD)/minigbm  \
 		CROSS_COMPILE=$(CROSS_COMPILE)
 
@@ -60,7 +60,7 @@ virglrenderer: minigbm $(BUILD)/meson-cross
 		$(SRC)/third_party/virglrenderer \
 		$(MESON_ARGS)
 
-	CPATH=$(SRC)/platform/minigbm \
+	CPATH=$(SRC)/third_party/minigbm \
 		meson compile -C $(BUILD)/virglrenderer
 
 
@@ -90,7 +90,7 @@ $(LIB):
 	# minigbm
 	ln -sf $(BUILD)/minigbm/libminigbm.so.1.0.0 $(LIB)/libgbm.so
 	ln -sf $(LIB)/libgbm.so $(LIB)/libgbm.so.1
-	ln -sf $(SRC)/platform/minigbm/gbm.pc $(LIB)/pkgconfig/
+	ln -sf $(SRC)/third_party/minigbm/gbm.pc $(LIB)/pkgconfig/
 
 	# virglrenderer
 	ln -sf $(BUILD)/virglrenderer/src/libvirglrenderer.so $(LIB)

ci/crosvm_aarch64_builder/Dockerfile

@@ -71,7 +71,7 @@ COPY --from=vm \
     /root/.ssh /root/.ssh
 
 # Setup entrypoint and interactive shell
-WORKDIR /workspace/src/platform/crosvm
+WORKDIR /workspace/src/crosvm
 COPY bashrc /root/.bashrc
 COPY entrypoint /workspace
 ENTRYPOINT ["/workspace/entrypoint"]

ci/crosvm_base/Dockerfile

@@ -80,4 +80,4 @@ ENV RUSTFLAGS='--cfg hermetic'
 ENV CROSVM_CROS_BUILD=1
 
 # All commands will be executed in the crosvm src directory.
-WORKDIR /workspace/src/platform/crosvm
+WORKDIR /workspace/src/crosvm

ci/crosvm_builder/Dockerfile

@@ -43,7 +43,7 @@ COPY --from=vm \
     /root/.ssh /root/.ssh
 
 # Setup entrypoint and interactive shell
-WORKDIR /workspace/src/platform/crosvm
+WORKDIR /workspace/src/crosvm
 COPY bashrc /root/.bashrc
 COPY entrypoint /workspace
 ENTRYPOINT ["/workspace/entrypoint"]

ci/image_tag

@@ -1 +1 @@
-r0008
+r0009

ci/kokoro/common.sh

@@ -3,8 +3,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-crosvm_root="${KOKORO_ARTIFACTS_DIR}"/git/crosvm
-
 # Enable SSH access to the kokoro builder.
 # Use the fusion2/ UI to trigger a build and set the DEBUG_SSH_KEY environment
 # variable to your public key, that will allow you to connect to the builder
@@ -21,42 +19,13 @@ if [[ ! -z "${DEBUG_SSH_KEY}" ]]; then
 fi
 
 setup_source() {
-  if [ -z "${KOKORO_ARTIFACTS_DIR}" ]; then
+  if [ -z "${KOKORO_ARTIFACTS_DIR}/git" ]; then
     echo "This script must be run in kokoro"
     exit 1
   fi
 
-  cd "${KOKORO_ARTIFACTS_DIR}"
-
-  echo ""
-  echo "Downloading crosvm dependencies to $(pwd)/cros..."
-  mkdir cros
-  cd cros
-
-  # repo gets confused by pyenv, make sure we select 3.6.1 as our default
-  # version.
-  if command -v pyenv >/dev/null; then
-    echo "Selecting Python 3.6.1"
-    pyenv global 3.6.1
-  fi
-  curl -s https://storage.googleapis.com/git-repo-downloads/repo >repo
-  chmod +x repo
-  ./repo init --depth 1 \
-    -u https://chromium.googlesource.com/chromiumos/manifest.git \
-    --repo-url https://chromium.googlesource.com/external/repo.git \
-    -g crosvm || return 1
-  ./repo sync -j8 -c || return 1
-
-  # Bind mount source into cros checkout.
-  echo ""
-  echo "Mounting crosvm source to $(pwd)/src/platform/crosvm..."
-  rm -rf src/platform/crosvm && mkdir -p src/platform/crosvm
-  if command -v bindfs >/dev/null; then
-    bindfs "${crosvm_root}" src/platform/crosvm || return 1
-  else
-    sudo mount --bind "${crosvm_root}" src/platform/crosvm || return 1
-  fi
-
+  cd "${KOKORO_ARTIFACTS_DIR}/git/crosvm"
+  git submodule update --init
 }
 
 cleanup() {
@@ -66,12 +35,6 @@ cleanup() {
     sleep 1h
   fi
 
-  if command -v bindfs >/dev/null; then
-    fusermount -uz "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"
-  else
-    sudo umount --lazy "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"
-  fi
-
   # List files in the logs directory which are uploaded to sponge.
   echo "Build Artifacts:"
   ls "${KOKORO_ARTIFACTS_DIR}/logs"
@@ -86,4 +49,4 @@ setup_source || {
 
 # Set logs directory so we can copy them to sponge
 export CROSVM_BUILDER_LOGS_DIR="${KOKORO_ARTIFACTS_DIR}/logs"
-cd "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm"
+cd "${KOKORO_ARTIFACTS_DIR}/git/crosvm"

ci/kokoro/simulate

@@ -19,7 +19,8 @@ cleanup() {
 main() {
   echo "Copying ${crosvm_src}/ to ${kokoro_src}"
   mkdir -p "${kokoro_src}"
-  rsync -arq --exclude "target" --exclude ".git" "${crosvm_src}/" "${kokoro_src}"
+  rsync -arq --exclude "target" --exclude "__pycache__" \
+    "${crosvm_src}/" "${kokoro_src}"
 
   # Run user-provided kokoro build script.
   export KOKORO_ARTIFACTS_DIR="${kokoro_root}/src"

ci/run_container.sh

@@ -11,14 +11,7 @@
 # CROSVM_BUILDER_SCRATCH_DIR or CROSVM_BUILDER_LOGS_DIR.
 
 crosvm_root=$(realpath "$(dirname $0)/..")
-cros_root=$(realpath "${crosvm_root}/../../..")
 
-if [ ! -d "${cros_root}/.repo" ]; then
-    echo "The CI builder must be run from a cros checkout. See ci/README.md"
-    exit 1
-fi
-
-# Parse parameters
 builder="$1"
 shift
 
@@ -43,14 +36,14 @@ fi
 version=$(cat $(dirname $0)/image_tag)
 echo "Using builder: ${builder}:${version}"
 
-src="${cros_root}/src"
-echo "Using source directory: ${src} (Available at /workspace/src)"
+echo "Using source directory: ${crosvm_root} \
+(Available at /workspace/src/crosvm)"
 
 docker_args=(
     --rm
     --device /dev/kvm
     --volume /dev/log:/dev/log
-    --volume "${src}":/workspace/src:rw
+    --volume "${crosvm_root}":/workspace/src/crosvm:rw
 )
 
 if [ ! -z "${CROSVM_BUILDER_SCRATCH_DIR}" ]; then

ci/vm_tools/sync_deps

@@ -8,7 +8,7 @@
 
 ${0%/*}/wait_for_vm_with_timeout || exit 1
 
-crosvm_root="/workspace/src/platform/crosvm"
+crosvm_root="/workspace/src/crosvm"
 rust_toolchain=$(cat ${crosvm_root}/rust-toolchain)
 target_dir=$(
     cargo metadata --no-deps --format-version 1 |

run_tests

@@ -78,7 +78,7 @@ FEATURE_REQUIREMENTS: Dict[str, List[Requirements]] = {
     "audio": [],
     "gpu": [Requirements.CROS_BUILD],
     "plugin": [Requirements.PRIVILEGED, Requirements.X86_64],
-    "power-monitor-powerd": [],
+    "power-monitor-powerd": [Requirements.DISABLED],
     "tpm": [Requirements.CROS_BUILD],
     "video-decoder": [Requirements.DISABLED],
     "video-encoder": [Requirements.DISABLED],

third_party/minigbm

@@ -0,0 +1 @@
+Subproject commit 2e63aaf616cdda26019d265989bd0d96ee11aab9

third_party/minijail

@@ -0,0 +1 @@
+Subproject commit 94cff17da47df59f567b64fd8e87795fcae055d4

third_party/virglrenderer

@@ -0,0 +1 @@
+Subproject commit a108be89e36ad427bf5f9affdbb4dd434e119b28

third_party/vmm_vhost

@@ -0,0 +1 @@
+Subproject commit 70810633abb3f7e1e3f11ba728b114ccdcec6f31