From 42b825f7407d63a04eab4b0bea3c4741369d347c Mon Sep 17 00:00:00 2001 From: Dennis Kempin Date: Fri, 23 Jul 2021 13:22:39 -0700 Subject: [PATCH] Switch to submodules based workflow Instead of requiring crosvm to be checked out via the chromeos manifest to access dependencies in the ChromeOS monorepo, this change adds git submodules to third_party/. The CI scripts and Cargo.toml are updated to use the new paths. BUG=b:194336213 TEST=git clone --recursive https://chromium.googlesource.com/chromiumos/platform/crosvm cd crosvm cargo test ./test_all ./ci/kokoro/simulate_all Change-Id: I9859d18176e21909ac3a140976fbd67cc14129bb Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3049003 Tested-by: kokoro Commit-Queue: Dennis Kempin Reviewed-by: Chirantan Ekbote --- .gitmodules | 12 ++++++++ Cargo.toml | 4 +-- README.md | 18 +++++------ bin/clippy | 1 - ci/README.md | 13 ++------ ci/build_environment/Makefile | 16 +++++----- ci/crosvm_aarch64_builder/Dockerfile | 2 +- ci/crosvm_base/Dockerfile | 2 +- ci/crosvm_builder/Dockerfile | 2 +- ci/image_tag | 2 +- ci/kokoro/common.sh | 45 +++------------------------- ci/kokoro/simulate | 3 +- ci/run_container.sh | 13 ++------ ci/vm_tools/sync_deps | 2 +- run_tests | 2 +- third_party/minigbm | 1 + third_party/minijail | 1 + third_party/virglrenderer | 1 + third_party/vmm_vhost | 1 + 19 files changed, 52 insertions(+), 89 deletions(-) create mode 160000 third_party/minigbm create mode 160000 third_party/minijail create mode 160000 third_party/virglrenderer create mode 160000 third_party/vmm_vhost diff --git a/.gitmodules b/.gitmodules index f04998daa1..a90256951e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,15 @@ [submodule "tpm2-sys/libtpm2"] path = tpm2-sys/libtpm2 url = https://chromium.googlesource.com/chromiumos/third_party/tpm2 +[submodule "third_party/minigbm"] + path = third_party/minigbm + url = https://chromium.googlesource.com/chromiumos/platform/minigbm +[submodule "third_party/virglrenderer"] + path = third_party/virglrenderer + url = https://chromium.googlesource.com/chromiumos/third_party/virglrenderer +[submodule "third_party/minijail"] + path = third_party/minijail + url = https://android.googlesource.com/platform/external/minijail +[submodule "third_party/vmm_vhost"] + path = third_party/vmm_vhost + url = https://chromium.googlesource.com/chromiumos/third_party/rust-vmm/vhost/ diff --git a/Cargo.toml b/Cargo.toml index 5e8bda854f..69f2220872 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,10 +120,10 @@ base = { path = "base" } cros_fuzz = { path = "common/cros-fuzz" } # ignored by ebuild data_model = { path = "data_model" } libcras = { path = "libcras_stub" } # ignored by ebuild -minijail = { path = "../../aosp/external/minijail/rust/minijail" } # ignored by ebuild p9 = { path = "common/p9" } # ignored by ebuild sync = { path = "sync" } sys_util = { path = "sys_util" } tempfile = { path = "tempfile" } -vmm_vhost = { path = "../../third_party/rust-vmm/vhost", features = ["vhost-user-master", "vhost-user-slave"] } # ignored by ebuild wire_format_derive = { path = "common/p9/wire_format_derive" } # ignored by ebuild +minijail = { path = "third_party/minijail/rust/minijail" } # ignored by ebuild +vmm_vhost = { path = "third_party/vmm_vhost", features = ["vhost-user-master", "vhost-user-slave"] } # ignored by ebuild diff --git a/README.md b/README.md index debe5096c3..68a17669a5 100644 --- a/README.md +++ b/README.md @@ -32,15 +32,17 @@ to build and test crosvm in enviroments outside of the Chrome OS chroot. >**NOTE:** Building for Linux natively is new and not fully supported. -First, [set up depot_tools] and use `repo` to sync down the crosvm source -tree. This is a subset of the entire Chromium OS manifest with just enough repos -to build crosvm. +Crosvm uses submodules to manage external dependencies. Initialize them via: ```sh -mkdir crosvm -cd crosvm -repo init -g crosvm -u https://chromium.googlesource.com/chromiumos/manifest.git --repo-url=https://chromium.googlesource.com/external/repo.git -repo sync +git submodule update --init +``` + +It is recommended to enable automatic recursive operations to keep the +submodules in sync with the main repository: + +```sh +git config --global submodule.recurse true ``` A basic crosvm build links against `libcap`. On a Debian-based system, @@ -77,8 +79,6 @@ Known issues: And that's it! You should be able to `cargo build/run/test`. -[set up depot_tools]: https://commondatastorage.googleapis.com/chrome-infra-docs/flat/depot_tools/docs/html/depot_tools_tutorial.html#_setting_up - ## Usage To see the usage information for your version of crosvm, run `crosvm` or `crosvm diff --git a/bin/clippy b/bin/clippy index d04e4dfd57..b017850cb0 100755 --- a/bin/clippy +++ b/bin/clippy @@ -71,7 +71,6 @@ FEATURES=( audio gpu plugin - power-monitor-powerd tpm usb video-decoder diff --git a/ci/README.md b/ci/README.md index 52ac317264..58ffa7e82f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -10,19 +10,10 @@ testing. ### Setting up the source -Since crosvm is part of chromiumos, and uses a couple of it's projects as -dependencies, you need a standard chromiumos checkout as described by the -[ChromiumOS Developer Guide](https://chromium.googlesource.com/chromiumos/docs/+/master/developer_guide.md#Get-the-Source). - -To reduce the number of repositories to download, you can use the `-g crosvm` -argument on `repo init`. This will be significantly faster: - -In summary: +Crosvm requires a bunch of dependencies that are checked out via submodules: ``` -$ repo init -u https://chromium.googlesource.com/chromiumos/manifest.git --repo-url https://chromium.googlesource.com/external/repo.git -g crosvm -$ repo sync -j4 -$ cd src/platform/crosvm +$ git submodule update --init ``` ### Installing Podman (or Docker) diff --git a/ci/build_environment/Makefile b/ci/build_environment/Makefile index 716daa40a8..619310e5f9 100644 --- a/ci/build_environment/Makefile +++ b/ci/build_environment/Makefile @@ -2,12 +2,12 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # -# This makefile is run by the ./ci/crosvm_* containers to build ChromiumOS -# dependencies required by crosvm. +# This makefile is run by the ./ci/crosvm_* containers to build native third +# party dependencies required by crosvm. # # Setting TARGET_ARCH=aarch64 enables cross-compilation for aarch64. -SRC ?= /workspace/src +SRC ?= /workspace/src/crosvm BUILD ?= /workspace/scratch/build LIB ?= /workspace/scratch/lib TARGET_ARCH ?= @@ -36,7 +36,7 @@ clean: tpm2: mkdir -p $(BUILD)/tpm2 - $(MAKE) -C $(SRC)/third_party/tpm2 \ + $(MAKE) -C $(SRC)/tpm2-sys/libtpm2 \ obj=$(BUILD)/tpm2 \ AR=$(CROSS_COMPILE)ar \ CC=$(CROSS_COMPILE)gcc \ @@ -44,13 +44,13 @@ tpm2: minijail: mkdir -p $(BUILD)/minijail - $(MAKE) -C $(SRC)/aosp/external/minijail \ + $(MAKE) -C $(SRC)/third_party/minijail \ OUT=$(BUILD)/minijail \ CROSS_COMPILE=$(CROSS_COMPILE) minigbm: mkdir -p $(BUILD)/minigbm - $(MAKE) -C $(SRC)/platform/minigbm \ + $(MAKE) -C $(SRC)/third_party/minigbm \ OUT=$(BUILD)/minigbm \ CROSS_COMPILE=$(CROSS_COMPILE) @@ -60,7 +60,7 @@ virglrenderer: minigbm $(BUILD)/meson-cross $(SRC)/third_party/virglrenderer \ $(MESON_ARGS) - CPATH=$(SRC)/platform/minigbm \ + CPATH=$(SRC)/third_party/minigbm \ meson compile -C $(BUILD)/virglrenderer @@ -90,7 +90,7 @@ $(LIB): # minigbm ln -sf $(BUILD)/minigbm/libminigbm.so.1.0.0 $(LIB)/libgbm.so ln -sf $(LIB)/libgbm.so $(LIB)/libgbm.so.1 - ln -sf $(SRC)/platform/minigbm/gbm.pc $(LIB)/pkgconfig/ + ln -sf $(SRC)/third_party/minigbm/gbm.pc $(LIB)/pkgconfig/ # virglrenderer ln -sf $(BUILD)/virglrenderer/src/libvirglrenderer.so $(LIB) diff --git a/ci/crosvm_aarch64_builder/Dockerfile b/ci/crosvm_aarch64_builder/Dockerfile index 42ba6284a2..fa6830f9ef 100644 --- a/ci/crosvm_aarch64_builder/Dockerfile +++ b/ci/crosvm_aarch64_builder/Dockerfile @@ -71,7 +71,7 @@ COPY --from=vm \ /root/.ssh /root/.ssh # Setup entrypoint and interactive shell -WORKDIR /workspace/src/platform/crosvm +WORKDIR /workspace/src/crosvm COPY bashrc /root/.bashrc COPY entrypoint /workspace ENTRYPOINT ["/workspace/entrypoint"] diff --git a/ci/crosvm_base/Dockerfile b/ci/crosvm_base/Dockerfile index fefeec2fed..c9d10918ab 100644 --- a/ci/crosvm_base/Dockerfile +++ b/ci/crosvm_base/Dockerfile @@ -80,4 +80,4 @@ ENV RUSTFLAGS='--cfg hermetic' ENV CROSVM_CROS_BUILD=1 # All commands will be executed in the crosvm src directory. -WORKDIR /workspace/src/platform/crosvm +WORKDIR /workspace/src/crosvm diff --git a/ci/crosvm_builder/Dockerfile b/ci/crosvm_builder/Dockerfile index 8c70dd8668..3b3e02b188 100644 --- a/ci/crosvm_builder/Dockerfile +++ b/ci/crosvm_builder/Dockerfile @@ -43,7 +43,7 @@ COPY --from=vm \ /root/.ssh /root/.ssh # Setup entrypoint and interactive shell -WORKDIR /workspace/src/platform/crosvm +WORKDIR /workspace/src/crosvm COPY bashrc /root/.bashrc COPY entrypoint /workspace ENTRYPOINT ["/workspace/entrypoint"] diff --git a/ci/image_tag b/ci/image_tag index fa0363b794..d9830c0eee 100644 --- a/ci/image_tag +++ b/ci/image_tag @@ -1 +1 @@ -r0008 +r0009 diff --git a/ci/kokoro/common.sh b/ci/kokoro/common.sh index c1e1e8b3b5..ac05184cfa 100755 --- a/ci/kokoro/common.sh +++ b/ci/kokoro/common.sh @@ -3,8 +3,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -crosvm_root="${KOKORO_ARTIFACTS_DIR}"/git/crosvm - # Enable SSH access to the kokoro builder. # Use the fusion2/ UI to trigger a build and set the DEBUG_SSH_KEY environment # variable to your public key, that will allow you to connect to the builder @@ -21,42 +19,13 @@ if [[ ! -z "${DEBUG_SSH_KEY}" ]]; then fi setup_source() { - if [ -z "${KOKORO_ARTIFACTS_DIR}" ]; then + if [ -z "${KOKORO_ARTIFACTS_DIR}/git" ]; then echo "This script must be run in kokoro" exit 1 fi - cd "${KOKORO_ARTIFACTS_DIR}" - - echo "" - echo "Downloading crosvm dependencies to $(pwd)/cros..." - mkdir cros - cd cros - - # repo gets confused by pyenv, make sure we select 3.6.1 as our default - # version. - if command -v pyenv >/dev/null; then - echo "Selecting Python 3.6.1" - pyenv global 3.6.1 - fi - curl -s https://storage.googleapis.com/git-repo-downloads/repo >repo - chmod +x repo - ./repo init --depth 1 \ - -u https://chromium.googlesource.com/chromiumos/manifest.git \ - --repo-url https://chromium.googlesource.com/external/repo.git \ - -g crosvm || return 1 - ./repo sync -j8 -c || return 1 - - # Bind mount source into cros checkout. - echo "" - echo "Mounting crosvm source to $(pwd)/src/platform/crosvm..." - rm -rf src/platform/crosvm && mkdir -p src/platform/crosvm - if command -v bindfs >/dev/null; then - bindfs "${crosvm_root}" src/platform/crosvm || return 1 - else - sudo mount --bind "${crosvm_root}" src/platform/crosvm || return 1 - fi - + cd "${KOKORO_ARTIFACTS_DIR}/git/crosvm" + git submodule update --init } cleanup() { @@ -66,12 +35,6 @@ cleanup() { sleep 1h fi - if command -v bindfs >/dev/null; then - fusermount -uz "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm" - else - sudo umount --lazy "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm" - fi - # List files in the logs directory which are uploaded to sponge. echo "Build Artifacts:" ls "${KOKORO_ARTIFACTS_DIR}/logs" @@ -86,4 +49,4 @@ setup_source || { # Set logs directory so we can copy them to sponge export CROSVM_BUILDER_LOGS_DIR="${KOKORO_ARTIFACTS_DIR}/logs" -cd "${KOKORO_ARTIFACTS_DIR}/cros/src/platform/crosvm" +cd "${KOKORO_ARTIFACTS_DIR}/git/crosvm" diff --git a/ci/kokoro/simulate b/ci/kokoro/simulate index bd192abbb5..2673a90e42 100755 --- a/ci/kokoro/simulate +++ b/ci/kokoro/simulate @@ -19,7 +19,8 @@ cleanup() { main() { echo "Copying ${crosvm_src}/ to ${kokoro_src}" mkdir -p "${kokoro_src}" - rsync -arq --exclude "target" --exclude ".git" "${crosvm_src}/" "${kokoro_src}" + rsync -arq --exclude "target" --exclude "__pycache__" \ + "${crosvm_src}/" "${kokoro_src}" # Run user-provided kokoro build script. export KOKORO_ARTIFACTS_DIR="${kokoro_root}/src" diff --git a/ci/run_container.sh b/ci/run_container.sh index 27ecc2a8a2..1ed298a235 100755 --- a/ci/run_container.sh +++ b/ci/run_container.sh @@ -11,14 +11,7 @@ # CROSVM_BUILDER_SCRATCH_DIR or CROSVM_BUILDER_LOGS_DIR. crosvm_root=$(realpath "$(dirname $0)/..") -cros_root=$(realpath "${crosvm_root}/../../..") -if [ ! -d "${cros_root}/.repo" ]; then - echo "The CI builder must be run from a cros checkout. See ci/README.md" - exit 1 -fi - -# Parse parameters builder="$1" shift @@ -43,14 +36,14 @@ fi version=$(cat $(dirname $0)/image_tag) echo "Using builder: ${builder}:${version}" -src="${cros_root}/src" -echo "Using source directory: ${src} (Available at /workspace/src)" +echo "Using source directory: ${crosvm_root} \ +(Available at /workspace/src/crosvm)" docker_args=( --rm --device /dev/kvm --volume /dev/log:/dev/log - --volume "${src}":/workspace/src:rw + --volume "${crosvm_root}":/workspace/src/crosvm:rw ) if [ ! -z "${CROSVM_BUILDER_SCRATCH_DIR}" ]; then diff --git a/ci/vm_tools/sync_deps b/ci/vm_tools/sync_deps index a97b019e0b..c0e0d04792 100755 --- a/ci/vm_tools/sync_deps +++ b/ci/vm_tools/sync_deps @@ -8,7 +8,7 @@ ${0%/*}/wait_for_vm_with_timeout || exit 1 -crosvm_root="/workspace/src/platform/crosvm" +crosvm_root="/workspace/src/crosvm" rust_toolchain=$(cat ${crosvm_root}/rust-toolchain) target_dir=$( cargo metadata --no-deps --format-version 1 | diff --git a/run_tests b/run_tests index ab5792fd1f..78c8744a3e 100755 --- a/run_tests +++ b/run_tests @@ -78,7 +78,7 @@ FEATURE_REQUIREMENTS: Dict[str, List[Requirements]] = { "audio": [], "gpu": [Requirements.CROS_BUILD], "plugin": [Requirements.PRIVILEGED, Requirements.X86_64], - "power-monitor-powerd": [], + "power-monitor-powerd": [Requirements.DISABLED], "tpm": [Requirements.CROS_BUILD], "video-decoder": [Requirements.DISABLED], "video-encoder": [Requirements.DISABLED], diff --git a/third_party/minigbm b/third_party/minigbm new file mode 160000 index 0000000000..2e63aaf616 --- /dev/null +++ b/third_party/minigbm @@ -0,0 +1 @@ +Subproject commit 2e63aaf616cdda26019d265989bd0d96ee11aab9 diff --git a/third_party/minijail b/third_party/minijail new file mode 160000 index 0000000000..94cff17da4 --- /dev/null +++ b/third_party/minijail @@ -0,0 +1 @@ +Subproject commit 94cff17da47df59f567b64fd8e87795fcae055d4 diff --git a/third_party/virglrenderer b/third_party/virglrenderer new file mode 160000 index 0000000000..a108be89e3 --- /dev/null +++ b/third_party/virglrenderer @@ -0,0 +1 @@ +Subproject commit a108be89e36ad427bf5f9affdbb4dd434e119b28 diff --git a/third_party/vmm_vhost b/third_party/vmm_vhost new file mode 160000 index 0000000000..70810633ab --- /dev/null +++ b/third_party/vmm_vhost @@ -0,0 +1 @@ +Subproject commit 70810633abb3f7e1e3f11ba728b114ccdcec6f31