From 12593ba64265b7a08ecae334109cad47d417a658 Mon Sep 17 00:00:00 2001 From: Dennis Kempin Date: Thu, 7 Jul 2022 22:22:13 +0000 Subject: [PATCH] infra: Add docs builder The new builder will generate the mdbook and api docs, then uploads them to GCS. BUG=b:233411583 TEST=luci-auth context ./infra/recipes.py run build_docs Change-Id: Ia22cdb20d73f9268db6299cd6f0875950d174b3f Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3751832 Reviewed-by: Daniel Verkamp Tested-by: kokoro --- infra/README.recipes.md | 14 ++ infra/config/generated/cr-buildbucket.cfg | 21 +++ infra/config/generated/luci-milo.cfg | 3 + infra/config/generated/luci-notify.cfg | 13 ++ infra/config/generated/luci-scheduler.cfg | 11 ++ infra/config/main.star | 8 ++ .../build_docs.expected/build_docs.json | 126 ++++++++++++++++++ infra/recipes/build_docs.py | 67 ++++++++++ 8 files changed, 263 insertions(+) create mode 100644 infra/recipes/build_docs.expected/build_docs.json create mode 100644 infra/recipes/build_docs.py diff --git a/infra/README.recipes.md b/infra/README.recipes.md index b6b2788b99..5f66af1e3e 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md @@ -7,6 +7,7 @@ **[Recipes](#Recipes)** * [build_chromeos](#recipes-build_chromeos) (Python3 ✅) + * [build_docs](#recipes-build_docs) (Python3 ✅) * [build_linux](#recipes-build_linux) (Python3 ✅) * [build_windows](#recipes-build_windows) (Python3 ✅) * [crosvm:examples/container_build_context](#recipes-crosvm_examples_container_build_context) (Python3 ✅) @@ -102,6 +103,18 @@ PYTHON_VERSION_COMPATIBILITY: PY3 — **def [RunSteps](/infra/recipes/build_chromeos.py#94)(api, properties):** — **def [SetupSource](/infra/recipes/build_chromeos.py#27)(api, workspace):** +### *recipes* / [build\_docs](/infra/recipes/build_docs.py) + +[DEPS](/infra/recipes/build_docs.py#9): [crosvm](#recipe_modules-crosvm), [depot\_tools/gsutil][depot_tools/recipe_modules/gsutil], [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +PYTHON_VERSION_COMPATIBILITY: PY3 + +— **def [RunSteps](/infra/recipes/build_docs.py#22)(api):** + +Builds crosvm mdbook and api docs, then uploads them to GCS. + +This recipe requires ambient luci authentication. To test locally run: + $ luci-auth context ./infra/recipes.py run build_docs ### *recipes* / [build\_linux](/infra/recipes/build_linux.py) [DEPS](/infra/recipes/build_linux.py#11): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] @@ -167,6 +180,7 @@ PYTHON_VERSION_COMPATIBILITY: PY3 [depot_tools/recipe_modules/depot_tools]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-depot_tools [depot_tools/recipe_modules/gclient]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-gclient [depot_tools/recipe_modules/git]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-git +[depot_tools/recipe_modules/gsutil]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-gsutil [recipe_engine/recipe_modules/buildbucket]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-buildbucket [recipe_engine/recipe_modules/cipd]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-cipd [recipe_engine/recipe_modules/context]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-context diff --git a/infra/config/generated/cr-buildbucket.cfg b/infra/config/generated/cr-buildbucket.cfg index dcdee29a15..faabc19b8e 100644 --- a/infra/config/generated/cr-buildbucket.cfg +++ b/infra/config/generated/cr-buildbucket.cfg @@ -14,6 +14,27 @@ buckets { group: "mdb/crosvm-acl-luci-admin" } swarming { + builders { + name: "build_docs" + swarming_host: "chromium-swarm.appspot.com" + dimensions: "cpu:x86-64" + dimensions: "os:Ubuntu" + dimensions: "pool:luci.crosvm.ci" + exe { + cipd_package: "infra/recipe_bundles/chromium.googlesource.com/crosvm/crosvm" + cipd_version: "refs/heads/main" + cmd: "luciexe" + } + properties: + '{' + ' "recipe": "build_docs"' + '}' + service_account: "crosvm-luci-ci-builder@crosvm-infra.iam.gserviceaccount.com" + experiments { + key: "luci.recipes.use_python3" + value: 100 + } + } builders { name: "chromeos_amd64-generic" swarming_host: "chromium-swarm.appspot.com" diff --git a/infra/config/generated/luci-milo.cfg b/infra/config/generated/luci-milo.cfg index efa2469fb4..01c834472d 100644 --- a/infra/config/generated/luci-milo.cfg +++ b/infra/config/generated/luci-milo.cfg @@ -45,6 +45,9 @@ consoles { builders { name: "buildbucket/luci.crosvm.ci/push_to_github" } + builders { + name: "buildbucket/luci.crosvm.ci/build_docs" + } builders { name: "buildbucket/luci.crosvm.ci/update_chromeos_merges" } diff --git a/infra/config/generated/luci-notify.cfg b/infra/config/generated/luci-notify.cfg index a1d6c6e96c..d2f84d474e 100644 --- a/infra/config/generated/luci-notify.cfg +++ b/infra/config/generated/luci-notify.cfg @@ -4,6 +4,19 @@ # For the schema of this file, see ProjectConfig message: # https://luci-config.appspot.com/schemas/projects:luci-notify.cfg +notifiers { + notifications { + on_change: true + email { + recipients: "denniskempin@google.com" + } + } + builders { + bucket: "ci" + name: "build_docs" + repository: "https://chromium.googlesource.com/crosvm/crosvm" + } +} notifiers { notifications { on_change: true diff --git a/infra/config/generated/luci-scheduler.cfg b/infra/config/generated/luci-scheduler.cfg index 8edd81f624..f847c374f1 100644 --- a/infra/config/generated/luci-scheduler.cfg +++ b/infra/config/generated/luci-scheduler.cfg @@ -4,6 +4,16 @@ # For the schema of this file, see ProjectConfig message: # https://luci-config.appspot.com/schemas/projects:luci-scheduler.cfg +job { + id: "build_docs" + realm: "ci" + acl_sets: "ci" + buildbucket { + server: "cr-buildbucket.appspot.com" + bucket: "ci" + builder: "build_docs" + } +} job { id: "chromeos_amd64-generic" realm: "ci" @@ -99,6 +109,7 @@ trigger { id: "main source" realm: "ci" acl_sets: "ci" + triggers: "build_docs" triggers: "chromeos_amd64-generic" triggers: "health_check" triggers: "linux_aarch64" diff --git a/infra/config/main.star b/infra/config/main.star index b6fcd539cf..df5d814332 100755 --- a/infra/config/main.star +++ b/infra/config/main.star @@ -361,6 +361,14 @@ infra_builder( postsubmit = True, ) +infra_builder( + name = "build_docs", + executable = luci.recipe( + name = "build_docs", + ), + postsubmit = True, +) + infra_builder( name = "update_chromeos_merges", executable = luci.recipe( diff --git a/infra/recipes/build_docs.expected/build_docs.json b/infra/recipes/build_docs.expected/build_docs.json new file mode 100644 index 0000000000..f3f479a89a --- /dev/null +++ b/infra/recipes/build_docs.expected/build_docs.json @@ -0,0 +1,126 @@ +[ + { + "cmd": [ + "vpython3", + "[CACHE]/builder/crosvm/tools/dev_container", + "--verbose", + "mdbook", + "build", + "docs/book/", + "--dest-dir", + "../target" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CROSVM_CONTAINER_CACHE": "[CACHE]/builder/dev_container" + }, + "luci_context": { + "realm": { + "name": "crosvm/crosvm:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Build mdbook" + }, + { + "cmd": [ + "vpython3", + "[CACHE]/builder/crosvm/tools/dev_container", + "--verbose", + "./tools/cargo-doc", + "--target-dir", + "docs/target" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CROSVM_CONTAINER_CACHE": "[CACHE]/builder/dev_container" + }, + "luci_context": { + "realm": { + "name": "crosvm/crosvm:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Run cargo docs" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "-m", + "----", + "rsync", + "-r", + "-d", + "./docs/target/html", + "gs://crosvm-dot-dev/book" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CROSVM_CONTAINER_CACHE": "[CACHE]/builder/dev_container" + }, + "infra_step": true, + "luci_context": { + "realm": { + "name": "crosvm/crosvm:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload book" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "-m", + "----", + "rsync", + "-r", + "-d", + "./docs/target/doc", + "gs://crosvm-dot-dev/api" + ], + "cwd": "[CACHE]/builder/crosvm", + "env": { + "CROSVM_CONTAINER_CACHE": "[CACHE]/builder/dev_container" + }, + "infra_step": true, + "luci_context": { + "realm": { + "name": "crosvm/crosvm:ci" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload docs" + } +] \ No newline at end of file diff --git a/infra/recipes/build_docs.py b/infra/recipes/build_docs.py new file mode 100644 index 0000000000..a1a5ae1847 --- /dev/null +++ b/infra/recipes/build_docs.py @@ -0,0 +1,67 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.post_process import Filter + +PYTHON_VERSION_COMPATIBILITY = "PY3" + +DEPS = [ + "crosvm", + "recipe_engine/file", + "recipe_engine/buildbucket", + "recipe_engine/context", + "recipe_engine/step", + "depot_tools/gsutil", +] + +BOOK_URL = "gs://crosvm-dot-dev/book" +DOCS_URL = "gs://crosvm-dot-dev/api" + + +def RunSteps(api): + """ + Builds crosvm mdbook and api docs, then uploads them to GCS. + + This recipe requires ambient luci authentication. To test locally run: + $ luci-auth context ./infra/recipes.py run build_docs + """ + with api.crosvm.container_build_context(): + api.crosvm.step_in_container( + "Build mdbook", ["mdbook", "build", "docs/book/", "--dest-dir", "../target"] + ) + api.crosvm.step_in_container( + "Run cargo docs", + ["./tools/cargo-doc", "--target-dir", "docs/target"], + ) + + # Container generated files are root-owned, we need to make sure they will be readable by + # gsutil (which has to run outside the container to run with proper authentication). + api.crosvm.step_in_container( + "Make docs readable by gsutil", + ["chmod", "-R", "o+r", "docs/target"], + ) + + api.gsutil( + ["rsync", "-r", "-d", "./docs/target/html", BOOK_URL], + name="Upload book", + multithreaded=True, + ) + api.gsutil( + ["rsync", "-r", "-d", "./docs/target/doc", DOCS_URL], + name="Upload docs", + multithreaded=True, + ) + + +def GenTests(api): + filter_steps = Filter( + "Build mdbook", "Run cargo docs", "gsutil Upload book", "gsutil Upload docs" + ) + yield ( + api.test( + "build_docs", + api.buildbucket.ci_build(project="crosvm/crosvm"), + ) + + api.post_process(filter_steps) + )