diff --git a/infra/README.recipes.md b/infra/README.recipes.md index 0626422daa..bdbb5f34d0 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md @@ -11,6 +11,7 @@ * [crosvm:examples/build_context](#recipes-crosvm_examples_build_context) (Python3 ✅) * [crosvm:examples/prepare_source](#recipes-crosvm_examples_prepare_source) (Python3 ✅) * [health_check](#recipes-health_check) (Python3 ✅) + * [push_to_github](#recipes-push_to_github) (Python3 ✅) ## Recipe Modules ### *recipe_modules* / [crosvm](/infra/recipe_modules/crosvm) @@ -96,11 +97,18 @@ PYTHON_VERSION_COMPATIBILITY: PY3 — **def [RunSteps](/infra/recipe_modules/crosvm/examples/prepare_source.py#18)(api):** ### *recipes* / [health\_check](/infra/recipes/health_check.py) -[DEPS](/infra/recipes/health_check.py#10): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] +[DEPS](/infra/recipes/health_check.py#9): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] PYTHON_VERSION_COMPATIBILITY: PY3 -— **def [RunSteps](/infra/recipes/health_check.py#19)(api):** +— **def [RunSteps](/infra/recipes/health_check.py#18)(api):** +### *recipes* / [push\_to\_github](/infra/recipes/push_to_github.py) + +[DEPS](/infra/recipes/push_to_github.py#9): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +PYTHON_VERSION_COMPATIBILITY: PY3 + +— **def [RunSteps](/infra/recipes/push_to_github.py#20)(api):** [depot_tools/recipe_modules/bot_update]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-bot_update [depot_tools/recipe_modules/depot_tools]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-depot_tools @@ -111,5 +119,6 @@ PYTHON_VERSION_COMPATIBILITY: PY3 [recipe_engine/recipe_modules/file]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-file [recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-path [recipe_engine/recipe_modules/properties]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-properties +[recipe_engine/recipe_modules/raw_io]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-raw_io [recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-step [recipe_engine/wkt/RecipeApi]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/recipe_engine/recipe_api.py#883 diff --git a/infra/config/generated/cr-buildbucket.cfg b/infra/config/generated/cr-buildbucket.cfg index 5c1e0e5529..e671d1cc57 100644 --- a/infra/config/generated/cr-buildbucket.cfg +++ b/infra/config/generated/cr-buildbucket.cfg @@ -123,6 +123,27 @@ buckets { value: 100 } } + builders { + name: "crosvm_push_to_github" + swarming_host: "chromium-swarm.appspot.com" + dimensions: "cpu:x86-64" + dimensions: "os:Ubuntu" + dimensions: "pool:luci.crosvm.ci" + exe { + cipd_package: "infra/recipe_bundles/chromium.googlesource.com/crosvm/crosvm" + cipd_version: "refs/heads/main" + cmd: "luciexe" + } + properties: + '{' + ' "recipe": "push_to_github"' + '}' + service_account: "crosvm-luci-ci-builder@crosvm-infra.iam.gserviceaccount.com" + experiments { + key: "luci.recipes.use_python3" + value: 100 + } + } } } buckets { diff --git a/infra/config/generated/luci-milo.cfg b/infra/config/generated/luci-milo.cfg index b0031a7ab2..93a73554c1 100644 --- a/infra/config/generated/luci-milo.cfg +++ b/infra/config/generated/luci-milo.cfg @@ -5,8 +5,8 @@ # https://luci-config.appspot.com/schemas/projects:luci-milo.cfg consoles { - id: "CI Console" - name: "CI Console" + id: "Postsubmit" + name: "Postsubmit" repo_url: "https://chromium.googlesource.com/crosvm/crosvm" refs: "regexp:refs/heads/main" manifest_name: "REVISION" @@ -31,3 +31,13 @@ consoles { category: "linux" } } +consoles { + id: "Infra" + name: "Infra" + repo_url: "https://chromium.googlesource.com/crosvm/crosvm" + refs: "regexp:refs/heads/main" + manifest_name: "REVISION" + builders { + name: "buildbucket/luci.crosvm.ci/crosvm_push_to_github" + } +} diff --git a/infra/config/generated/luci-scheduler.cfg b/infra/config/generated/luci-scheduler.cfg index edcfd825a0..e9cb0cb968 100644 --- a/infra/config/generated/luci-scheduler.cfg +++ b/infra/config/generated/luci-scheduler.cfg @@ -54,6 +54,16 @@ job { builder: "crosvm_linux_x86_64" } } +job { + id: "crosvm_push_to_github" + realm: "ci" + acl_sets: "ci" + buildbucket { + server: "cr-buildbucket.appspot.com" + bucket: "ci" + builder: "crosvm_push_to_github" + } +} trigger { id: "main source" realm: "ci" @@ -63,6 +73,7 @@ trigger { triggers: "crosvm_linux_aarch64" triggers: "crosvm_linux_armhf" triggers: "crosvm_linux_x86_64" + triggers: "crosvm_push_to_github" gitiles { repo: "https://chromium.googlesource.com/crosvm/crosvm" refs: "regexp:refs/heads/main" diff --git a/infra/config/main.star b/infra/config/main.star index ab3242eb71..2096800504 100755 --- a/infra/config/main.star +++ b/infra/config/main.star @@ -126,9 +126,15 @@ luci.cq_group( ), ) -# Configure postsubmit tests running in ci pool +# Console showing all postsubmit verify builders luci.console_view( - name = "CI Console", + name = "Postsubmit", + repo = "https://chromium.googlesource.com/crosvm/crosvm", +) + +# Console showing all postsubmit infra builders +luci.console_view( + name = "Infra", repo = "https://chromium.googlesource.com/crosvm/crosvm", ) @@ -162,7 +168,7 @@ def verify_builder(name, dimensions, presubmit = True, postsubmit = True, **args triggers = ["ci/%s" % name], ) luci.console_view_entry( - console_view = "CI Console", + console_view = "Postsubmit", builder = "ci/%s" % name, category = "linux", ) @@ -227,6 +233,37 @@ def verify_chromeos_builder(board, **kwargs): **kwargs ) +def infra_builder(name, **args): + """Creates a ci job to run infra recipes that are not involved in verifying changes. + + The builders are added to a separate infra dashboard. + + Args: + name: Name of the builder + **args: Passed to luci.builder + """ + luci.builder( + name = name, + bucket = "ci", + service_account = "crosvm-luci-ci-builder@crosvm-infra.iam.gserviceaccount.com", + dimensions = { + "pool": "luci.crosvm.ci", + "os": "Ubuntu", + "cpu": "x86-64", + }, + **args + ) + luci.gitiles_poller( + name = "main source", + bucket = "ci", + repo = "https://chromium.googlesource.com/crosvm/crosvm", + triggers = ["ci/%s" % name], + ) + luci.console_view_entry( + console_view = "Infra", + builder = "ci/%s" % name, + ) + verify_linux_builder("x86_64") verify_linux_builder("aarch64") verify_linux_builder("armhf") @@ -244,3 +281,9 @@ verify_builder( ), ) +infra_builder( + name = "crosvm_push_to_github", + executable = luci.recipe( + name = "push_to_github", + ), +) diff --git a/infra/recipes/health_check.py b/infra/recipes/health_check.py index cacf231688..b5637a4314 100644 --- a/infra/recipes/health_check.py +++ b/infra/recipes/health_check.py @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import re from recipe_engine.post_process import Filter PYTHON_VERSION_COMPATIBILITY = "PY3" diff --git a/infra/recipes/push_to_github.expected/basic.json b/infra/recipes/push_to_github.expected/basic.json new file mode 100644 index 0000000000..eed332dce1 --- /dev/null +++ b/infra/recipes/push_to_github.expected/basic.json @@ -0,0 +1,10 @@ +[ + { + "cmd": [ + "bash", + "RECIPE[crosvm::push_to_github].resources/push_to_github.sh" + ], + "cwd": "[CACHE]/builder/crosvm", + "name": "Pushing to github" + } +] \ No newline at end of file diff --git a/infra/recipes/push_to_github.py b/infra/recipes/push_to_github.py new file mode 100644 index 0000000000..73319b9d20 --- /dev/null +++ b/infra/recipes/push_to_github.py @@ -0,0 +1,29 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.post_process import Filter + +PYTHON_VERSION_COMPATIBILITY = "PY3" + +DEPS = [ + "crosvm", + "recipe_engine/buildbucket", + "recipe_engine/context", + "recipe_engine/raw_io", + "recipe_engine/step", + "recipe_engine/path", + "recipe_engine/file", +] + + +def RunSteps(api): + api.crosvm.prepare_source() + with api.context(cwd=api.crosvm.source_dir): + # Execute push in a bash script so there is no chance of leaking the github token via luci + # logs. + api.step("Pushing to github", ["bash", api.resource("push_to_github.sh")]) + + +def GenTests(api): + yield (api.test("basic") + api.post_process(Filter("Pushing to github"))) diff --git a/infra/recipes/push_to_github.resources/push_to_github.sh b/infra/recipes/push_to_github.resources/push_to_github.sh new file mode 100644 index 0000000000..9370d8354a --- /dev/null +++ b/infra/recipes/push_to_github.resources/push_to_github.sh @@ -0,0 +1,11 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Pushes the current repo to github using the secret provided by Secret Manager. +# See: https://console.cloud.google.com/security/secret-manager +# +# This script will only work if you have access to the above service via gcloud. +set -e +TOKEN=$(gcloud --project=crosvm-infra secrets versions access 1 --secret="github-crosvm-bot") +git push --force --all "https://crosvm-bot:${TOKEN}@github.com/google/crosvm.git"