From dd4b8f167682f05b049f9704bed150efa83c3eb0 Mon Sep 17 00:00:00 2001 From: Dennis Kempin Date: Thu, 26 May 2022 23:13:27 +0000 Subject: [PATCH] infra: Add push-to-github builder The builder runs as a post-submit and will force push changes to github. Authentication is done via an access token stored in gcloud secrets manager. For this to work in the bot pool, the bots need to be reconfigured to receive the scope needed for access to the secrets API as done in https://crrev.com/i/4782519 Once this bot is working, the kokoro one can be turned off. Since we mirror from platform/crosvm to the new repo, luci is triggered for all new changes. BUG=b:233913820 TEST=./recipe.py run push_to_github Change-Id: I45478a076b87767ec10f0d0148c0713da7656264 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3671131 Tested-by: kokoro Reviewed-by: Alexandre Courbot Commit-Queue: Dennis Kempin --- infra/README.recipes.md | 13 ++++- infra/config/generated/cr-buildbucket.cfg | 21 ++++++++ infra/config/generated/luci-milo.cfg | 14 +++++- infra/config/generated/luci-scheduler.cfg | 11 +++++ infra/config/main.star | 49 +++++++++++++++++-- infra/recipes/health_check.py | 1 - .../push_to_github.expected/basic.json | 10 ++++ infra/recipes/push_to_github.py | 29 +++++++++++ .../push_to_github.sh | 11 +++++ 9 files changed, 151 insertions(+), 8 deletions(-) create mode 100644 infra/recipes/push_to_github.expected/basic.json create mode 100644 infra/recipes/push_to_github.py create mode 100644 infra/recipes/push_to_github.resources/push_to_github.sh diff --git a/infra/README.recipes.md b/infra/README.recipes.md index 0626422daa..bdbb5f34d0 100644 --- a/infra/README.recipes.md +++ b/infra/README.recipes.md @@ -11,6 +11,7 @@ * [crosvm:examples/build_context](#recipes-crosvm_examples_build_context) (Python3 ✅) * [crosvm:examples/prepare_source](#recipes-crosvm_examples_prepare_source) (Python3 ✅) * [health_check](#recipes-health_check) (Python3 ✅) + * [push_to_github](#recipes-push_to_github) (Python3 ✅) ## Recipe Modules ### *recipe_modules* / [crosvm](/infra/recipe_modules/crosvm) @@ -96,11 +97,18 @@ PYTHON_VERSION_COMPATIBILITY: PY3 — **def [RunSteps](/infra/recipe_modules/crosvm/examples/prepare_source.py#18)(api):** ### *recipes* / [health\_check](/infra/recipes/health_check.py) -[DEPS](/infra/recipes/health_check.py#10): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] +[DEPS](/infra/recipes/health_check.py#9): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step] PYTHON_VERSION_COMPATIBILITY: PY3 -— **def [RunSteps](/infra/recipes/health_check.py#19)(api):** +— **def [RunSteps](/infra/recipes/health_check.py#18)(api):** +### *recipes* / [push\_to\_github](/infra/recipes/push_to_github.py) + +[DEPS](/infra/recipes/push_to_github.py#9): [crosvm](#recipe_modules-crosvm), [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/step][recipe_engine/recipe_modules/step] + +PYTHON_VERSION_COMPATIBILITY: PY3 + +— **def [RunSteps](/infra/recipes/push_to_github.py#20)(api):** [depot_tools/recipe_modules/bot_update]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-bot_update [depot_tools/recipe_modules/depot_tools]: https://chromium.googlesource.com/chromium/tools/depot_tools.git/+/8a87603683bda769d437e48cc1a7494a2e237ead/recipes/README.recipes.md#recipe_modules-depot_tools @@ -111,5 +119,6 @@ PYTHON_VERSION_COMPATIBILITY: PY3 [recipe_engine/recipe_modules/file]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-file [recipe_engine/recipe_modules/path]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-path [recipe_engine/recipe_modules/properties]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-properties +[recipe_engine/recipe_modules/raw_io]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-raw_io [recipe_engine/recipe_modules/step]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/README.recipes.md#recipe_modules-step [recipe_engine/wkt/RecipeApi]: https://chromium.googlesource.com/infra/luci/recipes-py.git/+/7b42800366a15f34b28e62f6bcb1cddcb2206db0/recipe_engine/recipe_api.py#883 diff --git a/infra/config/generated/cr-buildbucket.cfg b/infra/config/generated/cr-buildbucket.cfg index 5c1e0e5529..e671d1cc57 100644 --- a/infra/config/generated/cr-buildbucket.cfg +++ b/infra/config/generated/cr-buildbucket.cfg @@ -123,6 +123,27 @@ buckets { value: 100 } } + builders { + name: "crosvm_push_to_github" + swarming_host: "chromium-swarm.appspot.com" + dimensions: "cpu:x86-64" + dimensions: "os:Ubuntu" + dimensions: "pool:luci.crosvm.ci" + exe { + cipd_package: "infra/recipe_bundles/chromium.googlesource.com/crosvm/crosvm" + cipd_version: "refs/heads/main" + cmd: "luciexe" + } + properties: + '{' + ' "recipe": "push_to_github"' + '}' + service_account: "crosvm-luci-ci-builder@crosvm-infra.iam.gserviceaccount.com" + experiments { + key: "luci.recipes.use_python3" + value: 100 + } + } } } buckets { diff --git a/infra/config/generated/luci-milo.cfg b/infra/config/generated/luci-milo.cfg index b0031a7ab2..93a73554c1 100644 --- a/infra/config/generated/luci-milo.cfg +++ b/infra/config/generated/luci-milo.cfg @@ -5,8 +5,8 @@ # https://luci-config.appspot.com/schemas/projects:luci-milo.cfg consoles { - id: "CI Console" - name: "CI Console" + id: "Postsubmit" + name: "Postsubmit" repo_url: "https://chromium.googlesource.com/crosvm/crosvm" refs: "regexp:refs/heads/main" manifest_name: "REVISION" @@ -31,3 +31,13 @@ consoles { category: "linux" } } +consoles { + id: "Infra" + name: "Infra" + repo_url: "https://chromium.googlesource.com/crosvm/crosvm" + refs: "regexp:refs/heads/main" + manifest_name: "REVISION" + builders { + name: "buildbucket/luci.crosvm.ci/crosvm_push_to_github" + } +} diff --git a/infra/config/generated/luci-scheduler.cfg b/infra/config/generated/luci-scheduler.cfg index edcfd825a0..e9cb0cb968 100644 --- a/infra/config/generated/luci-scheduler.cfg +++ b/infra/config/generated/luci-scheduler.cfg @@ -54,6 +54,16 @@ job { builder: "crosvm_linux_x86_64" } } +job { + id: "crosvm_push_to_github" + realm: "ci" + acl_sets: "ci" + buildbucket { + server: "cr-buildbucket.appspot.com" + bucket: "ci" + builder: "crosvm_push_to_github" + } +} trigger { id: "main source" realm: "ci" @@ -63,6 +73,7 @@ trigger { triggers: "crosvm_linux_aarch64" triggers: "crosvm_linux_armhf" triggers: "crosvm_linux_x86_64" + triggers: "crosvm_push_to_github" gitiles { repo: "https://chromium.googlesource.com/crosvm/crosvm" refs: "regexp:refs/heads/main" diff --git a/infra/config/main.star b/infra/config/main.star index ab3242eb71..2096800504 100755 --- a/infra/config/main.star +++ b/infra/config/main.star @@ -126,9 +126,15 @@ luci.cq_group( ), ) -# Configure postsubmit tests running in ci pool +# Console showing all postsubmit verify builders luci.console_view( - name = "CI Console", + name = "Postsubmit", + repo = "https://chromium.googlesource.com/crosvm/crosvm", +) + +# Console showing all postsubmit infra builders +luci.console_view( + name = "Infra", repo = "https://chromium.googlesource.com/crosvm/crosvm", ) @@ -162,7 +168,7 @@ def verify_builder(name, dimensions, presubmit = True, postsubmit = True, **args triggers = ["ci/%s" % name], ) luci.console_view_entry( - console_view = "CI Console", + console_view = "Postsubmit", builder = "ci/%s" % name, category = "linux", ) @@ -227,6 +233,37 @@ def verify_chromeos_builder(board, **kwargs): **kwargs ) +def infra_builder(name, **args): + """Creates a ci job to run infra recipes that are not involved in verifying changes. + + The builders are added to a separate infra dashboard. + + Args: + name: Name of the builder + **args: Passed to luci.builder + """ + luci.builder( + name = name, + bucket = "ci", + service_account = "crosvm-luci-ci-builder@crosvm-infra.iam.gserviceaccount.com", + dimensions = { + "pool": "luci.crosvm.ci", + "os": "Ubuntu", + "cpu": "x86-64", + }, + **args + ) + luci.gitiles_poller( + name = "main source", + bucket = "ci", + repo = "https://chromium.googlesource.com/crosvm/crosvm", + triggers = ["ci/%s" % name], + ) + luci.console_view_entry( + console_view = "Infra", + builder = "ci/%s" % name, + ) + verify_linux_builder("x86_64") verify_linux_builder("aarch64") verify_linux_builder("armhf") @@ -244,3 +281,9 @@ verify_builder( ), ) +infra_builder( + name = "crosvm_push_to_github", + executable = luci.recipe( + name = "push_to_github", + ), +) diff --git a/infra/recipes/health_check.py b/infra/recipes/health_check.py index cacf231688..b5637a4314 100644 --- a/infra/recipes/health_check.py +++ b/infra/recipes/health_check.py @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import re from recipe_engine.post_process import Filter PYTHON_VERSION_COMPATIBILITY = "PY3" diff --git a/infra/recipes/push_to_github.expected/basic.json b/infra/recipes/push_to_github.expected/basic.json new file mode 100644 index 0000000000..eed332dce1 --- /dev/null +++ b/infra/recipes/push_to_github.expected/basic.json @@ -0,0 +1,10 @@ +[ + { + "cmd": [ + "bash", + "RECIPE[crosvm::push_to_github].resources/push_to_github.sh" + ], + "cwd": "[CACHE]/builder/crosvm", + "name": "Pushing to github" + } +] \ No newline at end of file diff --git a/infra/recipes/push_to_github.py b/infra/recipes/push_to_github.py new file mode 100644 index 0000000000..73319b9d20 --- /dev/null +++ b/infra/recipes/push_to_github.py @@ -0,0 +1,29 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.post_process import Filter + +PYTHON_VERSION_COMPATIBILITY = "PY3" + +DEPS = [ + "crosvm", + "recipe_engine/buildbucket", + "recipe_engine/context", + "recipe_engine/raw_io", + "recipe_engine/step", + "recipe_engine/path", + "recipe_engine/file", +] + + +def RunSteps(api): + api.crosvm.prepare_source() + with api.context(cwd=api.crosvm.source_dir): + # Execute push in a bash script so there is no chance of leaking the github token via luci + # logs. + api.step("Pushing to github", ["bash", api.resource("push_to_github.sh")]) + + +def GenTests(api): + yield (api.test("basic") + api.post_process(Filter("Pushing to github"))) diff --git a/infra/recipes/push_to_github.resources/push_to_github.sh b/infra/recipes/push_to_github.resources/push_to_github.sh new file mode 100644 index 0000000000..9370d8354a --- /dev/null +++ b/infra/recipes/push_to_github.resources/push_to_github.sh @@ -0,0 +1,11 @@ +# Copyright 2022 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Pushes the current repo to github using the secret provided by Secret Manager. +# See: https://console.cloud.google.com/security/secret-manager +# +# This script will only work if you have access to the above service via gcloud. +set -e +TOKEN=$(gcloud --project=crosvm-infra secrets versions access 1 --secret="github-crosvm-bot") +git push --force --all "https://crosvm-bot:${TOKEN}@github.com/google/crosvm.git"