From efa674503588eaf33121d15956264c7346c47d7a Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 19 Oct 2022 18:04:39 -0700 Subject: [PATCH 1/6] Add more paths to dockerignore --- .dockerignore | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0ac9c905ac..add07b4bf7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,11 @@ -/target -/manifest.yml -/migrate.yml +**/target +zed.xcworkspace +.DS_Store +plugins/bin +script/node_modules +styles/node_modules +crates/collab/static/styles.css +vendor/bin +assets/themes/*.json +assets/themes/internal/*.json +assets/themes/experiments/*.json From 9952f08cce1c2745fb213ea82130073ddb00f595 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Wed, 19 Oct 2022 15:31:47 -0700 Subject: [PATCH 2/6] Publish collab docker images on CI, deploy pre-built images --- .github/workflows/publish_collab_image.yml | 46 ++++++++++++++++++++++ crates/collab/src/main.rs | 14 +++++-- script/deploy | 41 ++++++++++--------- 3 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/publish_collab_image.yml diff --git a/.github/workflows/publish_collab_image.yml b/.github/workflows/publish_collab_image.yml new file mode 100644 index 0000000000..a257f25433 --- /dev/null +++ b/.github/workflows/publish_collab_image.yml @@ -0,0 +1,46 @@ +name: Publish Collab Server Image + +on: + push: + tags: + - collab-v* + +env: + DOCKER_BUILDKIT: 1 + DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} + +jobs: + publish: + name: Publish collab server image + runs-on: + - self-hosted + - deploy + steps: + - name: Add Rust to the PATH + run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Sign into DigitalOcean docker registry + run: doctl registry login + + - name: Checkout repo + uses: actions/checkout@v3 + with: + clean: false + + - name: Check that tag version matches package version + run: | + set -eu + package_version=$(cargo metadata --no-deps --format-version=1 | jq --raw-output '.packages[] | select(.name == "collab") | .version') + tag_version=$(echo $GITHUB_REF_NAME | sed -e 's/collab-v//') + if [[ $tag_version != $package_version ]]; then + echo "collab package version $package_version does not match git tag version $tag_version" + exit 1 + fi + echo "Publishing image version: $package_version" + echo "COLLAB_VERSION=$package_version" >> $GITHUB_ENV + + - name: Build docker image + run: docker build . --tag registry.digitalocean.com/zed/collab:v${COLLAB_VERSION} + + - name: Publish docker image + run: docker push registry.digitalocean.com/zed/collab:v${COLLAB_VERSION} diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index 1c3a5fa20d..2c0fb69b67 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -9,7 +9,7 @@ mod db_tests; #[cfg(test)] mod integration_tests; -use axum::{body::Body, Router}; +use axum::{routing::get, Router}; use collab::{Error, Result}; use db::{Db, PostgresDb}; use serde::Deserialize; @@ -22,6 +22,8 @@ use tracing_log::LogTracer; use tracing_subscriber::{filter::EnvFilter, fmt::format::JsonFields, Layer}; use util::ResultExt; +const VERSION: &'static str = env!("CARGO_PKG_VERSION"); + #[derive(Default, Deserialize)] pub struct Config { pub http_port: u16, @@ -67,9 +69,9 @@ async fn main() -> Result<()> { rpc_server.start_recording_project_activity(Duration::from_secs(5 * 60), rpc::RealExecutor); - let app = Router::::new() - .merge(api::routes(&rpc_server, state.clone())) - .merge(rpc::routes(rpc_server)); + let app = api::routes(&rpc_server, state.clone()) + .merge(rpc::routes(rpc_server)) + .merge(Router::new().route("/", get(handle_root))); axum::Server::from_tcp(listener)? .serve(app.into_make_service_with_connect_info::()) @@ -78,6 +80,10 @@ async fn main() -> Result<()> { Ok(()) } +async fn handle_root() -> String { + format!("collab v{VERSION}") +} + pub fn init_tracing(config: &Config) -> Option<()> { use std::str::FromStr; use tracing_subscriber::layer::SubscriberExt; diff --git a/script/deploy b/script/deploy index ce50737170..e352df345b 100755 --- a/script/deploy +++ b/script/deploy @@ -2,36 +2,41 @@ # Prerequisites: # -# - Log in to the DigitalOcean docker registry -# doctl registry login -# -# - Target the `zed-1` kubernetes cluster -# doctl kubernetes cluster kubeconfig save zed-1 +# - Log in to the DigitalOcean API, either interactively, by running +# `doctl auth init`, or by setting the `DIGITALOCEAN_ACCESS_TOKEN` +# environment variable. set -eu -if [[ $# < 1 ]]; then - echo "Usage: $0 [production|staging|...]" +if [[ $# < 2 ]]; then + echo "Usage: $0 " exit 1 fi - export ZED_KUBE_NAMESPACE=$1 +COLLAB_VERSION=$2 + ENV_FILE="crates/collab/k8s/environments/${ZED_KUBE_NAMESPACE}.sh" if [[ ! -f $ENV_FILE ]]; then echo "Invalid environment name '${ZED_KUBE_NAMESPACE}'" exit 1 fi - -if [[ $ZED_KUBE_NAMESPACE == "production" && -n $(git status --short) ]]; then - echo "Cannot deploy uncommited changes to production" - exit 1 -fi - -git_sha=$(git rev-parse HEAD) -export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:${ZED_KUBE_NAMESPACE}-${git_sha}" export $(cat $ENV_FILE) -docker build . --tag "$ZED_IMAGE_ID" -docker push "$ZED_IMAGE_ID" +if [[ ! $COLLAB_VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid version number '$COLLAB_VERSION'" + exit 1 +fi +TAG_NAMES=$(doctl registry repository list-tags collab --no-header --format Tag) +if ! $(echo "${TAG_NAMES}" | grep -Fqx v${COLLAB_VERSION}); then + echo "No such image tag: 'zed/collab:v${COLLAB_VERSION}'" + echo "Found tags" + echo "${TAG_NAMES}" + exit 1 +fi +export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:v${COLLAB_VERSION}" + +if [[ $(kubectl config current-context 2> /dev/null) != do-nyc1-zed-1 ]]; then + doctl kubernetes cluster kubeconfig save zed-1 +fi envsubst < crates/collab/k8s/manifest.template.yml | kubectl apply -f - From cedc0f64d5a7e2325b3efaad0294352d45772455 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 21 Oct 2022 13:15:34 -0700 Subject: [PATCH 3/6] Run migrations via a collab subcommand --- Dockerfile | 4 +- Dockerfile.migrator | 15 ----- Procfile | 2 +- crates/collab/Procfile | 2 - crates/collab/k8s/manifest.template.yml | 2 + crates/collab/k8s/migrate.template.yml | 2 + crates/collab/src/db.rs | 69 ++++++++++++++++++--- crates/collab/src/main.rs | 81 +++++++++++++++++-------- script/bootstrap | 2 +- script/deploy | 33 ++-------- script/deploy-migration | 38 +++--------- script/lib/deploy-helpers.sh | 43 +++++++++++++ 12 files changed, 180 insertions(+), 113 deletions(-) delete mode 100644 Dockerfile.migrator delete mode 100644 crates/collab/Procfile create mode 100644 script/lib/deploy-helpers.sh diff --git a/Dockerfile b/Dockerfile index 122600bf94..2f9c4ecbcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,5 +19,7 @@ FROM debian:bullseye-slim as runtime RUN apt-get update; \ apt-get install -y --no-install-recommends libcurl4-openssl-dev ca-certificates WORKDIR app -COPY --from=builder /app/collab /app +COPY --from=builder /app/collab /app/collab +COPY --from=builder /app/crates/collab/migrations /app/migrations +ENV MIGRATIONS_PATH=/app/migrations ENTRYPOINT ["/app/collab"] diff --git a/Dockerfile.migrator b/Dockerfile.migrator deleted file mode 100644 index 482228a2eb..0000000000 --- a/Dockerfile.migrator +++ /dev/null @@ -1,15 +0,0 @@ -# syntax = docker/dockerfile:1.2 - -FROM rust:1.64-bullseye as builder -WORKDIR app -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=./target \ - cargo install sqlx-cli --root=/app --target-dir=/app/target --version 0.5.7 - -FROM debian:bullseye-slim as runtime -RUN apt-get update; \ - apt-get install -y --no-install-recommends libssl1.1 -WORKDIR app -COPY --from=builder /app/bin/sqlx /app -COPY ./crates/collab/migrations /app/migrations -ENTRYPOINT ["/app/sqlx", "migrate", "run"] diff --git a/Procfile b/Procfile index e1b87dd48b..d5db6fbd68 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ web: cd ../zed.dev && PORT=3000 npx vercel dev -collab: cd crates/collab && cargo run +collab: cd crates/collab && cargo run serve diff --git a/crates/collab/Procfile b/crates/collab/Procfile deleted file mode 100644 index ef8914fcc0..0000000000 --- a/crates/collab/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -collab: ./target/release/collab -release: ./target/release/sqlx migrate run diff --git a/crates/collab/k8s/manifest.template.yml b/crates/collab/k8s/manifest.template.yml index 628cf92506..94af6cc152 100644 --- a/crates/collab/k8s/manifest.template.yml +++ b/crates/collab/k8s/manifest.template.yml @@ -54,6 +54,8 @@ spec: containers: - name: collab image: "${ZED_IMAGE_ID}" + args: + - serve ports: - containerPort: 8080 protocol: TCP diff --git a/crates/collab/k8s/migrate.template.yml b/crates/collab/k8s/migrate.template.yml index 9b1dc14d7e..3a848da85a 100644 --- a/crates/collab/k8s/migrate.template.yml +++ b/crates/collab/k8s/migrate.template.yml @@ -10,6 +10,8 @@ spec: containers: - name: migrator image: ${ZED_IMAGE_ID} + args: + - migrate env: - name: DATABASE_URL valueFrom: diff --git a/crates/collab/src/db.rs b/crates/collab/src/db.rs index b8c09d9afb..ce99b97348 100644 --- a/crates/collab/src/db.rs +++ b/crates/collab/src/db.rs @@ -6,8 +6,12 @@ use collections::HashMap; use futures::StreamExt; use serde::{Deserialize, Serialize}; pub use sqlx::postgres::PgPoolOptions as DbOptions; -use sqlx::{types::Uuid, FromRow, QueryBuilder}; -use std::{cmp, ops::Range, time::Duration}; +use sqlx::{ + migrate::{Migrate as _, Migration, MigrationSource}, + types::Uuid, + FromRow, QueryBuilder, +}; +use std::{cmp, ops::Range, path::Path, time::Duration}; use time::{OffsetDateTime, PrimitiveDateTime}; #[async_trait] @@ -173,6 +177,13 @@ pub trait Db: Send + Sync { fn as_fake(&self) -> Option<&FakeDb>; } +#[cfg(any(test, debug_assertions))] +pub const DEFAULT_MIGRATIONS_PATH: Option<&'static str> = + Some(concat!(env!("CARGO_MANIFEST_DIR"), "/migrations")); + +#[cfg(not(any(test, debug_assertions)))] +pub const DEFAULT_MIGRATIONS_PATH: Option<&'static str> = None; + pub struct PostgresDb { pool: sqlx::PgPool, } @@ -187,6 +198,47 @@ impl PostgresDb { Ok(Self { pool }) } + pub async fn migrate( + &self, + migrations_path: &Path, + ignore_checksum_mismatch: bool, + ) -> anyhow::Result> { + let migrations = MigrationSource::resolve(migrations_path) + .await + .map_err(|err| anyhow!("failed to load migrations: {err:?}"))?; + + let mut conn = self.pool.acquire().await?; + + conn.ensure_migrations_table().await?; + let applied_migrations: HashMap<_, _> = conn + .list_applied_migrations() + .await? + .into_iter() + .map(|m| (m.version, m)) + .collect(); + + let mut new_migrations = Vec::new(); + for migration in migrations { + match applied_migrations.get(&migration.version) { + Some(applied_migration) => { + if migration.checksum != applied_migration.checksum && !ignore_checksum_mismatch + { + Err(anyhow!( + "checksum mismatch for applied migration {}", + migration.description + ))?; + } + } + None => { + let elapsed = conn.apply(&migration).await?; + new_migrations.push((migration, elapsed)); + } + } + } + + Ok(new_migrations) + } + pub fn fuzzy_like_string(string: &str) -> String { let mut result = String::with_capacity(string.len() * 2 + 1); for c in string.chars() { @@ -1763,11 +1815,8 @@ mod test { use lazy_static::lazy_static; use parking_lot::Mutex; use rand::prelude::*; - use sqlx::{ - migrate::{MigrateDatabase, Migrator}, - Postgres, - }; - use std::{path::Path, sync::Arc}; + use sqlx::{migrate::MigrateDatabase, Postgres}; + use std::sync::Arc; use util::post_inc; pub struct FakeDb { @@ -2430,13 +2479,13 @@ mod test { let mut rng = StdRng::from_entropy(); let name = format!("zed-test-{}", rng.gen::()); let url = format!("postgres://postgres@localhost/{}", name); - let migrations_path = Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/migrations")); Postgres::create_database(&url) .await .expect("failed to create test db"); let db = PostgresDb::new(&url, 5).await.unwrap(); - let migrator = Migrator::new(migrations_path).await.unwrap(); - migrator.run(&db.pool).await.unwrap(); + db.migrate(Path::new(DEFAULT_MIGRATIONS_PATH.unwrap()), false) + .await + .unwrap(); Self { db: Some(Arc::new(db)), url, diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index 2c0fb69b67..0308b21e2b 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -9,12 +9,15 @@ mod db_tests; #[cfg(test)] mod integration_tests; +use anyhow::anyhow; use axum::{routing::get, Router}; use collab::{Error, Result}; use db::{Db, PostgresDb}; use serde::Deserialize; use std::{ + env::args, net::{SocketAddr, TcpListener}, + path::PathBuf, sync::Arc, time::Duration, }; @@ -34,22 +37,17 @@ pub struct Config { pub log_json: Option, } +#[derive(Default, Deserialize)] +pub struct MigrateConfig { + pub database_url: String, + pub migrations_path: Option, +} + pub struct AppState { db: Arc, config: Config, } -impl AppState { - async fn new(config: Config) -> Result> { - let db = PostgresDb::new(&config.database_url, 5).await?; - let this = Self { - db: Arc::new(db), - config, - }; - Ok(Arc::new(this)) - } -} - #[tokio::main] async fn main() -> Result<()> { if let Err(error) = env::load_dotenv() { @@ -59,24 +57,59 @@ async fn main() -> Result<()> { ); } - let config = envy::from_env::().expect("error loading config"); - init_tracing(&config); - let state = AppState::new(config).await?; + match args().skip(1).next().as_deref() { + Some("version") => { + println!("collab v{VERSION}"); + } + Some("migrate") => { + let config = envy::from_env::().expect("error loading config"); + let db = PostgresDb::new(&config.database_url, 5).await?; - let listener = TcpListener::bind(&format!("0.0.0.0:{}", state.config.http_port)) - .expect("failed to bind TCP listener"); - let rpc_server = rpc::Server::new(state.clone(), None); + let migrations_path = config + .migrations_path + .as_deref() + .or(db::DEFAULT_MIGRATIONS_PATH.map(|s| s.as_ref())) + .ok_or_else(|| anyhow!("missing MIGRATIONS_PATH environment variable"))?; - rpc_server.start_recording_project_activity(Duration::from_secs(5 * 60), rpc::RealExecutor); + let migrations = db.migrate(&migrations_path, false).await?; + for (migration, duration) in migrations { + println!( + "Ran {} {} {:?}", + migration.version, migration.description, duration + ); + } - let app = api::routes(&rpc_server, state.clone()) - .merge(rpc::routes(rpc_server)) - .merge(Router::new().route("/", get(handle_root))); + return Ok(()); + } + Some("serve") => { + let config = envy::from_env::().expect("error loading config"); + let db = PostgresDb::new(&config.database_url, 5).await?; - axum::Server::from_tcp(listener)? - .serve(app.into_make_service_with_connect_info::()) - .await?; + init_tracing(&config); + let state = Arc::new(AppState { + db: Arc::new(db), + config, + }); + let listener = TcpListener::bind(&format!("0.0.0.0:{}", state.config.http_port)) + .expect("failed to bind TCP listener"); + + let rpc_server = rpc::Server::new(state.clone(), None); + rpc_server + .start_recording_project_activity(Duration::from_secs(5 * 60), rpc::RealExecutor); + + let app = api::routes(&rpc_server, state.clone()) + .merge(rpc::routes(rpc_server)) + .merge(Router::new().route("/", get(handle_root))); + + axum::Server::from_tcp(listener)? + .serve(app.into_make_service_with_connect_info::()) + .await?; + } + _ => { + Err(anyhow!("usage: collab "))?; + } + } Ok(()) } diff --git a/script/bootstrap b/script/bootstrap index 4f6b9cc70d..e23f42e80e 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -7,7 +7,7 @@ echo "creating database..." script/sqlx database create echo "migrating database..." -script/sqlx migrate run +cargo run -p collab -- migrate echo "seeding database..." script/seed-db diff --git a/script/deploy b/script/deploy index e352df345b..f7da274841 100755 --- a/script/deploy +++ b/script/deploy @@ -1,12 +1,7 @@ #!/bin/bash -# Prerequisites: -# -# - Log in to the DigitalOcean API, either interactively, by running -# `doctl auth init`, or by setting the `DIGITALOCEAN_ACCESS_TOKEN` -# environment variable. - set -eu +source script/lib/deploy-helpers.sh if [[ $# < 2 ]]; then echo "Usage: $0 " @@ -15,28 +10,8 @@ fi export ZED_KUBE_NAMESPACE=$1 COLLAB_VERSION=$2 -ENV_FILE="crates/collab/k8s/environments/${ZED_KUBE_NAMESPACE}.sh" -if [[ ! -f $ENV_FILE ]]; then - echo "Invalid environment name '${ZED_KUBE_NAMESPACE}'" - exit 1 -fi -export $(cat $ENV_FILE) - -if [[ ! $COLLAB_VERSION =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Invalid version number '$COLLAB_VERSION'" - exit 1 -fi -TAG_NAMES=$(doctl registry repository list-tags collab --no-header --format Tag) -if ! $(echo "${TAG_NAMES}" | grep -Fqx v${COLLAB_VERSION}); then - echo "No such image tag: 'zed/collab:v${COLLAB_VERSION}'" - echo "Found tags" - echo "${TAG_NAMES}" - exit 1 -fi -export ZED_IMAGE_ID="registry.digitalocean.com/zed/collab:v${COLLAB_VERSION}" - -if [[ $(kubectl config current-context 2> /dev/null) != do-nyc1-zed-1 ]]; then - doctl kubernetes cluster kubeconfig save zed-1 -fi +export_vars_for_environment $ZED_KUBE_NAMESPACE +export ZED_IMAGE_ID=$(image_id_for_version $COLLAB_VERSION) +target_zed_kube_cluster envsubst < crates/collab/k8s/manifest.template.yml | kubectl apply -f - diff --git a/script/deploy-migration b/script/deploy-migration index 81a662db88..6812be3217 100755 --- a/script/deploy-migration +++ b/script/deploy-migration @@ -1,42 +1,20 @@ #!/bin/bash -# Prerequisites: -# -# - Log in to the DigitalOcean docker registry -# doctl registry login -# -# - Target the `zed-1` kubernetes cluster -# doctl kubernetes cluster kubeconfig save zed-1 - set -eu +source script/lib/deploy-helpers.sh -if [[ $# < 1 ]]; then - echo "Usage: $0 [production|staging|...]" +if [[ $# < 2 ]]; then + echo "Usage: $0 " exit 1 fi - export ZED_KUBE_NAMESPACE=$1 -ENV_FILE="crates/collab/k8s/environments/${ZED_KUBE_NAMESPACE}.sh" -if [[ ! -f $ENV_FILE ]]; then - echo "Invalid environment name '${ZED_KUBE_NAMESPACE}'" - exit 1 -fi +COLLAB_VERSION=$2 -if [[ -n $(git status --short) ]]; then - echo "Cannot deploy with uncommited changes" - exit 1 -fi - -git_sha=$(git rev-parse HEAD) -export ZED_IMAGE_ID=registry.digitalocean.com/zed/zed-migrator:${ZED_KUBE_NAMESPACE}-${git_sha} -export ZED_MIGRATE_JOB_NAME=zed-migrate-${git_sha} - -docker build . \ - --file ./Dockerfile.migrator \ - --tag $ZED_IMAGE_ID -docker push $ZED_IMAGE_ID +export_vars_for_environment $ZED_KUBE_NAMESPACE +export ZED_IMAGE_ID=$(image_id_for_version ${COLLAB_VERSION}) +export ZED_MIGRATE_JOB_NAME=zed-migrate-${COLLAB_VERSION} +target_zed_kube_cluster envsubst < crates/collab/k8s/migrate.template.yml | kubectl apply -f - - pod=$(kubectl --namespace=${ZED_KUBE_NAMESPACE} get pods --selector=job-name=${ZED_MIGRATE_JOB_NAME} --output=jsonpath='{.items[*].metadata.name}') echo "pod:" $pod diff --git a/script/lib/deploy-helpers.sh b/script/lib/deploy-helpers.sh new file mode 100644 index 0000000000..78ffa22671 --- /dev/null +++ b/script/lib/deploy-helpers.sh @@ -0,0 +1,43 @@ +# Prerequisites: +# +# - Log in to the DigitalOcean API, either interactively, by running +# `doctl auth init`, or by setting the `DIGITALOCEAN_ACCESS_TOKEN` +# environment variable. + +function export_vars_for_environment { + local environment=$1 + local env_file="crates/collab/k8s/environments/${environment}.sh" + if [[ ! -f $env_file ]]; then + echo "Invalid environment name '${environment}'" + exit 1 + fi + export $(cat $env_file) +} + +function image_id_for_version { + local version=$1 + if [[ ! ${version} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid version number '${version}'" + exit 1 + fi + TAG_NAMES=$(doctl registry repository list-tags collab --no-header --format Tag) + if ! $(echo "${TAG_NAMES}" | grep -Fqx v${version}); then + echo "No such image tag: 'zed/collab:v${version}'" + echo "Found tags" + echo "${TAG_NAMES}" + exit 1 + fi + + echo "registry.digitalocean.com/zed/collab:v${version}" +} + +function version_for_image_id { + local image_id=$1 + echo $image_id | cut -d: -f2 +} + +function target_zed_kube_cluster { + if [[ $(kubectl config current-context 2> /dev/null) != do-nyc1-zed-1 ]]; then + doctl kubernetes cluster kubeconfig save zed-1 + fi +} From 0c9ceb51e64eb3b721697d0456331243f515d6f9 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 21 Oct 2022 14:25:12 -0700 Subject: [PATCH 4/6] Add what-is-deployed-script --- script/what-is-deployed | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 script/what-is-deployed diff --git a/script/what-is-deployed b/script/what-is-deployed new file mode 100755 index 0000000000..384686493f --- /dev/null +++ b/script/what-is-deployed @@ -0,0 +1,17 @@ +#!/bin/bash + +set -eu +source script/lib/deploy-helpers.sh + +if [[ $# < 1 ]]; then + echo "Usage: $0 " + exit 1 +fi +ZED_KUBE_NAMESPACE=$1 + +export_vars_for_environment $ZED_KUBE_NAMESPACE +target_zed_kube_cluster + +IMAGE_ID=$(kubectl --namespace=${ZED_KUBE_NAMESPACE} get deployment collab -o 'jsonpath={.spec.template.spec.containers[0].image}') + +echo "Deployed image on ${ZED_KUBE_NAMESPACE}:" $(version_for_image_id $IMAGE_ID) \ No newline at end of file From 2f1ddc0d0f704d0ad343b67df50397d2cfd16928 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 21 Oct 2022 15:50:14 -0700 Subject: [PATCH 5/6] Improve deploy scripts --- crates/collab/k8s/migrate.template.yml | 1 + script/deploy | 15 ++++++++++----- script/deploy-migration | 22 ++++++++++++++-------- script/lib/deploy-helpers.sh | 23 ++++++++++------------- script/what-is-deployed | 26 ++++++++++++++++++++++---- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/crates/collab/k8s/migrate.template.yml b/crates/collab/k8s/migrate.template.yml index 3a848da85a..c890d7b330 100644 --- a/crates/collab/k8s/migrate.template.yml +++ b/crates/collab/k8s/migrate.template.yml @@ -9,6 +9,7 @@ spec: restartPolicy: Never containers: - name: migrator + imagePullPolicy: Always image: ${ZED_IMAGE_ID} args: - migrate diff --git a/script/deploy b/script/deploy index f7da274841..f675da6a99 100755 --- a/script/deploy +++ b/script/deploy @@ -7,11 +7,16 @@ if [[ $# < 2 ]]; then echo "Usage: $0 " exit 1 fi -export ZED_KUBE_NAMESPACE=$1 -COLLAB_VERSION=$2 +environment=$1 +version=$2 + +export_vars_for_environment ${environment} +image_id=$(image_id_for_version ${version}) + +export ZED_KUBE_NAMESPACE=${environment} +export ZED_IMAGE_ID=${image_id} -export_vars_for_environment $ZED_KUBE_NAMESPACE -export ZED_IMAGE_ID=$(image_id_for_version $COLLAB_VERSION) target_zed_kube_cluster - envsubst < crates/collab/k8s/manifest.template.yml | kubectl apply -f - + +echo "deployed collab v${version} to ${environment}" \ No newline at end of file diff --git a/script/deploy-migration b/script/deploy-migration index 6812be3217..a6b1574c04 100755 --- a/script/deploy-migration +++ b/script/deploy-migration @@ -7,14 +7,20 @@ if [[ $# < 2 ]]; then echo "Usage: $0 " exit 1 fi -export ZED_KUBE_NAMESPACE=$1 -COLLAB_VERSION=$2 +environment=$1 +version=$2 + +export_vars_for_environment ${environment} +image_id=$(image_id_for_version ${version}) + +export ZED_KUBE_NAMESPACE=${environment} +export ZED_IMAGE_ID=${image_id} +export ZED_MIGRATE_JOB_NAME=zed-migrate-${version} -export_vars_for_environment $ZED_KUBE_NAMESPACE -export ZED_IMAGE_ID=$(image_id_for_version ${COLLAB_VERSION}) -export ZED_MIGRATE_JOB_NAME=zed-migrate-${COLLAB_VERSION} target_zed_kube_cluster - envsubst < crates/collab/k8s/migrate.template.yml | kubectl apply -f - -pod=$(kubectl --namespace=${ZED_KUBE_NAMESPACE} get pods --selector=job-name=${ZED_MIGRATE_JOB_NAME} --output=jsonpath='{.items[*].metadata.name}') -echo "pod:" $pod + +pod=$(kubectl --namespace=${environment} get pods --selector=job-name=${ZED_MIGRATE_JOB_NAME} --output=jsonpath='{.items[0].metadata.name}') + +echo "Job pod:" $pod +kubectl --namespace=${environment} logs -f ${pod} \ No newline at end of file diff --git a/script/lib/deploy-helpers.sh b/script/lib/deploy-helpers.sh index 78ffa22671..705ae6e80c 100644 --- a/script/lib/deploy-helpers.sh +++ b/script/lib/deploy-helpers.sh @@ -1,14 +1,8 @@ -# Prerequisites: -# -# - Log in to the DigitalOcean API, either interactively, by running -# `doctl auth init`, or by setting the `DIGITALOCEAN_ACCESS_TOKEN` -# environment variable. - function export_vars_for_environment { local environment=$1 local env_file="crates/collab/k8s/environments/${environment}.sh" if [[ ! -f $env_file ]]; then - echo "Invalid environment name '${environment}'" + echo "Invalid environment name '${environment}'" >&2 exit 1 fi export $(cat $env_file) @@ -16,15 +10,18 @@ function export_vars_for_environment { function image_id_for_version { local version=$1 + + # Check that version is valid if [[ ! ${version} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "Invalid version number '${version}'" + echo "Invalid version number '${version}'" >&2 exit 1 fi - TAG_NAMES=$(doctl registry repository list-tags collab --no-header --format Tag) - if ! $(echo "${TAG_NAMES}" | grep -Fqx v${version}); then - echo "No such image tag: 'zed/collab:v${version}'" - echo "Found tags" - echo "${TAG_NAMES}" + + # Check that image exists for version + tag_names=$(doctl registry repository list-tags collab --no-header --format Tag) + if ! $(echo "${tag_names}" | grep -Fqx v${version}); then + echo "No docker image tagged for version '${version}'" >&2 + echo "Found images with these tags:" ${tag_names} >&2 exit 1 fi diff --git a/script/what-is-deployed b/script/what-is-deployed index 384686493f..6df2449b90 100755 --- a/script/what-is-deployed +++ b/script/what-is-deployed @@ -7,11 +7,29 @@ if [[ $# < 1 ]]; then echo "Usage: $0 " exit 1 fi -ZED_KUBE_NAMESPACE=$1 +environment=$1 -export_vars_for_environment $ZED_KUBE_NAMESPACE +export_vars_for_environment ${environment} target_zed_kube_cluster -IMAGE_ID=$(kubectl --namespace=${ZED_KUBE_NAMESPACE} get deployment collab -o 'jsonpath={.spec.template.spec.containers[0].image}') +deployed_image_id=$( + kubectl \ + --namespace=${environment} \ + get deployment collab \ + -o 'jsonpath={.spec.template.spec.containers[0].image}' \ + | cut -d: -f2 +) -echo "Deployed image on ${ZED_KUBE_NAMESPACE}:" $(version_for_image_id $IMAGE_ID) \ No newline at end of file +job_image_ids=$( + kubectl \ + --namespace=${environment} \ + get jobs \ + -o 'jsonpath={range .items[0:5]}{.spec.template.spec.containers[0].image}{"\n"}{end}' +) + +echo "Deployed image version:" +echo "$deployed_image_id" +echo +echo "Migration job image versions:" +echo "$job_image_ids" +echo From 95be2c60700cd81570a3d998db24bf0ee5441b85 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Fri, 21 Oct 2022 18:01:12 -0700 Subject: [PATCH 6/6] Add version bump scripts --- script/bump-app-version | 3 +++ script/bump-collab-version | 3 +++ script/lib/bump-version.sh | 39 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100755 script/bump-app-version create mode 100755 script/bump-collab-version create mode 100755 script/lib/bump-version.sh diff --git a/script/bump-app-version b/script/bump-app-version new file mode 100755 index 0000000000..88433b6c70 --- /dev/null +++ b/script/bump-app-version @@ -0,0 +1,3 @@ +#!/bin/bash + +exec script/lib/bump-version.sh zed v $@ diff --git a/script/bump-collab-version b/script/bump-collab-version new file mode 100755 index 0000000000..35f333f76a --- /dev/null +++ b/script/bump-collab-version @@ -0,0 +1,3 @@ +#!/bin/bash + +exec script/lib/bump-version.sh collab collab-v $@ diff --git a/script/lib/bump-version.sh b/script/lib/bump-version.sh new file mode 100755 index 0000000000..205fc168ef --- /dev/null +++ b/script/lib/bump-version.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -eu + +if [[ $# < 3 ]]; then + echo "Missing version increment (major, minor, or patch)" >&2 + exit 1 +fi + +package=$1 +tag_prefix=$2 +version_increment=$3 + +if [[ -n $(git status --short --untracked-files=no) ]]; then + echo "Can't push a new version with uncommitted changes" + exit 1 +fi + +which cargo-set-version > /dev/null || cargo install cargo-edit +cargo set-version --package $package --bump $version_increment +cargo check --quiet + +new_version=$(cargo metadata --no-deps --format-version=1 | jq --raw-output ".packages[] | select(.name == \"${package}\") | .version") +branch_name=$(git rev-parse --abbrev-ref HEAD) +old_sha=$(git rev-parse HEAD) +tag_name=${tag_prefix}${new_version} + +git commit --quiet --all --message "${package} ${new_version}" +git tag ${tag_name} + +cat <