remoting (#9680)

This PR provides some of the plumbing needed for a "remote" zed
instance.

The way this will work is:
* From zed on your laptop you'll be able to manage a set of dev servers,
each of which is identified by a token.
* You'll run `zed --dev-server-token XXXX` to boot a remotable dev
server.
* From the zed on your laptop you'll be able to open directories and
work on the projects on the remote server (exactly like collaboration
works today).

For now all this PR does is provide the ability for a zed instance to
sign in
using a "dev server token". The next steps will be:
* Adding support to the collaboration protocol to instruct a dev server
to "open" a directory and share it into a channel.
* Adding UI to manage these servers and tokens (manually for now)

Related #5347

Release Notes:

- N/A

---------

Co-authored-by: Nathan <nathan@zed.dev>

Cargo.lock

@@ -12562,6 +12562,7 @@ dependencies = [
  "call",
  "channel",
  "chrono",
+ "clap 4.4.4",
  "cli",
  "client",
  "clock",

crates/client/src/client.rs

@@ -27,8 +27,8 @@ use release_channel::{AppVersion, ReleaseChannel};
 use rpc::proto::{AnyTypedEnvelope, EntityMessage, EnvelopedMessage, PeerId, RequestMessage};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-
 use settings::{Settings, SettingsStore};
+use std::fmt;
 use std::{
     any::TypeId,
     convert::TryFrom,
@@ -52,6 +52,15 @@ pub use rpc::*;
 pub use telemetry_events::Event;
 pub use user::*;
 
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct DevServerToken(pub String);
+
+impl fmt::Display for DevServerToken {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
 lazy_static! {
     static ref ZED_SERVER_URL: Option<String> = std::env::var("ZED_SERVER_URL").ok();
     static ref ZED_RPC_URL: Option<String> = std::env::var("ZED_RPC_URL").ok();
@@ -277,10 +286,22 @@ enum WeakSubscriber {
     Pending(Vec<Box<dyn AnyTypedEnvelope>>),
 }
 
-#[derive(Clone, Debug)]
-pub struct Credentials {
-    pub user_id: u64,
-    pub access_token: String,
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum Credentials {
+    DevServer { token: DevServerToken },
+    User { user_id: u64, access_token: String },
+}
+
+impl Credentials {
+    pub fn authorization_header(&self) -> String {
+        match self {
+            Credentials::DevServer { token } => format!("dev-server-token {}", token),
+            Credentials::User {
+                user_id,
+                access_token,
+            } => format!("{} {}", user_id, access_token),
+        }
+    }
 }
 
 impl Default for ClientState {
@@ -497,11 +518,11 @@ impl Client {
     }
 
     pub fn user_id(&self) -> Option<u64> {
-        self.state
-            .read()
-            .credentials
-            .as_ref()
-            .map(|credentials| credentials.user_id)
+        if let Some(Credentials::User { user_id, .. }) = self.state.read().credentials.as_ref() {
+            Some(*user_id)
+        } else {
+            None
+        }
     }
 
     pub fn peer_id(&self) -> Option<PeerId> {
@@ -746,6 +767,10 @@ impl Client {
         read_credentials_from_keychain(cx).await.is_some()
     }
 
+    pub fn set_dev_server_token(&self, token: DevServerToken) {
+        self.state.write().credentials = Some(Credentials::DevServer { token });
+    }
+
     #[async_recursion(?Send)]
     pub async fn authenticate_and_connect(
         self: &Arc<Self>,
@@ -796,7 +821,9 @@ impl Client {
             }
         }
         let credentials = credentials.unwrap();
-        self.set_id(credentials.user_id);
+        if let Credentials::User { user_id, .. } = &credentials {
+            self.set_id(*user_id);
+        }
 
         if was_disconnected {
             self.set_status(Status::Connecting, cx);
@@ -812,7 +839,9 @@ impl Client {
                     Ok(conn) => {
                         self.state.write().credentials = Some(credentials.clone());
                         if !read_from_keychain && IMPERSONATE_LOGIN.is_none() {
-                            write_credentials_to_keychain(credentials, cx).await.log_err();
+                            if let Credentials::User{user_id, access_token} = credentials {
+                                write_credentials_to_keychain(user_id, access_token, cx).await.log_err();
+                            }
                         }
 
                         futures::select_biased! {
@@ -1020,10 +1049,7 @@ impl Client {
             .unwrap_or_default();
 
         let request = Request::builder()
-            .header(
-                "Authorization",
-                format!("{} {}", credentials.user_id, credentials.access_token),
-            )
+            .header("Authorization", credentials.authorization_header())
             .header("x-zed-protocol-version", rpc::PROTOCOL_VERSION)
             .header("x-zed-app-version", app_version)
             .header(
@@ -1176,7 +1202,7 @@ impl Client {
                         .decrypt_string(&access_token)
                         .context("failed to decrypt access token")?;
 
-                    Ok(Credentials {
+                    Ok(Credentials::User {
                         user_id: user_id.parse()?,
                         access_token,
                     })
@@ -1226,7 +1252,7 @@ impl Client {
 
         // Use the admin API token to authenticate as the impersonated user.
         api_token.insert_str(0, "ADMIN_TOKEN:");
-        Ok(Credentials {
+        Ok(Credentials::User {
             user_id: response.user.id,
             access_token: api_token,
         })
@@ -1439,21 +1465,22 @@ async fn read_credentials_from_keychain(cx: &AsyncAppContext) -> Option<Credenti
         .await
         .log_err()??;
 
-    Some(Credentials {
+    Some(Credentials::User {
         user_id: user_id.parse().ok()?,
         access_token: String::from_utf8(access_token).ok()?,
     })
 }
 
 async fn write_credentials_to_keychain(
-    credentials: Credentials,
+    user_id: u64,
+    access_token: String,
     cx: &AsyncAppContext,
 ) -> Result<()> {
     cx.update(move |cx| {
         cx.write_credentials(
             &ClientSettings::get_global(cx).server_url,
-            &credentials.user_id.to_string(),
-            credentials.access_token.as_bytes(),
+            &user_id.to_string(),
+            access_token.as_bytes(),
         )
     })?
     .await
@@ -1558,7 +1585,7 @@ mod tests {
         // Time out when client tries to connect.
         client.override_authenticate(move |cx| {
             cx.background_executor().spawn(async move {
-                Ok(Credentials {
+                Ok(Credentials::User {
                     user_id,
                     access_token: "token".into(),
                 })

crates/client/src/test.rs

@@ -48,7 +48,7 @@ impl FakeServer {
                         let mut state = state.lock();
                         state.auth_count += 1;
                         let access_token = state.access_token.to_string();
-                        Ok(Credentials {
+                        Ok(Credentials::User {
                             user_id: client_user_id,
                             access_token,
                         })
@@ -71,9 +71,12 @@ impl FakeServer {
                             )))?
                         }
 
-                        assert_eq!(credentials.user_id, client_user_id);
-
-                        if credentials.access_token != state.lock().access_token.to_string() {
+                        if credentials
+                            != (Credentials::User {
+                                user_id: client_user_id,
+                                access_token: state.lock().access_token.to_string(),
+                            })
+                        {
                             Err(EstablishConnectionError::Unauthorized)?
                         }
 

crates/collab/README.md

@@ -29,7 +29,7 @@ You can tell what is currently deployed with `./script/what-is-deployed`.
 To create a new migration:
 
 ```
-./script/sqlx migrate add <name>
+./script/create-migration <name>
 ```
 
 Migrations are run automatically on service start, so run `foreman start` again. The service will crash if the migrations fail.

crates/collab/migrations.sqlite/20221109000000_test_schema.sql

@@ -400,3 +400,11 @@ CREATE TABLE hosted_projects (
 );
 CREATE INDEX idx_hosted_projects_on_channel_id ON hosted_projects (channel_id);
 CREATE UNIQUE INDEX uix_hosted_projects_on_channel_id_and_name ON hosted_projects (channel_id, name) WHERE (deleted_at IS NULL);
+
+CREATE TABLE dev_servers (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    channel_id INTEGER NOT NULL REFERENCES channels(id),
+    name TEXT NOT NULL,
+    hashed_token TEXT NOT NULL
+);
+CREATE INDEX idx_dev_servers_on_channel_id ON dev_servers (channel_id);

crates/collab/migrations/20240321162658_add_devservers.sql

@@ -0,0 +1,7 @@
+CREATE TABLE dev_servers (
+  id INT PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
+  channel_id INT NOT NULL REFERENCES channels(id),
+  name TEXT NOT NULL,
+  hashed_token TEXT NOT NULL
+);
+CREATE INDEX idx_dev_servers_on_channel_id ON dev_servers (channel_id);

crates/collab/src/auth.rs

@@ -1,5 +1,6 @@
 use crate::{
-    db::{self, AccessTokenId, Database, UserId},
+    db::{self, dev_server, AccessTokenId, Database, DevServerId, UserId},
+    rpc::Principal,
     AppState, Error, Result,
 };
 use anyhow::{anyhow, Context};
@@ -19,11 +20,11 @@ use std::sync::OnceLock;
 use std::{sync::Arc, time::Instant};
 use subtle::ConstantTimeEq;
 
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct Impersonator(pub Option<db::User>);
-
-/// Validates the authorization header. This has two mechanisms, one for the ADMIN_TOKEN
-/// and one for the access tokens that we issue.
+/// Validates the authorization header and adds an Extension<Principal> to the request.
+/// Authorization: <user-id> <token>
+///   <token> can be an access_token attached to that user, or an access token of an admin
+///   or (in development) the string ADMIN:<config.api_token>.
+/// Authorization: "dev-server-token" <token>
 pub async fn validate_header<B>(mut req: Request<B>, next: Next<B>) -> impl IntoResponse {
     let mut auth_header = req
         .headers()
@@ -37,7 +38,26 @@ pub async fn validate_header<B>(mut req: Request<B>, next: Next<B>) -> impl Into
         })?
         .split_whitespace();
 
-    let user_id = UserId(auth_header.next().unwrap_or("").parse().map_err(|_| {
+    let state = req.extensions().get::<Arc<AppState>>().unwrap();
+
+    let first = auth_header.next().unwrap_or("");
+    if first == "dev-server-token" {
+        let dev_server_token = auth_header.next().ok_or_else(|| {
+            Error::Http(
+                StatusCode::BAD_REQUEST,
+                "missing dev-server-token token in authorization header".to_string(),
+            )
+        })?;
+        let dev_server = verify_dev_server_token(dev_server_token, &state.db)
+            .await
+            .map_err(|e| Error::Http(StatusCode::UNAUTHORIZED, format!("{}", e)))?;
+
+        req.extensions_mut()
+            .insert(Principal::DevServer(dev_server));
+        return Ok::<_, Error>(next.run(req).await);
+    }
+
+    let user_id = UserId(first.parse().map_err(|_| {
         Error::Http(
             StatusCode::BAD_REQUEST,
             "missing user id in authorization header".to_string(),
@@ -51,8 +71,6 @@ pub async fn validate_header<B>(mut req: Request<B>, next: Next<B>) -> impl Into
         )
     })?;
 
-    let state = req.extensions().get::<Arc<AppState>>().unwrap();
-
     // In development, allow impersonation using the admin API token.
     // Don't allow this in production because we can't tell who is doing
     // the impersonating.
@@ -76,18 +94,17 @@ pub async fn validate_header<B>(mut req: Request<B>, next: Next<B>) -> impl Into
                 .await?
                 .ok_or_else(|| anyhow!("user {} not found", user_id))?;
 
-            let impersonator = if let Some(impersonator_id) = validate_result.impersonator_id {
-                let impersonator = state
+            if let Some(impersonator_id) = validate_result.impersonator_id {
+                let admin = state
                     .db
                     .get_user_by_id(impersonator_id)
                     .await?
                     .ok_or_else(|| anyhow!("user {} not found", impersonator_id))?;
-                Some(impersonator)
+                req.extensions_mut()
+                    .insert(Principal::Impersonated { user, admin });
             } else {
-                None
+                req.extensions_mut().insert(Principal::User(user));
             };
-            req.extensions_mut().insert(user);
-            req.extensions_mut().insert(Impersonator(impersonator));
             return Ok::<_, Error>(next.run(req).await);
         }
     }
@@ -213,6 +230,33 @@ pub async fn verify_access_token(
     })
 }
 
+// a dev_server_token has the format <id>.<base64>. This is to make them
+// relatively easy to copy/paste around.
+pub async fn verify_dev_server_token(
+    dev_server_token: &str,
+    db: &Arc<Database>,
+) -> anyhow::Result<dev_server::Model> {
+    let mut parts = dev_server_token.splitn(2, '.');
+    let id = DevServerId(parts.next().unwrap_or_default().parse()?);
+    let token = parts
+        .next()
+        .ok_or_else(|| anyhow!("invalid dev server token format"))?;
+
+    let token_hash = hash_access_token(&token);
+    let server = db.get_dev_server(id).await?;
+
+    if server
+        .hashed_token
+        .as_bytes()
+        .ct_eq(token_hash.as_ref())
+        .into()
+    {
+        Ok(server)
+    } else {
+        Err(anyhow!("wrong token for dev server"))
+    }
+}
+
 #[cfg(test)]
 mod test {
     use rand::thread_rng;

crates/collab/src/db/ids.rs

@@ -67,28 +67,29 @@ macro_rules! id_type {
     };
 }
 
-id_type!(BufferId);
 id_type!(AccessTokenId);
+id_type!(BufferId);
+id_type!(ChannelBufferCollaboratorId);
 id_type!(ChannelChatParticipantId);
 id_type!(ChannelId);
 id_type!(ChannelMemberId);
-id_type!(MessageId);
 id_type!(ContactId);
+id_type!(DevServerId);
+id_type!(ExtensionId);
+id_type!(FlagId);
 id_type!(FollowerId);
+id_type!(HostedProjectId);
+id_type!(MessageId);
+id_type!(NotificationId);
+id_type!(NotificationKindId);
+id_type!(ProjectCollaboratorId);
+id_type!(ProjectId);
+id_type!(ReplicaId);
 id_type!(RoomId);
 id_type!(RoomParticipantId);
-id_type!(ProjectId);
-id_type!(ProjectCollaboratorId);
-id_type!(ReplicaId);
 id_type!(ServerId);
 id_type!(SignupId);
 id_type!(UserId);
-id_type!(ChannelBufferCollaboratorId);
-id_type!(FlagId);
-id_type!(ExtensionId);
-id_type!(NotificationId);
-id_type!(NotificationKindId);
-id_type!(HostedProjectId);
 
 /// ChannelRole gives you permissions for both channels and calls.
 #[derive(

crates/collab/src/db/queries.rs

@@ -5,6 +5,7 @@ pub mod buffers;
 pub mod channels;
 pub mod contacts;
 pub mod contributors;
+pub mod dev_servers;
 pub mod extensions;
 pub mod hosted_projects;
 pub mod messages;

crates/collab/src/db/queries/dev_servers.rs

@@ -0,0 +1,18 @@
+use sea_orm::EntityTrait;
+
+use super::{dev_server, Database, DevServerId};
+
+impl Database {
+    pub async fn get_dev_server(
+        &self,
+        dev_server_id: DevServerId,
+    ) -> crate::Result<dev_server::Model> {
+        self.transaction(|tx| async move {
+            Ok(dev_server::Entity::find_by_id(dev_server_id)
+                .one(&*tx)
+                .await?
+                .ok_or_else(|| anyhow::anyhow!("no dev server with id {}", dev_server_id))?)
+        })
+        .await
+    }
+}

crates/collab/src/db/tables.rs

@@ -10,6 +10,7 @@ pub mod channel_message;
 pub mod channel_message_mention;
 pub mod contact;
 pub mod contributor;
+pub mod dev_server;
 pub mod extension;
 pub mod extension_version;
 pub mod feature_flag;

crates/collab/src/db/tables/dev_server.rs

@@ -0,0 +1,17 @@
+use crate::db::{ChannelId, DevServerId};
+use sea_orm::entity::prelude::*;
+
+#[derive(Clone, Debug, PartialEq, Eq, DeriveEntityModel)]
+#[sea_orm(table_name = "dev_servers")]
+pub struct Model {
+    #[sea_orm(primary_key)]
+    pub id: DevServerId,
+    pub name: String,
+    pub channel_id: ChannelId,
+    pub hashed_token: String,
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}

crates/collab/src/tests/test_server.rs

@@ -1,7 +1,7 @@
 use crate::{
     db::{tests::TestDb, NewUserParams, UserId},
     executor::Executor,
-    rpc::{Server, ZedVersion, CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
+    rpc::{Principal, Server, ZedVersion, CLEANUP_TIMEOUT, RECONNECT_TIMEOUT},
     AppState, Config, RateLimiter,
 };
 use anyhow::anyhow;
@@ -197,15 +197,20 @@ impl TestServer {
             .override_authenticate(move |cx| {
                 cx.spawn(|_| async move {
                     let access_token = "the-token".to_string();
-                    Ok(Credentials {
+                    Ok(Credentials::User {
                         user_id: user_id.to_proto(),
                         access_token,
                     })
                 })
             })
             .override_establish_connection(move |credentials, cx| {
-                assert_eq!(credentials.user_id, user_id.0 as u64);
-                assert_eq!(credentials.access_token, "the-token");
+                assert_eq!(
+                    credentials,
+                    &Credentials::User {
+                        user_id: user_id.0 as u64,
+                        access_token: "the-token".into()
+                    }
+                );
 
                 let server = server.clone();
                 let db = db.clone();
@@ -230,9 +235,8 @@ impl TestServer {
                             .spawn(server.handle_connection(
                                 server_conn,
                                 client_name,
-                                user,
+                                Principal::User(user),
                                 ZedVersion(SemanticVersion::new(1, 0, 0)),
-                                None,
                                 Some(connection_id_tx),
                                 Executor::Deterministic(cx.background_executor().clone()),
                             ))

crates/theme/theme.md

@@ -1,15 +1,15 @@
- # Theme
+# Theme
 
- This crate provides the theme system for Zed.
+This crate provides the theme system for Zed.
 
- ## Overview
+## Overview
 
- A theme is a collection of colors used to build a consistent appearance for UI components across the application.
- To produce a theme in Zed,
+A theme is a collection of colors used to build a consistent appearance for UI components across the application.
+To produce a theme in Zed,
 
- A theme is made of of two parts: A [ThemeFamily] and one or more [Theme]s.
+A theme is made of of two parts: A [ThemeFamily] and one or more [Theme]s.
 
 //
- A [ThemeFamily] contains metadata like theme name, author, and theme-specific [ColorScales] as well as a series of themes.
+A [ThemeFamily] contains metadata like theme name, author, and theme-specific [ColorScales] as well as a series of themes.
 
- - [ThemeColors] - A set of colors that are used to style the UI. Refer to the [ThemeColors] documentation for more information.
+- [ThemeColors] - A set of colors that are used to style the UI. Refer to the [ThemeColors] documentation for more information.

crates/zed/Cargo.toml

@@ -26,6 +26,7 @@ breadcrumbs.workspace = true
 call.workspace = true
 channel.workspace = true
 chrono.workspace = true
+clap.workspace = true
 cli.workspace = true
 client.workspace = true
 clock.workspace = true

crates/zed/src/main.rs

@@ -6,8 +6,9 @@ mod zed;
 use anyhow::{anyhow, Context as _, Result};
 use backtrace::Backtrace;
 use chrono::Utc;
+use clap::{command, Parser};
 use cli::FORCE_CLI_MODE_ENV_VAR_NAME;
-use client::{parse_zed_link, Client, UserStore};
+use client::{parse_zed_link, Client, ClientSettings, DevServerToken, UserStore};
 use collab_ui::channel_view::ChannelView;
 use db::kvp::KEY_VALUE_STORE;
 use editor::Editor;
@@ -270,9 +271,28 @@ fn main() {
 
         cx.activate(true);
 
-        let urls = collect_url_args(cx);
-        if !urls.is_empty() {
-            listener.open_urls(urls)
+        let mut args = Args::parse();
+        if let Some(dev_server_token) = args.dev_server_token.take() {
+            let dev_server_token = DevServerToken(dev_server_token);
+            let server_url = ClientSettings::get_global(&cx).server_url.clone();
+            let client = client.clone();
+            client.set_dev_server_token(dev_server_token);
+            cx.spawn(|cx| async move {
+                client.authenticate_and_connect(false, &cx).await?;
+                log::info!("Connected to {}", server_url);
+                anyhow::Ok(())
+            })
+            .detach_and_log_err(cx);
+        } else {
+            let urls: Vec<_> = args
+                .paths_or_urls
+                .iter()
+                .filter_map(|arg| parse_url_arg(arg, cx).log_err())
+                .collect();
+
+            if !urls.is_empty() {
+                listener.open_urls(urls)
+            }
         }
 
         let mut triggered_authentication = false;
@@ -898,23 +918,35 @@ fn stdout_is_a_pty() -> bool {
     std::env::var(FORCE_CLI_MODE_ENV_VAR_NAME).ok().is_none() && std::io::stdout().is_terminal()
 }
 
-fn collect_url_args(cx: &AppContext) -> Vec<String> {
-    env::args()
-        .skip(1)
-        .filter_map(|arg| match std::fs::canonicalize(Path::new(&arg)) {
-            Ok(path) => Some(format!("file://{}", path.to_string_lossy())),
-            Err(error) => {
-                if arg.starts_with("file://") || arg.starts_with("zed-cli://") {
-                    Some(arg)
-                } else if let Some(_) = parse_zed_link(&arg, cx) {
-                    Some(arg)
-                } else {
-                    log::error!("error parsing path argument: {}", error);
-                    None
-                }
+#[derive(Parser, Debug)]
+#[command(name = "zed", disable_version_flag = true)]
+struct Args {
+    /// A sequence of space-separated paths or urls that you want to open.
+    ///
+    /// Use `path:line:row` syntax to open a file at a specific location.
+    /// Non-existing paths and directories will ignore `:line:row` suffix.
+    ///
+    /// URLs can either be file:// or zed:// scheme, or relative to https://zed.dev.
+    paths_or_urls: Vec<String>,
+
+    /// Instructs zed to run as a dev server on this machine. (not implemented)
+    #[arg(long)]
+    dev_server_token: Option<String>,
+}
+
+fn parse_url_arg(arg: &str, cx: &AppContext) -> Result<String> {
+    match std::fs::canonicalize(Path::new(&arg)) {
+        Ok(path) => Ok(format!("file://{}", path.to_string_lossy())),
+        Err(error) => {
+            if arg.starts_with("file://") || arg.starts_with("zed-cli://") {
+                Ok(arg.into())
+            } else if let Some(_) = parse_zed_link(&arg, cx) {
+                Ok(arg.into())
+            } else {
+                Err(anyhow!("error parsing path argument: {}", error))
             }
-        })
-        .collect()
+        }
+    }
 }
 
 fn load_embedded_fonts(cx: &AppContext) {

script/create-migration

@@ -0,0 +1,3 @@
+zed . \
+    "crates/collab/migrations.sqlite/20221109000000_test_schema.sql" \
+    "crates/collab/migrations/$(date -u +%Y%m%d%H%M%S)_$(echo $1 | sed 's/[^a-z0-9]/_/g').sql"

script/eula/eula.rtf

@@ -182,4 +182,4 @@
 
 \f0\b \cf2 DATE: April 5, 2023
 \f1\b0 \
-}
+}