From 0bbba90f30fc923997bb08433ef3176b9ead0d44 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 13 Oct 2022 18:02:08 -0600 Subject: [PATCH 01/63] Use ScreenCaptureKit-enabled LiveKit SDK and add display_sources function Co-Authored-By: Antonio Scandurra --- crates/capture/src/main.rs | 5 +++ .../live_kit/LiveKitBridge/Package.resolved | 6 +-- crates/live_kit/LiveKitBridge/Package.swift | 2 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 9 +++++ crates/live_kit/src/live_kit.rs | 39 ++++++++++++++++++- 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index c34f451e41..9398322c69 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -38,6 +38,11 @@ fn main() { let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); cx.spawn(|mut cx| async move { + match live_kit::display_sources().await { + Ok(sources) => println!("found {} sources", sources.len()), + Err(error) => println!("error finding display sources {}", error), + } + let user1_token = live_kit_token::create_token( &live_kit_key, &live_kit_secret, diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved index b19e2980a4..93c1738b06 100644 --- a/crates/live_kit/LiveKitBridge/Package.resolved +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -6,7 +6,7 @@ "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", "state": { "branch": null, - "revision": "5cc3c001779ab147199ce3ea0dce465b846368b4", + "revision": "a90ecba800f65bebfc31f5c9c59f635c8587ec7e", "version": null } }, @@ -42,8 +42,8 @@ "repositoryURL": "https://github.com/apple/swift-protobuf.git", "state": { "branch": null, - "revision": "b8230909dedc640294d7324d37f4c91ad3dcf177", - "version": "1.20.1" + "revision": "88c7d15e1242fdb6ecbafbc7926426a19be1e98a", + "version": "1.20.2" } } ] diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift index 76e528bda9..850e3459b4 100644 --- a/crates/live_kit/LiveKitBridge/Package.swift +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -15,7 +15,7 @@ let package = Package( targets: ["LiveKitBridge"]), ], dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "5cc3c001779ab147199ce3ea0dce465b846368b4"), + .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "a90ecba800f65bebfc31f5c9c59f635c8587ec7e"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index f59b829203..c718f27761 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -103,3 +103,12 @@ public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRaw let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() track.add(videoRenderer: renderer) } + +@_cdecl("LKDisplaySources") +public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { + MacOSScreenCapturer.displaySources().then { displaySources in + callback(data, displaySources as CFArray, nil) + }.catch { error in + callback(data, nil, error.localizedDescription as CFString) + } +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 59ce860a78..79ee645041 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, Context, Result}; use core_foundation::{ - array::CFArray, + array::{CFArray, CFArrayRef}, base::{TCFType, TCFTypeRef}, dictionary::CFDictionary, number::CFNumber, @@ -56,6 +56,14 @@ extern "C" { fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; + fn LKDisplaySources( + callback_data: *mut c_void, + callback: extern "C" fn( + callback_data: *mut c_void, + sources: CFArrayRef, + error: CFStringRef, + ), + ); } pub struct Room { @@ -274,3 +282,32 @@ pub fn list_windows() -> Vec { .collect() } } + +pub struct MacOSDisplay(*const c_void); + +pub fn display_sources() -> impl Future>> { + extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { + unsafe { + let tx = Box::from_raw(tx as *mut oneshot::Sender>>); + + if sources.is_null() { + let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); + } else { + let sources = CFArray::wrap_under_get_rule(sources); + let sources = sources + .into_iter() + .map(|source| MacOSDisplay(*source)) + .collect(); + let _ = tx.send(Ok(sources)); + } + } + } + + let (tx, rx) = oneshot::channel(); + + unsafe { + LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); + } + + async move { rx.await.unwrap() } +} From 9569323f93ecfd45ae50f4c5dd7df40d15184200 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 13 Oct 2022 19:00:20 -0600 Subject: [PATCH 02/63] WIP: Getting a big black window, then a crash --- crates/capture/src/main.rs | 17 ++++++------- .../Sources/LiveKitBridge/LiveKitBridge.swift | 8 +++++++ crates/live_kit/src/live_kit.rs | 24 ++++++++++++++----- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 9398322c69..b248f071f1 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -38,11 +38,6 @@ fn main() { let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); cx.spawn(|mut cx| async move { - match live_kit::display_sources().await { - Ok(sources) => println!("found {} sources", sources.len()), - Err(error) => println!("error finding display sources {}", error), - } - let user1_token = live_kit_token::create_token( &live_kit_key, &live_kit_secret, @@ -64,12 +59,14 @@ fn main() { room2.connect(&live_kit_url, &user2_token).await.unwrap(); cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); - let windows = live_kit::list_windows(); - let window = windows - .iter() - .find(|w| w.owner_name.as_deref() == Some("Safari")) + let display = live_kit::display_sources() + .await + .unwrap() + .into_iter() + .next() .unwrap(); - let track = LocalVideoTrack::screen_share_for_window(window.id); + + let track = LocalVideoTrack::screen_share_for_display(display); room1.publish_video_track(&track).await.unwrap(); }) .detach(); diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index c718f27761..2fc35e4e8c 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -92,6 +92,14 @@ public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutable return Unmanaged.passRetained(track).toOpaque() } +@_cdecl("LKCreateScreenShareTrackForDisplay") +public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { + let display = Unmanaged.fromOpaque(display).takeRetainedValue() + print("!!!!!!!!!! display id", display.displayID) + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display) + return Unmanaged.passRetained(track).toOpaque() +} + @_cdecl("LKVideoRendererCreate") public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 79ee645041..8cec1e2f65 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -55,7 +55,6 @@ extern "C" { fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); - fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; fn LKDisplaySources( callback_data: *mut c_void, callback: extern "C" fn( @@ -64,6 +63,8 @@ extern "C" { error: CFStringRef, ), ); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; + fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void; } pub struct Room { @@ -196,6 +197,13 @@ impl LocalVideoTrack { pub fn screen_share_for_window(window_id: u32) -> Self { Self(unsafe { LKCreateScreenShareTrackForWindow(window_id) }) } + + pub fn screen_share_for_display(display: MacOSDisplay) -> Self { + let ptr = display.0; + let this = Self(unsafe { LKCreateScreenShareTrackForDisplay(ptr) }); + std::mem::forget(display); + this + } } impl Drop for LocalVideoTrack { @@ -285,6 +293,12 @@ pub fn list_windows() -> Vec { pub struct MacOSDisplay(*const c_void); +impl Drop for MacOSDisplay { + fn drop(&mut self) { + unsafe { LKRelease(self.0) } + } +} + pub fn display_sources() -> impl Future>> { extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { unsafe { @@ -294,11 +308,9 @@ pub fn display_sources() -> impl Future>> { let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); } else { let sources = CFArray::wrap_under_get_rule(sources); - let sources = sources - .into_iter() - .map(|source| MacOSDisplay(*source)) - .collect(); - let _ = tx.send(Ok(sources)); + let sources_vec = sources.iter().map(|source| MacOSDisplay(*source)).collect(); + let _ = tx.send(Ok(sources_vec)); + std::mem::forget(sources); // HACK: If I drop the CFArray, all the objects inside it get dropped and we get issues accessing the display later. } } } From 4222f86537170c202ff7f13e8f69b7e642a979ea Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 14 Oct 2022 10:31:02 +0200 Subject: [PATCH 03/63] Temporarily use legacy screen capture API --- .../Sources/LiveKitBridge/LiveKitBridge.swift | 9 +--- crates/live_kit/src/live_kit.rs | 53 +------------------ 2 files changed, 2 insertions(+), 60 deletions(-) diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 2fc35e4e8c..7109e6dcf3 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -86,17 +86,10 @@ public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPoin } } -@_cdecl("LKCreateScreenShareTrackForWindow") -public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutableRawPointer { - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)) - return Unmanaged.passRetained(track).toOpaque() -} - @_cdecl("LKCreateScreenShareTrackForDisplay") public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { let display = Unmanaged.fromOpaque(display).takeRetainedValue() - print("!!!!!!!!!! display id", display.displayID) - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display) + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) return Unmanaged.passRetained(track).toOpaque() } diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 8cec1e2f65..f2d61c261a 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -1,15 +1,9 @@ use anyhow::{anyhow, Context, Result}; use core_foundation::{ array::{CFArray, CFArrayRef}, - base::{TCFType, TCFTypeRef}, - dictionary::CFDictionary, - number::CFNumber, + base::TCFType, string::{CFString, CFStringRef}, }; -use core_graphics::window::{ - kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, - kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, -}; use futures::{ channel::{mpsc, oneshot}, Future, @@ -63,7 +57,6 @@ extern "C" { error: CFStringRef, ), ); - fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void; } @@ -194,10 +187,6 @@ impl Drop for RoomDelegate { pub struct LocalVideoTrack(*const c_void); impl LocalVideoTrack { - pub fn screen_share_for_window(window_id: u32) -> Self { - Self(unsafe { LKCreateScreenShareTrackForWindow(window_id) }) - } - pub fn screen_share_for_display(display: MacOSDisplay) -> Self { let ptr = display.0; let this = Self(unsafe { LKCreateScreenShareTrackForDisplay(ptr) }); @@ -251,46 +240,6 @@ impl Drop for RemoteVideoTrack { } } -#[derive(Debug)] -pub struct WindowInfo { - pub id: u32, - pub owner_pid: i32, - pub owner_name: Option, -} - -pub fn list_windows() -> Vec { - unsafe { - let dicts = CFArray::::wrap_under_get_rule(CGWindowListCopyWindowInfo( - kCGWindowListOptionOnScreenOnly | kCGWindowListOptionExcludeDesktopElements, - kCGNullWindowID, - )); - - dicts - .iter() - .map(|dict| { - let id = - CFNumber::wrap_under_get_rule(*dict.get(kCGWindowNumber.as_void_ptr()) as _) - .to_i64() - .unwrap() as u32; - - let owner_pid = - CFNumber::wrap_under_get_rule(*dict.get(kCGWindowOwnerPID.as_void_ptr()) as _) - .to_i32() - .unwrap(); - - let owner_name = dict - .find(kCGWindowOwnerName.as_void_ptr()) - .map(|name| CFString::wrap_under_get_rule(*name as _).to_string()); - WindowInfo { - id, - owner_pid, - owner_name, - } - }) - .collect() - } -} - pub struct MacOSDisplay(*const c_void); impl Drop for MacOSDisplay { From c25acc155d869332e497e612c197e5a4f2f23e57 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 14 Oct 2022 10:37:10 +0200 Subject: [PATCH 04/63] Move ownership of `MacOSDisplay` to the rust side --- crates/capture/src/main.rs | 10 ++-------- .../Sources/LiveKitBridge/LiveKitBridge.swift | 2 +- crates/live_kit/src/live_kit.rs | 8 ++------ 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index b248f071f1..e56bdf8fea 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -59,14 +59,8 @@ fn main() { room2.connect(&live_kit_url, &user2_token).await.unwrap(); cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); - let display = live_kit::display_sources() - .await - .unwrap() - .into_iter() - .next() - .unwrap(); - - let track = LocalVideoTrack::screen_share_for_display(display); + let display_sources = live_kit::display_sources().await.unwrap(); + let track = LocalVideoTrack::screen_share_for_display(display_sources.first().unwrap()); room1.publish_video_track(&track).await.unwrap(); }) .detach(); diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 7109e6dcf3..b33b0ecdb4 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -88,7 +88,7 @@ public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPoin @_cdecl("LKCreateScreenShareTrackForDisplay") public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { - let display = Unmanaged.fromOpaque(display).takeRetainedValue() + let display = Unmanaged.fromOpaque(display).takeUnretainedValue() let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) return Unmanaged.passRetained(track).toOpaque() } diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index f2d61c261a..f8a94f6ae5 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -187,11 +187,8 @@ impl Drop for RoomDelegate { pub struct LocalVideoTrack(*const c_void); impl LocalVideoTrack { - pub fn screen_share_for_display(display: MacOSDisplay) -> Self { - let ptr = display.0; - let this = Self(unsafe { LKCreateScreenShareTrackForDisplay(ptr) }); - std::mem::forget(display); - this + pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { + Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) }) } } @@ -259,7 +256,6 @@ pub fn display_sources() -> impl Future>> { let sources = CFArray::wrap_under_get_rule(sources); let sources_vec = sources.iter().map(|source| MacOSDisplay(*source)).collect(); let _ = tx.send(Ok(sources_vec)); - std::mem::forget(sources); // HACK: If I drop the CFArray, all the objects inside it get dropped and we get issues accessing the display later. } } } From caeae38e3a80846255699df380b5ead3567eeef2 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 14 Oct 2022 14:51:13 +0200 Subject: [PATCH 05/63] Move `live_kit` to `live_kit_client` and add `live_kit_server` --- Cargo.lock | 20 +++++++++++++------ crates/capture/Cargo.toml | 7 ++----- crates/capture/src/main.rs | 10 ++++------ .../{live_kit => live_kit_client}/Cargo.toml | 4 ++-- .../LiveKitBridge/.gitignore | 0 .../LiveKitBridge/Package.resolved | 0 .../LiveKitBridge/Package.swift | 0 .../LiveKitBridge/README.md | 0 .../Sources/LiveKitBridge/LiveKitBridge.swift | 0 crates/{live_kit => live_kit_client}/build.rs | 0 .../src/live_kit_client.rs} | 0 crates/live_kit_server/Cargo.toml | 16 +++++++++++++++ .../src/live_kit_server.rs} | 0 13 files changed, 38 insertions(+), 19 deletions(-) rename crates/{live_kit => live_kit_client}/Cargo.toml (88%) rename crates/{live_kit => live_kit_client}/LiveKitBridge/.gitignore (100%) rename crates/{live_kit => live_kit_client}/LiveKitBridge/Package.resolved (100%) rename crates/{live_kit => live_kit_client}/LiveKitBridge/Package.swift (100%) rename crates/{live_kit => live_kit_client}/LiveKitBridge/README.md (100%) rename crates/{live_kit => live_kit_client}/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift (100%) rename crates/{live_kit => live_kit_client}/build.rs (100%) rename crates/{live_kit/src/live_kit.rs => live_kit_client/src/live_kit_client.rs} (100%) create mode 100644 crates/live_kit_server/Cargo.toml rename crates/{capture/src/live_kit_token.rs => live_kit_server/src/live_kit_server.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index 1a399ad4ad..0e32c86091 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,16 +817,13 @@ dependencies = [ "foreign-types", "futures 0.3.24", "gpui", - "hmac 0.12.1", - "jwt", - "live_kit", + "live_kit_client", + "live_kit_server", "log", "media", "objc", "parking_lot 0.11.2", "postage", - "serde", - "sha2 0.10.6", "simplelog", ] @@ -3187,7 +3184,7 @@ dependencies = [ ] [[package]] -name = "live_kit" +name = "live_kit_client" version = "0.1.0" dependencies = [ "anyhow", @@ -3200,6 +3197,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "live_kit_server" +version = "0.1.0" +dependencies = [ + "anyhow", + "hmac 0.12.1", + "jwt", + "serde", + "sha2 0.10.6", +] + [[package]] name = "lock_api" version = "0.4.9" diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index f8ed31097a..a5ec131ff1 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -6,7 +6,8 @@ description = "An example of screen capture" [dependencies] gpui = { path = "../gpui" } -live_kit = { path = "../live_kit" } +live_kit_client = { path = "../live_kit_client" } +live_kit_server = { path = "../live_kit_server" } media = { path = "../media" } anyhow = "1.0.38" @@ -18,14 +19,10 @@ core-foundation = "0.9.3" core-graphics = "0.22.3" foreign-types = "0.3" futures = "0.3" -hmac = "0.12" -jwt = "0.16" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" parking_lot = "0.11.1" postage = { version = "0.4.1", features = ["futures-traits"] } -serde = { version = "1.0", features = ["derive", "rc"] } -sha2 = "0.10" simplelog = "0.9" [build-dependencies] diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index e56bdf8fea..00f3eac2df 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,5 +1,3 @@ -mod live_kit_token; - use futures::StreamExt; use gpui::{ actions, @@ -8,7 +6,7 @@ use gpui::{ platform::current::Surface, Menu, MenuItem, ViewContext, }; -use live_kit::{LocalVideoTrack, Room}; +use live_kit_client::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; use postage::watch; @@ -38,7 +36,7 @@ fn main() { let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); cx.spawn(|mut cx| async move { - let user1_token = live_kit_token::create_token( + let user1_token = live_kit_server::create_token( &live_kit_key, &live_kit_secret, "test-room", @@ -48,7 +46,7 @@ fn main() { let room1 = Room::new(); room1.connect(&live_kit_url, &user1_token).await.unwrap(); - let user2_token = live_kit_token::create_token( + let user2_token = live_kit_server::create_token( &live_kit_key, &live_kit_secret, "test-room", @@ -59,7 +57,7 @@ fn main() { room2.connect(&live_kit_url, &user2_token).await.unwrap(); cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); - let display_sources = live_kit::display_sources().await.unwrap(); + let display_sources = live_kit_client::display_sources().await.unwrap(); let track = LocalVideoTrack::screen_share_for_display(display_sources.first().unwrap()); room1.publish_video_track(&track).await.unwrap(); }) diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit_client/Cargo.toml similarity index 88% rename from crates/live_kit/Cargo.toml rename to crates/live_kit_client/Cargo.toml index e88d4f7b24..1344ecbcdb 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "live_kit" +name = "live_kit_client" version = "0.1.0" edition = "2021" description = "Bindings to LiveKit Swift client SDK" [lib] -path = "src/live_kit.rs" +path = "src/live_kit_client.rs" doctest = false [dependencies] diff --git a/crates/live_kit/LiveKitBridge/.gitignore b/crates/live_kit_client/LiveKitBridge/.gitignore similarity index 100% rename from crates/live_kit/LiveKitBridge/.gitignore rename to crates/live_kit_client/LiveKitBridge/.gitignore diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit_client/LiveKitBridge/Package.resolved similarity index 100% rename from crates/live_kit/LiveKitBridge/Package.resolved rename to crates/live_kit_client/LiveKitBridge/Package.resolved diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit_client/LiveKitBridge/Package.swift similarity index 100% rename from crates/live_kit/LiveKitBridge/Package.swift rename to crates/live_kit_client/LiveKitBridge/Package.swift diff --git a/crates/live_kit/LiveKitBridge/README.md b/crates/live_kit_client/LiveKitBridge/README.md similarity index 100% rename from crates/live_kit/LiveKitBridge/README.md rename to crates/live_kit_client/LiveKitBridge/README.md diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift similarity index 100% rename from crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift rename to crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift diff --git a/crates/live_kit/build.rs b/crates/live_kit_client/build.rs similarity index 100% rename from crates/live_kit/build.rs rename to crates/live_kit_client/build.rs diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit_client/src/live_kit_client.rs similarity index 100% rename from crates/live_kit/src/live_kit.rs rename to crates/live_kit_client/src/live_kit_client.rs diff --git a/crates/live_kit_server/Cargo.toml b/crates/live_kit_server/Cargo.toml new file mode 100644 index 0000000000..dc9d56074e --- /dev/null +++ b/crates/live_kit_server/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "live_kit_server" +version = "0.1.0" +edition = "2021" +description = "SDK for the LiveKit server API" + +[lib] +path = "src/live_kit_server.rs" +doctest = false + +[dependencies] +anyhow = "1.0.38" +hmac = "0.12" +jwt = "0.16" +serde = { version = "1.0", features = ["derive", "rc"] } +sha2 = "0.10" \ No newline at end of file diff --git a/crates/capture/src/live_kit_token.rs b/crates/live_kit_server/src/live_kit_server.rs similarity index 100% rename from crates/capture/src/live_kit_token.rs rename to crates/live_kit_server/src/live_kit_server.rs From 5d433b16662baf8a77bc00df438d570bc0347229 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 14 Oct 2022 17:00:38 +0200 Subject: [PATCH 06/63] WIP: start on live_kit_server --- .gitmodules | 3 + Cargo.lock | 16 +++- crates/capture/src/main.rs | 4 +- crates/live_kit_server/Cargo.toml | 8 +- crates/live_kit_server/build.rs | 5 ++ crates/live_kit_server/protocol | 1 + crates/live_kit_server/src/api.rs | 36 +++++++++ crates/live_kit_server/src/live_kit_server.rs | 74 +------------------ crates/live_kit_server/src/proto.rs | 1 + crates/live_kit_server/src/token.rs | 71 ++++++++++++++++++ 10 files changed, 144 insertions(+), 75 deletions(-) create mode 100644 .gitmodules create mode 100644 crates/live_kit_server/build.rs create mode 160000 crates/live_kit_server/protocol create mode 100644 crates/live_kit_server/src/api.rs create mode 100644 crates/live_kit_server/src/proto.rs create mode 100644 crates/live_kit_server/src/token.rs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..b1dad4cbbe --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "crates/live_kit_server/protocol"] + path = crates/live_kit_server/protocol + url = https://github.com/livekit/protocol diff --git a/Cargo.lock b/Cargo.lock index 0e32c86091..16417f6906 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3203,7 +3203,11 @@ version = "0.1.0" dependencies = [ "anyhow", "hmac 0.12.1", + "hyper", "jwt", + "prost 0.8.0", + "prost-build", + "prost-types 0.8.0", "serde", "sha2 0.10.6", ] @@ -4363,7 +4367,7 @@ dependencies = [ "multimap", "petgraph", "prost 0.9.0", - "prost-types", + "prost-types 0.9.0", "regex", "tempfile", "which", @@ -4395,6 +4399,16 @@ dependencies = [ "syn", ] +[[package]] +name = "prost-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" +dependencies = [ + "bytes 1.2.1", + "prost 0.8.0", +] + [[package]] name = "prost-types" version = "0.9.0" diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 00f3eac2df..912e1e0be2 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -36,7 +36,7 @@ fn main() { let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); cx.spawn(|mut cx| async move { - let user1_token = live_kit_server::create_token( + let user1_token = live_kit_server::token::create( &live_kit_key, &live_kit_secret, "test-room", @@ -46,7 +46,7 @@ fn main() { let room1 = Room::new(); room1.connect(&live_kit_url, &user1_token).await.unwrap(); - let user2_token = live_kit_server::create_token( + let user2_token = live_kit_server::token::create( &live_kit_key, &live_kit_secret, "test-room", diff --git a/crates/live_kit_server/Cargo.toml b/crates/live_kit_server/Cargo.toml index dc9d56074e..17eb83f83e 100644 --- a/crates/live_kit_server/Cargo.toml +++ b/crates/live_kit_server/Cargo.toml @@ -12,5 +12,11 @@ doctest = false anyhow = "1.0.38" hmac = "0.12" jwt = "0.16" +hyper = "0.14" +prost = "0.8" +prost-types = "0.8" serde = { version = "1.0", features = ["derive", "rc"] } -sha2 = "0.10" \ No newline at end of file +sha2 = "0.10" + +[build-dependencies] +prost-build = "0.9" diff --git a/crates/live_kit_server/build.rs b/crates/live_kit_server/build.rs new file mode 100644 index 0000000000..fa1bde69d6 --- /dev/null +++ b/crates/live_kit_server/build.rs @@ -0,0 +1,5 @@ +fn main() { + prost_build::Config::new() + .compile_protos(&["protocol/livekit_room.proto"], &["protocol"]) + .unwrap(); +} diff --git a/crates/live_kit_server/protocol b/crates/live_kit_server/protocol new file mode 160000 index 0000000000..8645a138fb --- /dev/null +++ b/crates/live_kit_server/protocol @@ -0,0 +1 @@ +Subproject commit 8645a138fb2ea72c4dab13e739b1f3c9ea29ac84 diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs new file mode 100644 index 0000000000..acdb80aaf9 --- /dev/null +++ b/crates/live_kit_server/src/api.rs @@ -0,0 +1,36 @@ +use crate::token; +use hyper::{client::HttpConnector, Request, Uri}; + +pub struct Client { + http: hyper::Client, + uri: Uri, + key: String, + secret: String, +} + +impl Client { + pub fn new(uri: Uri, key: String, secret: String) -> Self { + assert!(uri.scheme().is_some(), "base uri must have a scheme"); + assert!(uri.authority().is_some(), "base uri must have an authority"); + Self { + http: hyper::Client::new(), + uri: uri, + key, + secret, + } + } + + pub fn create_room(&self) { + // let mut uri = url.clone(); + // uri.set_path_and_query() + + let uri = Uri::builder() + .scheme(self.uri.scheme().unwrap().clone()) + .authority(self.uri.authority().unwrap().clone()) + .path_and_query("twirp/livekit.RoomService/CreateRoom") + .build(); + + // token::create(api_key, secret_key, room_name, participant_name) + self.http.request(req) + } +} diff --git a/crates/live_kit_server/src/live_kit_server.rs b/crates/live_kit_server/src/live_kit_server.rs index be4fc4f4a2..7b4a741355 100644 --- a/crates/live_kit_server/src/live_kit_server.rs +++ b/crates/live_kit_server/src/live_kit_server.rs @@ -1,71 +1,3 @@ -use anyhow::Result; -use hmac::{Hmac, Mac}; -use jwt::SignWithKey; -use serde::Serialize; -use sha2::Sha256; -use std::{ - ops::Add, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -static DEFAULT_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours - -#[derive(Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct ClaimGrants<'a> { - iss: &'a str, - sub: &'a str, - iat: u64, - exp: u64, - nbf: u64, - jwtid: &'a str, - video: VideoGrant<'a>, -} - -#[derive(Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct VideoGrant<'a> { - room_create: Option, - room_join: Option, - room_list: Option, - room_record: Option, - room_admin: Option, - room: Option<&'a str>, - can_publish: Option, - can_subscribe: Option, - can_publish_data: Option, - hidden: Option, - recorder: Option, -} - -pub fn create_token( - api_key: &str, - secret_key: &str, - room_name: &str, - participant_name: &str, -) -> Result { - let secret_key: Hmac = Hmac::new_from_slice(secret_key.as_bytes())?; - - let now = SystemTime::now(); - - let claims = ClaimGrants { - iss: api_key, - sub: participant_name, - iat: now.duration_since(UNIX_EPOCH).unwrap().as_secs(), - exp: now - .add(DEFAULT_TTL) - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(), - nbf: 0, - jwtid: participant_name, - video: VideoGrant { - room: Some(room_name), - room_join: Some(true), - can_publish: Some(true), - can_subscribe: Some(true), - ..Default::default() - }, - }; - Ok(claims.sign_with_key(&secret_key)?) -} +mod api; +mod proto; +mod token; diff --git a/crates/live_kit_server/src/proto.rs b/crates/live_kit_server/src/proto.rs new file mode 100644 index 0000000000..a304705c59 --- /dev/null +++ b/crates/live_kit_server/src/proto.rs @@ -0,0 +1 @@ +include!(concat!(env!("OUT_DIR"), "/livekit.rs")); diff --git a/crates/live_kit_server/src/token.rs b/crates/live_kit_server/src/token.rs new file mode 100644 index 0000000000..f6ac2945b5 --- /dev/null +++ b/crates/live_kit_server/src/token.rs @@ -0,0 +1,71 @@ +use anyhow::Result; +use hmac::{Hmac, Mac}; +use jwt::SignWithKey; +use serde::Serialize; +use sha2::Sha256; +use std::{ + ops::Add, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +static DEFAULT_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ClaimGrants<'a> { + iss: &'a str, + sub: &'a str, + iat: u64, + exp: u64, + nbf: u64, + jwtid: &'a str, + video: VideoGrant<'a>, +} + +#[derive(Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct VideoGrant<'a> { + room_create: Option, + room_join: Option, + room_list: Option, + room_record: Option, + room_admin: Option, + room: Option<&'a str>, + can_publish: Option, + can_subscribe: Option, + can_publish_data: Option, + hidden: Option, + recorder: Option, +} + +pub fn create( + api_key: &str, + secret_key: &str, + room_name: &str, + participant_name: &str, +) -> Result { + let secret_key: Hmac = Hmac::new_from_slice(secret_key.as_bytes())?; + + let now = SystemTime::now(); + + let claims = ClaimGrants { + iss: api_key, + sub: participant_name, + iat: now.duration_since(UNIX_EPOCH).unwrap().as_secs(), + exp: now + .add(DEFAULT_TTL) + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + nbf: 0, + jwtid: participant_name, + video: VideoGrant { + room: Some(room_name), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + }, + }; + Ok(claims.sign_with_key(&secret_key)?) +} From 19a275267410dbb370ee21b73b66a62c6adc3bc4 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Fri, 14 Oct 2022 10:00:50 -0600 Subject: [PATCH 07/63] WIP: Update token module to support server api --- crates/capture/src/main.rs | 21 ++++++-- crates/live_kit_server/Cargo.toml | 2 +- crates/live_kit_server/src/api.rs | 2 +- crates/live_kit_server/src/live_kit_server.rs | 4 +- crates/live_kit_server/src/token.rs | 52 +++++++++---------- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index 912e1e0be2..79cb0df922 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -39,8 +39,14 @@ fn main() { let user1_token = live_kit_server::token::create( &live_kit_key, &live_kit_secret, - "test-room", - "test-participant-1", + Some("test-participant-1"), + live_kit_server::token::VideoGrant { + room: Some("test-room"), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + }, ) .unwrap(); let room1 = Room::new(); @@ -49,10 +55,17 @@ fn main() { let user2_token = live_kit_server::token::create( &live_kit_key, &live_kit_secret, - "test-room", - "test-participant-2", + Some("test-participant-2"), + live_kit_server::token::VideoGrant { + room: Some("test-room"), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + }, ) .unwrap(); + let room2 = Room::new(); room2.connect(&live_kit_url, &user2_token).await.unwrap(); cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); diff --git a/crates/live_kit_server/Cargo.toml b/crates/live_kit_server/Cargo.toml index 17eb83f83e..51d134ca95 100644 --- a/crates/live_kit_server/Cargo.toml +++ b/crates/live_kit_server/Cargo.toml @@ -12,7 +12,7 @@ doctest = false anyhow = "1.0.38" hmac = "0.12" jwt = "0.16" -hyper = "0.14" +hyper = { version = "0.14", features = ["client", "http1"] } prost = "0.8" prost-types = "0.8" serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index acdb80aaf9..cec360a30d 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -31,6 +31,6 @@ impl Client { .build(); // token::create(api_key, secret_key, room_name, participant_name) - self.http.request(req) + // self.http.request(req) } } diff --git a/crates/live_kit_server/src/live_kit_server.rs b/crates/live_kit_server/src/live_kit_server.rs index 7b4a741355..7471a96ec4 100644 --- a/crates/live_kit_server/src/live_kit_server.rs +++ b/crates/live_kit_server/src/live_kit_server.rs @@ -1,3 +1,3 @@ -mod api; +pub mod api; mod proto; -mod token; +pub mod token; diff --git a/crates/live_kit_server/src/token.rs b/crates/live_kit_server/src/token.rs index f6ac2945b5..ae03cb3469 100644 --- a/crates/live_kit_server/src/token.rs +++ b/crates/live_kit_server/src/token.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use hmac::{Hmac, Mac}; use jwt::SignWithKey; use serde::Serialize; @@ -14,43 +14,49 @@ static DEFAULT_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours #[serde(rename_all = "camelCase")] struct ClaimGrants<'a> { iss: &'a str, - sub: &'a str, + sub: Option<&'a str>, iat: u64, exp: u64, nbf: u64, - jwtid: &'a str, + jwtid: Option<&'a str>, video: VideoGrant<'a>, } #[derive(Default, Serialize)] #[serde(rename_all = "camelCase")] -struct VideoGrant<'a> { - room_create: Option, - room_join: Option, - room_list: Option, - room_record: Option, - room_admin: Option, - room: Option<&'a str>, - can_publish: Option, - can_subscribe: Option, - can_publish_data: Option, - hidden: Option, - recorder: Option, +pub struct VideoGrant<'a> { + pub room_create: Option, + pub room_join: Option, + pub room_list: Option, + pub room_record: Option, + pub room_admin: Option, + pub room: Option<&'a str>, + pub can_publish: Option, + pub can_subscribe: Option, + pub can_publish_data: Option, + pub hidden: Option, + pub recorder: Option, } pub fn create( api_key: &str, secret_key: &str, - room_name: &str, - participant_name: &str, + identity: Option<&str>, + video_grant: VideoGrant, ) -> Result { + if video_grant.room_join.is_some() && identity.is_none() { + Err(anyhow!( + "identity is required for room_join grant, but it is none" + ))?; + } + let secret_key: Hmac = Hmac::new_from_slice(secret_key.as_bytes())?; let now = SystemTime::now(); let claims = ClaimGrants { iss: api_key, - sub: participant_name, + sub: identity, iat: now.duration_since(UNIX_EPOCH).unwrap().as_secs(), exp: now .add(DEFAULT_TTL) @@ -58,14 +64,8 @@ pub fn create( .unwrap() .as_secs(), nbf: 0, - jwtid: participant_name, - video: VideoGrant { - room: Some(room_name), - room_join: Some(true), - can_publish: Some(true), - can_subscribe: Some(true), - ..Default::default() - }, + jwtid: identity, + video: video_grant, }; Ok(claims.sign_with_key(&secret_key)?) } From f09d6b7b9502d78d4c3ed7e0b234498d7a8bc45c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 14 Oct 2022 18:31:03 +0200 Subject: [PATCH 08/63] WIP --- crates/live_kit_server/src/api.rs | 43 ++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index cec360a30d..52a911c593 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -1,5 +1,7 @@ -use crate::token; -use hyper::{client::HttpConnector, Request, Uri}; +use crate::{proto, token}; +use anyhow::{anyhow, Result}; +use hyper::{client::HttpConnector, header::AUTHORIZATION, Method, Request, Uri}; +use std::future::Future; pub struct Client { http: hyper::Client, @@ -14,23 +16,46 @@ impl Client { assert!(uri.authority().is_some(), "base uri must have an authority"); Self { http: hyper::Client::new(), - uri: uri, + uri, key, secret, } } - pub fn create_room(&self) { - // let mut uri = url.clone(); - // uri.set_path_and_query() + pub fn create_room(&self, name: String) -> impl Future> { + let token = token::create( + &self.key, + &self.secret, + None, + token::VideoGrant { + room_create: Some(true), + ..Default::default() + }, + ); + let client = self.http.clone(); let uri = Uri::builder() .scheme(self.uri.scheme().unwrap().clone()) .authority(self.uri.authority().unwrap().clone()) .path_and_query("twirp/livekit.RoomService/CreateRoom") .build(); - - // token::create(api_key, secret_key, room_name, participant_name) - // self.http.request(req) + async move { + let token = token?; + let uri = uri?; + let body = proto::CreateRoomRequest { + name: todo!(), + empty_timeout: todo!(), + max_participants: todo!(), + node_id: todo!(), + metadata: todo!(), + egress: todo!(), + }; + let mut request = Request::builder() + .uri(uri) + .method(Method::POST) + .header(AUTHORIZATION, format!("Bearer {}", token)) + .body(body); + Err(anyhow!("yeah")) + } } } From e39c7c62e4c53783e46f57f85400e11b6cbaacf5 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 17 Oct 2022 10:48:09 +0200 Subject: [PATCH 09/63] Update livekit_client --- crates/live_kit_client/LiveKitBridge/Package.resolved | 2 +- crates/live_kit_client/LiveKitBridge/Package.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/live_kit_client/LiveKitBridge/Package.resolved b/crates/live_kit_client/LiveKitBridge/Package.resolved index 93c1738b06..402bd635ce 100644 --- a/crates/live_kit_client/LiveKitBridge/Package.resolved +++ b/crates/live_kit_client/LiveKitBridge/Package.resolved @@ -6,7 +6,7 @@ "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", "state": { "branch": null, - "revision": "a90ecba800f65bebfc31f5c9c59f635c8587ec7e", + "revision": "42258f5d3467ec3981323e33200b7403ac637ece", "version": null } }, diff --git a/crates/live_kit_client/LiveKitBridge/Package.swift b/crates/live_kit_client/LiveKitBridge/Package.swift index 850e3459b4..e4227d8b64 100644 --- a/crates/live_kit_client/LiveKitBridge/Package.swift +++ b/crates/live_kit_client/LiveKitBridge/Package.swift @@ -15,7 +15,7 @@ let package = Package( targets: ["LiveKitBridge"]), ], dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "a90ecba800f65bebfc31f5c9c59f635c8587ec7e"), + .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "42258f5d3467ec3981323e33200b7403ac637ece"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. From 75c339851fd8e8898ce02739abe7eb2061a4ea93 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 17 Oct 2022 11:24:09 +0200 Subject: [PATCH 10/63] Add `live_kit_server::api::Client::{create,delete}_room` --- Cargo.lock | 3 +- crates/live_kit_server/Cargo.toml | 3 +- crates/live_kit_server/src/api.rs | 94 ++++++++++++++++++++----------- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16417f6906..51e037e748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3202,12 +3202,13 @@ name = "live_kit_server" version = "0.1.0" dependencies = [ "anyhow", + "futures 0.3.24", "hmac 0.12.1", - "hyper", "jwt", "prost 0.8.0", "prost-build", "prost-types 0.8.0", + "reqwest", "serde", "sha2 0.10.6", ] diff --git a/crates/live_kit_server/Cargo.toml b/crates/live_kit_server/Cargo.toml index 51d134ca95..7cc9d82333 100644 --- a/crates/live_kit_server/Cargo.toml +++ b/crates/live_kit_server/Cargo.toml @@ -10,11 +10,12 @@ doctest = false [dependencies] anyhow = "1.0.38" +futures = "0.3" hmac = "0.12" jwt = "0.16" -hyper = { version = "0.14", features = ["client", "http1"] } prost = "0.8" prost-types = "0.8" +reqwest = "0.11" serde = { version = "1.0", features = ["derive", "rc"] } sha2 = "0.10" diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index 52a911c593..9abf3bc7c6 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -1,21 +1,24 @@ use crate::{proto, token}; use anyhow::{anyhow, Result}; -use hyper::{client::HttpConnector, header::AUTHORIZATION, Method, Request, Uri}; +use prost::Message; +use reqwest::header::CONTENT_TYPE; use std::future::Future; pub struct Client { - http: hyper::Client, - uri: Uri, + http: reqwest::Client, + uri: String, key: String, secret: String, } impl Client { - pub fn new(uri: Uri, key: String, secret: String) -> Self { - assert!(uri.scheme().is_some(), "base uri must have a scheme"); - assert!(uri.authority().is_some(), "base uri must have an authority"); + pub fn new(mut uri: String, key: String, secret: String) -> Self { + if uri.ends_with('/') { + uri.pop(); + } + Self { - http: hyper::Client::new(), + http: reqwest::Client::new(), uri, key, secret, @@ -23,39 +26,66 @@ impl Client { } pub fn create_room(&self, name: String) -> impl Future> { - let token = token::create( - &self.key, - &self.secret, - None, + self.request( + "twirp/livekit.RoomService/CreateRoom", token::VideoGrant { room_create: Some(true), ..Default::default() }, - ); + proto::CreateRoomRequest { + name, + ..Default::default() + }, + ) + } + pub fn delete_room(&self, name: String) -> impl Future> { + let response = self.request( + "twirp/livekit.RoomService/DeleteRoom", + token::VideoGrant { + room_create: Some(true), + ..Default::default() + }, + proto::DeleteRoomRequest { room: name }, + ); + async move { + response.await?; + Ok(()) + } + } + + fn request( + &self, + path: &str, + grant: token::VideoGrant, + body: Req, + ) -> impl Future> + where + Req: Message, + Res: Default + Message, + { let client = self.http.clone(); - let uri = Uri::builder() - .scheme(self.uri.scheme().unwrap().clone()) - .authority(self.uri.authority().unwrap().clone()) - .path_and_query("twirp/livekit.RoomService/CreateRoom") - .build(); + let token = token::create(&self.key, &self.secret, None, grant); + let uri = format!("{}/{}", self.uri, path); async move { let token = token?; - let uri = uri?; - let body = proto::CreateRoomRequest { - name: todo!(), - empty_timeout: todo!(), - max_participants: todo!(), - node_id: todo!(), - metadata: todo!(), - egress: todo!(), - }; - let mut request = Request::builder() - .uri(uri) - .method(Method::POST) - .header(AUTHORIZATION, format!("Bearer {}", token)) - .body(body); - Err(anyhow!("yeah")) + let response = client + .post(&uri) + .header(CONTENT_TYPE, "application/protobuf") + .bearer_auth(token) + .body(body.encode_to_vec()) + .send() + .await?; + if response.status().is_success() { + Ok(Res::decode(response.bytes().await?)?) + } else { + Err(anyhow!( + "POST {} failed with status code {:?}, {:?}", + uri, + response.status(), + response.text().await + )) + } } } } From c9225bb87c269d51fa0cbc058a13d6b4a51e232b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 17 Oct 2022 12:20:55 +0200 Subject: [PATCH 11/63] WIP: Start integrating with LiveKit when creating/joining rooms --- Cargo.lock | 1 + crates/call/src/room.rs | 3 +- crates/collab/Cargo.toml | 2 + crates/collab/src/integration_tests.rs | 1 + crates/collab/src/main.rs | 20 +++++++++ crates/collab/src/rpc.rs | 59 +++++++++++++++++++++++--- crates/collab/src/rpc/store.rs | 33 ++++++++------ crates/live_kit_server/src/api.rs | 15 +++++++ crates/rpc/proto/zed.proto | 10 +++-- 9 files changed, 120 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51e037e748..4ea784749d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1100,6 +1100,7 @@ dependencies = [ "language", "lazy_static", "lipsum", + "live_kit_server", "log", "lsp", "nanoid", diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 09b49716e0..258400da92 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -94,7 +94,8 @@ impl Room { ) -> Task>> { cx.spawn(|mut cx| async move { let response = client.request(proto::CreateRoom {}).await?; - let room = cx.add_model(|cx| Self::new(response.id, client, user_store, cx)); + let room_proto = response.room.ok_or_else(|| anyhow!("invalid room"))?; + let room = cx.add_model(|cx| Self::new(room_proto.id, client, user_store, cx)); let initial_project_id = if let Some(initial_project) = initial_project { let initial_project_id = room diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index de41e8a1f3..8ce42afb64 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -14,8 +14,10 @@ required-features = ["seed-support"] [dependencies] collections = { path = "../collections" } +live_kit_server = { path = "../live_kit_server" } rpc = { path = "../rpc" } util = { path = "../util" } + anyhow = "1.0.40" async-trait = "0.1.50" async-tungstenite = "0.16" diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index 90d7b6d4b5..44a70e71e9 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -6321,6 +6321,7 @@ impl TestServer { async fn build_app_state(test_db: &TestDb) -> Arc { Arc::new(AppState { db: test_db.db().clone(), + live_kit_client: None, api_token: Default::default(), invite_link_prefix: Default::default(), }) diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index 272d52cc95..49a82bb926 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -30,6 +30,9 @@ pub struct Config { pub invite_link_prefix: String, pub honeycomb_api_key: Option, pub honeycomb_dataset: Option, + pub live_kit_server: Option, + pub live_kit_key: Option, + pub live_kit_secret: Option, pub rust_log: Option, pub log_json: Option, } @@ -38,13 +41,30 @@ pub struct AppState { db: Arc, api_token: String, invite_link_prefix: String, + live_kit_client: Option, } impl AppState { async fn new(config: &Config) -> Result> { let db = PostgresDb::new(&config.database_url, 5).await?; + let live_kit_client = if let Some(((server, key), secret)) = config + .live_kit_server + .as_ref() + .zip(config.live_kit_key.as_ref()) + .zip(config.live_kit_secret.as_ref()) + { + Some(live_kit_server::api::Client::new( + server.clone(), + key.clone(), + secret.clone(), + )) + } else { + None + }; + let this = Self { db: Arc::new(db), + live_kit_client, api_token: config.api_token.clone(), invite_link_prefix: config.invite_link_prefix.clone(), }; diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index 564e173fec..8983f69f9b 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -5,7 +5,7 @@ use crate::{ db::{self, ChannelId, MessageId, ProjectId, User, UserId}, AppState, Result, }; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_tungstenite::tungstenite::{ protocol::CloseFrame as TungsteniteCloseFrame, Message as TungsteniteMessage, }; @@ -49,6 +49,7 @@ use std::{ }, time::Duration, }; +pub use store::{Store, Worktree}; use time::OffsetDateTime; use tokio::{ sync::{Mutex, MutexGuard}, @@ -57,8 +58,6 @@ use tokio::{ use tower::ServiceBuilder; use tracing::{info_span, instrument, Instrument}; -pub use store::{Store, Worktree}; - lazy_static! { static ref METRIC_CONNECTIONS: IntGauge = register_int_gauge!("connections", "number of connections").unwrap(); @@ -597,13 +596,45 @@ impl Server { response: Response, ) -> Result<()> { let user_id; - let room_id; + let room; { let mut store = self.store().await; user_id = store.user_id_for_connection(request.sender_id)?; - room_id = store.create_room(request.sender_id)?; + room = store.create_room(request.sender_id)?.clone(); } - response.send(proto::CreateRoomResponse { id: room_id })?; + + let live_kit_token = if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { + if let Some(_) = live_kit + .create_room(room.live_kit_room.clone()) + .await + .with_context(|| { + format!( + "error creating LiveKit room (LiveKit room: {}, Zed room: {})", + room.live_kit_room, room.id + ) + }) + .trace_err() + { + live_kit + .room_token_for_user(&room.live_kit_room, &user_id.to_string()) + .with_context(|| { + format!( + "error creating LiveKit access token (LiveKit room: {}, Zed room: {})", + room.live_kit_room, room.id + ) + }) + .trace_err() + } else { + None + } + } else { + None + }; + + response.send(proto::CreateRoomResponse { + room: Some(room), + live_kit_token, + })?; self.update_user_contacts(user_id).await?; Ok(()) } @@ -624,8 +655,24 @@ impl Server { .send(recipient_id, proto::CallCanceled {}) .trace_err(); } + + let live_kit_token = if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { + live_kit + .room_token_for_user(&room.live_kit_room, &user_id.to_string()) + .with_context(|| { + format!( + "error creating LiveKit access token (LiveKit room: {}, Zed room: {})", + room.live_kit_room, room.id + ) + }) + .trace_err() + } else { + None + }; + response.send(proto::JoinRoomResponse { room: Some(room.clone()), + live_kit_token, })?; self.room_updated(room); } diff --git a/crates/collab/src/rpc/store.rs b/crates/collab/src/rpc/store.rs index b7dd39cff1..096ac0fa06 100644 --- a/crates/collab/src/rpc/store.rs +++ b/crates/collab/src/rpc/store.rs @@ -1,6 +1,7 @@ use crate::db::{self, ChannelId, ProjectId, UserId}; use anyhow::{anyhow, Result}; use collections::{btree_map, BTreeMap, BTreeSet, HashMap, HashSet}; +use nanoid::nanoid; use rpc::{proto, ConnectionId}; use serde::Serialize; use std::{mem, path::PathBuf, str, time::Duration}; @@ -339,7 +340,7 @@ impl Store { } } - pub fn create_room(&mut self, creator_connection_id: ConnectionId) -> Result { + pub fn create_room(&mut self, creator_connection_id: ConnectionId) -> Result<&proto::Room> { let connection = self .connections .get_mut(&creator_connection_id) @@ -353,19 +354,23 @@ impl Store { "can't create a room with an active call" ); - let mut room = proto::Room::default(); - room.participants.push(proto::Participant { - user_id: connection.user_id.to_proto(), - peer_id: creator_connection_id.0, - projects: Default::default(), - location: Some(proto::ParticipantLocation { - variant: Some(proto::participant_location::Variant::External( - proto::participant_location::External {}, - )), - }), - }); - let room_id = post_inc(&mut self.next_room_id); + let room = proto::Room { + id: room_id, + participants: vec![proto::Participant { + user_id: connection.user_id.to_proto(), + peer_id: creator_connection_id.0, + projects: Default::default(), + location: Some(proto::ParticipantLocation { + variant: Some(proto::participant_location::Variant::External( + proto::participant_location::External {}, + )), + }), + }], + pending_participant_user_ids: Default::default(), + live_kit_room: nanoid!(30), + }; + self.rooms.insert(room_id, room); connected_user.active_call = Some(Call { caller_user_id: connection.user_id, @@ -373,7 +378,7 @@ impl Store { connection_id: Some(creator_connection_id), initial_project_id: None, }); - Ok(room_id) + Ok(self.rooms.get(&room_id).unwrap()) } pub fn join_room( diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index 9abf3bc7c6..2bbad785c3 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -54,6 +54,21 @@ impl Client { } } + pub fn room_token_for_user(&self, room: &str, identity: &str) -> Result { + token::create( + &self.key, + &self.secret, + Some(identity), + token::VideoGrant { + room: Some(room), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + }, + ) + } + fn request( &self, path: &str, diff --git a/crates/rpc/proto/zed.proto b/crates/rpc/proto/zed.proto index 1248bb0551..ee612e0499 100644 --- a/crates/rpc/proto/zed.proto +++ b/crates/rpc/proto/zed.proto @@ -140,7 +140,8 @@ message Test { message CreateRoom {} message CreateRoomResponse { - uint64 id = 1; + Room room = 1; + optional string live_kit_token = 2; } message JoinRoom { @@ -149,6 +150,7 @@ message JoinRoom { message JoinRoomResponse { Room room = 1; + optional string live_kit_token = 2; } message LeaveRoom { @@ -156,8 +158,10 @@ message LeaveRoom { } message Room { - repeated Participant participants = 1; - repeated uint64 pending_participant_user_ids = 2; + uint64 id = 1; + repeated Participant participants = 2; + repeated uint64 pending_participant_user_ids = 3; + string live_kit_room = 4; } message Participant { From cce00526b9a65afb415dcacd06c47b47fcac2af5 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 17 Oct 2022 14:03:44 +0200 Subject: [PATCH 12/63] Remove participants from live-kit rooms when they leave zed rooms --- crates/collab/.env.toml | 3 +++ crates/collab/src/rpc.rs | 41 ++++++++++++++++++++++--------- crates/collab/src/rpc/store.rs | 20 ++++++++------- crates/live_kit_server/src/api.rs | 36 +++++++++++++++++++++------ 4 files changed, 71 insertions(+), 29 deletions(-) diff --git a/crates/collab/.env.toml b/crates/collab/.env.toml index 98198eb775..8a9e28a7c8 100644 --- a/crates/collab/.env.toml +++ b/crates/collab/.env.toml @@ -2,6 +2,9 @@ DATABASE_URL = "postgres://postgres@localhost/zed" HTTP_PORT = 8080 API_TOKEN = "secret" INVITE_LINK_PREFIX = "http://localhost:3000/invites/" +LIVE_KIT_SERVER = "http://localhost:7880" +LIVE_KIT_KEY = "devkey" +LIVE_KIT_SECRET = "secret" # HONEYCOMB_API_KEY= # HONEYCOMB_DATASET= diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index 8983f69f9b..888217f5aa 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -473,6 +473,7 @@ impl Server { let mut projects_to_unshare = Vec::new(); let mut contacts_to_update = HashSet::default(); + let mut room_left = None; { let mut store = self.store().await; let removed_connection = store.remove_connection(connection_id)?; @@ -501,23 +502,24 @@ impl Server { }); } + if let Some(room) = removed_connection.room { + self.room_updated(&room); + room_left = Some(self.room_left(&room, removed_connection.user_id)); + } + + contacts_to_update.insert(removed_connection.user_id); for connection_id in removed_connection.canceled_call_connection_ids { self.peer .send(connection_id, proto::CallCanceled {}) .trace_err(); contacts_to_update.extend(store.user_id_for_connection(connection_id).ok()); } - - if let Some(room) = removed_connection - .room_id - .and_then(|room_id| store.room(room_id)) - { - self.room_updated(room); - } - - contacts_to_update.insert(removed_connection.user_id); }; + if let Some(room_left) = room_left { + room_left.await.trace_err(); + } + for user_id in contacts_to_update { self.update_user_contacts(user_id).await.trace_err(); } @@ -682,6 +684,7 @@ impl Server { async fn leave_room(self: Arc, message: TypedEnvelope) -> Result<()> { let mut contacts_to_update = HashSet::default(); + let room_left; { let mut store = self.store().await; let user_id = store.user_id_for_connection(message.sender_id)?; @@ -720,9 +723,8 @@ impl Server { } } - if let Some(room) = left_room.room { - self.room_updated(room); - } + self.room_updated(&left_room.room); + room_left = self.room_left(&left_room.room, user_id); for connection_id in left_room.canceled_call_connection_ids { self.peer @@ -732,6 +734,7 @@ impl Server { } } + room_left.await.trace_err(); for user_id in contacts_to_update { self.update_user_contacts(user_id).await?; } @@ -880,6 +883,20 @@ impl Server { } } + fn room_left(&self, room: &proto::Room, user_id: UserId) -> impl Future> { + let client = self.app_state.live_kit_client.clone(); + let room_name = room.live_kit_room.clone(); + async move { + if let Some(client) = client { + client + .remove_participant(room_name, user_id.to_string()) + .await?; + } + + Ok(()) + } + } + async fn share_project( self: Arc, request: TypedEnvelope, diff --git a/crates/collab/src/rpc/store.rs b/crates/collab/src/rpc/store.rs index 096ac0fa06..df8be453b1 100644 --- a/crates/collab/src/rpc/store.rs +++ b/crates/collab/src/rpc/store.rs @@ -4,7 +4,7 @@ use collections::{btree_map, BTreeMap, BTreeSet, HashMap, HashSet}; use nanoid::nanoid; use rpc::{proto, ConnectionId}; use serde::Serialize; -use std::{mem, path::PathBuf, str, time::Duration}; +use std::{borrow::Cow, mem, path::PathBuf, str, time::Duration}; use time::OffsetDateTime; use tracing::instrument; use util::post_inc; @@ -85,12 +85,12 @@ pub struct Channel { pub type ReplicaId = u16; #[derive(Default)] -pub struct RemovedConnectionState { +pub struct RemovedConnectionState<'a> { pub user_id: UserId, pub hosted_projects: Vec, pub guest_projects: Vec, pub contact_ids: HashSet, - pub room_id: Option, + pub room: Option>, pub canceled_call_connection_ids: Vec, } @@ -103,7 +103,7 @@ pub struct LeftProject { } pub struct LeftRoom<'a> { - pub room: Option<&'a proto::Room>, + pub room: Cow<'a, proto::Room>, pub unshared_projects: Vec, pub left_projects: Vec, pub canceled_call_connection_ids: Vec, @@ -218,7 +218,7 @@ impl Store { let left_room = self.leave_room(room_id, connection_id)?; result.hosted_projects = left_room.unshared_projects; result.guest_projects = left_room.left_projects; - result.room_id = Some(room_id); + result.room = Some(Cow::Owned(left_room.room.into_owned())); result.canceled_call_connection_ids = left_room.canceled_call_connection_ids; } @@ -495,12 +495,14 @@ impl Store { } }); - if room.participants.is_empty() && room.pending_participant_user_ids.is_empty() { - self.rooms.remove(&room_id); - } + let room = if room.participants.is_empty() && room.pending_participant_user_ids.is_empty() { + Cow::Owned(self.rooms.remove(&room_id).unwrap()) + } else { + Cow::Borrowed(self.rooms.get(&room_id).unwrap()) + }; Ok(LeftRoom { - room: self.rooms.get(&room_id), + room, unshared_projects, left_projects, canceled_call_connection_ids, diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index 2bbad785c3..cc235c15be 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -2,13 +2,14 @@ use crate::{proto, token}; use anyhow::{anyhow, Result}; use prost::Message; use reqwest::header::CONTENT_TYPE; -use std::future::Future; +use std::{future::Future, sync::Arc}; +#[derive(Clone)] pub struct Client { http: reqwest::Client, - uri: String, - key: String, - secret: String, + uri: Arc, + key: Arc, + secret: Arc, } impl Client { @@ -19,9 +20,9 @@ impl Client { Self { http: reqwest::Client::new(), - uri, - key, - secret, + uri: uri.into(), + key: key.into(), + secret: secret.into(), } } @@ -49,7 +50,26 @@ impl Client { proto::DeleteRoomRequest { room: name }, ); async move { - response.await?; + let _: proto::DeleteRoomResponse = response.await?; + Ok(()) + } + } + + pub fn remove_participant( + &self, + room: String, + identity: String, + ) -> impl Future> { + let response = self.request( + "twirp/livekit.RoomService/RemoveParticipant", + token::VideoGrant { + room_admin: Some(true), + ..Default::default() + }, + proto::RoomParticipantIdentity { room, identity }, + ); + async move { + let _: proto::RemoveParticipantResponse = response.await?; Ok(()) } } From 81d83841ab5b8a62dac6458d74637a38d629e918 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 17 Oct 2022 14:50:05 +0200 Subject: [PATCH 13/63] WIP: Start integrating screen-sharing --- Cargo.lock | 1 + crates/call/Cargo.toml | 1 + crates/call/src/room.rs | 64 ++++++++++++++- crates/collab/src/rpc.rs | 80 +++++++++---------- .../Sources/LiveKitBridge/LiveKitBridge.swift | 2 +- crates/live_kit_server/src/api.rs | 20 +++-- crates/rpc/proto/zed.proto | 9 ++- crates/zed/build.rs | 6 ++ 8 files changed, 130 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ea784749d..67160b4d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -731,6 +731,7 @@ dependencies = [ "collections", "futures 0.3.24", "gpui", + "live_kit_client", "postage", "project", "util", diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index e725c7cfe3..663b561f29 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -19,6 +19,7 @@ test-support = [ [dependencies] client = { path = "../client" } collections = { path = "../collections" } +live_kit_client = { path = "../live_kit_client" } gpui = { path = "../gpui" } project = { path = "../project" } util = { path = "../util" } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 258400da92..e10f55f13e 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -7,6 +7,7 @@ use client::{proto, Client, PeerId, TypedEnvelope, User, UserStore}; use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; +use live_kit_client::LocalVideoTrack; use project::Project; use std::sync::Arc; use util::ResultExt; @@ -26,6 +27,7 @@ pub enum Event { pub struct Room { id: u64, + live_kit_room: Option>, status: RoomStatus, local_participant: LocalParticipant, remote_participants: BTreeMap, @@ -50,6 +52,7 @@ impl Entity for Room { impl Room { fn new( id: u64, + live_kit_connection_info: Option, client: Arc, user_store: ModelHandle, cx: &mut ModelContext, @@ -69,8 +72,27 @@ impl Room { }) .detach(); + let live_kit_room = if let Some(connection_info) = live_kit_connection_info { + let room = live_kit_client::Room::new(); + let mut tracks = room.remote_video_tracks(); + cx.foreground() + .spawn(async move { + while let Some(track) = tracks.next().await { + dbg!("received track"); + } + }) + .detach(); + cx.foreground() + .spawn(room.connect(&connection_info.server_url, &connection_info.token)) + .detach_and_log_err(cx); + Some(room) + } else { + None + }; + Self { id, + live_kit_room, status: RoomStatus::Online, participant_user_ids: Default::default(), local_participant: Default::default(), @@ -95,7 +117,15 @@ impl Room { cx.spawn(|mut cx| async move { let response = client.request(proto::CreateRoom {}).await?; let room_proto = response.room.ok_or_else(|| anyhow!("invalid room"))?; - let room = cx.add_model(|cx| Self::new(room_proto.id, client, user_store, cx)); + let room = cx.add_model(|cx| { + Self::new( + room_proto.id, + response.live_kit_connection_info, + client, + user_store, + cx, + ) + }); let initial_project_id = if let Some(initial_project) = initial_project { let initial_project_id = room @@ -131,7 +161,15 @@ impl Room { cx.spawn(|mut cx| async move { let response = client.request(proto::JoinRoom { id: room_id }).await?; let room_proto = response.room.ok_or_else(|| anyhow!("invalid room"))?; - let room = cx.add_model(|cx| Self::new(room_id, client, user_store, cx)); + let room = cx.add_model(|cx| { + Self::new( + room_id, + response.live_kit_connection_info, + client, + user_store, + cx, + ) + }); room.update(&mut cx, |room, cx| { room.leave_when_empty = true; room.apply_room_update(room_proto, cx)?; @@ -458,6 +496,28 @@ impl Room { Ok(()) }) } + + pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { + if self.status.is_offline() { + return Task::ready(Err(anyhow!("room is offline"))); + } + + let room = if let Some(room) = self.live_kit_room.as_ref() { + room.clone() + } else { + return Task::ready(Err(anyhow!("not connected to LiveKit"))); + }; + + cx.foreground().spawn(async move { + let displays = live_kit_client::display_sources().await?; + let display = displays + .first() + .ok_or_else(|| anyhow!("no display found"))?; + let track = LocalVideoTrack::screen_share_for_display(display); + room.publish_video_track(&track).await?; + Ok(()) + }) + } } #[derive(Copy, Clone, PartialEq, Eq)] diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index 888217f5aa..6ba056d03a 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -5,7 +5,7 @@ use crate::{ db::{self, ChannelId, MessageId, ProjectId, User, UserId}, AppState, Result, }; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; use async_tungstenite::tungstenite::{ protocol::CloseFrame as TungsteniteCloseFrame, Message as TungsteniteMessage, }; @@ -605,37 +605,34 @@ impl Server { room = store.create_room(request.sender_id)?.clone(); } - let live_kit_token = if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { - if let Some(_) = live_kit - .create_room(room.live_kit_room.clone()) - .await - .with_context(|| { - format!( - "error creating LiveKit room (LiveKit room: {}, Zed room: {})", - room.live_kit_room, room.id - ) - }) - .trace_err() - { - live_kit - .room_token_for_user(&room.live_kit_room, &user_id.to_string()) - .with_context(|| { - format!( - "error creating LiveKit access token (LiveKit room: {}, Zed room: {})", - room.live_kit_room, room.id - ) - }) + let live_kit_connection_info = + if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { + if let Some(_) = live_kit + .create_room(room.live_kit_room.clone()) + .await .trace_err() + { + if let Some(token) = live_kit + .room_token_for_user(&room.live_kit_room, &user_id.to_string()) + .trace_err() + { + Some(proto::LiveKitConnectionInfo { + server_url: live_kit.url().into(), + token, + }) + } else { + None + } + } else { + None + } } else { None - } - } else { - None - }; + }; response.send(proto::CreateRoomResponse { room: Some(room), - live_kit_token, + live_kit_connection_info, })?; self.update_user_contacts(user_id).await?; Ok(()) @@ -658,23 +655,26 @@ impl Server { .trace_err(); } - let live_kit_token = if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { - live_kit - .room_token_for_user(&room.live_kit_room, &user_id.to_string()) - .with_context(|| { - format!( - "error creating LiveKit access token (LiveKit room: {}, Zed room: {})", - room.live_kit_room, room.id - ) - }) - .trace_err() - } else { - None - }; + let live_kit_connection_info = + if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { + if let Some(token) = live_kit + .room_token_for_user(&room.live_kit_room, &user_id.to_string()) + .trace_err() + { + Some(proto::LiveKitConnectionInfo { + server_url: live_kit.url().into(), + token, + }) + } else { + None + } + } else { + None + }; response.send(proto::JoinRoomResponse { room: Some(room.clone()), - live_kit_token, + live_kit_connection_info, })?; self.room_updated(room); } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index b33b0ecdb4..9a7b25114e 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -107,7 +107,7 @@ public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRaw @_cdecl("LKDisplaySources") public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { - MacOSScreenCapturer.displaySources().then { displaySources in + MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in callback(data, displaySources as CFArray, nil) }.catch { error in callback(data, nil, error.localizedDescription as CFString) diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index cc235c15be..c21f586434 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -7,25 +7,29 @@ use std::{future::Future, sync::Arc}; #[derive(Clone)] pub struct Client { http: reqwest::Client, - uri: Arc, + url: Arc, key: Arc, secret: Arc, } impl Client { - pub fn new(mut uri: String, key: String, secret: String) -> Self { - if uri.ends_with('/') { - uri.pop(); + pub fn new(mut url: String, key: String, secret: String) -> Self { + if url.ends_with('/') { + url.pop(); } Self { http: reqwest::Client::new(), - uri: uri.into(), + url: url.into(), key: key.into(), secret: secret.into(), } } + pub fn url(&self) -> &str { + &self.url + } + pub fn create_room(&self, name: String) -> impl Future> { self.request( "twirp/livekit.RoomService/CreateRoom", @@ -101,11 +105,11 @@ impl Client { { let client = self.http.clone(); let token = token::create(&self.key, &self.secret, None, grant); - let uri = format!("{}/{}", self.uri, path); + let url = format!("{}/{}", self.url, path); async move { let token = token?; let response = client - .post(&uri) + .post(&url) .header(CONTENT_TYPE, "application/protobuf") .bearer_auth(token) .body(body.encode_to_vec()) @@ -116,7 +120,7 @@ impl Client { } else { Err(anyhow!( "POST {} failed with status code {:?}, {:?}", - uri, + url, response.status(), response.text().await )) diff --git a/crates/rpc/proto/zed.proto b/crates/rpc/proto/zed.proto index ee612e0499..3f6bf6149e 100644 --- a/crates/rpc/proto/zed.proto +++ b/crates/rpc/proto/zed.proto @@ -141,7 +141,7 @@ message CreateRoom {} message CreateRoomResponse { Room room = 1; - optional string live_kit_token = 2; + optional LiveKitConnectionInfo live_kit_connection_info = 2; } message JoinRoom { @@ -150,7 +150,7 @@ message JoinRoom { message JoinRoomResponse { Room room = 1; - optional string live_kit_token = 2; + optional LiveKitConnectionInfo live_kit_connection_info = 2; } message LeaveRoom { @@ -225,6 +225,11 @@ message RoomUpdated { Room room = 1; } +message LiveKitConnectionInfo { + string server_url = 1; + string token = 2; +} + message ShareProject { uint64 room_id = 1; repeated WorktreeMetadata worktrees = 2; diff --git a/crates/zed/build.rs b/crates/zed/build.rs index d3167851a0..ed83137f95 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -7,6 +7,12 @@ fn main() { println!("cargo:rustc-env=ZED_AMPLITUDE_API_KEY={api_key}"); } + // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); + + // Register exported Objective-C selectors, protocols, etc + println!("cargo:rustc-link-arg=-Wl,-ObjC"); + let output = Command::new("npm") .current_dir("../../styles") .args(["install", "--no-save"]) From 499b8f5f5536e76d292e0746da37f26f02fd697d Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 17 Oct 2022 18:00:54 +0200 Subject: [PATCH 14/63] WIP --- Cargo.lock | 26 +-- crates/call/Cargo.toml | 3 +- crates/call/src/call.rs | 2 + crates/call/src/participant.rs | 20 ++- crates/call/src/room.rs | 111 +++++++++++-- crates/capture/Cargo.toml | 29 ---- crates/capture/build.rs | 7 - crates/capture/src/main.rs | 150 ------------------ crates/collab/src/rpc.rs | 16 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 25 ++- crates/live_kit_client/src/live_kit_client.rs | 85 ++++++++-- crates/live_kit_server/src/api.rs | 2 +- crates/workspace/src/pane_group.rs | 32 ++-- 13 files changed, 245 insertions(+), 263 deletions(-) delete mode 100644 crates/capture/Cargo.toml delete mode 100644 crates/capture/build.rs delete mode 100644 crates/capture/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 67160b4d0e..c896fdf7ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -732,6 +732,7 @@ dependencies = [ "futures 0.3.24", "gpui", "live_kit_client", + "media", "postage", "project", "util", @@ -803,31 +804,6 @@ dependencies = [ "winx", ] -[[package]] -name = "capture" -version = "0.1.0" -dependencies = [ - "anyhow", - "bindgen", - "block", - "byteorder", - "bytes 1.2.1", - "cocoa", - "core-foundation", - "core-graphics", - "foreign-types", - "futures 0.3.24", - "gpui", - "live_kit_client", - "live_kit_server", - "log", - "media", - "objc", - "parking_lot 0.11.2", - "postage", - "simplelog", -] - [[package]] name = "castaway" version = "0.1.2" diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index 663b561f29..bd592e1de1 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -19,8 +19,9 @@ test-support = [ [dependencies] client = { path = "../client" } collections = { path = "../collections" } -live_kit_client = { path = "../live_kit_client" } gpui = { path = "../gpui" } +live_kit_client = { path = "../live_kit_client" } +media = { path = "../media" } project = { path = "../project" } util = { path = "../util" } diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 6b06d04375..056c6f1260 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -132,6 +132,8 @@ impl ActiveCall { Room::create(recipient_user_id, initial_project, client, user_store, cx) }) .await?; + room.update(&mut cx, |room, cx| room.share_screen(cx)) + .await?; this.update(&mut cx, |this, cx| this.set_room(Some(room), cx)); }; diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index a5be5b4af2..09174df793 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -1,6 +1,8 @@ use anyhow::{anyhow, Result}; use client::{proto, User}; -use gpui::WeakModelHandle; +use collections::HashMap; +use gpui::{Task, WeakModelHandle}; +use media::core_video::CVImageBuffer; use project::Project; use std::sync::Arc; @@ -34,9 +36,23 @@ pub struct LocalParticipant { pub active_project: Option>, } -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct RemoteParticipant { pub user: Arc, pub projects: Vec, pub location: ParticipantLocation, + pub tracks: HashMap, +} + +#[derive(Clone)] +pub struct RemoteVideoTrack { + pub(crate) frame: Option, + pub(crate) _live_kit_track: Arc, + pub(crate) _maintain_frame: Arc>, +} + +impl RemoteVideoTrack { + pub fn frame(&self) -> Option<&CVImageBuffer> { + self.frame.as_ref() + } } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index e10f55f13e..dd1e4598db 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -1,5 +1,5 @@ use crate::{ - participant::{LocalParticipant, ParticipantLocation, RemoteParticipant}, + participant::{LocalParticipant, ParticipantLocation, RemoteParticipant, RemoteVideoTrack}, IncomingCall, }; use anyhow::{anyhow, Result}; @@ -7,7 +7,8 @@ use client::{proto, Client, PeerId, TypedEnvelope, User, UserStore}; use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; -use live_kit_client::LocalVideoTrack; +use live_kit_client::{LocalVideoTrack, RemoteVideoTrackChange}; +use postage::watch; use project::Project; use std::sync::Arc; use util::ResultExt; @@ -27,7 +28,7 @@ pub enum Event { pub struct Room { id: u64, - live_kit_room: Option>, + live_kit_room: Option<(Arc, Task<()>)>, status: RoomStatus, local_participant: LocalParticipant, remote_participants: BTreeMap, @@ -75,17 +76,23 @@ impl Room { let live_kit_room = if let Some(connection_info) = live_kit_connection_info { let room = live_kit_client::Room::new(); let mut tracks = room.remote_video_tracks(); - cx.foreground() - .spawn(async move { - while let Some(track) = tracks.next().await { - dbg!("received track"); - } - }) - .detach(); + let maintain_room = cx.spawn_weak(|this, mut cx| async move { + while let Some(track_change) = tracks.next().await { + let this = if let Some(this) = this.upgrade(&cx) { + this + } else { + break; + }; + + this.update(&mut cx, |this, cx| { + this.remote_video_track_changed(track_change, cx).log_err() + }); + } + }); cx.foreground() .spawn(room.connect(&connection_info.server_url, &connection_info.token)) .detach_and_log_err(cx); - Some(room) + Some((room, maintain_room)) } else { None }; @@ -318,8 +325,20 @@ impl Room { projects: participant.projects, location: ParticipantLocation::from_proto(participant.location) .unwrap_or(ParticipantLocation::External), + tracks: Default::default(), }, ); + + if let Some((room, _)) = this.live_kit_room.as_ref() { + for track in + room.video_tracks_for_remote_participant(peer_id.0.to_string()) + { + this.remote_video_track_changed( + RemoteVideoTrackChange::Subscribed(track), + cx, + ); + } + } } this.remote_participants.retain(|_, participant| { @@ -357,6 +376,74 @@ impl Room { Ok(()) } + fn remote_video_track_changed( + &mut self, + change: RemoteVideoTrackChange, + cx: &mut ModelContext, + ) -> Result<()> { + match change { + RemoteVideoTrackChange::Subscribed(track) => { + let peer_id = PeerId(track.publisher_id().parse()?); + let track_id = track.id().to_string(); + let participant = self + .remote_participants + .get_mut(&peer_id) + .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; + let (mut tx, mut rx) = watch::channel(); + track.add_renderer(move |frame| *tx.borrow_mut() = Some(frame)); + participant.tracks.insert( + track_id.clone(), + RemoteVideoTrack { + frame: None, + _live_kit_track: track, + _maintain_frame: Arc::new(cx.spawn_weak(|this, mut cx| async move { + while let Some(frame) = rx.next().await { + let this = if let Some(this) = this.upgrade(&cx) { + this + } else { + break; + }; + + let done = this.update(&mut cx, |this, cx| { + // TODO: replace this with an emit. + cx.notify(); + if let Some(track) = + this.remote_participants.get_mut(&peer_id).and_then( + |participant| participant.tracks.get_mut(&track_id), + ) + { + track.frame = frame; + false + } else { + true + } + }); + + if done { + break; + } + } + })), + }, + ); + } + RemoteVideoTrackChange::Unsubscribed { + publisher_id, + track_id, + } => { + let peer_id = PeerId(publisher_id.parse()?); + let participant = self + .remote_participants + .get_mut(&peer_id) + .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; + participant.tracks.remove(&track_id); + } + } + + cx.notify(); + Ok(()) + } + fn check_invariants(&self) { #[cfg(any(test, feature = "test-support"))] { @@ -502,7 +589,7 @@ impl Room { return Task::ready(Err(anyhow!("room is offline"))); } - let room = if let Some(room) = self.live_kit_room.as_ref() { + let room = if let Some((room, _)) = self.live_kit_room.as_ref() { room.clone() } else { return Task::ready(Err(anyhow!("not connected to LiveKit"))); diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml deleted file mode 100644 index a5ec131ff1..0000000000 --- a/crates/capture/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "capture" -version = "0.1.0" -edition = "2021" -description = "An example of screen capture" - -[dependencies] -gpui = { path = "../gpui" } -live_kit_client = { path = "../live_kit_client" } -live_kit_server = { path = "../live_kit_server" } -media = { path = "../media" } - -anyhow = "1.0.38" -block = "0.1" -bytes = "1.2" -byteorder = "1.4" -cocoa = "0.24" -core-foundation = "0.9.3" -core-graphics = "0.22.3" -foreign-types = "0.3" -futures = "0.3" -log = { version = "0.4.16", features = ["kv_unstable_serde"] } -objc = "0.2" -parking_lot = "0.11.1" -postage = { version = "0.4.1", features = ["futures-traits"] } -simplelog = "0.9" - -[build-dependencies] -bindgen = "0.59.2" diff --git a/crates/capture/build.rs b/crates/capture/build.rs deleted file mode 100644 index 41f60ff486..0000000000 --- a/crates/capture/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - - // Register exported Objective-C selectors, protocols, etc - println!("cargo:rustc-link-arg=-Wl,-ObjC"); -} diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs deleted file mode 100644 index 79cb0df922..0000000000 --- a/crates/capture/src/main.rs +++ /dev/null @@ -1,150 +0,0 @@ -use futures::StreamExt; -use gpui::{ - actions, - elements::{Canvas, *}, - keymap::Binding, - platform::current::Surface, - Menu, MenuItem, ViewContext, -}; -use live_kit_client::{LocalVideoTrack, Room}; -use log::LevelFilter; -use media::core_video::CVImageBuffer; -use postage::watch; -use simplelog::SimpleLogger; -use std::sync::Arc; - -actions!(capture, [Quit]); - -fn main() { - SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); - - gpui::App::new(()).unwrap().run(|cx| { - cx.platform().activate(true); - cx.add_global_action(quit); - - cx.add_bindings([Binding::new("cmd-q", Quit, None)]); - cx.set_menus(vec![Menu { - name: "Zed", - items: vec![MenuItem::Action { - name: "Quit", - action: Box::new(Quit), - }], - }]); - - let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap(); - let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); - let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); - - cx.spawn(|mut cx| async move { - let user1_token = live_kit_server::token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-1"), - live_kit_server::token::VideoGrant { - room: Some("test-room"), - room_join: Some(true), - can_publish: Some(true), - can_subscribe: Some(true), - ..Default::default() - }, - ) - .unwrap(); - let room1 = Room::new(); - room1.connect(&live_kit_url, &user1_token).await.unwrap(); - - let user2_token = live_kit_server::token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-2"), - live_kit_server::token::VideoGrant { - room: Some("test-room"), - room_join: Some(true), - can_publish: Some(true), - can_subscribe: Some(true), - ..Default::default() - }, - ) - .unwrap(); - - let room2 = Room::new(); - room2.connect(&live_kit_url, &user2_token).await.unwrap(); - cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); - - let display_sources = live_kit_client::display_sources().await.unwrap(); - let track = LocalVideoTrack::screen_share_for_display(display_sources.first().unwrap()); - room1.publish_video_track(&track).await.unwrap(); - }) - .detach(); - }); -} - -struct ScreenCaptureView { - image_buffer: Option, - _room: Arc, -} - -impl gpui::Entity for ScreenCaptureView { - type Event = (); -} - -impl ScreenCaptureView { - pub fn new(room: Arc, cx: &mut ViewContext) -> Self { - let mut remote_video_tracks = room.remote_video_tracks(); - cx.spawn_weak(|this, mut cx| async move { - if let Some(video_track) = remote_video_tracks.next().await { - let (mut frames_tx, mut frames_rx) = watch::channel_with(None); - video_track.add_renderer(move |frame| *frames_tx.borrow_mut() = Some(frame)); - - while let Some(frame) = frames_rx.next().await { - if let Some(this) = this.upgrade(&cx) { - this.update(&mut cx, |this, cx| { - this.image_buffer = frame; - cx.notify(); - }); - } else { - break; - } - } - } - }) - .detach(); - - Self { - image_buffer: None, - _room: room, - } - } -} - -impl gpui::View for ScreenCaptureView { - fn ui_name() -> &'static str { - "View" - } - - fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - let image_buffer = self.image_buffer.clone(); - let canvas = Canvas::new(move |bounds, _, cx| { - if let Some(image_buffer) = image_buffer.clone() { - cx.scene.push_surface(Surface { - bounds, - image_buffer, - }); - } - }); - - if let Some(image_buffer) = self.image_buffer.as_ref() { - canvas - .constrained() - .with_width(image_buffer.width() as f32) - .with_height(image_buffer.height() as f32) - .aligned() - .boxed() - } else { - canvas.boxed() - } - } -} - -fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { - cx.platform().quit(); -} diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index 6ba056d03a..70e47a755e 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -504,7 +504,7 @@ impl Server { if let Some(room) = removed_connection.room { self.room_updated(&room); - room_left = Some(self.room_left(&room, removed_connection.user_id)); + room_left = Some(self.room_left(&room, connection_id)); } contacts_to_update.insert(removed_connection.user_id); @@ -613,7 +613,7 @@ impl Server { .trace_err() { if let Some(token) = live_kit - .room_token_for_user(&room.live_kit_room, &user_id.to_string()) + .room_token(&room.live_kit_room, &request.sender_id.to_string()) .trace_err() { Some(proto::LiveKitConnectionInfo { @@ -658,7 +658,7 @@ impl Server { let live_kit_connection_info = if let Some(live_kit) = self.app_state.live_kit_client.as_ref() { if let Some(token) = live_kit - .room_token_for_user(&room.live_kit_room, &user_id.to_string()) + .room_token(&room.live_kit_room, &request.sender_id.to_string()) .trace_err() { Some(proto::LiveKitConnectionInfo { @@ -724,7 +724,7 @@ impl Server { } self.room_updated(&left_room.room); - room_left = self.room_left(&left_room.room, user_id); + room_left = self.room_left(&left_room.room, message.sender_id); for connection_id in left_room.canceled_call_connection_ids { self.peer @@ -883,13 +883,17 @@ impl Server { } } - fn room_left(&self, room: &proto::Room, user_id: UserId) -> impl Future> { + fn room_left( + &self, + room: &proto::Room, + connection_id: ConnectionId, + ) -> impl Future> { let client = self.app_state.live_kit_client.clone(); let room_name = room.live_kit_room.clone(); async move { if let Some(client) = client { client - .remove_participant(room_name, user_id.to_string()) + .remove_participant(room_name, connection_id.to_string()) .await?; } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 9a7b25114e..b39f359737 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -4,16 +4,24 @@ import WebRTC class LKRoomDelegate: RoomDelegate { var data: UnsafeRawPointer - var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void + var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void + var onDidUnsubscribeFromRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { + init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) { self.data = data self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack + self.onDidUnsubscribeFromRemoteVideoTrack = onDidUnsubscribeFromRemoteVideoTrack } func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + self.onDidSubscribeToRemoteVideoTrack(self.data, participant.sid as CFString, track.id as CFString, Unmanaged.passRetained(track).toOpaque()) + } + } + + func room(_ room: Room, participant: RemoteParticipant, didUnsubscribe publication: RemoteTrackPublication, track: Track) { + if track.kind == .video { + self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.sid as CFString, track.id as CFString) } } } @@ -53,8 +61,8 @@ public func LKRelease(ptr: UnsafeRawPointer) { } @_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack) +public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack) return Unmanaged.passRetained(delegate).toOpaque() } @@ -86,6 +94,13 @@ public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPoin } } +@_cdecl("LKRoomVideoTracksForRemoteParticipant") +public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() + let tracks = room.remoteParticipants[participantId as Sid]?.videoTracks.compactMap { $0.track as? RemoteVideoTrack } + return tracks as CFArray? +} + @_cdecl("LKCreateScreenShareTrackForDisplay") public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { let display = Unmanaged.fromOpaque(display).takeUnretainedValue() diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index f8a94f6ae5..6bd5ce47e6 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -22,8 +22,15 @@ extern "C" { callback_data: *mut c_void, on_did_subscribe_to_remote_video_track: extern "C" fn( callback_data: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, remote_track: *const c_void, ), + on_did_unsubscribe_from_remote_video_track: extern "C" fn( + callback_data: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + ), ) -> *const c_void; fn LKRoomCreate(delegate: *const c_void) -> *const c_void; @@ -62,7 +69,7 @@ extern "C" { pub struct Room { native_room: *const c_void, - remote_video_track_subscribers: Mutex>>>, + remote_video_track_subscribers: Mutex>>, _delegate: RoomDelegate, } @@ -103,7 +110,7 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } - pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver> { + pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver { let (tx, rx) = mpsc::unbounded(); self.remote_video_track_subscribers.lock().push(tx); rx @@ -111,9 +118,20 @@ impl Room { fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { let track = Arc::new(track); - self.remote_video_track_subscribers - .lock() - .retain(|tx| tx.unbounded_send(track.clone()).is_ok()); + self.remote_video_track_subscribers.lock().retain(|tx| { + tx.unbounded_send(RemoteVideoTrackChange::Subscribed(track.clone())) + .is_ok() + }); + } + + fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { + self.remote_video_track_subscribers.lock().retain(|tx| { + tx.unbounded_send(RemoteVideoTrackChange::Unsubscribed { + publisher_id: publisher_id.clone(), + track_id: track_id.clone(), + }) + .is_ok() + }); } fn build_done_callback() -> ( @@ -157,6 +175,7 @@ impl RoomDelegate { LKRoomDelegateCreate( weak_room as *mut c_void, Self::on_did_subscribe_to_remote_video_track, + Self::on_did_unsubscribe_from_remote_video_track, ) }; Self { @@ -165,14 +184,39 @@ impl RoomDelegate { } } - extern "C" fn on_did_subscribe_to_remote_video_track(room: *mut c_void, track: *const c_void) { + extern "C" fn on_did_subscribe_to_remote_video_track( + room: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + track: *const c_void, + ) { let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track = RemoteVideoTrack(track); + let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; + let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; + let track = RemoteVideoTrack { + id: track_id, + native_track: track, + publisher_id, + }; if let Some(room) = room.upgrade() { room.did_subscribe_to_remote_video_track(track); } let _ = Weak::into_raw(room); } + + extern "C" fn on_did_unsubscribe_from_remote_video_track( + room: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + ) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; + let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; + if let Some(room) = room.upgrade() { + room.did_unsubscribe_from_remote_video_track(publisher_id, track_id); + } + let _ = Weak::into_raw(room); + } } impl Drop for RoomDelegate { @@ -198,9 +242,22 @@ impl Drop for LocalVideoTrack { } } -pub struct RemoteVideoTrack(*const c_void); +#[derive(Debug)] +pub struct RemoteVideoTrack { + id: String, + native_track: *const c_void, + publisher_id: String, +} impl RemoteVideoTrack { + pub fn id(&self) -> &str { + &self.id + } + + pub fn publisher_id(&self) -> &str { + &self.publisher_id + } + pub fn add_renderer(&self, callback: F) where F: 'static + FnMut(CVImageBuffer), @@ -226,17 +283,25 @@ impl RemoteVideoTrack { unsafe { let renderer = LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); - LKVideoTrackAddRenderer(self.0, renderer); + LKVideoTrackAddRenderer(self.native_track, renderer); } } } impl Drop for RemoteVideoTrack { fn drop(&mut self) { - unsafe { LKRelease(self.0) } + unsafe { LKRelease(self.native_track) } } } +pub enum RemoteVideoTrackChange { + Subscribed(Arc), + Unsubscribed { + publisher_id: String, + track_id: String, + }, +} + pub struct MacOSDisplay(*const c_void); impl Drop for MacOSDisplay { diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index c21f586434..bd98d27069 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -78,7 +78,7 @@ impl Client { } } - pub fn room_token_for_user(&self, room: &str, identity: &str) -> Result { + pub fn room_token(&self, room: &str, identity: &str) -> Result { token::create( &self.key, &self.secret, diff --git a/crates/workspace/src/pane_group.rs b/crates/workspace/src/pane_group.rs index 10fac09fff..c1c4aef096 100644 --- a/crates/workspace/src/pane_group.rs +++ b/crates/workspace/src/pane_group.rs @@ -201,21 +201,23 @@ impl Member { .right() .boxed(), ), - call::ParticipantLocation::External => Some( - Label::new( - format!( - "{} is viewing a window outside of Zed", - leader.user.github_login - ), - theme.workspace.external_location_message.text.clone(), - ) - .contained() - .with_style(theme.workspace.external_location_message.container) - .aligned() - .bottom() - .right() - .boxed(), - ), + call::ParticipantLocation::External => { + let frame = leader + .tracks + .values() + .next() + .and_then(|track| track.frame()) + .cloned(); + return Canvas::new(move |bounds, _, cx| { + if let Some(frame) = frame.clone() { + cx.scene.push_surface(gpui::mac::Surface { + bounds, + image_buffer: frame, + }); + } + }) + .boxed(); + } } } else { None From 8c1c98a0bf4b2822d844e2a382920b69b7aac519 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 17 Oct 2022 23:24:48 -0600 Subject: [PATCH 15/63] WIP --- Cargo.lock | 14 ++ crates/call/src/room.rs | 42 ++--- crates/live_kit_client/.cargo/config.toml | 2 + crates/live_kit_client/Cargo.toml | 27 +++ .../Sources/LiveKitBridge/LiveKitBridge.swift | 33 +++- crates/live_kit_client/build.rs | 5 + crates/live_kit_client/examples/test_app.rs | 160 ++++++++++++++++++ crates/live_kit_client/src/live_kit_client.rs | 88 +++++++--- crates/live_kit_server/src/token.rs | 12 ++ 9 files changed, 336 insertions(+), 47 deletions(-) create mode 100644 crates/live_kit_client/.cargo/config.toml create mode 100644 crates/live_kit_client/examples/test_app.rs diff --git a/Cargo.lock b/Cargo.lock index c896fdf7ce..4814befcb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3166,13 +3166,27 @@ name = "live_kit_client" version = "0.1.0" dependencies = [ "anyhow", + "block", + "byteorder", + "bytes 1.2.1", + "cocoa", "core-foundation", "core-graphics", + "foreign-types", "futures 0.3.24", + "gpui", + "hmac 0.12.1", + "jwt", + "live_kit_server", + "log", "media", + "objc", "parking_lot 0.11.2", + "postage", "serde", "serde_json", + "sha2 0.10.6", + "simplelog", ] [[package]] diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index dd1e4598db..4434aa77e0 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -7,7 +7,7 @@ use client::{proto, Client, PeerId, TypedEnvelope, User, UserStore}; use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; -use live_kit_client::{LocalVideoTrack, RemoteVideoTrackChange}; +use live_kit_client::{LocalVideoTrack, RemoteVideoTrackUpdate}; use postage::watch; use project::Project; use std::sync::Arc; @@ -75,9 +75,9 @@ impl Room { let live_kit_room = if let Some(connection_info) = live_kit_connection_info { let room = live_kit_client::Room::new(); - let mut tracks = room.remote_video_tracks(); + let mut track_changes = room.remote_video_track_updates(); let maintain_room = cx.spawn_weak(|this, mut cx| async move { - while let Some(track_change) = tracks.next().await { + while let Some(track_change) = track_changes.next().await { let this = if let Some(this) = this.upgrade(&cx) { this } else { @@ -85,7 +85,7 @@ impl Room { }; this.update(&mut cx, |this, cx| { - this.remote_video_track_changed(track_change, cx).log_err() + this.remote_video_track_updated(track_change, cx).log_err() }); } }); @@ -330,13 +330,16 @@ impl Room { ); if let Some((room, _)) = this.live_kit_room.as_ref() { - for track in - room.video_tracks_for_remote_participant(peer_id.0.to_string()) - { - this.remote_video_track_changed( - RemoteVideoTrackChange::Subscribed(track), + println!("getting video tracks for peer id {}", peer_id.0.to_string()); + let tracks = room.remote_video_tracks(&peer_id.0.to_string()); + dbg!(tracks.len()); + for track in tracks { + dbg!(track.id(), track.publisher_id()); + this.remote_video_track_updated( + RemoteVideoTrackUpdate::Subscribed(track), cx, - ); + ) + .log_err(); } } } @@ -376,13 +379,14 @@ impl Room { Ok(()) } - fn remote_video_track_changed( + fn remote_video_track_updated( &mut self, - change: RemoteVideoTrackChange, + change: RemoteVideoTrackUpdate, cx: &mut ModelContext, ) -> Result<()> { match change { - RemoteVideoTrackChange::Subscribed(track) => { + RemoteVideoTrackUpdate::Subscribed(track) => { + dbg!(track.publisher_id(), track.id()); let peer_id = PeerId(track.publisher_id().parse()?); let track_id = track.id().to_string(); let participant = self @@ -427,7 +431,7 @@ impl Room { }, ); } - RemoteVideoTrackChange::Unsubscribed { + RemoteVideoTrackUpdate::Unsubscribed { publisher_id, track_id, } => { @@ -596,11 +600,11 @@ impl Room { }; cx.foreground().spawn(async move { - let displays = live_kit_client::display_sources().await?; - let display = displays - .first() - .ok_or_else(|| anyhow!("no display found"))?; - let track = LocalVideoTrack::screen_share_for_display(display); + let display = live_kit_client::display_source().await?; + // let display = displays + // .first() + // .ok_or_else(|| anyhow!("no display found"))?; + let track = LocalVideoTrack::screen_share_for_display(&display); room.publish_video_track(&track).await?; Ok(()) }) diff --git a/crates/live_kit_client/.cargo/config.toml b/crates/live_kit_client/.cargo/config.toml new file mode 100644 index 0000000000..b33fe211bd --- /dev/null +++ b/crates/live_kit_client/.cargo/config.toml @@ -0,0 +1,2 @@ +[live_kit_client_test] +rustflags = ["-C", "link-args=-ObjC"] diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index 1344ecbcdb..af7ddb67c0 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -8,6 +8,9 @@ description = "Bindings to LiveKit Swift client SDK" path = "src/live_kit_client.rs" doctest = false +[[example]] +name = "test_app" + [dependencies] media = { path = "../media" } @@ -17,6 +20,30 @@ core-graphics = "0.22.3" futures = "0.3" parking_lot = "0.11.1" +[dev-dependencies] +gpui = { path = "../gpui" } +live_kit_server = { path = "../live_kit_server" } +media = { path = "../media" } + +anyhow = "1.0.38" +block = "0.1" +bytes = "1.2" +byteorder = "1.4" +cocoa = "0.24" +core-foundation = "0.9.3" +core-graphics = "0.22.3" +foreign-types = "0.3" +futures = "0.3" +hmac = "0.12" +jwt = "0.16" +log = { version = "0.4.16", features = ["kv_unstable_serde"] } +objc = "0.2" +parking_lot = "0.11.1" +postage = { version = "0.4.1", features = ["futures-traits"] } +serde = { version = "1.0", features = ["derive", "rc"] } +sha2 = "0.10" +simplelog = "0.9" + [build-dependencies] serde = { version = "1.0", features = ["derive", "rc"] } serde_json = { version = "1.0", features = ["preserve_order"] } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index b39f359737..4401b06b30 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -15,7 +15,7 @@ class LKRoomDelegate: RoomDelegate { func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, participant.sid as CFString, track.id as CFString, Unmanaged.passRetained(track).toOpaque()) + self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.id as CFString, Unmanaged.passRetained(track).toOpaque()) } } @@ -97,8 +97,22 @@ public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPoin @_cdecl("LKRoomVideoTracksForRemoteParticipant") public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let tracks = room.remoteParticipants[participantId as Sid]?.videoTracks.compactMap { $0.track as? RemoteVideoTrack } - return tracks as CFArray? + + for (_, participant) in room.remoteParticipants { + if participant.identity == participantId as String { + var tracks = [UnsafeMutableRawPointer]() + for publication in participant.videoTracks { + let track = publication.track as? RemoteVideoTrack + if track != nil { + tracks.append(Unmanaged.passRetained(track!).toOpaque()) + } + + } + return tracks as CFArray? + } + } + + return nil; } @_cdecl("LKCreateScreenShareTrackForDisplay") @@ -120,10 +134,17 @@ public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRaw track.add(videoRenderer: renderer) } -@_cdecl("LKDisplaySources") -public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { +@_cdecl("LKRemoteVideoTrackGetSid") +public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { + let track = Unmanaged.fromOpaque(track).takeUnretainedValue() + return track.sid! as CFString +} + +@_cdecl("LKDisplaySource") +public func LKDisplaySource(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer?, CFString?) -> Void) { MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in - callback(data, displaySources as CFArray, nil) + let displaySource = displaySources.first.map { Unmanaged.passRetained($0).toOpaque() } + callback(data, displaySource, nil) }.catch { error in callback(data, nil, error.localizedDescription as CFString) } diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 79d7d84cdd..872454a489 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -40,6 +40,9 @@ fn main() { build_bridge(&swift_target); link_swift_stdlib(&swift_target); link_webrtc_framework(&swift_target); + + // Register exported Objective-C selectors, protocols, etc when building example binaries. + println!("cargo:rustc-link-arg=-Wl,-ObjC"); } fn build_bridge(swift_target: &SwiftTarget) { @@ -94,6 +97,8 @@ fn link_webrtc_framework(swift_target: &SwiftTarget) { ); // Find WebRTC.framework as a sibling of the executable when running tests. println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); + // Find WebRTC.framework in parent directory of the executable when running examples. + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/.."); let source_path = swift_out_dir_path.join("WebRTC.framework"); let deps_dir_path = diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs new file mode 100644 index 0000000000..370d20354f --- /dev/null +++ b/crates/live_kit_client/examples/test_app.rs @@ -0,0 +1,160 @@ +use core_foundation::base::CFRetain; +use futures::StreamExt; +use gpui::{ + actions, + elements::{Canvas, *}, + keymap::Binding, + platform::current::Surface, + Menu, MenuItem, ViewContext, +}; +use live_kit_client::{LocalVideoTrack, RemoteVideoTrackUpdate, Room}; +use live_kit_server::token::{self, VideoGrant}; +use log::LevelFilter; +use media::core_video::CVImageBuffer; +use postage::watch; +use simplelog::SimpleLogger; +use std::sync::Arc; + +actions!(capture, [Quit]); + +fn main() { + SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); + + gpui::App::new(()).unwrap().run(|cx| { + cx.platform().activate(true); + cx.add_global_action(quit); + + cx.add_bindings([Binding::new("cmd-q", Quit, None)]); + cx.set_menus(vec![Menu { + name: "Zed", + items: vec![MenuItem::Action { + name: "Quit", + action: Box::new(Quit), + }], + }]); + + let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap_or("http://localhost:7880".into()); + let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap_or("devkey".into()); + let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap_or("secret".into()); + + cx.spawn(|cx| async move { + let user_a_token = token::create( + &live_kit_key, + &live_kit_secret, + Some("test-participant-1"), + VideoGrant::to_join("test-room"), + ) + .unwrap(); + let room_a = Room::new(); + room_a.connect(&live_kit_url, &user_a_token).await.unwrap(); + + let user2_token = token::create( + &live_kit_key, + &live_kit_secret, + Some("test-participant-2"), + VideoGrant::to_join("test-room"), + ) + .unwrap(); + let room_b = Room::new(); + room_b.connect(&live_kit_url, &user2_token).await.unwrap(); + + let mut track_changes = room_b.remote_video_track_updates(); + + let display = live_kit_client::display_source().await.unwrap(); + + let track_a = LocalVideoTrack::screen_share_for_display(&display); + room_a.publish_video_track(&track_a).await.unwrap(); + + let next_update = track_changes.next().await.unwrap(); + + if let RemoteVideoTrackUpdate::Subscribed(track) = next_update { + println!("A !!!!!!!!!!!!"); + let remote_tracks = room_b.remote_video_tracks("test-participant-1"); + println!("B !!!!!!!!!!!!"); + assert_eq!(remote_tracks.len(), 1); + println!("C !!!!!!!!!!!!"); + assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); + println!("D !!!!!!!!!!!!"); + // dbg!(track.id()); + // assert_eq!(track.id(), "test-participant-1"); + } else { + panic!("unexpected message") + } + println!("E !!!!!!!!!!!!"); + + cx.platform().quit(); + }) + .detach(); + }); +} + +struct ScreenCaptureView { + image_buffer: Option, + _room: Arc, +} + +impl gpui::Entity for ScreenCaptureView { + type Event = (); +} + +impl ScreenCaptureView { + pub fn new(room: Arc, cx: &mut ViewContext) -> Self { + let mut remote_video_tracks = room.remote_video_track_updates(); + cx.spawn_weak(|this, mut cx| async move { + if let Some(video_track) = remote_video_tracks.next().await { + let (mut frames_tx, mut frames_rx) = watch::channel_with(None); + // video_track.add_renderer(move |frame| *frames_tx.borrow_mut() = Some(frame)); + + while let Some(frame) = frames_rx.next().await { + if let Some(this) = this.upgrade(&cx) { + this.update(&mut cx, |this, cx| { + this.image_buffer = frame; + cx.notify(); + }); + } else { + break; + } + } + } + }) + .detach(); + + Self { + image_buffer: None, + _room: room, + } + } +} + +impl gpui::View for ScreenCaptureView { + fn ui_name() -> &'static str { + "View" + } + + fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { + let image_buffer = self.image_buffer.clone(); + let canvas = Canvas::new(move |bounds, _, cx| { + if let Some(image_buffer) = image_buffer.clone() { + cx.scene.push_surface(Surface { + bounds, + image_buffer, + }); + } + }); + + if let Some(image_buffer) = self.image_buffer.as_ref() { + canvas + .constrained() + .with_width(image_buffer.width() as f32) + .with_height(image_buffer.height() as f32) + .aligned() + .boxed() + } else { + canvas.boxed() + } + } +} + +fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { + cx.platform().quit(); +} diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 6bd5ce47e6..2d871d31b4 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -15,6 +15,10 @@ use std::{ sync::{Arc, Weak}, }; +pub type Sid = String; +#[allow(non_camel_case_types)] +pub type sid = str; + extern "C" { fn LKRelease(object: *const c_void); @@ -47,6 +51,10 @@ extern "C" { callback: extern "C" fn(*mut c_void, CFStringRef), callback_data: *mut c_void, ); + fn LKRoomVideoTracksForRemoteParticipant( + room: *const c_void, + participant_id: CFStringRef, + ) -> CFArrayRef; fn LKVideoRendererCreate( callback_data: *mut c_void, @@ -55,12 +63,13 @@ extern "C" { ) -> *const c_void; fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); + fn LKRemoteVideoTrackGetSid(track: *const c_void) -> CFStringRef; - fn LKDisplaySources( + fn LKDisplaySource( callback_data: *mut c_void, callback: extern "C" fn( callback_data: *mut c_void, - sources: CFArrayRef, + source: *mut c_void, error: CFStringRef, ), ); @@ -69,7 +78,7 @@ extern "C" { pub struct Room { native_room: *const c_void, - remote_video_track_subscribers: Mutex>>, + remote_video_track_subscribers: Mutex>>, _delegate: RoomDelegate, } @@ -110,7 +119,40 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } - pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver { + pub fn remote_video_tracks(&self, participant_sid: &sid) -> Vec> { + unsafe { + let tracks = LKRoomVideoTracksForRemoteParticipant( + self.native_room, + CFString::new(participant_sid).as_concrete_TypeRef(), + ); + + if tracks.is_null() { + Vec::new() + } else { + println!("aaaa >>>>>>>>>>>>>>>"); + let tracks = CFArray::wrap_under_get_rule(tracks); + println!("bbbb >>>>>>>>>>>>>>>"); + tracks + .into_iter() + .map(|native_track| { + let native_track = *native_track; + println!("cccc >>>>>>>>>>>>>>>"); + let id = + CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) + .to_string(); + println!("dddd >>>>>>>>>>>>>>>"); + Arc::new(RemoteVideoTrack { + native_track, + publisher_id: participant_sid.into(), + id, + }) + }) + .collect() + } + } + } + + pub fn remote_video_track_updates(&self) -> mpsc::UnboundedReceiver { let (tx, rx) = mpsc::unbounded(); self.remote_video_track_subscribers.lock().push(tx); rx @@ -119,14 +161,14 @@ impl Room { fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { let track = Arc::new(track); self.remote_video_track_subscribers.lock().retain(|tx| { - tx.unbounded_send(RemoteVideoTrackChange::Subscribed(track.clone())) + tx.unbounded_send(RemoteVideoTrackUpdate::Subscribed(track.clone())) .is_ok() }); } fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { self.remote_video_track_subscribers.lock().retain(|tx| { - tx.unbounded_send(RemoteVideoTrackChange::Unsubscribed { + tx.unbounded_send(RemoteVideoTrackUpdate::Unsubscribed { publisher_id: publisher_id.clone(), track_id: track_id.clone(), }) @@ -201,7 +243,7 @@ impl RoomDelegate { if let Some(room) = room.upgrade() { room.did_subscribe_to_remote_video_track(track); } - let _ = Weak::into_raw(room); + // let _ = Weak::into_raw(room); } extern "C" fn on_did_unsubscribe_from_remote_video_track( @@ -244,9 +286,9 @@ impl Drop for LocalVideoTrack { #[derive(Debug)] pub struct RemoteVideoTrack { - id: String, + id: Sid, native_track: *const c_void, - publisher_id: String, + publisher_id: Sid, } impl RemoteVideoTrack { @@ -294,12 +336,9 @@ impl Drop for RemoteVideoTrack { } } -pub enum RemoteVideoTrackChange { +pub enum RemoteVideoTrackUpdate { Subscribed(Arc), - Unsubscribed { - publisher_id: String, - track_id: String, - }, + Unsubscribed { publisher_id: Sid, track_id: Sid }, } pub struct MacOSDisplay(*const c_void); @@ -310,17 +349,16 @@ impl Drop for MacOSDisplay { } } -pub fn display_sources() -> impl Future>> { - extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { +pub fn display_source() -> impl Future> { + extern "C" fn callback(tx: *mut c_void, source: *mut c_void, error: CFStringRef) { unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>>); + let tx = Box::from_raw(tx as *mut oneshot::Sender>); - if sources.is_null() { + if source.is_null() { let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); } else { - let sources = CFArray::wrap_under_get_rule(sources); - let sources_vec = sources.iter().map(|source| MacOSDisplay(*source)).collect(); - let _ = tx.send(Ok(sources_vec)); + let source = MacOSDisplay(source); + let _ = tx.send(Ok(source)); } } } @@ -328,8 +366,14 @@ pub fn display_sources() -> impl Future>> { let (tx, rx) = oneshot::channel(); unsafe { - LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); + LKDisplaySource(Box::into_raw(Box::new(tx)) as *mut _, callback); } async move { rx.await.unwrap() } } + +#[cfg(test)] +mod tests { + #[test] + fn test_client() {} +} diff --git a/crates/live_kit_server/src/token.rs b/crates/live_kit_server/src/token.rs index ae03cb3469..9956ad006c 100644 --- a/crates/live_kit_server/src/token.rs +++ b/crates/live_kit_server/src/token.rs @@ -38,6 +38,18 @@ pub struct VideoGrant<'a> { pub recorder: Option, } +impl<'a> VideoGrant<'a> { + pub fn to_join(room: &'a str) -> Self { + Self { + room: Some(room), + room_join: Some(true), + can_publish: Some(true), + can_subscribe: Some(true), + ..Default::default() + } + } +} + pub fn create( api_key: &str, secret_key: &str, From c73e2c2d0f869292385df35fa93dea00ab7772a9 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 17 Oct 2022 23:38:43 -0600 Subject: [PATCH 16/63] Get test_app running without crashing --- crates/call/src/room.rs | 2 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 17 ++--- crates/live_kit_client/examples/test_app.rs | 11 +-- crates/live_kit_client/src/live_kit_client.rs | 69 +++++++++++-------- 4 files changed, 50 insertions(+), 49 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 4434aa77e0..32f22d41e0 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -600,7 +600,7 @@ impl Room { }; cx.foreground().spawn(async move { - let display = live_kit_client::display_source().await?; + let display = live_kit_client::display_sources().await?; // let display = displays // .first() // .ok_or_else(|| anyhow!("no display found"))?; diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 4401b06b30..109b9fb84a 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -100,15 +100,7 @@ public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, partic for (_, participant) in room.remoteParticipants { if participant.identity == participantId as String { - var tracks = [UnsafeMutableRawPointer]() - for publication in participant.videoTracks { - let track = publication.track as? RemoteVideoTrack - if track != nil { - tracks.append(Unmanaged.passRetained(track!).toOpaque()) - } - - } - return tracks as CFArray? + return participant.videoTracks.compactMap { $0.track as? RemoteVideoTrack } as CFArray? } } @@ -140,11 +132,10 @@ public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { return track.sid! as CFString } -@_cdecl("LKDisplaySource") -public func LKDisplaySource(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer?, CFString?) -> Void) { +@_cdecl("LKDisplaySources") +public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in - let displaySource = displaySources.first.map { Unmanaged.passRetained($0).toOpaque() } - callback(data, displaySource, nil) + callback(data, displaySources as CFArray, nil) }.catch { error in callback(data, nil, error.localizedDescription as CFString) } diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index 370d20354f..d35e9fc549 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -60,7 +60,8 @@ fn main() { let mut track_changes = room_b.remote_video_track_updates(); - let display = live_kit_client::display_source().await.unwrap(); + let displays = live_kit_client::display_sources().await.unwrap(); + let display = displays.into_iter().next().unwrap(); let track_a = LocalVideoTrack::screen_share_for_display(&display); room_a.publish_video_track(&track_a).await.unwrap(); @@ -68,19 +69,13 @@ fn main() { let next_update = track_changes.next().await.unwrap(); if let RemoteVideoTrackUpdate::Subscribed(track) = next_update { - println!("A !!!!!!!!!!!!"); let remote_tracks = room_b.remote_video_tracks("test-participant-1"); - println!("B !!!!!!!!!!!!"); assert_eq!(remote_tracks.len(), 1); - println!("C !!!!!!!!!!!!"); assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); - println!("D !!!!!!!!!!!!"); - // dbg!(track.id()); - // assert_eq!(track.id(), "test-participant-1"); + assert_eq!(track.publisher_id(), "test-participant-1"); } else { panic!("unexpected message") } - println!("E !!!!!!!!!!!!"); cx.platform().quit(); }) diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 2d871d31b4..ccd0ba6775 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use core_foundation::{ array::{CFArray, CFArrayRef}, - base::TCFType, + base::{CFRetain, TCFType}, string::{CFString, CFStringRef}, }; use futures::{ @@ -65,11 +65,11 @@ extern "C" { fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); fn LKRemoteVideoTrackGetSid(track: *const c_void) -> CFStringRef; - fn LKDisplaySource( + fn LKDisplaySources( callback_data: *mut c_void, callback: extern "C" fn( callback_data: *mut c_void, - source: *mut c_void, + sources: CFArrayRef, error: CFStringRef, ), ); @@ -119,33 +119,29 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } - pub fn remote_video_tracks(&self, participant_sid: &sid) -> Vec> { + pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { unsafe { let tracks = LKRoomVideoTracksForRemoteParticipant( self.native_room, - CFString::new(participant_sid).as_concrete_TypeRef(), + CFString::new(participant_id).as_concrete_TypeRef(), ); if tracks.is_null() { Vec::new() } else { - println!("aaaa >>>>>>>>>>>>>>>"); let tracks = CFArray::wrap_under_get_rule(tracks); - println!("bbbb >>>>>>>>>>>>>>>"); tracks .into_iter() .map(|native_track| { let native_track = *native_track; - println!("cccc >>>>>>>>>>>>>>>"); let id = CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) .to_string(); - println!("dddd >>>>>>>>>>>>>>>"); - Arc::new(RemoteVideoTrack { + Arc::new(RemoteVideoTrack::new( native_track, - publisher_id: participant_sid.into(), id, - }) + participant_id.into(), + )) }) .collect() } @@ -235,15 +231,10 @@ impl RoomDelegate { let room = unsafe { Weak::from_raw(room as *mut Room) }; let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteVideoTrack { - id: track_id, - native_track: track, - publisher_id, - }; + let track = RemoteVideoTrack::new(track, track_id, publisher_id); if let Some(room) = room.upgrade() { room.did_subscribe_to_remote_video_track(track); } - // let _ = Weak::into_raw(room); } extern "C" fn on_did_unsubscribe_from_remote_video_track( @@ -286,12 +277,23 @@ impl Drop for LocalVideoTrack { #[derive(Debug)] pub struct RemoteVideoTrack { - id: Sid, native_track: *const c_void, - publisher_id: Sid, + id: Sid, + publisher_id: String, } impl RemoteVideoTrack { + pub fn new(native_track: *const c_void, id: Sid, publisher_id: String) -> Self { + unsafe { + CFRetain(native_track); + } + Self { + native_track, + id, + publisher_id, + } + } + pub fn id(&self) -> &str { &self.id } @@ -343,22 +345,35 @@ pub enum RemoteVideoTrackUpdate { pub struct MacOSDisplay(*const c_void); +impl MacOSDisplay { + fn new(ptr: *const c_void) -> Self { + unsafe { + CFRetain(ptr); + } + Self(ptr) + } +} + impl Drop for MacOSDisplay { fn drop(&mut self) { unsafe { LKRelease(self.0) } } } -pub fn display_source() -> impl Future> { - extern "C" fn callback(tx: *mut c_void, source: *mut c_void, error: CFStringRef) { +pub fn display_sources() -> impl Future>> { + extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>); + let tx = Box::from_raw(tx as *mut oneshot::Sender>>); - if source.is_null() { + if sources.is_null() { let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); } else { - let source = MacOSDisplay(source); - let _ = tx.send(Ok(source)); + let sources = CFArray::wrap_under_get_rule(sources) + .into_iter() + .map(|source| MacOSDisplay::new(*source)) + .collect(); + + let _ = tx.send(Ok(sources)); } } } @@ -366,7 +381,7 @@ pub fn display_source() -> impl Future> { let (tx, rx) = oneshot::channel(); unsafe { - LKDisplaySource(Box::into_raw(Box::new(tx)) as *mut _, callback); + LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); } async move { rx.await.unwrap() } From 59fab0bb2d19b969a80f257961a77bd1c90a2e68 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 17 Oct 2022 23:47:55 -0600 Subject: [PATCH 17/63] WIP --- crates/call/src/room.rs | 14 +++++++------- crates/live_kit_client/examples/test_app.rs | 1 - crates/live_kit_client/src/live_kit_client.rs | 12 +++++------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 32f22d41e0..d1d2c5433e 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -334,7 +334,7 @@ impl Room { let tracks = room.remote_video_tracks(&peer_id.0.to_string()); dbg!(tracks.len()); for track in tracks { - dbg!(track.id(), track.publisher_id()); + dbg!(track.sid(), track.publisher_id()); this.remote_video_track_updated( RemoteVideoTrackUpdate::Subscribed(track), cx, @@ -386,9 +386,9 @@ impl Room { ) -> Result<()> { match change { RemoteVideoTrackUpdate::Subscribed(track) => { - dbg!(track.publisher_id(), track.id()); + dbg!(track.publisher_id(), track.sid()); let peer_id = PeerId(track.publisher_id().parse()?); - let track_id = track.id().to_string(); + let track_id = track.sid().to_string(); let participant = self .remote_participants .get_mut(&peer_id) @@ -600,10 +600,10 @@ impl Room { }; cx.foreground().spawn(async move { - let display = live_kit_client::display_sources().await?; - // let display = displays - // .first() - // .ok_or_else(|| anyhow!("no display found"))?; + let displays = live_kit_client::display_sources().await?; + let display = displays + .first() + .ok_or_else(|| anyhow!("no display found"))?; let track = LocalVideoTrack::screen_share_for_display(&display); room.publish_video_track(&track).await?; Ok(()) diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index d35e9fc549..94c324becf 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -1,4 +1,3 @@ -use core_foundation::base::CFRetain; use futures::StreamExt; use gpui::{ actions, diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index ccd0ba6775..af46fbb2e0 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -16,8 +16,6 @@ use std::{ }; pub type Sid = String; -#[allow(non_camel_case_types)] -pub type sid = str; extern "C" { fn LKRelease(object: *const c_void); @@ -278,24 +276,24 @@ impl Drop for LocalVideoTrack { #[derive(Debug)] pub struct RemoteVideoTrack { native_track: *const c_void, - id: Sid, + sid: Sid, publisher_id: String, } impl RemoteVideoTrack { - pub fn new(native_track: *const c_void, id: Sid, publisher_id: String) -> Self { + fn new(native_track: *const c_void, sid: Sid, publisher_id: String) -> Self { unsafe { CFRetain(native_track); } Self { native_track, - id, + sid, publisher_id, } } - pub fn id(&self) -> &str { - &self.id + pub fn sid(&self) -> &str { + &self.sid } pub fn publisher_id(&self) -> &str { From a42a703b35caa7a132beefdf8306bbc6efa28bf3 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 17 Oct 2022 23:56:41 -0600 Subject: [PATCH 18/63] Pass tracks to Rust unretained We always call CFRetain when constructing a track on the Rust side. --- .../LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 109b9fb84a..80032752a9 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -15,7 +15,7 @@ class LKRoomDelegate: RoomDelegate { func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.id as CFString, Unmanaged.passRetained(track).toOpaque()) + self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.id as CFString, Unmanaged.passUnretained(track).toOpaque()) } } From 8c6de991591db11fcbb27d18e1f8dfdb2d0f96b9 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 12:05:59 +0200 Subject: [PATCH 19/63] Use participant identity and track sid everywhere --- crates/call/src/room.rs | 46 ++++++++++--------- .../Sources/LiveKitBridge/LiveKitBridge.swift | 4 +- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index d1d2c5433e..6787a072f3 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -318,28 +318,32 @@ impl Room { }); } - this.remote_participants.insert( - peer_id, - RemoteParticipant { - user: user.clone(), - projects: participant.projects, - location: ParticipantLocation::from_proto(participant.location) - .unwrap_or(ParticipantLocation::External), - tracks: Default::default(), - }, - ); + let location = ParticipantLocation::from_proto(participant.location) + .unwrap_or(ParticipantLocation::External); + if let Some(remote_participant) = this.remote_participants.get_mut(&peer_id) + { + remote_participant.projects = participant.projects; + remote_participant.location = location; + } else { + this.remote_participants.insert( + peer_id, + RemoteParticipant { + user: user.clone(), + projects: participant.projects, + location, + tracks: Default::default(), + }, + ); - if let Some((room, _)) = this.live_kit_room.as_ref() { - println!("getting video tracks for peer id {}", peer_id.0.to_string()); - let tracks = room.remote_video_tracks(&peer_id.0.to_string()); - dbg!(tracks.len()); - for track in tracks { - dbg!(track.sid(), track.publisher_id()); - this.remote_video_track_updated( - RemoteVideoTrackUpdate::Subscribed(track), - cx, - ) - .log_err(); + if let Some((room, _)) = this.live_kit_room.as_ref() { + let tracks = room.remote_video_tracks(&peer_id.0.to_string()); + for track in tracks { + this.remote_video_track_updated( + RemoteVideoTrackUpdate::Subscribed(track), + cx, + ) + .log_err(); + } } } } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 80032752a9..1c49109fc5 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -15,13 +15,13 @@ class LKRoomDelegate: RoomDelegate { func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.id as CFString, Unmanaged.passUnretained(track).toOpaque()) + self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque()) } } func room(_ room: Room, participant: RemoteParticipant, didUnsubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { - self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.sid as CFString, track.id as CFString) + self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString) } } } From 46635956f43ec42af674e61c67a1ead2c5da1074 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 12:18:49 +0200 Subject: [PATCH 20/63] Emit `Frame` event when new frames are generated for a remote track --- crates/call/src/participant.rs | 2 +- crates/call/src/room.rs | 11 +++++--- .../src/project_shared_notification.rs | 1 + crates/workspace/src/workspace.rs | 25 +++++++++++++------ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index 09174df793..f335689050 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -41,7 +41,7 @@ pub struct RemoteParticipant { pub user: Arc, pub projects: Vec, pub location: ParticipantLocation, - pub tracks: HashMap, + pub tracks: HashMap, } #[derive(Clone)] diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 6787a072f3..909fe4d977 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -15,6 +15,10 @@ use util::ResultExt; #[derive(Clone, Debug, PartialEq, Eq)] pub enum Event { + Frame { + participant_id: PeerId, + track_id: live_kit_client::Sid, + }, RemoteProjectShared { owner: Arc, project_id: u64, @@ -390,7 +394,6 @@ impl Room { ) -> Result<()> { match change { RemoteVideoTrackUpdate::Subscribed(track) => { - dbg!(track.publisher_id(), track.sid()); let peer_id = PeerId(track.publisher_id().parse()?); let track_id = track.sid().to_string(); let participant = self @@ -413,14 +416,16 @@ impl Room { }; let done = this.update(&mut cx, |this, cx| { - // TODO: replace this with an emit. - cx.notify(); if let Some(track) = this.remote_participants.get_mut(&peer_id).and_then( |participant| participant.tracks.get_mut(&track_id), ) { track.frame = frame; + cx.emit(Event::Frame { + participant_id: peer_id, + track_id: track_id.clone(), + }); false } else { true diff --git a/crates/collab_ui/src/project_shared_notification.rs b/crates/collab_ui/src/project_shared_notification.rs index a17e11b079..e5ded819af 100644 --- a/crates/collab_ui/src/project_shared_notification.rs +++ b/crates/collab_ui/src/project_shared_notification.rs @@ -62,6 +62,7 @@ pub fn init(cx: &mut MutableAppContext) { cx.remove_window(window_id); } } + _ => {} }) .detach(); } diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index ece8cedfb1..5ec76f5130 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -980,9 +980,8 @@ pub struct Workspace { follower_states_by_leader: FollowerStatesByLeader, last_leaders_by_pane: HashMap, PeerId>, window_edited: bool, - active_call: Option>, + active_call: Option<(ModelHandle, Vec)>, _observe_current_user: Task<()>, - _active_call_observation: Option, } #[derive(Default)] @@ -1092,11 +1091,18 @@ impl Workspace { }); let mut active_call = None; - let mut active_call_observation = None; if cx.has_global::>() { let call = cx.global::>().clone(); - active_call_observation = Some(cx.observe(&call, |_, _, cx| cx.notify())); - active_call = Some(call); + let mut subscriptions = Vec::new(); + subscriptions.push(cx.observe(&call, |_, _, cx| cx.notify())); + subscriptions.push(cx.subscribe(&call, |this, _, event, cx| { + if let call::room::Event::Frame { participant_id, .. } = event { + if this.follower_states_by_leader.contains_key(&participant_id) { + cx.notify(); + } + } + })); + active_call = Some((call, subscriptions)); } let mut this = Workspace { @@ -1127,7 +1133,6 @@ impl Workspace { window_edited: false, active_call, _observe_current_user, - _active_call_observation: active_call_observation, }; this.project_remote_id_changed(this.project.read(cx).remote_id(), cx); cx.defer(|this, cx| this.update_window_title(cx)); @@ -1262,7 +1267,7 @@ impl Workspace { quitting: bool, cx: &mut ViewContext, ) -> Task> { - let active_call = self.active_call.clone(); + let active_call = self.active_call().cloned(); let window_id = cx.window_id(); let workspace_count = cx .window_ids() @@ -2549,6 +2554,10 @@ impl Workspace { } } } + + fn active_call(&self) -> Option<&ModelHandle> { + self.active_call.as_ref().map(|(call, _)| call) + } } impl Entity for Workspace { @@ -2590,7 +2599,7 @@ impl View for Workspace { &project, &theme, &self.follower_states_by_leader, - self.active_call.as_ref(), + self.active_call(), cx, )) .flex(1., true) From bf983005473eb70b6de4ec7d539b59ba30ac2fbb Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 14:16:19 +0200 Subject: [PATCH 21/63] Render remote participant's screen preserving aspect ratio --- crates/gpui/src/elements.rs | 2 +- crates/workspace/src/pane_group.rs | 86 +++++++++++++++++++----------- styles/src/styleTree/workspace.ts | 2 +- 3 files changed, 56 insertions(+), 34 deletions(-) diff --git a/crates/gpui/src/elements.rs b/crates/gpui/src/elements.rs index 59269f8af6..68c0b25f20 100644 --- a/crates/gpui/src/elements.rs +++ b/crates/gpui/src/elements.rs @@ -495,7 +495,7 @@ pub trait ParentElement<'a>: Extend + Sized { impl<'a, T> ParentElement<'a> for T where T: Extend {} -fn constrain_size_preserving_aspect_ratio(max_size: Vector2F, size: Vector2F) -> Vector2F { +pub fn constrain_size_preserving_aspect_ratio(max_size: Vector2F, size: Vector2F) -> Vector2F { if max_size.x().is_infinite() && max_size.y().is_infinite() { size } else if max_size.x().is_infinite() || max_size.x() / max_size.y() > size.x() / size.y() { diff --git a/crates/workspace/src/pane_group.rs b/crates/workspace/src/pane_group.rs index c1c4aef096..b99aba3b3a 100644 --- a/crates/workspace/src/pane_group.rs +++ b/crates/workspace/src/pane_group.rs @@ -1,8 +1,10 @@ use crate::{FollowerStatesByLeader, JoinProject, Pane, Workspace}; use anyhow::{anyhow, Result}; -use call::ActiveCall; +use call::{ActiveCall, ParticipantLocation}; use gpui::{ - elements::*, Axis, Border, CursorStyle, ModelHandle, MouseButton, RenderContext, ViewHandle, + elements::*, + geometry::{rect::RectF, vector::vec2f}, + Axis, Border, CursorStyle, ModelHandle, MouseButton, RenderContext, ViewHandle, }; use project::Project; use serde::Deserialize; @@ -130,18 +132,45 @@ impl Member { Some((collaborator.replica_id, participant)) }); - let mut border = Border::default(); - - let prompt = if let Some((replica_id, leader)) = leader { - let leader_color = theme.editor.replica_selection_style(replica_id).cursor; - border = Border::all(theme.workspace.leader_border_width, leader_color); + let border = if let Some((replica_id, _)) = leader.as_ref() { + let leader_color = theme.editor.replica_selection_style(*replica_id).cursor; + let mut border = Border::all(theme.workspace.leader_border_width, leader_color); border .color .fade_out(1. - theme.workspace.leader_border_opacity); border.overlay = true; + border + } else { + Border::default() + }; + let content = if leader.as_ref().map_or(false, |(_, leader)| { + leader.location == ParticipantLocation::External && !leader.tracks.is_empty() + }) { + let (_, leader) = leader.unwrap(); + let track = leader.tracks.values().next().unwrap(); + let frame = track.frame().cloned(); + Canvas::new(move |bounds, _, cx| { + if let Some(frame) = frame.clone() { + let size = constrain_size_preserving_aspect_ratio( + bounds.size(), + vec2f(frame.width() as f32, frame.height() as f32), + ); + let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.; + cx.scene.push_surface(gpui::mac::Surface { + bounds: RectF::new(origin, size), + image_buffer: frame, + }); + } + }) + .boxed() + } else { + ChildView::new(pane, cx).boxed() + }; + + let prompt = if let Some((_, leader)) = leader { match leader.location { - call::ParticipantLocation::SharedProject { + ParticipantLocation::SharedProject { project_id: leader_project_id, } => { if Some(leader_project_id) == project.read(cx).remote_id() { @@ -186,7 +215,7 @@ impl Member { ) } } - call::ParticipantLocation::UnsharedProject => Some( + ParticipantLocation::UnsharedProject => Some( Label::new( format!( "{} is viewing an unshared Zed project", @@ -201,35 +230,28 @@ impl Member { .right() .boxed(), ), - call::ParticipantLocation::External => { - let frame = leader - .tracks - .values() - .next() - .and_then(|track| track.frame()) - .cloned(); - return Canvas::new(move |bounds, _, cx| { - if let Some(frame) = frame.clone() { - cx.scene.push_surface(gpui::mac::Surface { - bounds, - image_buffer: frame, - }); - } - }) - .boxed(); - } + ParticipantLocation::External => Some( + Label::new( + format!( + "{} is viewing a window outside of Zed", + leader.user.github_login + ), + theme.workspace.external_location_message.text.clone(), + ) + .contained() + .with_style(theme.workspace.external_location_message.container) + .aligned() + .bottom() + .right() + .boxed(), + ), } } else { None }; Stack::new() - .with_child( - ChildView::new(pane, cx) - .contained() - .with_border(border) - .boxed(), - ) + .with_child(Container::new(content).with_border(border).boxed()) .with_children(prompt) .boxed() } diff --git a/styles/src/styleTree/workspace.ts b/styles/src/styleTree/workspace.ts index cfbda49056..c8d32ae1a8 100644 --- a/styles/src/styleTree/workspace.ts +++ b/styles/src/styleTree/workspace.ts @@ -49,7 +49,7 @@ export default function workspace(theme: Theme) { ...text(theme, "sans", "primary", { size: "lg" }), }, externalLocationMessage: { - background: backgroundColor(theme, "info"), + background: backgroundColor(theme, "on500Info"), border: border(theme, "secondary"), cornerRadius: 6, padding: 12, From 47be340cac190393e2f5e3ef649da0445f09c19d Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 14:35:06 +0200 Subject: [PATCH 22/63] Fix invoking `RemoveParticipant` on live-kit server --- crates/live_kit_server/src/api.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index bd98d27069..d14cedd64a 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -68,9 +68,13 @@ impl Client { "twirp/livekit.RoomService/RemoveParticipant", token::VideoGrant { room_admin: Some(true), + room: Some(&room), ..Default::default() }, - proto::RoomParticipantIdentity { room, identity }, + proto::RoomParticipantIdentity { + room: room.clone(), + identity, + }, ); async move { let _: proto::RemoveParticipantResponse = response.await?; From 9cf39b1da6fe8024f07298ee19a5e2db3163faea Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 14:50:03 +0200 Subject: [PATCH 23/63] Disconnect from live-kit `Room` on drop --- crates/call/src/room.rs | 9 ++++++++- .../Sources/LiveKitBridge/LiveKitBridge.swift | 6 ++++++ crates/live_kit_client/src/live_kit_client.rs | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 909fe4d977..1bd86944bf 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -50,7 +50,9 @@ impl Entity for Room { type Event = Event; fn release(&mut self, _: &mut MutableAppContext) { - self.client.send(proto::LeaveRoom { id: self.id }).log_err(); + if self.status.is_online() { + self.client.send(proto::LeaveRoom { id: self.id }).log_err(); + } } } @@ -210,6 +212,7 @@ impl Room { self.pending_participants.clear(); self.participant_user_ids.clear(); self.subscriptions.clear(); + self.live_kit_room.take(); self.client.send(proto::LeaveRoom { id: self.id })?; Ok(()) } @@ -630,4 +633,8 @@ impl RoomStatus { pub fn is_offline(&self) -> bool { matches!(self, RoomStatus::Offline) } + + pub fn is_online(&self) -> bool { + matches!(self, RoomStatus::Online) + } } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 1c49109fc5..dadc2fdb10 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -83,6 +83,12 @@ public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString } } +@_cdecl("LKRoomDisconnect") +public func LKRoomDisconnect(room: UnsafeRawPointer) { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() + room.disconnect() +} + @_cdecl("LKRoomPublishVideoTrack") public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue() diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index af46fbb2e0..07075dcee8 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -43,6 +43,7 @@ extern "C" { callback: extern "C" fn(*mut c_void, CFStringRef), callback_data: *mut c_void, ); + fn LKRoomDisconnect(room: *const c_void); fn LKRoomPublishVideoTrack( room: *const c_void, track: *const c_void, @@ -195,7 +196,10 @@ impl Room { impl Drop for Room { fn drop(&mut self) { - unsafe { LKRelease(self.native_room) } + unsafe { + LKRoomDisconnect(self.native_room); + LKRelease(self.native_room); + } } } From 48a1dd1588962b82d0d8f686f2d8b5e6590d379d Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 14:57:31 +0200 Subject: [PATCH 24/63] Delete room when no participants are left --- Cargo.lock | 1 + crates/collab/src/rpc.rs | 7 ++++++- crates/collab/src/rpc/store.rs | 2 +- crates/live_kit_server/Cargo.toml | 1 + crates/live_kit_server/src/api.rs | 4 ++++ 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4814befcb4..1b946b4084 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3197,6 +3197,7 @@ dependencies = [ "futures 0.3.24", "hmac 0.12.1", "jwt", + "log", "prost 0.8.0", "prost-build", "prost-types 0.8.0", diff --git a/crates/collab/src/rpc.rs b/crates/collab/src/rpc.rs index 70e47a755e..bca9cad12f 100644 --- a/crates/collab/src/rpc.rs +++ b/crates/collab/src/rpc.rs @@ -890,11 +890,16 @@ impl Server { ) -> impl Future> { let client = self.app_state.live_kit_client.clone(); let room_name = room.live_kit_room.clone(); + let participant_count = room.participants.len(); async move { if let Some(client) = client { client - .remove_participant(room_name, connection_id.to_string()) + .remove_participant(room_name.clone(), connection_id.to_string()) .await?; + + if participant_count == 0 { + client.delete_room(room_name).await?; + } } Ok(()) diff --git a/crates/collab/src/rpc/store.rs b/crates/collab/src/rpc/store.rs index df8be453b1..6fa84d7f10 100644 --- a/crates/collab/src/rpc/store.rs +++ b/crates/collab/src/rpc/store.rs @@ -495,7 +495,7 @@ impl Store { } }); - let room = if room.participants.is_empty() && room.pending_participant_user_ids.is_empty() { + let room = if room.participants.is_empty() { Cow::Owned(self.rooms.remove(&room_id).unwrap()) } else { Cow::Borrowed(self.rooms.get(&room_id).unwrap()) diff --git a/crates/live_kit_server/Cargo.toml b/crates/live_kit_server/Cargo.toml index 7cc9d82333..8cd0b36c7c 100644 --- a/crates/live_kit_server/Cargo.toml +++ b/crates/live_kit_server/Cargo.toml @@ -12,6 +12,7 @@ doctest = false anyhow = "1.0.38" futures = "0.3" hmac = "0.12" +log = "0.4" jwt = "0.16" prost = "0.8" prost-types = "0.8" diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index d14cedd64a..8ba94be43a 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -110,6 +110,7 @@ impl Client { let client = self.http.clone(); let token = token::create(&self.key, &self.secret, None, grant); let url = format!("{}/{}", self.url, path); + log::info!("Request {}: {:?}", url, body); async move { let token = token?; let response = client @@ -119,9 +120,12 @@ impl Client { .body(body.encode_to_vec()) .send() .await?; + if response.status().is_success() { + log::info!("Response {}: {:?}", url, response.status()); Ok(Res::decode(response.bytes().await?)?) } else { + log::error!("Response {}: {:?}", url, response.status()); Err(anyhow!( "POST {} failed with status code {:?}, {:?}", url, From 29b9651ebdcaaa837518de4167ea5d20e5af68e3 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 15:47:56 +0200 Subject: [PATCH 25/63] Use `CFRelease` instead of a custom `LKRelease` --- .../Sources/LiveKitBridge/LiveKitBridge.swift | 5 ----- crates/live_kit_client/src/live_kit_client.rs | 14 ++++++-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index dadc2fdb10..c7ce2178ab 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -55,11 +55,6 @@ class LKVideoRenderer: NSObject, VideoRenderer { } } -@_cdecl("LKRelease") -public func LKRelease(ptr: UnsafeRawPointer) { - let _ = Unmanaged.fromOpaque(ptr).takeRetainedValue() -} - @_cdecl("LKRoomDelegateCreate") public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) -> UnsafeMutableRawPointer { let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack) diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 07075dcee8..8eaa9a6f6a 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use core_foundation::{ array::{CFArray, CFArrayRef}, - base::{CFRetain, TCFType}, + base::{CFRelease, CFRetain, TCFType}, string::{CFString, CFStringRef}, }; use futures::{ @@ -18,8 +18,6 @@ use std::{ pub type Sid = String; extern "C" { - fn LKRelease(object: *const c_void); - fn LKRoomDelegateCreate( callback_data: *mut c_void, on_did_subscribe_to_remote_video_track: extern "C" fn( @@ -198,7 +196,7 @@ impl Drop for Room { fn drop(&mut self) { unsafe { LKRoomDisconnect(self.native_room); - LKRelease(self.native_room); + CFRelease(self.native_room); } } } @@ -257,7 +255,7 @@ impl RoomDelegate { impl Drop for RoomDelegate { fn drop(&mut self) { unsafe { - LKRelease(self.native_delegate); + CFRelease(self.native_delegate); let _ = Weak::from_raw(self.weak_room); } } @@ -273,7 +271,7 @@ impl LocalVideoTrack { impl Drop for LocalVideoTrack { fn drop(&mut self) { - unsafe { LKRelease(self.0) } + unsafe { CFRelease(self.0) } } } @@ -336,7 +334,7 @@ impl RemoteVideoTrack { impl Drop for RemoteVideoTrack { fn drop(&mut self) { - unsafe { LKRelease(self.native_track) } + unsafe { CFRelease(self.native_track) } } } @@ -358,7 +356,7 @@ impl MacOSDisplay { impl Drop for MacOSDisplay { fn drop(&mut self) { - unsafe { LKRelease(self.0) } + unsafe { CFRelease(self.0) } } } From 0c3c1e1f68f9ebda86c0b5ec65e9ce3500791a5d Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Tue, 18 Oct 2022 19:30:45 +0200 Subject: [PATCH 26/63] WIP --- crates/call/src/room.rs | 51 ++++++++++++------- .../Sources/LiveKitBridge/LiveKitBridge.swift | 8 +-- crates/live_kit_client/src/live_kit_client.rs | 34 +++++++++++-- 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 1bd86944bf..7670a0e239 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -7,7 +7,7 @@ use client::{proto, Client, PeerId, TypedEnvelope, User, UserStore}; use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; -use live_kit_client::{LocalVideoTrack, RemoteVideoTrackUpdate}; +use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate}; use postage::watch; use project::Project; use std::sync::Arc; @@ -32,7 +32,7 @@ pub enum Event { pub struct Room { id: u64, - live_kit_room: Option<(Arc, Task<()>)>, + live_kit: Option, status: RoomStatus, local_participant: LocalParticipant, remote_participants: BTreeMap, @@ -82,7 +82,7 @@ impl Room { let live_kit_room = if let Some(connection_info) = live_kit_connection_info { let room = live_kit_client::Room::new(); let mut track_changes = room.remote_video_track_updates(); - let maintain_room = cx.spawn_weak(|this, mut cx| async move { + let _maintain_room = cx.spawn_weak(|this, mut cx| async move { while let Some(track_change) = track_changes.next().await { let this = if let Some(this) = this.upgrade(&cx) { this @@ -98,14 +98,18 @@ impl Room { cx.foreground() .spawn(room.connect(&connection_info.server_url, &connection_info.token)) .detach_and_log_err(cx); - Some((room, maintain_room)) + Some(LiveKitRoom { + room, + screen_track: None, + _maintain_room, + }) } else { None }; Self { id, - live_kit_room, + live_kit: live_kit_room, status: RoomStatus::Online, participant_user_ids: Default::default(), local_participant: Default::default(), @@ -212,7 +216,7 @@ impl Room { self.pending_participants.clear(); self.participant_user_ids.clear(); self.subscriptions.clear(); - self.live_kit_room.take(); + self.live_kit.take(); self.client.send(proto::LeaveRoom { id: self.id })?; Ok(()) } @@ -342,8 +346,9 @@ impl Room { }, ); - if let Some((room, _)) = this.live_kit_room.as_ref() { - let tracks = room.remote_video_tracks(&peer_id.0.to_string()); + if let Some(live_kit) = this.live_kit.as_ref() { + let tracks = + live_kit.room.remote_video_tracks(&peer_id.0.to_string()); for track in tracks { this.remote_video_track_updated( RemoteVideoTrackUpdate::Subscribed(track), @@ -605,24 +610,36 @@ impl Room { return Task::ready(Err(anyhow!("room is offline"))); } - let room = if let Some((room, _)) = self.live_kit_room.as_ref() { - room.clone() - } else { - return Task::ready(Err(anyhow!("not connected to LiveKit"))); - }; - - cx.foreground().spawn(async move { + cx.spawn_weak(|this, mut cx| async move { let displays = live_kit_client::display_sources().await?; let display = displays .first() .ok_or_else(|| anyhow!("no display found"))?; let track = LocalVideoTrack::screen_share_for_display(&display); - room.publish_video_track(&track).await?; - Ok(()) + + let publication = this + .upgrade(&cx)? + .read_with(&cx, |this, _| { + this.live_kit + .as_ref() + .map(|live_kit| live_kit.room.publish_video_track(&track)) + })? + .await?; + + this.upgrade(&cx)?.update(cx, |this, _| { + this.live_kit.as_mut()?.screen_track = Some(publication); + Some(()) + }) }) } } +struct LiveKitRoom { + room: Arc, + screen_track: Option, + _maintain_room: Task<()>, +} + #[derive(Copy, Clone, PartialEq, Eq)] pub enum RoomStatus { Online, diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index c7ce2178ab..7488eb9444 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -85,13 +85,13 @@ public func LKRoomDisconnect(room: UnsafeRawPointer) { } @_cdecl("LKRoomPublishVideoTrack") -public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { +public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue() let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishVideoTrack(track: track).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?) + room.localParticipant?.publishVideoTrack(track: track).then { publication in + callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) }.catch { error in - callback(callback_data, error.localizedDescription as CFString) + callback(callback_data, nil, error.localizedDescription as CFString) } } diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 8eaa9a6f6a..b3724e91db 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -45,7 +45,7 @@ extern "C" { fn LKRoomPublishVideoTrack( room: *const c_void, track: *const c_void, - callback: extern "C" fn(*mut c_void, CFStringRef), + callback: extern "C" fn(*mut c_void, *mut c_void, CFStringRef), callback_data: *mut c_void, ); fn LKRoomVideoTracksForRemoteParticipant( @@ -108,10 +108,28 @@ impl Room { async { rx.await.unwrap().context("error connecting to room") } } - pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future> { - let (did_publish, tx, rx) = Self::build_done_callback(); + pub fn publish_video_track( + &self, + track: &LocalVideoTrack, + ) -> impl Future> { + let (tx, rx) = oneshot::channel::>(); + extern "C" fn callback(tx: *mut c_void, publication: *mut c_void, error: CFStringRef) { + let tx = + unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; + if error.is_null() { + let _ = tx.send(Ok(LocalTrackPublication(publication))); + } else { + let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; + let _ = tx.send(Err(anyhow!(error))); + } + } unsafe { - LKRoomPublishVideoTrack(self.native_room, track.0, did_publish, tx); + LKRoomPublishVideoTrack( + self.native_room, + track.0, + callback, + Box::into_raw(Box::new(tx)) as *mut c_void, + ); } async { rx.await.unwrap().context("error publishing video track") } } @@ -275,6 +293,14 @@ impl Drop for LocalVideoTrack { } } +pub struct LocalTrackPublication(*const c_void); + +impl Drop for LocalTrackPublication { + fn drop(&mut self) { + unsafe { CFRelease(self.0) } + } +} + #[derive(Debug)] pub struct RemoteVideoTrack { native_track: *const c_void, From 773f569385fabae114fd83edbe1bfbd7d3cdb75c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 10:19:20 +0200 Subject: [PATCH 27/63] Add control to toggle screen-sharing --- assets/icons/disable_screen_sharing_12.svg | 3 + assets/icons/enable_screen_sharing_12.svg | 3 + crates/call/src/call.rs | 2 - crates/call/src/room.rs | 47 +++++++++++--- crates/collab_ui/src/collab_titlebar_item.rs | 62 ++++++++++++++++++- .../Sources/LiveKitBridge/LiveKitBridge.swift | 7 +++ crates/live_kit_client/src/live_kit_client.rs | 7 +++ crates/theme/src/theme.rs | 1 + styles/src/styleTree/workspace.ts | 11 ++++ 9 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 assets/icons/disable_screen_sharing_12.svg create mode 100644 assets/icons/enable_screen_sharing_12.svg diff --git a/assets/icons/disable_screen_sharing_12.svg b/assets/icons/disable_screen_sharing_12.svg new file mode 100644 index 0000000000..c2a4edd45b --- /dev/null +++ b/assets/icons/disable_screen_sharing_12.svg @@ -0,0 +1,3 @@ + + + diff --git a/assets/icons/enable_screen_sharing_12.svg b/assets/icons/enable_screen_sharing_12.svg new file mode 100644 index 0000000000..6ae37649d2 --- /dev/null +++ b/assets/icons/enable_screen_sharing_12.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 056c6f1260..6b06d04375 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -132,8 +132,6 @@ impl ActiveCall { Room::create(recipient_user_id, initial_project, client, user_store, cx) }) .await?; - room.update(&mut cx, |room, cx| room.share_screen(cx)) - .await?; this.update(&mut cx, |this, cx| this.set_room(Some(room), cx)); }; diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 86946b7974..4553772095 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -606,6 +606,12 @@ impl Room { }) } + pub fn is_screen_sharing(&self) -> bool { + self.live_kit + .as_ref() + .map_or(false, |live_kit| live_kit.screen_track.is_some()) + } + pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); @@ -617,22 +623,49 @@ impl Room { .first() .ok_or_else(|| anyhow!("no display found"))?; let track = LocalVideoTrack::screen_share_for_display(&display); - let publication = this - .upgrade(&cx)? + .upgrade(&cx) + .ok_or_else(|| anyhow!("room was dropped"))? .read_with(&cx, |this, _| { this.live_kit .as_ref() .map(|live_kit| live_kit.room.publish_video_track(&track)) - })? + }) + .ok_or_else(|| anyhow!("live-kit was not initialized"))? .await?; - this.upgrade(&cx)?.update(cx, |this, _| { - this.live_kit.as_mut()?.screen_track = Some(publication); - Some(()) - }) + this.upgrade(&cx) + .ok_or_else(|| anyhow!("room was dropped"))? + .update(&mut cx, |this, cx| { + let live_kit = this + .live_kit + .as_mut() + .ok_or_else(|| anyhow!("live-kit was not initialized"))?; + live_kit.screen_track = Some(publication); + cx.notify(); + Ok(()) + }) }) } + + pub fn unshare_screen(&mut self, cx: &mut ModelContext) -> Result<()> { + if self.status.is_offline() { + return Err(anyhow!("room is offline")); + } + + let live_kit = self + .live_kit + .as_mut() + .ok_or_else(|| anyhow!("live-kit was not initialized"))?; + let track = live_kit + .screen_track + .take() + .ok_or_else(|| anyhow!("screen was not shared"))?; + live_kit.room.unpublish_track(track); + cx.notify(); + + Ok(()) + } } struct LiveKitRoom { diff --git a/crates/collab_ui/src/collab_titlebar_item.rs b/crates/collab_ui/src/collab_titlebar_item.rs index 1279d30437..43d9bc0dbf 100644 --- a/crates/collab_ui/src/collab_titlebar_item.rs +++ b/crates/collab_ui/src/collab_titlebar_item.rs @@ -10,17 +10,21 @@ use gpui::{ geometry::{rect::RectF, vector::vec2f, PathBuilder}, json::{self, ToJson}, Border, CursorStyle, Entity, ModelHandle, MouseButton, MutableAppContext, RenderContext, - Subscription, View, ViewContext, ViewHandle, WeakViewHandle, + Subscription, Task, View, ViewContext, ViewHandle, WeakViewHandle, }; use settings::Settings; use std::ops::Range; use theme::Theme; use workspace::{FollowNextCollaborator, JoinProject, ToggleFollow, Workspace}; -actions!(collab, [ToggleCollaborationMenu, ShareProject]); +actions!( + collab, + [ToggleCollaborationMenu, ToggleScreenSharing, ShareProject] +); pub fn init(cx: &mut MutableAppContext) { cx.add_action(CollabTitlebarItem::toggle_contacts_popover); + cx.add_action(CollabTitlebarItem::toggle_screen_sharing); cx.add_action(CollabTitlebarItem::share_project); } @@ -48,10 +52,12 @@ impl View for CollabTitlebarItem { }; let theme = cx.global::().theme.clone(); - let project = workspace.read(cx).project().read(cx); let mut container = Flex::row(); + container.add_children(self.render_toggle_screen_sharing_button(&theme, cx)); + if workspace.read(cx).client().status().borrow().is_connected() { + let project = workspace.read(cx).project().read(cx); if project.is_shared() || project.is_remote() || ActiveCall::global(cx).read(cx).room().is_none() @@ -169,6 +175,19 @@ impl CollabTitlebarItem { cx.notify(); } + pub fn toggle_screen_sharing(&mut self, _: &ToggleScreenSharing, cx: &mut ViewContext) { + if let Some(room) = ActiveCall::global(cx).read(cx).room().cloned() { + let toggle_screen_sharing = room.update(cx, |room, cx| { + if room.is_screen_sharing() { + Task::ready(room.unshare_screen(cx)) + } else { + room.share_screen(cx) + } + }); + toggle_screen_sharing.detach_and_log_err(cx); + } + } + fn render_toggle_contacts_button( &self, theme: &Theme, @@ -237,6 +256,43 @@ impl CollabTitlebarItem { .boxed() } + fn render_toggle_screen_sharing_button( + &self, + theme: &Theme, + cx: &mut RenderContext, + ) -> Option { + let active_call = ActiveCall::global(cx); + let room = active_call.read(cx).room().cloned()?; + let icon = if room.read(cx).is_screen_sharing() { + "icons/disable_screen_sharing_12.svg" + } else { + "icons/enable_screen_sharing_12.svg" + }; + let titlebar = &theme.workspace.titlebar; + Some( + MouseEventHandler::::new(0, cx, |state, _| { + let style = titlebar.call_control.style_for(state, false); + Svg::new(icon) + .with_color(style.color) + .constrained() + .with_width(style.icon_width) + .aligned() + .constrained() + .with_width(style.button_width) + .with_height(style.button_width) + .contained() + .with_style(style.container) + .boxed() + }) + .with_cursor_style(CursorStyle::PointingHand) + .on_click(MouseButton::Left, move |_, cx| { + cx.dispatch_action(ToggleScreenSharing); + }) + .aligned() + .boxed(), + ) + } + fn render_share_button(&self, theme: &Theme, cx: &mut RenderContext) -> ElementBox { enum Share {} diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 7488eb9444..8bfa98b522 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -95,6 +95,13 @@ public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPoin } } +@_cdecl("LKRoomUnpublishTrack") +public func LKRoomUnpublishTrack(room: UnsafeRawPointer, publication: UnsafeRawPointer) { + let room = Unmanaged.fromOpaque(room).takeUnretainedValue() + let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() + room.localParticipant?.unpublish(publication: publication) +} + @_cdecl("LKRoomVideoTracksForRemoteParticipant") public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { let room = Unmanaged.fromOpaque(room).takeUnretainedValue() diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index b3724e91db..6532ce110a 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -48,6 +48,7 @@ extern "C" { callback: extern "C" fn(*mut c_void, *mut c_void, CFStringRef), callback_data: *mut c_void, ); + fn LKRoomUnpublishTrack(room: *const c_void, publication: *const c_void); fn LKRoomVideoTracksForRemoteParticipant( room: *const c_void, participant_id: CFStringRef, @@ -134,6 +135,12 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } + pub fn unpublish_track(&self, publication: LocalTrackPublication) { + unsafe { + LKRoomUnpublishTrack(self.native_room, publication.0); + } + } + pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { unsafe { let tracks = LKRoomVideoTracksForRemoteParticipant( diff --git a/crates/theme/src/theme.rs b/crates/theme/src/theme.rs index 51aa3232d4..e517f8ced0 100644 --- a/crates/theme/src/theme.rs +++ b/crates/theme/src/theme.rs @@ -79,6 +79,7 @@ pub struct Titlebar { pub sign_in_prompt: Interactive, pub outdated_warning: ContainedText, pub share_button: Interactive, + pub call_control: Interactive, pub toggle_contacts_button: Interactive, pub toggle_contacts_badge: ContainerStyle, } diff --git a/styles/src/styleTree/workspace.ts b/styles/src/styleTree/workspace.ts index c8d32ae1a8..0defabbd7f 100644 --- a/styles/src/styleTree/workspace.ts +++ b/styles/src/styleTree/workspace.ts @@ -138,7 +138,18 @@ export default function workspace(theme: Theme) { }, cornerRadius: 6, }, + callControl: { + cornerRadius: 6, + color: iconColor(theme, "secondary"), + iconWidth: 12, + buttonWidth: 20, + hover: { + background: backgroundColor(theme, "on300", "hovered"), + color: iconColor(theme, "active"), + }, + }, toggleContactsButton: { + margin: { left: 6 }, cornerRadius: 6, color: iconColor(theme, "secondary"), iconWidth: 8, From ed6f482e68d0f2969a838fd30ec9625635dfd1ae Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 10:29:03 +0200 Subject: [PATCH 28/63] Exercise `unpublish_track` in `live_kit_client` --- crates/live_kit_client/examples/test_app.rs | 25 +++++++++++++++---- crates/live_kit_client/src/live_kit_client.rs | 1 + 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index 94c324becf..63f15c2685 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -63,17 +63,32 @@ fn main() { let display = displays.into_iter().next().unwrap(); let track_a = LocalVideoTrack::screen_share_for_display(&display); - room_a.publish_video_track(&track_a).await.unwrap(); + let track_a_publication = room_a.publish_video_track(&track_a).await.unwrap(); - let next_update = track_changes.next().await.unwrap(); - - if let RemoteVideoTrackUpdate::Subscribed(track) = next_update { + if let RemoteVideoTrackUpdate::Subscribed(track) = track_changes.next().await.unwrap() { let remote_tracks = room_b.remote_video_tracks("test-participant-1"); assert_eq!(remote_tracks.len(), 1); assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); assert_eq!(track.publisher_id(), "test-participant-1"); } else { - panic!("unexpected message") + panic!("unexpected message"); + } + + let remote_track = room_b + .remote_video_tracks("test-participant-1") + .pop() + .unwrap(); + room_a.unpublish_track(track_a_publication); + if let RemoteVideoTrackUpdate::Unsubscribed { + publisher_id, + track_id, + } = track_changes.next().await.unwrap() + { + assert_eq!(publisher_id, "test-participant-1"); + assert_eq!(remote_track.sid(), track_id); + assert_eq!(room_b.remote_video_tracks("test-participant-1").len(), 0); + } else { + panic!("unexpected message"); } cx.platform().quit(); diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 6532ce110a..ab4afb1941 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -260,6 +260,7 @@ impl RoomDelegate { if let Some(room) = room.upgrade() { room.did_subscribe_to_remote_video_track(track); } + let _ = Weak::into_raw(room); } extern "C" fn on_did_unsubscribe_from_remote_video_track( From e49fc9f4b1a14963d28ee1936a0e5ca677df3be3 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 10:45:51 +0200 Subject: [PATCH 29/63] Prevent `Room` from screen-sharing twice --- crates/call/src/room.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 4553772095..d3d462e54a 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -615,6 +615,8 @@ impl Room { pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); + } else if self.is_screen_sharing() { + return Task::ready(Err(anyhow!("screen was already shared"))); } cx.spawn_weak(|this, mut cx| async move { From 3160d07b9c5389cb6d5b6a34d39a52da07490968 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 11:38:24 +0200 Subject: [PATCH 30/63] Model pending screen share in `Room` --- crates/call/src/room.rs | 124 +++++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 34 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index d3d462e54a..f09b255df5 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -10,8 +10,8 @@ use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate}; use postage::watch; use project::Project; -use std::{os::unix::prelude::OsStrExt, sync::Arc}; -use util::ResultExt; +use std::{mem, os::unix::prelude::OsStrExt, sync::Arc}; +use util::{post_inc, ResultExt}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum Event { @@ -100,7 +100,8 @@ impl Room { .detach_and_log_err(cx); Some(LiveKitRoom { room, - screen_track: None, + screen_track: ScreenTrack::None, + next_publish_id: 0, _maintain_room, }) } else { @@ -607,9 +608,9 @@ impl Room { } pub fn is_screen_sharing(&self) -> bool { - self.live_kit - .as_ref() - .map_or(false, |live_kit| live_kit.screen_track.is_some()) + self.live_kit.as_ref().map_or(false, |live_kit| { + !matches!(live_kit.screen_track, ScreenTrack::None) + }) } pub fn share_screen(&mut self, cx: &mut ModelContext) -> Task> { @@ -619,23 +620,34 @@ impl Room { return Task::ready(Err(anyhow!("screen was already shared"))); } - cx.spawn_weak(|this, mut cx| async move { - let displays = live_kit_client::display_sources().await?; - let display = displays - .first() - .ok_or_else(|| anyhow!("no display found"))?; - let track = LocalVideoTrack::screen_share_for_display(&display); - let publication = this - .upgrade(&cx) - .ok_or_else(|| anyhow!("room was dropped"))? - .read_with(&cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_video_track(&track)) - }) - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await?; + let publish_id = if let Some(live_kit) = self.live_kit.as_mut() { + let publish_id = post_inc(&mut live_kit.next_publish_id); + live_kit.screen_track = ScreenTrack::Pending { publish_id }; + cx.notify(); + publish_id + } else { + return Task::ready(Err(anyhow!("live-kit was not initialized"))); + }; + cx.spawn_weak(|this, mut cx| async move { + let publish_track = async { + let displays = live_kit_client::display_sources().await?; + let display = displays + .first() + .ok_or_else(|| anyhow!("no display found"))?; + let track = LocalVideoTrack::screen_share_for_display(&display); + this.upgrade(&cx) + .ok_or_else(|| anyhow!("room was dropped"))? + .read_with(&cx, |this, _| { + this.live_kit + .as_ref() + .map(|live_kit| live_kit.room.publish_video_track(&track)) + }) + .ok_or_else(|| anyhow!("live-kit was not initialized"))? + .await + }; + + let publication = publish_track.await; this.upgrade(&cx) .ok_or_else(|| anyhow!("room was dropped"))? .update(&mut cx, |this, cx| { @@ -643,9 +655,36 @@ impl Room { .live_kit .as_mut() .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - live_kit.screen_track = Some(publication); - cx.notify(); - Ok(()) + + let canceled = if let ScreenTrack::Pending { + publish_id: cur_publish_id, + } = &live_kit.screen_track + { + *cur_publish_id != publish_id + } else { + true + }; + + match publication { + Ok(publication) => { + if canceled { + live_kit.room.unpublish_track(publication); + } else { + live_kit.screen_track = ScreenTrack::Published(publication); + cx.notify(); + } + Ok(()) + } + Err(error) => { + if canceled { + Ok(()) + } else { + live_kit.screen_track = ScreenTrack::None; + cx.notify(); + Err(error) + } + } + } }) }) } @@ -659,23 +698,40 @@ impl Room { .live_kit .as_mut() .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - let track = live_kit - .screen_track - .take() - .ok_or_else(|| anyhow!("screen was not shared"))?; - live_kit.room.unpublish_track(track); - cx.notify(); - - Ok(()) + match mem::take(&mut live_kit.screen_track) { + ScreenTrack::None => Err(anyhow!("screen was not shared")), + ScreenTrack::Pending { .. } => { + cx.notify(); + Ok(()) + } + ScreenTrack::Published(track) => { + live_kit.room.unpublish_track(track); + cx.notify(); + Ok(()) + } + } } } struct LiveKitRoom { room: Arc, - screen_track: Option, + screen_track: ScreenTrack, + next_publish_id: usize, _maintain_room: Task<()>, } +pub enum ScreenTrack { + None, + Pending { publish_id: usize }, + Published(LocalTrackPublication), +} + +impl Default for ScreenTrack { + fn default() -> Self { + Self::None + } +} + #[derive(Copy, Clone, PartialEq, Eq)] pub enum RoomStatus { Online, From fb5c6493cfa8adf0f9c440d50a418da14734c6ed Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 13:53:40 +0200 Subject: [PATCH 31/63] WIP: Start on a fake implementation of live-kit --- crates/call/Cargo.toml | 2 + crates/live_kit_client/Cargo.toml | 3 + crates/live_kit_client/build.rs | 14 +- crates/live_kit_client/src/live_kit_client.rs | 432 +----------------- crates/live_kit_client/src/prod.rs | 422 +++++++++++++++++ crates/live_kit_client/src/test.rs | 77 ++++ 6 files changed, 518 insertions(+), 432 deletions(-) create mode 100644 crates/live_kit_client/src/prod.rs create mode 100644 crates/live_kit_client/src/test.rs diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index bd592e1de1..7556be6f77 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -12,6 +12,7 @@ test-support = [ "client/test-support", "collections/test-support", "gpui/test-support", + "live_kit_client/test-support", "project/test-support", "util/test-support" ] @@ -33,5 +34,6 @@ postage = { version = "0.4.1", features = ["futures-traits"] } client = { path = "../client", features = ["test-support"] } collections = { path = "../collections", features = ["test-support"] } gpui = { path = "../gpui", features = ["test-support"] } +live_kit_client = { path = "../live_kit_client", features = ["test-support"] } project = { path = "../project", features = ["test-support"] } util = { path = "../util", features = ["test-support"] } diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index af7ddb67c0..43aabf46e6 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -11,6 +11,9 @@ doctest = false [[example]] name = "test_app" +[features] +test-support = [] + [dependencies] media = { path = "../media" } diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 872454a489..7d5b0944f1 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -35,14 +35,16 @@ pub struct SwiftTarget { const MACOS_TARGET_VERSION: &str = "10.15"; fn main() { - let swift_target = get_swift_target(); + if cfg!(not(any(test, feature = "test-support"))) { + let swift_target = get_swift_target(); - build_bridge(&swift_target); - link_swift_stdlib(&swift_target); - link_webrtc_framework(&swift_target); + build_bridge(&swift_target); + link_swift_stdlib(&swift_target); + link_webrtc_framework(&swift_target); - // Register exported Objective-C selectors, protocols, etc when building example binaries. - println!("cargo:rustc-link-arg=-Wl,-ObjC"); + // Register exported Objective-C selectors, protocols, etc when building example binaries. + println!("cargo:rustc-link-arg=-Wl,-ObjC"); + } } fn build_bridge(swift_target: &SwiftTarget) { diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index ab4afb1941..6a1c775157 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -1,428 +1,8 @@ -use anyhow::{anyhow, Context, Result}; -use core_foundation::{ - array::{CFArray, CFArrayRef}, - base::{CFRelease, CFRetain, TCFType}, - string::{CFString, CFStringRef}, -}; -use futures::{ - channel::{mpsc, oneshot}, - Future, -}; -use media::core_video::{CVImageBuffer, CVImageBufferRef}; -use parking_lot::Mutex; -use std::{ - ffi::c_void, - sync::{Arc, Weak}, -}; +pub mod prod; +pub mod test; -pub type Sid = String; +#[cfg(not(any(test, feature = "test-support")))] +pub use prod::*; -extern "C" { - fn LKRoomDelegateCreate( - callback_data: *mut c_void, - on_did_subscribe_to_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: *const c_void, - ), - on_did_unsubscribe_from_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - ) -> *const c_void; - - fn LKRoomCreate(delegate: *const c_void) -> *const c_void; - fn LKRoomConnect( - room: *const c_void, - url: CFStringRef, - token: CFStringRef, - callback: extern "C" fn(*mut c_void, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomDisconnect(room: *const c_void); - fn LKRoomPublishVideoTrack( - room: *const c_void, - track: *const c_void, - callback: extern "C" fn(*mut c_void, *mut c_void, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomUnpublishTrack(room: *const c_void, publication: *const c_void); - fn LKRoomVideoTracksForRemoteParticipant( - room: *const c_void, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKVideoRendererCreate( - callback_data: *mut c_void, - on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef), - on_drop: extern "C" fn(callback_data: *mut c_void), - ) -> *const c_void; - - fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); - fn LKRemoteVideoTrackGetSid(track: *const c_void) -> CFStringRef; - - fn LKDisplaySources( - callback_data: *mut c_void, - callback: extern "C" fn( - callback_data: *mut c_void, - sources: CFArrayRef, - error: CFStringRef, - ), - ); - fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void; -} - -pub struct Room { - native_room: *const c_void, - remote_video_track_subscribers: Mutex>>, - _delegate: RoomDelegate, -} - -impl Room { - pub fn new() -> Arc { - Arc::new_cyclic(|weak_room| { - let delegate = RoomDelegate::new(weak_room.clone()); - Self { - native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, - remote_video_track_subscribers: Default::default(), - _delegate: delegate, - } - }) - } - - pub fn connect(&self, url: &str, token: &str) -> impl Future> { - let url = CFString::new(url); - let token = CFString::new(token); - let (did_connect, tx, rx) = Self::build_done_callback(); - unsafe { - LKRoomConnect( - self.native_room, - url.as_concrete_TypeRef(), - token.as_concrete_TypeRef(), - did_connect, - tx, - ) - } - - async { rx.await.unwrap().context("error connecting to room") } - } - - pub fn publish_video_track( - &self, - track: &LocalVideoTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback(tx: *mut c_void, publication: *mut c_void, error: CFStringRef) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishVideoTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing video track") } - } - - pub fn unpublish_track(&self, publication: LocalTrackPublication) { - unsafe { - LKRoomUnpublishTrack(self.native_room, publication.0); - } - } - - pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomVideoTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = *native_track; - let id = - CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteVideoTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_video_track_updates(&self) -> mpsc::UnboundedReceiver { - let (tx, rx) = mpsc::unbounded(); - self.remote_video_track_subscribers.lock().push(tx); - rx - } - - fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { - let track = Arc::new(track); - self.remote_video_track_subscribers.lock().retain(|tx| { - tx.unbounded_send(RemoteVideoTrackUpdate::Subscribed(track.clone())) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { - self.remote_video_track_subscribers.lock().retain(|tx| { - tx.unbounded_send(RemoteVideoTrackUpdate::Unsubscribed { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn build_done_callback() -> ( - extern "C" fn(*mut c_void, CFStringRef), - *mut c_void, - oneshot::Receiver>, - ) { - let (tx, rx) = oneshot::channel(); - extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(())); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - ( - done_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - rx, - ) - } -} - -impl Drop for Room { - fn drop(&mut self) { - unsafe { - LKRoomDisconnect(self.native_room); - CFRelease(self.native_room); - } - } -} - -struct RoomDelegate { - native_delegate: *const c_void, - weak_room: *const Room, -} - -impl RoomDelegate { - fn new(weak_room: Weak) -> Self { - let weak_room = Weak::into_raw(weak_room); - let native_delegate = unsafe { - LKRoomDelegateCreate( - weak_room as *mut c_void, - Self::on_did_subscribe_to_remote_video_track, - Self::on_did_unsubscribe_from_remote_video_track, - ) - }; - Self { - native_delegate, - weak_room, - } - } - - extern "C" fn on_did_subscribe_to_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: *const c_void, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteVideoTrack::new(track, track_id, publisher_id); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_video_track(track); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_video_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } -} - -impl Drop for RoomDelegate { - fn drop(&mut self) { - unsafe { - CFRelease(self.native_delegate); - let _ = Weak::from_raw(self.weak_room); - } - } -} - -pub struct LocalVideoTrack(*const c_void); - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) }) - } -} - -impl Drop for LocalVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0) } - } -} - -pub struct LocalTrackPublication(*const c_void); - -impl Drop for LocalTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0) } - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - native_track: *const c_void, - sid: Sid, - publisher_id: String, -} - -impl RemoteVideoTrack { - fn new(native_track: *const c_void, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn add_renderer(&self, callback: F) - where - F: 'static + FnMut(CVImageBuffer), - { - extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) - where - F: FnMut(CVImageBuffer), - { - unsafe { - let buffer = CVImageBuffer::wrap_under_get_rule(frame); - let callback = &mut *(callback_data as *mut F); - callback(buffer); - } - } - - extern "C" fn on_drop(callback_data: *mut c_void) { - unsafe { - let _ = Box::from_raw(callback_data as *mut F); - } - } - - let callback_data = Box::into_raw(Box::new(callback)); - unsafe { - let renderer = - LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); - LKVideoTrackAddRenderer(self.native_track, renderer); - } - } -} - -impl Drop for RemoteVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.native_track) } - } -} - -pub enum RemoteVideoTrackUpdate { - Subscribed(Arc), - Unsubscribed { publisher_id: Sid, track_id: Sid }, -} - -pub struct MacOSDisplay(*const c_void); - -impl MacOSDisplay { - fn new(ptr: *const c_void) -> Self { - unsafe { - CFRetain(ptr); - } - Self(ptr) - } -} - -impl Drop for MacOSDisplay { - fn drop(&mut self) { - unsafe { CFRelease(self.0) } - } -} - -pub fn display_sources() -> impl Future>> { - extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { - unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>>); - - if sources.is_null() { - let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); - } else { - let sources = CFArray::wrap_under_get_rule(sources) - .into_iter() - .map(|source| MacOSDisplay::new(*source)) - .collect(); - - let _ = tx.send(Ok(sources)); - } - } - } - - let (tx, rx) = oneshot::channel(); - - unsafe { - LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); - } - - async move { rx.await.unwrap() } -} - -#[cfg(test)] -mod tests { - #[test] - fn test_client() {} -} +#[cfg(any(test, feature = "test-support"))] +pub use test::*; diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs new file mode 100644 index 0000000000..427f45fe72 --- /dev/null +++ b/crates/live_kit_client/src/prod.rs @@ -0,0 +1,422 @@ +use anyhow::{anyhow, Context, Result}; +use core_foundation::{ + array::{CFArray, CFArrayRef}, + base::{CFRelease, CFRetain, TCFType}, + string::{CFString, CFStringRef}, +}; +use futures::{ + channel::{mpsc, oneshot}, + Future, +}; +use media::core_video::{CVImageBuffer, CVImageBufferRef}; +use parking_lot::Mutex; +use std::{ + ffi::c_void, + sync::{Arc, Weak}, +}; + +extern "C" { + fn LKRoomDelegateCreate( + callback_data: *mut c_void, + on_did_subscribe_to_remote_video_track: extern "C" fn( + callback_data: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + remote_track: *const c_void, + ), + on_did_unsubscribe_from_remote_video_track: extern "C" fn( + callback_data: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + ), + ) -> *const c_void; + + fn LKRoomCreate(delegate: *const c_void) -> *const c_void; + fn LKRoomConnect( + room: *const c_void, + url: CFStringRef, + token: CFStringRef, + callback: extern "C" fn(*mut c_void, CFStringRef), + callback_data: *mut c_void, + ); + fn LKRoomDisconnect(room: *const c_void); + fn LKRoomPublishVideoTrack( + room: *const c_void, + track: *const c_void, + callback: extern "C" fn(*mut c_void, *mut c_void, CFStringRef), + callback_data: *mut c_void, + ); + fn LKRoomUnpublishTrack(room: *const c_void, publication: *const c_void); + fn LKRoomVideoTracksForRemoteParticipant( + room: *const c_void, + participant_id: CFStringRef, + ) -> CFArrayRef; + + fn LKVideoRendererCreate( + callback_data: *mut c_void, + on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef), + on_drop: extern "C" fn(callback_data: *mut c_void), + ) -> *const c_void; + + fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); + fn LKRemoteVideoTrackGetSid(track: *const c_void) -> CFStringRef; + + fn LKDisplaySources( + callback_data: *mut c_void, + callback: extern "C" fn( + callback_data: *mut c_void, + sources: CFArrayRef, + error: CFStringRef, + ), + ); + fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void; +} + +pub type Sid = String; + +pub struct Room { + native_room: *const c_void, + remote_video_track_subscribers: Mutex>>, + _delegate: RoomDelegate, +} + +impl Room { + pub fn new() -> Arc { + Arc::new_cyclic(|weak_room| { + let delegate = RoomDelegate::new(weak_room.clone()); + Self { + native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + remote_video_track_subscribers: Default::default(), + _delegate: delegate, + } + }) + } + + pub fn connect(&self, url: &str, token: &str) -> impl Future> { + let url = CFString::new(url); + let token = CFString::new(token); + let (did_connect, tx, rx) = Self::build_done_callback(); + unsafe { + LKRoomConnect( + self.native_room, + url.as_concrete_TypeRef(), + token.as_concrete_TypeRef(), + did_connect, + tx, + ) + } + + async { rx.await.unwrap().context("error connecting to room") } + } + + pub fn publish_video_track( + &self, + track: &LocalVideoTrack, + ) -> impl Future> { + let (tx, rx) = oneshot::channel::>(); + extern "C" fn callback(tx: *mut c_void, publication: *mut c_void, error: CFStringRef) { + let tx = + unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; + if error.is_null() { + let _ = tx.send(Ok(LocalTrackPublication(publication))); + } else { + let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; + let _ = tx.send(Err(anyhow!(error))); + } + } + unsafe { + LKRoomPublishVideoTrack( + self.native_room, + track.0, + callback, + Box::into_raw(Box::new(tx)) as *mut c_void, + ); + } + async { rx.await.unwrap().context("error publishing video track") } + } + + pub fn unpublish_track(&self, publication: LocalTrackPublication) { + unsafe { + LKRoomUnpublishTrack(self.native_room, publication.0); + } + } + + pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { + unsafe { + let tracks = LKRoomVideoTracksForRemoteParticipant( + self.native_room, + CFString::new(participant_id).as_concrete_TypeRef(), + ); + + if tracks.is_null() { + Vec::new() + } else { + let tracks = CFArray::wrap_under_get_rule(tracks); + tracks + .into_iter() + .map(|native_track| { + let native_track = *native_track; + let id = + CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) + .to_string(); + Arc::new(RemoteVideoTrack::new( + native_track, + id, + participant_id.into(), + )) + }) + .collect() + } + } + } + + pub fn remote_video_track_updates(&self) -> mpsc::UnboundedReceiver { + let (tx, rx) = mpsc::unbounded(); + self.remote_video_track_subscribers.lock().push(tx); + rx + } + + fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { + let track = Arc::new(track); + self.remote_video_track_subscribers.lock().retain(|tx| { + tx.unbounded_send(RemoteVideoTrackUpdate::Subscribed(track.clone())) + .is_ok() + }); + } + + fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { + self.remote_video_track_subscribers.lock().retain(|tx| { + tx.unbounded_send(RemoteVideoTrackUpdate::Unsubscribed { + publisher_id: publisher_id.clone(), + track_id: track_id.clone(), + }) + .is_ok() + }); + } + + fn build_done_callback() -> ( + extern "C" fn(*mut c_void, CFStringRef), + *mut c_void, + oneshot::Receiver>, + ) { + let (tx, rx) = oneshot::channel(); + extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { + let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; + if error.is_null() { + let _ = tx.send(Ok(())); + } else { + let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; + let _ = tx.send(Err(anyhow!(error))); + } + } + ( + done_callback, + Box::into_raw(Box::new(tx)) as *mut c_void, + rx, + ) + } +} + +impl Drop for Room { + fn drop(&mut self) { + unsafe { + LKRoomDisconnect(self.native_room); + CFRelease(self.native_room); + } + } +} + +struct RoomDelegate { + native_delegate: *const c_void, + weak_room: *const Room, +} + +impl RoomDelegate { + fn new(weak_room: Weak) -> Self { + let weak_room = Weak::into_raw(weak_room); + let native_delegate = unsafe { + LKRoomDelegateCreate( + weak_room as *mut c_void, + Self::on_did_subscribe_to_remote_video_track, + Self::on_did_unsubscribe_from_remote_video_track, + ) + }; + Self { + native_delegate, + weak_room, + } + } + + extern "C" fn on_did_subscribe_to_remote_video_track( + room: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + track: *const c_void, + ) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; + let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; + let track = RemoteVideoTrack::new(track, track_id, publisher_id); + if let Some(room) = room.upgrade() { + room.did_subscribe_to_remote_video_track(track); + } + let _ = Weak::into_raw(room); + } + + extern "C" fn on_did_unsubscribe_from_remote_video_track( + room: *mut c_void, + publisher_id: CFStringRef, + track_id: CFStringRef, + ) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; + let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; + if let Some(room) = room.upgrade() { + room.did_unsubscribe_from_remote_video_track(publisher_id, track_id); + } + let _ = Weak::into_raw(room); + } +} + +impl Drop for RoomDelegate { + fn drop(&mut self) { + unsafe { + CFRelease(self.native_delegate); + let _ = Weak::from_raw(self.weak_room); + } + } +} + +pub struct LocalVideoTrack(*const c_void); + +impl LocalVideoTrack { + pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { + Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) }) + } +} + +impl Drop for LocalVideoTrack { + fn drop(&mut self) { + unsafe { CFRelease(self.0) } + } +} + +pub struct LocalTrackPublication(*const c_void); + +impl Drop for LocalTrackPublication { + fn drop(&mut self) { + unsafe { CFRelease(self.0) } + } +} + +#[derive(Debug)] +pub struct RemoteVideoTrack { + native_track: *const c_void, + sid: Sid, + publisher_id: String, +} + +impl RemoteVideoTrack { + fn new(native_track: *const c_void, sid: Sid, publisher_id: String) -> Self { + unsafe { + CFRetain(native_track); + } + Self { + native_track, + sid, + publisher_id, + } + } + + pub fn sid(&self) -> &str { + &self.sid + } + + pub fn publisher_id(&self) -> &str { + &self.publisher_id + } + + pub fn add_renderer(&self, callback: F) + where + F: 'static + FnMut(CVImageBuffer), + { + extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) + where + F: FnMut(CVImageBuffer), + { + unsafe { + let buffer = CVImageBuffer::wrap_under_get_rule(frame); + let callback = &mut *(callback_data as *mut F); + callback(buffer); + } + } + + extern "C" fn on_drop(callback_data: *mut c_void) { + unsafe { + let _ = Box::from_raw(callback_data as *mut F); + } + } + + let callback_data = Box::into_raw(Box::new(callback)); + unsafe { + let renderer = + LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); + LKVideoTrackAddRenderer(self.native_track, renderer); + } + } +} + +impl Drop for RemoteVideoTrack { + fn drop(&mut self) { + unsafe { CFRelease(self.native_track) } + } +} + +pub enum RemoteVideoTrackUpdate { + Subscribed(Arc), + Unsubscribed { publisher_id: Sid, track_id: Sid }, +} + +pub struct MacOSDisplay(*const c_void); + +impl MacOSDisplay { + fn new(ptr: *const c_void) -> Self { + unsafe { + CFRetain(ptr); + } + Self(ptr) + } +} + +impl Drop for MacOSDisplay { + fn drop(&mut self) { + unsafe { CFRelease(self.0) } + } +} + +pub fn display_sources() -> impl Future>> { + extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { + unsafe { + let tx = Box::from_raw(tx as *mut oneshot::Sender>>); + + if sources.is_null() { + let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); + } else { + let sources = CFArray::wrap_under_get_rule(sources) + .into_iter() + .map(|source| MacOSDisplay::new(*source)) + .collect(); + + let _ = tx.send(Ok(sources)); + } + } + } + + let (tx, rx) = oneshot::channel(); + + unsafe { + LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); + } + + async move { rx.await.unwrap() } +} diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs new file mode 100644 index 0000000000..09e4733d92 --- /dev/null +++ b/crates/live_kit_client/src/test.rs @@ -0,0 +1,77 @@ +use anyhow::Result; +use futures::{channel::mpsc, future}; +use media::core_video::CVImageBuffer; +use std::{future::Future, sync::Arc}; + +pub type Sid = String; + +pub struct Room; + +impl Room { + pub fn new() -> Arc { + Arc::new(Self) + } + + pub fn connect(&self, url: &str, token: &str) -> impl Future> { + future::pending() + } + + pub fn publish_video_track( + &self, + track: &LocalVideoTrack, + ) -> impl Future> { + future::pending() + } + + pub fn unpublish_track(&self, publication: LocalTrackPublication) {} + + pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { + Default::default() + } + + pub fn remote_video_track_updates(&self) -> mpsc::UnboundedReceiver { + mpsc::unbounded().1 + } +} + +pub struct LocalTrackPublication; + +pub struct LocalVideoTrack; + +impl LocalVideoTrack { + pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { + Self + } +} + +pub struct RemoteVideoTrack { + sid: Sid, + publisher_id: Sid, +} + +impl RemoteVideoTrack { + pub fn sid(&self) -> &str { + &self.sid + } + + pub fn publisher_id(&self) -> &str { + &self.publisher_id + } + + pub fn add_renderer(&self, callback: F) + where + F: 'static + FnMut(CVImageBuffer), + { + } +} + +pub enum RemoteVideoTrackUpdate { + Subscribed(Arc), + Unsubscribed { publisher_id: Sid, track_id: Sid }, +} + +pub struct MacOSDisplay; + +pub fn display_sources() -> impl Future>> { + future::pending() +} From 288c0399296ea23bec73c6684f245b520b2f3446 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 14:58:50 +0200 Subject: [PATCH 32/63] Start on implementing a fake live-kit server --- Cargo.lock | 2 + crates/live_kit_client/Cargo.toml | 10 +- crates/live_kit_client/src/live_kit_client.rs | 4 +- crates/live_kit_client/src/test.rs | 91 ++++++++++++++++++- crates/live_kit_server/src/api.rs | 14 +-- crates/live_kit_server/src/token.rs | 48 ++++++---- 6 files changed, 134 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68516021b1..68a7ed2b82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3167,6 +3167,7 @@ dependencies = [ "byteorder", "bytes 1.2.1", "cocoa", + "collections", "core-foundation", "core-graphics", "foreign-types", @@ -3174,6 +3175,7 @@ dependencies = [ "gpui", "hmac 0.12.1", "jwt", + "lazy_static", "live_kit_server", "log", "media", diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index 43aabf46e6..7b77386fde 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -12,19 +12,25 @@ doctest = false name = "test_app" [features] -test-support = [] +test-support = ["collections/test-support", "gpui/test-support", "lazy_static", "live_kit_server"] [dependencies] +collections = { path = "../collections", optional = true } +gpui = { path = "../gpui", optional = true } +live_kit_server = { path = "../live_kit_server", optional = true } media = { path = "../media" } anyhow = "1.0.38" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" +lazy_static = { version = "1.4", optional = true } parking_lot = "0.11.1" [dev-dependencies] -gpui = { path = "../gpui" } +collections = { path = "../collections", features = ["test-support"] } +gpui = { path = "../gpui", features = ["test-support"] } +lazy_static = "1.4" live_kit_server = { path = "../live_kit_server" } media = { path = "../media" } diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 6a1c775157..2ded570828 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -1,8 +1,10 @@ pub mod prod; -pub mod test; #[cfg(not(any(test, feature = "test-support")))] pub use prod::*; +#[cfg(any(test, feature = "test-support"))] +mod test; + #[cfg(any(test, feature = "test-support"))] pub use test::*; diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 09e4733d92..50be090c80 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -1,8 +1,80 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; +use collections::HashMap; use futures::{channel::mpsc, future}; +use gpui::executor::Background; +use lazy_static::lazy_static; use media::core_video::CVImageBuffer; +use parking_lot::Mutex; use std::{future::Future, sync::Arc}; +lazy_static! { + static ref SERVERS: Mutex>> = Default::default(); +} + +pub struct FakeServer { + url: String, + secret_key: String, + rooms: Mutex>, + background: Arc, +} + +impl FakeServer { + pub fn create( + url: String, + secret_key: String, + background: Arc, + ) -> Result> { + let mut servers = SERVERS.lock(); + if servers.contains_key(&url) { + Err(anyhow!("a server with url {:?} already exists", url)) + } else { + let server = Arc::new(FakeServer { + url: url.clone(), + secret_key, + rooms: Default::default(), + background, + }); + servers.insert(url, server.clone()); + Ok(server) + } + } + + fn get(url: &str) -> Result> { + Ok(SERVERS + .lock() + .get(url) + .ok_or_else(|| anyhow!("no server found for url"))? + .clone()) + } + + pub fn teardown(&self) -> Result<()> { + SERVERS + .lock() + .remove(&self.url) + .ok_or_else(|| anyhow!("server with url {:?} does not exist", self.url))?; + Ok(()) + } + + async fn join_room(&self, token: String, client_room: Arc) -> Result<()> { + self.background.simulate_random_delay().await; + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; + let identity = claims.sub.unwrap().to_string(); + let room = claims.video.room.unwrap(); + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room) + .ok_or_else(|| anyhow!("room {} does not exist", room))?; + room.clients.insert(identity, client_room); + Ok(()) + } +} + +struct FakeServerRoom { + clients: HashMap>, +} + +impl FakeServerRoom {} + pub type Sid = String; pub struct Room; @@ -12,8 +84,15 @@ impl Room { Arc::new(Self) } - pub fn connect(&self, url: &str, token: &str) -> impl Future> { - future::pending() + pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { + let this = self.clone(); + let url = url.to_string(); + let token = token.to_string(); + async move { + let server = FakeServer::get(&url)?; + server.join_room(token, this).await?; + Ok(()) + } } pub fn publish_video_track( @@ -34,6 +113,12 @@ impl Room { } } +impl Drop for Room { + fn drop(&mut self) { + todo!() + } +} + pub struct LocalTrackPublication; pub struct LocalVideoTrack; diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index 8ba94be43a..2dafef253b 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -66,11 +66,7 @@ impl Client { ) -> impl Future> { let response = self.request( "twirp/livekit.RoomService/RemoveParticipant", - token::VideoGrant { - room_admin: Some(true), - room: Some(&room), - ..Default::default() - }, + token::VideoGrant::to_admin(&room), proto::RoomParticipantIdentity { room: room.clone(), identity, @@ -87,13 +83,7 @@ impl Client { &self.key, &self.secret, Some(identity), - token::VideoGrant { - room: Some(room), - room_join: Some(true), - can_publish: Some(true), - can_subscribe: Some(true), - ..Default::default() - }, + token::VideoGrant::to_join(room), ) } diff --git a/crates/live_kit_server/src/token.rs b/crates/live_kit_server/src/token.rs index 9956ad006c..072a8be0c9 100644 --- a/crates/live_kit_server/src/token.rs +++ b/crates/live_kit_server/src/token.rs @@ -1,28 +1,29 @@ use anyhow::{anyhow, Result}; use hmac::{Hmac, Mac}; -use jwt::SignWithKey; -use serde::Serialize; +use jwt::{SignWithKey, VerifyWithKey}; +use serde::{Deserialize, Serialize}; use sha2::Sha256; use std::{ + borrow::Cow, ops::Add, time::{Duration, SystemTime, UNIX_EPOCH}, }; static DEFAULT_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6 hours -#[derive(Default, Serialize)] +#[derive(Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -struct ClaimGrants<'a> { - iss: &'a str, - sub: Option<&'a str>, - iat: u64, - exp: u64, - nbf: u64, - jwtid: Option<&'a str>, - video: VideoGrant<'a>, +pub struct ClaimGrants<'a> { + pub iss: Cow<'a, str>, + pub sub: Option>, + pub iat: u64, + pub exp: u64, + pub nbf: u64, + pub jwtid: Option>, + pub video: VideoGrant<'a>, } -#[derive(Default, Serialize)] +#[derive(Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VideoGrant<'a> { pub room_create: Option, @@ -30,7 +31,7 @@ pub struct VideoGrant<'a> { pub room_list: Option, pub room_record: Option, pub room_admin: Option, - pub room: Option<&'a str>, + pub room: Option>, pub can_publish: Option, pub can_subscribe: Option, pub can_publish_data: Option, @@ -39,9 +40,17 @@ pub struct VideoGrant<'a> { } impl<'a> VideoGrant<'a> { + pub fn to_admin(room: &'a str) -> Self { + Self { + room_admin: Some(true), + room: Some(Cow::Borrowed(room)), + ..Default::default() + } + } + pub fn to_join(room: &'a str) -> Self { Self { - room: Some(room), + room: Some(Cow::Borrowed(room)), room_join: Some(true), can_publish: Some(true), can_subscribe: Some(true), @@ -67,8 +76,8 @@ pub fn create( let now = SystemTime::now(); let claims = ClaimGrants { - iss: api_key, - sub: identity, + iss: Cow::Borrowed(api_key), + sub: identity.map(Cow::Borrowed), iat: now.duration_since(UNIX_EPOCH).unwrap().as_secs(), exp: now .add(DEFAULT_TTL) @@ -76,8 +85,13 @@ pub fn create( .unwrap() .as_secs(), nbf: 0, - jwtid: identity, + jwtid: identity.map(Cow::Borrowed), video: video_grant, }; Ok(claims.sign_with_key(&secret_key)?) } + +pub fn validate<'a>(token: &'a str, secret_key: &str) -> Result> { + let secret_key: Hmac = Hmac::new_from_slice(secret_key.as_bytes())?; + Ok(token.verify_with_key(&secret_key)?) +} From b6e5aa3bb0c31f9d8d907d27505b25a39cdd07fa Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 19 Oct 2022 16:35:34 +0200 Subject: [PATCH 33/63] Use `live_kit_client::TestServer` in integration tests Co-Authored-By: Nathan Sobo --- Cargo.lock | 3 + crates/collab/Cargo.toml | 4 +- crates/collab/src/integration_tests.rs | 24 ++- crates/collab/src/main.rs | 6 +- crates/live_kit_client/Cargo.toml | 12 +- crates/live_kit_client/examples/test_app.rs | 5 +- crates/live_kit_client/src/test.rs | 171 +++++++++++++++++--- crates/live_kit_server/Cargo.toml | 1 + crates/live_kit_server/src/api.rs | 137 ++++++++-------- 9 files changed, 269 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68a7ed2b82..fa167e245f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1066,6 +1066,7 @@ dependencies = [ "language", "lazy_static", "lipsum", + "live_kit_client", "live_kit_server", "log", "lsp", @@ -3163,6 +3164,7 @@ name = "live_kit_client" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "block", "byteorder", "bytes 1.2.1", @@ -3193,6 +3195,7 @@ name = "live_kit_server" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "futures 0.3.24", "hmac 0.12.1", "jwt", diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index 8ce42afb64..6145afad48 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -62,15 +62,17 @@ editor = { path = "../editor", features = ["test-support"] } language = { path = "../language", features = ["test-support"] } fs = { path = "../fs", features = ["test-support"] } git = { path = "../git", features = ["test-support"] } -log = { version = "0.4.16", features = ["kv_unstable_serde"] } +live_kit_client = { path = "../live_kit_client", features = ["test-support"] } lsp = { path = "../lsp", features = ["test-support"] } project = { path = "../project", features = ["test-support"] } rpc = { path = "../rpc", features = ["test-support"] } settings = { path = "../settings", features = ["test-support"] } theme = { path = "../theme" } workspace = { path = "../workspace", features = ["test-support"] } + ctor = "0.1" env_logger = "0.9" +log = { version = "0.4.16", features = ["kv_unstable_serde"] } util = { path = "../util" } lazy_static = "1.4" serde_json = { version = "1.0", features = ["preserve_order"] } diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index cd08dd8632..7cd964cd8a 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -47,7 +47,7 @@ use std::{ path::{Path, PathBuf}, rc::Rc, sync::{ - atomic::{AtomicBool, Ordering::SeqCst}, + atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}, Arc, }, time::Duration, @@ -6138,6 +6138,7 @@ struct TestServer { connection_killers: Arc>>>, forbid_connections: Arc, _test_db: TestDb, + test_live_kit_server: Arc, } impl TestServer { @@ -6145,8 +6146,18 @@ impl TestServer { foreground: Rc, background: Arc, ) -> Self { + static NEXT_LIVE_KIT_SERVER_ID: AtomicUsize = AtomicUsize::new(0); + let test_db = TestDb::fake(background.clone()); - let app_state = Self::build_app_state(&test_db).await; + let live_kit_server_id = NEXT_LIVE_KIT_SERVER_ID.fetch_add(1, SeqCst); + let live_kit_server = live_kit_client::TestServer::create( + format!("http://livekit.{}.test", live_kit_server_id), + format!("devkey-{}", live_kit_server_id), + format!("secret-{}", live_kit_server_id), + background.clone(), + ) + .unwrap(); + let app_state = Self::build_app_state(&test_db, &live_kit_server).await; let peer = Peer::new(); let notifications = mpsc::unbounded(); let server = Server::new(app_state.clone(), Some(notifications.0)); @@ -6159,6 +6170,7 @@ impl TestServer { connection_killers: Default::default(), forbid_connections: Default::default(), _test_db: test_db, + test_live_kit_server: live_kit_server, } } @@ -6354,10 +6366,13 @@ impl TestServer { } } - async fn build_app_state(test_db: &TestDb) -> Arc { + async fn build_app_state( + test_db: &TestDb, + fake_server: &live_kit_client::TestServer, + ) -> Arc { Arc::new(AppState { db: test_db.db().clone(), - live_kit_client: None, + live_kit_client: Some(Arc::new(fake_server.create_api_client())), api_token: Default::default(), invite_link_prefix: Default::default(), }) @@ -6390,6 +6405,7 @@ impl Deref for TestServer { impl Drop for TestServer { fn drop(&mut self) { self.peer.reset(); + self.test_live_kit_server.teardown().unwrap(); } } diff --git a/crates/collab/src/main.rs b/crates/collab/src/main.rs index 49a82bb926..bc860abf62 100644 --- a/crates/collab/src/main.rs +++ b/crates/collab/src/main.rs @@ -41,7 +41,7 @@ pub struct AppState { db: Arc, api_token: String, invite_link_prefix: String, - live_kit_client: Option, + live_kit_client: Option>, } impl AppState { @@ -53,11 +53,11 @@ impl AppState { .zip(config.live_kit_key.as_ref()) .zip(config.live_kit_secret.as_ref()) { - Some(live_kit_server::api::Client::new( + Some(Arc::new(live_kit_server::api::LiveKitClient::new( server.clone(), key.clone(), secret.clone(), - )) + )) as Arc) } else { None }; diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index 7b77386fde..ee2c3f88dc 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -12,7 +12,13 @@ doctest = false name = "test_app" [features] -test-support = ["collections/test-support", "gpui/test-support", "lazy_static", "live_kit_server"] +test-support = [ + "async-trait", + "collections/test-support", + "gpui/test-support", + "lazy_static", + "live_kit_server" +] [dependencies] collections = { path = "../collections", optional = true } @@ -21,6 +27,7 @@ live_kit_server = { path = "../live_kit_server", optional = true } media = { path = "../media" } anyhow = "1.0.38" +async-trait = { version = "0.1", optional = true } core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" @@ -30,11 +37,11 @@ parking_lot = "0.11.1" [dev-dependencies] collections = { path = "../collections", features = ["test-support"] } gpui = { path = "../gpui", features = ["test-support"] } -lazy_static = "1.4" live_kit_server = { path = "../live_kit_server" } media = { path = "../media" } anyhow = "1.0.38" +async-trait = "0.1" block = "0.1" bytes = "1.2" byteorder = "1.4" @@ -45,6 +52,7 @@ foreign-types = "0.3" futures = "0.3" hmac = "0.12" jwt = "0.16" +lazy_static = "1.4" log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" parking_lot = "0.11.1" diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index 63f15c2685..b4f037cead 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -7,7 +7,10 @@ use gpui::{ Menu, MenuItem, ViewContext, }; use live_kit_client::{LocalVideoTrack, RemoteVideoTrackUpdate, Room}; -use live_kit_server::token::{self, VideoGrant}; +use live_kit_server::{ + api::Client, + token::{self, VideoGrant}, +}; use log::LevelFilter; use media::core_video::CVImageBuffer; use postage::watch; diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 50be090c80..050cfd0a47 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -1,35 +1,40 @@ use anyhow::{anyhow, Result}; +use async_trait::async_trait; use collections::HashMap; use futures::{channel::mpsc, future}; use gpui::executor::Background; use lazy_static::lazy_static; +use live_kit_server::token; use media::core_video::CVImageBuffer; use parking_lot::Mutex; use std::{future::Future, sync::Arc}; lazy_static! { - static ref SERVERS: Mutex>> = Default::default(); + static ref SERVERS: Mutex>> = Default::default(); } -pub struct FakeServer { - url: String, - secret_key: String, - rooms: Mutex>, +pub struct TestServer { + pub url: String, + pub api_key: String, + pub secret_key: String, + rooms: Mutex>, background: Arc, } -impl FakeServer { +impl TestServer { pub fn create( url: String, + api_key: String, secret_key: String, background: Arc, - ) -> Result> { + ) -> Result> { let mut servers = SERVERS.lock(); if servers.contains_key(&url) { Err(anyhow!("a server with url {:?} already exists", url)) } else { - let server = Arc::new(FakeServer { + let server = Arc::new(TestServer { url: url.clone(), + api_key, secret_key, rooms: Default::default(), background, @@ -39,7 +44,7 @@ impl FakeServer { } } - fn get(url: &str) -> Result> { + fn get(url: &str) -> Result> { Ok(SERVERS .lock() .get(url) @@ -55,33 +60,151 @@ impl FakeServer { Ok(()) } + pub fn create_api_client(&self) -> TestApiClient { + TestApiClient { + url: self.url.clone(), + } + } + + async fn create_room(&self, room: String) -> Result<()> { + self.background.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); + if server_rooms.contains_key(&room) { + Err(anyhow!("room {:?} already exists", room)) + } else { + server_rooms.insert(room, Default::default()); + Ok(()) + } + } + + async fn delete_room(&self, room: String) -> Result<()> { + // TODO: clear state associated with all `Room`s. + self.background.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); + server_rooms + .remove(&room) + .ok_or_else(|| anyhow!("room {:?} does not exist", room))?; + Ok(()) + } + async fn join_room(&self, token: String, client_room: Arc) -> Result<()> { self.background.simulate_random_delay().await; let claims = live_kit_server::token::validate(&token, &self.secret_key)?; let identity = claims.sub.unwrap().to_string(); - let room = claims.video.room.unwrap(); + let room_name = claims.video.room.unwrap(); let mut server_rooms = self.rooms.lock(); let room = server_rooms - .get_mut(&*room) - .ok_or_else(|| anyhow!("room {} does not exist", room))?; - room.clients.insert(identity, client_room); + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {:?} does not exist", room_name))?; + if room.clients.contains_key(&identity) { + Err(anyhow!( + "{:?} attempted to join room {:?} twice", + identity, + room_name + )) + } else { + room.clients.insert(identity, client_room); + Ok(()) + } + } + + async fn leave_room(&self, token: String) -> Result<()> { + self.background.simulate_random_delay().await; + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; + let identity = claims.sub.unwrap().to_string(); + let room_name = claims.video.room.unwrap(); + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + room.clients.remove(&identity).ok_or_else(|| { + anyhow!( + "{:?} attempted to leave room {:?} before joining it", + identity, + room_name + ) + })?; + Ok(()) + } + + async fn remove_participant(&self, room_name: String, identity: String) -> Result<()> { + // TODO: clear state associated with the `Room`. + + self.background.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + room.clients.remove(&identity).ok_or_else(|| { + anyhow!( + "participant {:?} did not join room {:?}", + identity, + room_name + ) + })?; Ok(()) } } -struct FakeServerRoom { +#[derive(Default)] +struct TestServerRoom { clients: HashMap>, } -impl FakeServerRoom {} +impl TestServerRoom {} + +pub struct TestApiClient { + url: String, +} + +#[async_trait] +impl live_kit_server::api::Client for TestApiClient { + fn url(&self) -> &str { + &self.url + } + + async fn create_room(&self, name: String) -> Result<()> { + let server = TestServer::get(&self.url)?; + server.create_room(name).await?; + Ok(()) + } + + async fn delete_room(&self, name: String) -> Result<()> { + let server = TestServer::get(&self.url)?; + server.delete_room(name).await?; + Ok(()) + } + + async fn remove_participant(&self, room: String, identity: String) -> Result<()> { + let server = TestServer::get(&self.url)?; + server.remove_participant(room, identity).await?; + Ok(()) + } + + fn room_token(&self, room: &str, identity: &str) -> Result { + let server = TestServer::get(&self.url)?; + token::create( + &server.api_key, + &server.secret_key, + Some(identity), + token::VideoGrant::to_join(room), + ) + } +} pub type Sid = String; -pub struct Room; +#[derive(Default)] +struct RoomState { + token: Option, +} + +#[derive(Default)] +pub struct Room(Mutex); impl Room { pub fn new() -> Arc { - Arc::new(Self) + Default::default() } pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { @@ -89,8 +212,9 @@ impl Room { let url = url.to_string(); let token = token.to_string(); async move { - let server = FakeServer::get(&url)?; - server.join_room(token, this).await?; + let server = TestServer::get(&url)?; + server.join_room(token.clone(), this.clone()).await?; + this.0.lock().token = Some(token); Ok(()) } } @@ -115,7 +239,14 @@ impl Room { impl Drop for Room { fn drop(&mut self) { - todo!() + if let Some(token) = self.0.lock().token.take() { + if let Ok(server) = TestServer::get(&token) { + let background = server.background.clone(); + background + .spawn(async move { server.leave_room(token).await.unwrap() }) + .detach(); + } + } } } diff --git a/crates/live_kit_server/Cargo.toml b/crates/live_kit_server/Cargo.toml index 8cd0b36c7c..64267f62d1 100644 --- a/crates/live_kit_server/Cargo.toml +++ b/crates/live_kit_server/Cargo.toml @@ -10,6 +10,7 @@ doctest = false [dependencies] anyhow = "1.0.38" +async-trait = "0.1" futures = "0.3" hmac = "0.12" log = "0.4" diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index 2dafef253b..43e01ce880 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -1,18 +1,28 @@ use crate::{proto, token}; use anyhow::{anyhow, Result}; +use async_trait::async_trait; use prost::Message; use reqwest::header::CONTENT_TYPE; use std::{future::Future, sync::Arc}; +#[async_trait] +pub trait Client: Send + Sync { + fn url(&self) -> &str; + async fn create_room(&self, name: String) -> Result<()>; + async fn delete_room(&self, name: String) -> Result<()>; + async fn remove_participant(&self, room: String, identity: String) -> Result<()>; + fn room_token(&self, room: &str, identity: &str) -> Result; +} + #[derive(Clone)] -pub struct Client { +pub struct LiveKitClient { http: reqwest::Client, url: Arc, key: Arc, secret: Arc, } -impl Client { +impl LiveKitClient { pub fn new(mut url: String, key: String, secret: String) -> Self { if url.ends_with('/') { url.pop(); @@ -26,67 +36,6 @@ impl Client { } } - pub fn url(&self) -> &str { - &self.url - } - - pub fn create_room(&self, name: String) -> impl Future> { - self.request( - "twirp/livekit.RoomService/CreateRoom", - token::VideoGrant { - room_create: Some(true), - ..Default::default() - }, - proto::CreateRoomRequest { - name, - ..Default::default() - }, - ) - } - - pub fn delete_room(&self, name: String) -> impl Future> { - let response = self.request( - "twirp/livekit.RoomService/DeleteRoom", - token::VideoGrant { - room_create: Some(true), - ..Default::default() - }, - proto::DeleteRoomRequest { room: name }, - ); - async move { - let _: proto::DeleteRoomResponse = response.await?; - Ok(()) - } - } - - pub fn remove_participant( - &self, - room: String, - identity: String, - ) -> impl Future> { - let response = self.request( - "twirp/livekit.RoomService/RemoveParticipant", - token::VideoGrant::to_admin(&room), - proto::RoomParticipantIdentity { - room: room.clone(), - identity, - }, - ); - async move { - let _: proto::RemoveParticipantResponse = response.await?; - Ok(()) - } - } - - pub fn room_token(&self, room: &str, identity: &str) -> Result { - token::create( - &self.key, - &self.secret, - Some(identity), - token::VideoGrant::to_join(room), - ) - } - fn request( &self, path: &str, @@ -126,3 +75,65 @@ impl Client { } } } + +#[async_trait] +impl Client for LiveKitClient { + fn url(&self) -> &str { + &self.url + } + + async fn create_room(&self, name: String) -> Result<()> { + let x: proto::Room = self + .request( + "twirp/livekit.RoomService/CreateRoom", + token::VideoGrant { + room_create: Some(true), + ..Default::default() + }, + proto::CreateRoomRequest { + name, + ..Default::default() + }, + ) + .await?; + dbg!(x); + Ok(()) + } + + async fn delete_room(&self, name: String) -> Result<()> { + let _: proto::DeleteRoomResponse = self + .request( + "twirp/livekit.RoomService/DeleteRoom", + token::VideoGrant { + room_create: Some(true), + ..Default::default() + }, + proto::DeleteRoomRequest { room: name }, + ) + .await?; + Ok(()) + } + + async fn remove_participant(&self, room: String, identity: String) -> Result<()> { + let _: proto::RemoveParticipantResponse = self + .request( + "twirp/livekit.RoomService/RemoveParticipant", + token::VideoGrant::to_admin(&room), + proto::RoomParticipantIdentity { + room: room.clone(), + identity, + }, + ) + .await?; + Ok(()) + } + + fn room_token(&self, room: &str, identity: &str) -> Result { + token::create( + &self.key, + &self.secret, + Some(identity), + token::VideoGrant::to_join(room), + ) + } +} From 723fa839098163e4aa0bc552fa2deb343e7e0152 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 19 Oct 2022 11:20:31 -0600 Subject: [PATCH 34/63] Use fake LiveKit server to test we can send frames when screen sharing --- Cargo.lock | 15 +- crates/call/src/participant.rs | 6 +- crates/call/src/room.rs | 63 ++++--- crates/collab/src/integration_tests.rs | 73 +++++-- crates/live_kit_client/Cargo.toml | 11 +- crates/live_kit_client/examples/test_app.rs | 86 +-------- crates/live_kit_client/src/prod.rs | 67 ++++--- crates/live_kit_client/src/test.rs | 199 ++++++++++++++++---- crates/workspace/src/pane_group.rs | 2 +- 9 files changed, 334 insertions(+), 188 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa167e245f..2188d2879f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,6 +181,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-broadcast" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d26004fe83b2d1cd3a97609b21e39f9a31535822210fe83205d2ce48866ea61" +dependencies = [ + "event-listener", + "futures-core", + "parking_lot 0.12.1", +] + [[package]] name = "async-channel" version = "1.7.1" @@ -2991,7 +3002,7 @@ name = "language" version = "0.1.0" dependencies = [ "anyhow", - "async-broadcast", + "async-broadcast 0.3.4", "async-trait", "client", "clock", @@ -3164,6 +3175,7 @@ name = "live_kit_client" version = "0.1.0" dependencies = [ "anyhow", + "async-broadcast 0.4.1", "async-trait", "block", "byteorder", @@ -3181,6 +3193,7 @@ dependencies = [ "live_kit_server", "log", "media", + "nanoid", "objc", "parking_lot 0.11.2", "postage", diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index f335689050..c045bd77dc 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -2,7 +2,7 @@ use anyhow::{anyhow, Result}; use client::{proto, User}; use collections::HashMap; use gpui::{Task, WeakModelHandle}; -use media::core_video::CVImageBuffer; +use live_kit_client::Frame; use project::Project; use std::sync::Arc; @@ -46,13 +46,13 @@ pub struct RemoteParticipant { #[derive(Clone)] pub struct RemoteVideoTrack { - pub(crate) frame: Option, + pub(crate) frame: Option, pub(crate) _live_kit_track: Arc, pub(crate) _maintain_frame: Arc>, } impl RemoteVideoTrack { - pub fn frame(&self) -> Option<&CVImageBuffer> { + pub fn frame(&self) -> Option<&Frame> { self.frame.as_ref() } } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index f09b255df5..399b0e857e 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -418,31 +418,33 @@ impl Room { _live_kit_track: track, _maintain_frame: Arc::new(cx.spawn_weak(|this, mut cx| async move { while let Some(frame) = rx.next().await { - let this = if let Some(this) = this.upgrade(&cx) { - this - } else { - break; - }; - - let done = this.update(&mut cx, |this, cx| { - if let Some(track) = - this.remote_participants.get_mut(&peer_id).and_then( - |participant| participant.tracks.get_mut(&track_id), - ) - { - track.frame = frame; - cx.emit(Event::Frame { - participant_id: peer_id, - track_id: track_id.clone(), - }); - false + if let Some(frame) = frame { + let this = if let Some(this) = this.upgrade(&cx) { + this } else { - true - } - }); + break; + }; - if done { - break; + let done = this.update(&mut cx, |this, cx| { + if let Some(track) = + this.remote_participants.get_mut(&peer_id).and_then( + |participant| participant.tracks.get_mut(&track_id), + ) + { + track.frame = Some(frame); + cx.emit(Event::Frame { + participant_id: peer_id, + track_id: track_id.clone(), + }); + false + } else { + true + } + }); + + if done { + break; + } } } })), @@ -620,18 +622,18 @@ impl Room { return Task::ready(Err(anyhow!("screen was already shared"))); } - let publish_id = if let Some(live_kit) = self.live_kit.as_mut() { + let (displays, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { let publish_id = post_inc(&mut live_kit.next_publish_id); live_kit.screen_track = ScreenTrack::Pending { publish_id }; cx.notify(); - publish_id + (live_kit.room.display_sources(), publish_id) } else { return Task::ready(Err(anyhow!("live-kit was not initialized"))); }; cx.spawn_weak(|this, mut cx| async move { let publish_track = async { - let displays = live_kit_client::display_sources().await?; + let displays = displays.await?; let display = displays .first() .ok_or_else(|| anyhow!("no display found"))?; @@ -711,6 +713,15 @@ impl Room { } } } + + #[cfg(any(test, feature = "test-support"))] + pub fn set_display_sources(&self, sources: Vec) { + self.live_kit + .as_ref() + .unwrap() + .room + .set_display_sources(sources); + } } struct LiveKitRoom { diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index 7cd964cd8a..8ac898a4ad 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -5,7 +5,10 @@ use crate::{ }; use ::rpc::Peer; use anyhow::anyhow; -use call::{room, ActiveCall, ParticipantLocation, Room}; +use call::{ + room::{self, Event}, + ActiveCall, ParticipantLocation, Room, +}; use client::{ self, test::FakeHttpClient, Channel, ChannelDetails, ChannelList, Client, Connection, Credentials, EstablishConnectionError, PeerId, User, UserStore, RECEIVE_TIMEOUT, @@ -30,6 +33,7 @@ use language::{ range_to_lsp, tree_sitter_rust, Diagnostic, DiagnosticEntry, FakeLspAdapter, Language, LanguageConfig, LanguageRegistry, OffsetRangeExt, Point, Rope, }; +use live_kit_client::{Frame, MacOSDisplay}; use lsp::{self, FakeLanguageServer}; use parking_lot::Mutex; use project::{ @@ -185,6 +189,45 @@ async fn test_basic_calls( } ); + // User A shares their screen + let display = MacOSDisplay::new(); + let events_b = active_call_events(cx_b); + active_call_a + .update(cx_a, |call, cx| { + call.room().unwrap().update(cx, |room, cx| { + room.set_display_sources(vec![display.clone()]); + room.share_screen(cx) + }) + }) + .await + .unwrap(); + + let frame = Frame { + width: 800, + height: 600, + label: "a".into(), + }; + display.send_frame(frame.clone()); + deterministic.run_until_parked(); + + assert_eq!(events_b.borrow().len(), 1); + let event = events_b.borrow().first().unwrap().clone(); + if let Event::Frame { + participant_id, + track_id, + } = event + { + assert_eq!(participant_id, client_a.peer_id().unwrap()); + room_b.read_with(cx_b, |room, _| { + assert_eq!( + room.remote_participants()[&client_a.peer_id().unwrap()].tracks[&track_id].frame(), + Some(&frame) + ); + }); + } else { + panic!("unexpected event") + } + // User A leaves the room. active_call_a.update(cx_a, |call, cx| { call.hang_up(cx).unwrap(); @@ -954,21 +997,21 @@ async fn test_active_call_events( deterministic.run_until_parked(); assert_eq!(mem::take(&mut *events_a.borrow_mut()), vec![]); assert_eq!(mem::take(&mut *events_b.borrow_mut()), vec![]); +} - fn active_call_events(cx: &mut TestAppContext) -> Rc>> { - let events = Rc::new(RefCell::new(Vec::new())); - let active_call = cx.read(ActiveCall::global); - cx.update({ - let events = events.clone(); - |cx| { - cx.subscribe(&active_call, move |_, event, _| { - events.borrow_mut().push(event.clone()) - }) - .detach() - } - }); - events - } +fn active_call_events(cx: &mut TestAppContext) -> Rc>> { + let events = Rc::new(RefCell::new(Vec::new())); + let active_call = cx.read(ActiveCall::global); + cx.update({ + let events = events.clone(); + |cx| { + cx.subscribe(&active_call, move |_, event, _| { + events.borrow_mut().push(event.clone()) + }) + .detach() + } + }); + events } #[gpui::test(iterations = 10)] diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index ee2c3f88dc..cd8006314c 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -13,11 +13,13 @@ name = "test_app" [features] test-support = [ + "async-broadcast", "async-trait", "collections/test-support", "gpui/test-support", "lazy_static", - "live_kit_server" + "live_kit_server", + "nanoid", ] [dependencies] @@ -27,13 +29,16 @@ live_kit_server = { path = "../live_kit_server", optional = true } media = { path = "../media" } anyhow = "1.0.38" -async-trait = { version = "0.1", optional = true } core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" -lazy_static = { version = "1.4", optional = true } parking_lot = "0.11.1" +async-broadcast = { version = "0.4", optional = true } +async-trait = { version = "0.1", optional = true } +lazy_static = { version = "1.4", optional = true } +nanoid = { version ="0.4", optional = true} + [dev-dependencies] collections = { path = "../collections", features = ["test-support"] } gpui = { path = "../gpui", features = ["test-support"] } diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index b4f037cead..7ad1eee967 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -1,21 +1,9 @@ use futures::StreamExt; -use gpui::{ - actions, - elements::{Canvas, *}, - keymap::Binding, - platform::current::Surface, - Menu, MenuItem, ViewContext, -}; +use gpui::{actions, keymap::Binding, Menu, MenuItem}; use live_kit_client::{LocalVideoTrack, RemoteVideoTrackUpdate, Room}; -use live_kit_server::{ - api::Client, - token::{self, VideoGrant}, -}; +use live_kit_server::token::{self, VideoGrant}; use log::LevelFilter; -use media::core_video::CVImageBuffer; -use postage::watch; use simplelog::SimpleLogger; -use std::sync::Arc; actions!(capture, [Quit]); @@ -62,7 +50,7 @@ fn main() { let mut track_changes = room_b.remote_video_track_updates(); - let displays = live_kit_client::display_sources().await.unwrap(); + let displays = room_a.display_sources().await.unwrap(); let display = displays.into_iter().next().unwrap(); let track_a = LocalVideoTrack::screen_share_for_display(&display); @@ -72,6 +60,7 @@ fn main() { let remote_tracks = room_b.remote_video_tracks("test-participant-1"); assert_eq!(remote_tracks.len(), 1); assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); + dbg!(track.sid()); assert_eq!(track.publisher_id(), "test-participant-1"); } else { panic!("unexpected message"); @@ -100,73 +89,6 @@ fn main() { }); } -struct ScreenCaptureView { - image_buffer: Option, - _room: Arc, -} - -impl gpui::Entity for ScreenCaptureView { - type Event = (); -} - -impl ScreenCaptureView { - pub fn new(room: Arc, cx: &mut ViewContext) -> Self { - let mut remote_video_tracks = room.remote_video_track_updates(); - cx.spawn_weak(|this, mut cx| async move { - if let Some(video_track) = remote_video_tracks.next().await { - let (mut frames_tx, mut frames_rx) = watch::channel_with(None); - // video_track.add_renderer(move |frame| *frames_tx.borrow_mut() = Some(frame)); - - while let Some(frame) = frames_rx.next().await { - if let Some(this) = this.upgrade(&cx) { - this.update(&mut cx, |this, cx| { - this.image_buffer = frame; - cx.notify(); - }); - } else { - break; - } - } - } - }) - .detach(); - - Self { - image_buffer: None, - _room: room, - } - } -} - -impl gpui::View for ScreenCaptureView { - fn ui_name() -> &'static str { - "View" - } - - fn render(&mut self, _: &mut gpui::RenderContext) -> gpui::ElementBox { - let image_buffer = self.image_buffer.clone(); - let canvas = Canvas::new(move |bounds, _, cx| { - if let Some(image_buffer) = image_buffer.clone() { - cx.scene.push_surface(Surface { - bounds, - image_buffer, - }); - } - }); - - if let Some(image_buffer) = self.image_buffer.as_ref() { - canvas - .constrained() - .with_width(image_buffer.width() as f32) - .with_height(image_buffer.height() as f32) - .aligned() - .boxed() - } else { - canvas.boxed() - } - } -} - fn quit(_: &Quit, cx: &mut gpui::MutableAppContext) { cx.platform().quit(); } diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs index 427f45fe72..15f5faacca 100644 --- a/crates/live_kit_client/src/prod.rs +++ b/crates/live_kit_client/src/prod.rs @@ -8,7 +8,8 @@ use futures::{ channel::{mpsc, oneshot}, Future, }; -use media::core_video::{CVImageBuffer, CVImageBufferRef}; +pub use media::core_video::CVImageBuffer; +use media::core_video::CVImageBufferRef; use parking_lot::Mutex; use std::{ ffi::c_void, @@ -109,8 +110,35 @@ impl Room { async { rx.await.unwrap().context("error connecting to room") } } + pub fn display_sources(self: &Arc) -> impl Future>> { + extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { + unsafe { + let tx = Box::from_raw(tx as *mut oneshot::Sender>>); + + if sources.is_null() { + let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); + } else { + let sources = CFArray::wrap_under_get_rule(sources) + .into_iter() + .map(|source| MacOSDisplay::new(*source)) + .collect(); + + let _ = tx.send(Ok(sources)); + } + } + } + + let (tx, rx) = oneshot::channel(); + + unsafe { + LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); + } + + async move { rx.await.unwrap() } + } + pub fn publish_video_track( - &self, + self: &Arc, track: &LocalVideoTrack, ) -> impl Future> { let (tx, rx) = oneshot::channel::>(); @@ -338,16 +366,16 @@ impl RemoteVideoTrack { pub fn add_renderer(&self, callback: F) where - F: 'static + FnMut(CVImageBuffer), + F: 'static + Send + Sync + FnMut(Frame), { extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) where - F: FnMut(CVImageBuffer), + F: FnMut(Frame), { unsafe { let buffer = CVImageBuffer::wrap_under_get_rule(frame); let callback = &mut *(callback_data as *mut F); - callback(buffer); + callback(Frame(buffer)); } } @@ -394,29 +422,18 @@ impl Drop for MacOSDisplay { } } -pub fn display_sources() -> impl Future>> { - extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { - unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>>); +pub struct Frame(CVImageBuffer); - if sources.is_null() { - let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); - } else { - let sources = CFArray::wrap_under_get_rule(sources) - .into_iter() - .map(|source| MacOSDisplay::new(*source)) - .collect(); - - let _ = tx.send(Ok(sources)); - } - } +impl Frame { + pub fn width(&self) -> usize { + self.0.width() } - let (tx, rx) = oneshot::channel(); - - unsafe { - LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); + pub fn height(&self) -> usize { + self.0.height() } - async move { rx.await.unwrap() } + pub fn image(&self) -> CVImageBuffer { + self.0.clone() + } } diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 050cfd0a47..6a1a6ee2d6 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -1,8 +1,8 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use collections::HashMap; -use futures::{channel::mpsc, future}; -use gpui::executor::Background; +use futures::{Stream, StreamExt}; +use gpui::executor::{self, Background}; use lazy_static::lazy_static; use live_kit_server::token; use media::core_video::CVImageBuffer; @@ -96,14 +96,14 @@ impl TestServer { let room = server_rooms .get_mut(&*room_name) .ok_or_else(|| anyhow!("room {:?} does not exist", room_name))?; - if room.clients.contains_key(&identity) { + if room.client_rooms.contains_key(&identity) { Err(anyhow!( "{:?} attempted to join room {:?} twice", identity, room_name )) } else { - room.clients.insert(identity, client_room); + room.client_rooms.insert(identity, client_room); Ok(()) } } @@ -117,7 +117,7 @@ impl TestServer { let room = server_rooms .get_mut(&*room_name) .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.clients.remove(&identity).ok_or_else(|| { + room.client_rooms.remove(&identity).ok_or_else(|| { anyhow!( "{:?} attempted to leave room {:?} before joining it", identity, @@ -135,7 +135,7 @@ impl TestServer { let room = server_rooms .get_mut(&room_name) .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.clients.remove(&identity).ok_or_else(|| { + room.client_rooms.remove(&identity).ok_or_else(|| { anyhow!( "participant {:?} did not join room {:?}", identity, @@ -144,11 +144,44 @@ impl TestServer { })?; Ok(()) } + + async fn publish_video_track(&self, token: String, local_track: LocalVideoTrack) -> Result<()> { + self.background.simulate_random_delay().await; + let claims = live_kit_server::token::validate(&token, &self.secret_key)?; + let identity = claims.sub.unwrap().to_string(); + let room_name = claims.video.room.unwrap(); + + let mut server_rooms = self.rooms.lock(); + let room = server_rooms + .get_mut(&*room_name) + .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; + + let update = RemoteVideoTrackUpdate::Subscribed(Arc::new(RemoteVideoTrack { + sid: nanoid::nanoid!(17), + publisher_id: identity.clone(), + frames_rx: local_track.frames_rx.clone(), + background: self.background.clone(), + })); + + for (id, client_room) in &room.client_rooms { + if *id != identity { + let _ = client_room + .0 + .lock() + .video_track_updates + .0 + .try_broadcast(update.clone()) + .unwrap(); + } + } + + Ok(()) + } } #[derive(Default)] struct TestServerRoom { - clients: HashMap>, + client_rooms: HashMap>, } impl TestServerRoom {} @@ -194,17 +227,29 @@ impl live_kit_server::api::Client for TestApiClient { pub type Sid = String; -#[derive(Default)] struct RoomState { - token: Option, + connection: Option, + display_sources: Vec, + video_track_updates: ( + async_broadcast::Sender, + async_broadcast::Receiver, + ), +} + +struct ConnectionState { + url: String, + token: String, } -#[derive(Default)] pub struct Room(Mutex); impl Room { pub fn new() -> Arc { - Default::default() + Arc::new(Self(Mutex::new(RoomState { + connection: None, + display_sources: Default::default(), + video_track_updates: async_broadcast::broadcast(128), + }))) } pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { @@ -214,36 +259,75 @@ impl Room { async move { let server = TestServer::get(&url)?; server.join_room(token.clone(), this.clone()).await?; - this.0.lock().token = Some(token); + this.0.lock().connection = Some(ConnectionState { url, token }); Ok(()) } } - pub fn publish_video_track( - &self, - track: &LocalVideoTrack, - ) -> impl Future> { - future::pending() + pub fn display_sources(self: &Arc) -> impl Future>> { + let this = self.clone(); + async move { + let server = this.test_server(); + server.background.simulate_random_delay().await; + Ok(this.0.lock().display_sources.clone()) + } } - pub fn unpublish_track(&self, publication: LocalTrackPublication) {} + pub fn publish_video_track( + self: &Arc, + track: &LocalVideoTrack, + ) -> impl Future> { + let this = self.clone(); + let track = track.clone(); + async move { + this.test_server() + .publish_video_track(this.token(), track) + .await?; + Ok(LocalTrackPublication) + } + } - pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { + pub fn unpublish_track(&self, _: LocalTrackPublication) {} + + pub fn remote_video_tracks(&self, _: &str) -> Vec> { Default::default() } - pub fn remote_video_track_updates(&self) -> mpsc::UnboundedReceiver { - mpsc::unbounded().1 + pub fn remote_video_track_updates(&self) -> impl Stream { + self.0.lock().video_track_updates.1.clone() + } + + pub fn set_display_sources(&self, sources: Vec) { + self.0.lock().display_sources = sources; + } + + fn test_server(&self) -> Arc { + let this = self.0.lock(); + let connection = this + .connection + .as_ref() + .expect("must be connected to call this method"); + TestServer::get(&connection.url).unwrap() + } + + fn token(&self) -> String { + self.0 + .lock() + .connection + .as_ref() + .expect("must be connected to call this method") + .token + .clone() } } impl Drop for Room { fn drop(&mut self) { - if let Some(token) = self.0.lock().token.take() { - if let Ok(server) = TestServer::get(&token) { + if let Some(connection) = self.0.lock().connection.take() { + if let Ok(server) = TestServer::get(&connection.token) { let background = server.background.clone(); background - .spawn(async move { server.leave_room(token).await.unwrap() }) + .spawn(async move { server.leave_room(connection.token).await.unwrap() }) .detach(); } } @@ -252,17 +336,24 @@ impl Drop for Room { pub struct LocalTrackPublication; -pub struct LocalVideoTrack; +#[derive(Clone)] +pub struct LocalVideoTrack { + frames_rx: async_broadcast::Receiver, +} impl LocalVideoTrack { pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self + Self { + frames_rx: display.frames.1.clone(), + } } } pub struct RemoteVideoTrack { sid: Sid, publisher_id: Sid, + frames_rx: async_broadcast::Receiver, + background: Arc, } impl RemoteVideoTrack { @@ -274,20 +365,64 @@ impl RemoteVideoTrack { &self.publisher_id } - pub fn add_renderer(&self, callback: F) + pub fn add_renderer(&self, mut callback: F) where - F: 'static + FnMut(CVImageBuffer), + F: 'static + Send + Sync + FnMut(Frame), { + let mut frames_rx = self.frames_rx.clone(); + self.background + .spawn(async move { + while let Some(frame) = frames_rx.next().await { + callback(frame) + } + }) + .detach(); } } +#[derive(Clone)] pub enum RemoteVideoTrackUpdate { Subscribed(Arc), Unsubscribed { publisher_id: Sid, track_id: Sid }, } -pub struct MacOSDisplay; - -pub fn display_sources() -> impl Future>> { - future::pending() +#[derive(Clone)] +pub struct MacOSDisplay { + frames: ( + async_broadcast::Sender, + async_broadcast::Receiver, + ), +} + +impl MacOSDisplay { + pub fn new() -> Self { + Self { + frames: async_broadcast::broadcast(128), + } + } + + pub fn send_frame(&self, frame: Frame) { + self.frames.0.try_broadcast(frame).unwrap(); + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Frame { + pub label: String, + pub width: usize, + pub height: usize, +} + +impl Frame { + pub fn width(&self) -> usize { + self.width + } + + pub fn height(&self) -> usize { + self.height + } + + pub fn image(&self) -> CVImageBuffer { + unimplemented!("you can't call this in test mode") + } } diff --git a/crates/workspace/src/pane_group.rs b/crates/workspace/src/pane_group.rs index b99aba3b3a..c778115d91 100644 --- a/crates/workspace/src/pane_group.rs +++ b/crates/workspace/src/pane_group.rs @@ -159,7 +159,7 @@ impl Member { let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.; cx.scene.push_surface(gpui::mac::Surface { bounds: RectF::new(origin, size), - image_buffer: frame, + image_buffer: frame.image(), }); } }) From 69472f78236cb5e6302854854ff4761c4252508e Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Wed, 19 Oct 2022 19:21:09 -0600 Subject: [PATCH 35/63] Ensure we can send a second frame --- crates/collab/src/integration_tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index 8ac898a4ad..37dfedfe55 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -228,6 +228,10 @@ async fn test_basic_calls( panic!("unexpected event") } + display.send_frame(frame.clone()); + deterministic.run_until_parked(); + assert_eq!(events_b.borrow().len(), 2); + // User A leaves the room. active_call_a.update(cx_a, |call, cx| { call.hang_up(cx).unwrap(); From 99aa1219d2177fe8c8ba869d4189635c77c7ef96 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 09:51:55 +0200 Subject: [PATCH 36/63] Simplify renderer interface for live-kit-client --- crates/call/src/room.rs | 54 +++++++++---------- crates/live_kit_client/Cargo.toml | 5 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 16 ++++-- crates/live_kit_client/examples/test_app.rs | 1 - crates/live_kit_client/src/prod.rs | 44 +++++++++------ crates/live_kit_client/src/test.rs | 20 ++----- 6 files changed, 70 insertions(+), 70 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 399b0e857e..98b223281c 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -8,7 +8,6 @@ use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate}; -use postage::watch; use project::Project; use std::{mem, os::unix::prelude::OsStrExt, sync::Arc}; use util::{post_inc, ResultExt}; @@ -409,42 +408,39 @@ impl Room { .remote_participants .get_mut(&peer_id) .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - let (mut tx, mut rx) = watch::channel(); - track.add_renderer(move |frame| *tx.borrow_mut() = Some(frame)); + let mut frames = track.frames(); participant.tracks.insert( track_id.clone(), RemoteVideoTrack { frame: None, _live_kit_track: track, _maintain_frame: Arc::new(cx.spawn_weak(|this, mut cx| async move { - while let Some(frame) = rx.next().await { - if let Some(frame) = frame { - let this = if let Some(this) = this.upgrade(&cx) { - this + while let Some(frame) = frames.next().await { + let this = if let Some(this) = this.upgrade(&cx) { + this + } else { + break; + }; + + let done = this.update(&mut cx, |this, cx| { + if let Some(track) = + this.remote_participants.get_mut(&peer_id).and_then( + |participant| participant.tracks.get_mut(&track_id), + ) + { + track.frame = Some(frame); + cx.emit(Event::Frame { + participant_id: peer_id, + track_id: track_id.clone(), + }); + false } else { - break; - }; - - let done = this.update(&mut cx, |this, cx| { - if let Some(track) = - this.remote_participants.get_mut(&peer_id).and_then( - |participant| participant.tracks.get_mut(&track_id), - ) - { - track.frame = Some(frame); - cx.emit(Event::Frame { - participant_id: peer_id, - track_id: track_id.clone(), - }); - false - } else { - true - } - }); - - if done { - break; + true } + }); + + if done { + break; } } })), diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index cd8006314c..ce555ccbc3 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -13,7 +13,6 @@ name = "test_app" [features] test-support = [ - "async-broadcast", "async-trait", "collections/test-support", "gpui/test-support", @@ -29,12 +28,13 @@ live_kit_server = { path = "../live_kit_server", optional = true } media = { path = "../media" } anyhow = "1.0.38" +async-broadcast = "0.4" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" +log = { version = "0.4.16", features = ["kv_unstable_serde"] } parking_lot = "0.11.1" -async-broadcast = { version = "0.4", optional = true } async-trait = { version = "0.1", optional = true } lazy_static = { version = "1.4", optional = true } nanoid = { version ="0.4", optional = true} @@ -58,7 +58,6 @@ futures = "0.3" hmac = "0.12" jwt = "0.16" lazy_static = "1.4" -log = { version = "0.4.16", features = ["kv_unstable_serde"] } objc = "0.2" parking_lot = "0.11.1" postage = { version = "0.4.1", features = ["futures-traits"] } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 8bfa98b522..af6485ce69 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -28,12 +28,13 @@ class LKRoomDelegate: RoomDelegate { class LKVideoRenderer: NSObject, VideoRenderer { var data: UnsafeRawPointer - var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void + var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool var onDrop: @convention(c) (UnsafeRawPointer) -> Void var adaptiveStreamIsEnabled: Bool = false var adaptiveStreamSize: CGSize = .zero + weak var track: VideoTrack? - init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { + init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { self.data = data self.onFrame = onFrame self.onDrop = onDrop @@ -50,7 +51,11 @@ class LKVideoRenderer: NSObject, VideoRenderer { func renderFrame(_ frame: RTCVideoFrame?) { let buffer = frame?.buffer as? RTCCVPixelBuffer if let pixelBuffer = buffer?.pixelBuffer { - self.onFrame(self.data, pixelBuffer) + if !self.onFrame(self.data, pixelBuffer) { + DispatchQueue.main.async { + self.track?.remove(videoRenderer: self) + } + } } } } @@ -99,7 +104,7 @@ public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPoin public func LKRoomUnpublishTrack(room: UnsafeRawPointer, publication: UnsafeRawPointer) { let room = Unmanaged.fromOpaque(room).takeUnretainedValue() let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - room.localParticipant?.unpublish(publication: publication) + let _ = room.localParticipant?.unpublish(publication: publication) } @_cdecl("LKRoomVideoTracksForRemoteParticipant") @@ -123,7 +128,7 @@ public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) } @_cdecl("LKVideoRendererCreate") -public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { +public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() } @@ -131,6 +136,7 @@ public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @co public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() + renderer.track = track track.add(videoRenderer: renderer) } diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs index 7ad1eee967..eddee785bc 100644 --- a/crates/live_kit_client/examples/test_app.rs +++ b/crates/live_kit_client/examples/test_app.rs @@ -60,7 +60,6 @@ fn main() { let remote_tracks = room_b.remote_video_tracks("test-participant-1"); assert_eq!(remote_tracks.len(), 1); assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); - dbg!(track.sid()); assert_eq!(track.publisher_id(), "test-participant-1"); } else { panic!("unexpected message"); diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs index 15f5faacca..35a5705a24 100644 --- a/crates/live_kit_client/src/prod.rs +++ b/crates/live_kit_client/src/prod.rs @@ -55,7 +55,7 @@ extern "C" { fn LKVideoRendererCreate( callback_data: *mut c_void, - on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef), + on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool, on_drop: extern "C" fn(callback_data: *mut c_void), ) -> *const c_void; @@ -364,32 +364,43 @@ impl RemoteVideoTrack { &self.publisher_id } - pub fn add_renderer(&self, callback: F) - where - F: 'static + Send + Sync + FnMut(Frame), - { - extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) - where - F: FnMut(Frame), - { + pub fn frames(&self) -> async_broadcast::Receiver { + extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool { unsafe { + let tx = Box::from_raw(callback_data as *mut async_broadcast::Sender); let buffer = CVImageBuffer::wrap_under_get_rule(frame); - let callback = &mut *(callback_data as *mut F); - callback(Frame(buffer)); + let result = tx.try_broadcast(Frame(buffer)); + let _ = Box::into_raw(tx); + match result { + Ok(_) => true, + Err(async_broadcast::TrySendError::Closed(_)) + | Err(async_broadcast::TrySendError::Inactive(_)) => { + log::warn!("no active receiver for frame"); + false + } + Err(async_broadcast::TrySendError::Full(_)) => { + log::warn!("skipping frame as receiver is not keeping up"); + true + } + } } } - extern "C" fn on_drop(callback_data: *mut c_void) { + extern "C" fn on_drop(callback_data: *mut c_void) { unsafe { - let _ = Box::from_raw(callback_data as *mut F); + let _ = Box::from_raw(callback_data as *mut async_broadcast::Sender); } } - let callback_data = Box::into_raw(Box::new(callback)); + let (tx, rx) = async_broadcast::broadcast(64); unsafe { - let renderer = - LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); + let renderer = LKVideoRendererCreate( + Box::into_raw(Box::new(tx)) as *mut c_void, + on_frame, + on_drop, + ); LKVideoTrackAddRenderer(self.native_track, renderer); + rx } } } @@ -422,6 +433,7 @@ impl Drop for MacOSDisplay { } } +#[derive(Clone)] pub struct Frame(CVImageBuffer); impl Frame { diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 6a1a6ee2d6..23076e13a5 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -1,8 +1,8 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use collections::HashMap; -use futures::{Stream, StreamExt}; -use gpui::executor::{self, Background}; +use futures::Stream; +use gpui::executor::Background; use lazy_static::lazy_static; use live_kit_server::token; use media::core_video::CVImageBuffer; @@ -160,7 +160,6 @@ impl TestServer { sid: nanoid::nanoid!(17), publisher_id: identity.clone(), frames_rx: local_track.frames_rx.clone(), - background: self.background.clone(), })); for (id, client_room) in &room.client_rooms { @@ -353,7 +352,6 @@ pub struct RemoteVideoTrack { sid: Sid, publisher_id: Sid, frames_rx: async_broadcast::Receiver, - background: Arc, } impl RemoteVideoTrack { @@ -365,18 +363,8 @@ impl RemoteVideoTrack { &self.publisher_id } - pub fn add_renderer(&self, mut callback: F) - where - F: 'static + Send + Sync + FnMut(Frame), - { - let mut frames_rx = self.frames_rx.clone(); - self.background - .spawn(async move { - while let Some(frame) = frames_rx.next().await { - callback(frame) - } - }) - .detach(); + pub fn frames(&self) -> async_broadcast::Receiver { + self.frames_rx.clone() } } From 76a1b81e45c286f941000c9666e3608ad1ed0e4c Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 14:03:26 +0200 Subject: [PATCH 37/63] Update live-kit to the latest version --- crates/live_kit_client/LiveKitBridge/Package.resolved | 6 +++--- crates/live_kit_client/LiveKitBridge/Package.swift | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/live_kit_client/LiveKitBridge/Package.resolved b/crates/live_kit_client/LiveKitBridge/Package.resolved index 402bd635ce..9318cc0184 100644 --- a/crates/live_kit_client/LiveKitBridge/Package.resolved +++ b/crates/live_kit_client/LiveKitBridge/Package.resolved @@ -6,7 +6,7 @@ "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", "state": { "branch": null, - "revision": "42258f5d3467ec3981323e33200b7403ac637ece", + "revision": "f6ca534eb334e99acb8e82cc99b491717df28d8a", "version": null } }, @@ -24,8 +24,8 @@ "repositoryURL": "https://github.com/webrtc-sdk/Specs.git", "state": { "branch": null, - "revision": "5225f2de4b6d0098803b3a0e55b255a41f293dad", - "version": "104.5112.2" + "revision": "38ac06261e62f980652278c69b70284324c769e0", + "version": "104.5112.5" } }, { diff --git a/crates/live_kit_client/LiveKitBridge/Package.swift b/crates/live_kit_client/LiveKitBridge/Package.swift index e4227d8b64..bdd664c6fb 100644 --- a/crates/live_kit_client/LiveKitBridge/Package.swift +++ b/crates/live_kit_client/LiveKitBridge/Package.swift @@ -15,7 +15,7 @@ let package = Package( targets: ["LiveKitBridge"]), ], dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "42258f5d3467ec3981323e33200b7403ac637ece"), + .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "f6ca534eb334e99acb8e82cc99b491717df28d8a"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. From 5dc82d3df8d0b32ed7c327b25b019d6d6c4d5d12 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 14:34:05 +0200 Subject: [PATCH 38/63] Delete all live-kit rooms when server is shut down --- crates/collab/src/api.rs | 4 +-- crates/collab/src/main.rs | 56 ++++++++++++++++++++++++++++++++-- crates/collab/src/rpc/store.rs | 4 +++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/collab/src/api.rs b/crates/collab/src/api.rs index 08dfa91ba9..0dd5498978 100644 --- a/crates/collab/src/api.rs +++ b/crates/collab/src/api.rs @@ -22,7 +22,7 @@ use time::OffsetDateTime; use tower::ServiceBuilder; use tracing::instrument; -pub fn routes(rpc_server: &Arc, state: Arc) -> Router { +pub fn routes(rpc_server: Arc, state: Arc) -> Router { Router::new() .route("/user", get(get_authenticated_user)) .route("/users", get(get_users).post(create_user)) @@ -50,7 +50,7 @@ pub fn routes(rpc_server: &Arc, state: Arc) -> Router Result<()> { rpc_server.start_recording_project_activity(Duration::from_secs(5 * 60), rpc::RealExecutor); let app = Router::::new() - .merge(api::routes(&rpc_server, state.clone())) - .merge(rpc::routes(rpc_server)); + .merge(api::routes(rpc_server.clone(), state.clone())) + .merge(rpc::routes(rpc_server.clone())); axum::Server::from_tcp(listener)? .serve(app.into_make_service_with_connect_info::()) + .with_graceful_shutdown(graceful_shutdown(rpc_server, state)) .await?; Ok(()) @@ -133,3 +136,52 @@ pub fn init_tracing(config: &Config) -> Option<()> { None } + +async fn graceful_shutdown(rpc_server: Arc, state: Arc) { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + + if let Some(live_kit) = state.live_kit_client.as_ref() { + let deletions = rpc_server + .store() + .await + .rooms() + .values() + .map(|room| { + let name = room.live_kit_room.clone(); + async { + live_kit.delete_room(name).await.trace_err(); + } + }) + .collect::>(); + + tracing::info!("deleting all live-kit rooms"); + if let Err(_) = tokio::time::timeout( + Duration::from_secs(10), + futures::future::join_all(deletions), + ) + .await + { + tracing::error!("timed out waiting for live-kit room deletion"); + } + } +} diff --git a/crates/collab/src/rpc/store.rs b/crates/collab/src/rpc/store.rs index 9a2e8cbdda..a7abce7094 100644 --- a/crates/collab/src/rpc/store.rs +++ b/crates/collab/src/rpc/store.rs @@ -519,6 +519,10 @@ impl Store { self.rooms.get(&room_id) } + pub fn rooms(&self) -> &BTreeMap { + &self.rooms + } + pub fn call( &mut self, room_id: RoomId, From 629d3d473c1d2faaee19a7ec087532ba24e50893 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 15:38:54 +0200 Subject: [PATCH 39/63] Copy WebRTC into `Zed.app/Contents/Frameworks` when bundling the app --- crates/zed/build.rs | 9 +++++++-- script/bundle | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/zed/build.rs b/crates/zed/build.rs index ed83137f95..9b486cbb8b 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -7,8 +7,13 @@ fn main() { println!("cargo:rustc-env=ZED_AMPLITUDE_API_KEY={api_key}"); } - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); + if std::env::var("ZED_BUNDLE").ok().as_deref() == Some("true") { + // Find WebRTC.framework in the Frameworks folder when running as part of an application bundle. + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../Frameworks"); + } else { + // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); + } // Register exported Objective-C selectors, protocols, etc println!("cargo:rustc-link-arg=-Wl,-ObjC"); diff --git a/script/bundle b/script/bundle index f3fc4e7434..a4a1391f2b 100755 --- a/script/bundle +++ b/script/bundle @@ -33,6 +33,11 @@ lipo \ -output \ target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/MacOS/cli +echo "Copying WebRTC.framework into the frameworks folder" +mkdir target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/Frameworks +cp -R target/x86_64-apple-darwin/release/WebRTC.framework target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/Frameworks/ +rm -rf target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/Frameworks/WebRTC.framework/Versions + if [[ -n $MACOS_CERTIFICATE && -n $MACOS_CERTIFICATE_PASSWORD && -n $APPLE_NOTARIZATION_USERNAME && -n $APPLE_NOTARIZATION_PASSWORD ]]; then echo "Signing bundle with Apple-issued certificate" security create-keychain -p $MACOS_CERTIFICATE_PASSWORD zed.keychain || echo "" From de24b4b4e89da24c440d2045660252daab434496 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 16:55:39 +0200 Subject: [PATCH 40/63] Bump minimum macOS version to 10.15.7 This solves an issue with loading Swift libraries when running the x86_64 binary. Co-Authored-By: Nathan Sobo --- crates/gpui/build.rs | 2 +- crates/live_kit_client/build.rs | 12 ++++-------- crates/zed/Cargo.toml | 2 +- crates/zed/build.rs | 2 +- script/bundle | 7 +++++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/crates/gpui/build.rs b/crates/gpui/build.rs index 836d586c26..3fb0119782 100644 --- a/crates/gpui/build.rs +++ b/crates/gpui/build.rs @@ -57,7 +57,7 @@ fn compile_metal_shaders() { "macosx", "metal", "-gline-tables-only", - "-mmacosx-version-min=10.14", + "-mmacosx-version-min=10.15.7", "-MO", "-c", shader_path, diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 7d5b0944f1..4bbd61ac10 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -32,7 +32,7 @@ pub struct SwiftTarget { pub paths: SwiftPaths, } -const MACOS_TARGET_VERSION: &str = "10.15"; +const MACOS_TARGET_VERSION: &str = "10.15.7"; fn main() { if cfg!(not(any(test, feature = "test-support"))) { @@ -81,13 +81,9 @@ fn build_bridge(swift_target: &SwiftTarget) { } fn link_swift_stdlib(swift_target: &SwiftTarget) { - swift_target - .paths - .runtime_library_paths - .iter() - .for_each(|path| { - println!("cargo:rustc-link-search=native={}", path); - }); + for path in &swift_target.paths.runtime_library_paths { + println!("cargo:rustc-link-search=native={}", path); + } } fn link_webrtc_framework(swift_target: &SwiftTarget) { diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.toml index e35bfabe10..9a9c2fc5f1 100644 --- a/crates/zed/Cargo.toml +++ b/crates/zed/Cargo.toml @@ -127,4 +127,4 @@ unindent = "0.1.7" icon = ["app-icon@2x.png", "app-icon.png"] identifier = "dev.zed.Zed" name = "Zed" -osx_minimum_system_version = "10.14" +osx_minimum_system_version = "10.15.7" diff --git a/crates/zed/build.rs b/crates/zed/build.rs index 9b486cbb8b..c140f333f4 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -1,7 +1,7 @@ use std::process::Command; fn main() { - println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.14"); + println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.15.7"); if let Ok(api_key) = std::env::var("ZED_AMPLITUDE_API_KEY") { println!("cargo:rustc-env=ZED_AMPLITUDE_API_KEY={api_key}"); diff --git a/script/bundle b/script/bundle index a4a1391f2b..c9b663751e 100755 --- a/script/bundle +++ b/script/bundle @@ -3,7 +3,7 @@ set -e export ZED_BUNDLE=true -export MACOSX_DEPLOYMENT_TARGET=10.14 +export MACOSX_DEPLOYMENT_TARGET=10.15.7 echo "Installing cargo bundle" cargo install cargo-bundle --version 0.5.0 @@ -12,10 +12,13 @@ rustup target add wasm32-wasi # Deal with versions of macOS that don't include libstdc++ headers export CXXFLAGS="-stdlib=libc++" -echo "Compiling binaries" +echo "Compiling zed binary for aarch64-apple-darwin" cargo build --release --package zed --target aarch64-apple-darwin +echo "Compiling zed binary for x86_64-apple-darwin" cargo build --release --package zed --target x86_64-apple-darwin +echo "Compiling cli binary for aarch64-apple-darwin" cargo build --release --package cli --target aarch64-apple-darwin +echo "Compiling cli binary for x86_64-apple-darwin" cargo build --release --package cli --target x86_64-apple-darwin echo "Creating application bundle" From be1dc01d9e972b2ddcf06a3b50c758e55ca443bf Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 16:57:47 +0200 Subject: [PATCH 41/63] Add 5s timeout to LiveKit API requests Co-Authored-By: Nathan Sobo --- crates/live_kit_server/src/api.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index 43e01ce880..f78cd8f6fb 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use prost::Message; use reqwest::header::CONTENT_TYPE; -use std::{future::Future, sync::Arc}; +use std::{future::Future, sync::Arc, time::Duration}; #[async_trait] pub trait Client: Send + Sync { @@ -29,7 +29,10 @@ impl LiveKitClient { } Self { - http: reqwest::Client::new(), + http: reqwest::ClientBuilder::new() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(), url: url.into(), key: key.into(), secret: secret.into(), From 9b8e6cce02f9ce89497dc543e6bb251df198049d Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 19:28:21 +0200 Subject: [PATCH 42/63] WIP: Try using the new ScreenCaptureKit API when possible --- .../LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index af6485ce69..716d96644c 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -123,7 +123,7 @@ public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, partic @_cdecl("LKCreateScreenShareTrackForDisplay") public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { let display = Unmanaged.fromOpaque(display).takeUnretainedValue() - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display) return Unmanaged.passRetained(track).toOpaque() } @@ -148,7 +148,7 @@ public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { @_cdecl("LKDisplaySources") public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { - MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in + MacOSScreenCapturer.displaySources().then { displaySources in callback(data, displaySources as CFArray, nil) }.catch { error in callback(data, nil, error.localizedDescription as CFString) From db8b8ef66bd2cb62c180dbd485637e42b5274101 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Thu, 20 Oct 2022 20:17:54 +0200 Subject: [PATCH 43/63] WIP --- crates/call/src/call.rs | 1 + .../Sources/LiveKitBridge/LiveKitBridge.swift | 17 +++++++++++++++++ crates/live_kit_client/build.rs | 1 + crates/live_kit_client/src/prod.rs | 7 +++++++ crates/live_kit_client/src/test.rs | 4 ++++ 5 files changed, 30 insertions(+) diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 6b06d04375..1153259932 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -14,6 +14,7 @@ pub use room::Room; use std::sync::Arc; pub fn init(client: Arc, user_store: ModelHandle, cx: &mut MutableAppContext) { + live_kit_client::displays(); let active_call = cx.add_model(|cx| ActiveCall::new(client, user_store, cx)); cx.set_global(active_call); } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 716d96644c..9a21b17800 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,6 +1,7 @@ import Foundation import LiveKit import WebRTC +import ScreenCaptureKit class LKRoomDelegate: RoomDelegate { var data: UnsafeRawPointer @@ -154,3 +155,19 @@ public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @conven callback(data, nil, error.localizedDescription as CFString) } } + +@_cdecl("LKDisplays") +public func LKDisplays() { + if #available(macOS 12.3, *) { + Task.init { + let content = try await SCShareableContent.current + print(content.displays.count) + } + +// SCShareableContent.getWithCompletionHandler { content, error in +// print(content!.displays.count) +// } + } else { + print("OOOPS") + } +} diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 4bbd61ac10..6c5704b9db 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -81,6 +81,7 @@ fn build_bridge(swift_target: &SwiftTarget) { } fn link_swift_stdlib(swift_target: &SwiftTarget) { + panic!("{:?}", swift_target.paths.runtime_library_paths); for path in &swift_target.paths.runtime_library_paths { println!("cargo:rustc-link-search=native={}", path); } diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs index 35a5705a24..a0879f016a 100644 --- a/crates/live_kit_client/src/prod.rs +++ b/crates/live_kit_client/src/prod.rs @@ -62,6 +62,7 @@ extern "C" { fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); fn LKRemoteVideoTrackGetSid(track: *const c_void) -> CFStringRef; + fn LKDisplays(); fn LKDisplaySources( callback_data: *mut c_void, callback: extern "C" fn( @@ -73,6 +74,12 @@ extern "C" { fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void; } +pub fn displays() { + unsafe { + LKDisplays(); + } +} + pub type Sid = String; pub struct Room { diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 23076e13a5..e68bff1bf9 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -9,6 +9,10 @@ use media::core_video::CVImageBuffer; use parking_lot::Mutex; use std::{future::Future, sync::Arc}; +pub fn displays() { + panic!() +} + lazy_static! { static ref SERVERS: Mutex>> = Default::default(); } From 6bdb08ab9c0d95c14c93cec595503d1ee9c846cb Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 20 Oct 2022 13:18:53 -0600 Subject: [PATCH 44/63] Fix crash loading Swift symbol (I think associated with concurrency) I add /usr/lib/swift as an rpath, which seems to fix the issue even though there doesn't seem to be a relevant library at that location on my machine. Based on my research, wondering if `-Wl,-weak-lswiftCompatibilityConcurrency` is also required for this to work on older OSes, but holding back for now. --- crates/live_kit_client/build.rs | 1 - crates/zed/build.rs | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 6c5704b9db..4bbd61ac10 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -81,7 +81,6 @@ fn build_bridge(swift_target: &SwiftTarget) { } fn link_swift_stdlib(swift_target: &SwiftTarget) { - panic!("{:?}", swift_target.paths.runtime_library_paths); for path in &swift_target.paths.runtime_library_paths { println!("cargo:rustc-link-search=native={}", path); } diff --git a/crates/zed/build.rs b/crates/zed/build.rs index c140f333f4..3d75886f1c 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -15,6 +15,9 @@ fn main() { println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); } + // Seems to be required to enable Swift concurrency + println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); + // Register exported Objective-C selectors, protocols, etc println!("cargo:rustc-link-arg=-Wl,-ObjC"); From 0ef62fc334741b966b84e362cb849fe6e5957001 Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Thu, 20 Oct 2022 14:37:04 -0600 Subject: [PATCH 45/63] Preserve symlinks in WebRTC.framework to avoid bundle signing failure --- crates/live_kit_client/build.rs | 13 ++++++++++++- script/bundle | 1 - 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 4bbd61ac10..50ed8418f7 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -128,9 +128,20 @@ fn swift_package_root() -> PathBuf { } fn copy_dir(source: &Path, destination: &Path) { + assert!( + Command::new("rm") + .arg("-rf") + .arg(destination) + .status() + .unwrap() + .success(), + "could not remove {:?} before copying", + destination + ); + assert!( Command::new("cp") - .arg("-r") + .arg("-R") .args(&[source, destination]) .status() .unwrap() diff --git a/script/bundle b/script/bundle index c9b663751e..e69413bf93 100755 --- a/script/bundle +++ b/script/bundle @@ -39,7 +39,6 @@ lipo \ echo "Copying WebRTC.framework into the frameworks folder" mkdir target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/Frameworks cp -R target/x86_64-apple-darwin/release/WebRTC.framework target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/Frameworks/ -rm -rf target/x86_64-apple-darwin/release/bundle/osx/Zed.app/Contents/Frameworks/WebRTC.framework/Versions if [[ -n $MACOS_CERTIFICATE && -n $MACOS_CERTIFICATE_PASSWORD && -n $APPLE_NOTARIZATION_USERNAME && -n $APPLE_NOTARIZATION_PASSWORD ]]; then echo "Signing bundle with Apple-issued certificate" From ae44a3828512579455311174fb20352f898b4d6a Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 21 Oct 2022 10:12:24 +0200 Subject: [PATCH 46/63] Remove unused `LKDisplays` API --- crates/call/src/call.rs | 1 - .../Sources/LiveKitBridge/LiveKitBridge.swift | 16 ---------------- crates/live_kit_client/src/prod.rs | 7 ------- crates/live_kit_client/src/test.rs | 4 ---- 4 files changed, 28 deletions(-) diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 1153259932..6b06d04375 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -14,7 +14,6 @@ pub use room::Room; use std::sync::Arc; pub fn init(client: Arc, user_store: ModelHandle, cx: &mut MutableAppContext) { - live_kit_client::displays(); let active_call = cx.add_model(|cx| ActiveCall::new(client, user_store, cx)); cx.set_global(active_call); } diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 9a21b17800..f9380d747e 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -155,19 +155,3 @@ public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @conven callback(data, nil, error.localizedDescription as CFString) } } - -@_cdecl("LKDisplays") -public func LKDisplays() { - if #available(macOS 12.3, *) { - Task.init { - let content = try await SCShareableContent.current - print(content.displays.count) - } - -// SCShareableContent.getWithCompletionHandler { content, error in -// print(content!.displays.count) -// } - } else { - print("OOOPS") - } -} diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs index a0879f016a..35a5705a24 100644 --- a/crates/live_kit_client/src/prod.rs +++ b/crates/live_kit_client/src/prod.rs @@ -62,7 +62,6 @@ extern "C" { fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); fn LKRemoteVideoTrackGetSid(track: *const c_void) -> CFStringRef; - fn LKDisplays(); fn LKDisplaySources( callback_data: *mut c_void, callback: extern "C" fn( @@ -74,12 +73,6 @@ extern "C" { fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void; } -pub fn displays() { - unsafe { - LKDisplays(); - } -} - pub type Sid = String; pub struct Room { diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index e68bff1bf9..23076e13a5 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -9,10 +9,6 @@ use media::core_video::CVImageBuffer; use parking_lot::Mutex; use std::{future::Future, sync::Arc}; -pub fn displays() { - panic!() -} - lazy_static! { static ref SERVERS: Mutex>> = Default::default(); } From bac3dc1ccd5e2e6d5ca9aa618a25ec99eb84ff93 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 21 Oct 2022 10:12:35 +0200 Subject: [PATCH 47/63] Re-build live_kit_client when `MACOSX_DEPLOYMENT_TARGET` changes --- crates/live_kit_client/build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/live_kit_client/build.rs b/crates/live_kit_client/build.rs index 50ed8418f7..bceb4fb927 100644 --- a/crates/live_kit_client/build.rs +++ b/crates/live_kit_client/build.rs @@ -48,6 +48,7 @@ fn main() { } fn build_bridge(swift_target: &SwiftTarget) { + println!("cargo:rerun-if-env-changed=MACOSX_DEPLOYMENT_TARGET"); println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME); println!( "cargo:rerun-if-changed={}/Package.swift", From 78969d0938178a4f0a027ef8389a78858d4fd8c5 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 21 Oct 2022 11:54:52 +0200 Subject: [PATCH 48/63] Switch back to using the legacy screen capturing API The new API is buggy and inconsistent, so I think we should move on for now. --- .../LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index f9380d747e..c3f0e64449 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -124,7 +124,7 @@ public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, partic @_cdecl("LKCreateScreenShareTrackForDisplay") public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { let display = Unmanaged.fromOpaque(display).takeUnretainedValue() - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display) + let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) return Unmanaged.passRetained(track).toOpaque() } @@ -149,7 +149,7 @@ public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { @_cdecl("LKDisplaySources") public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { - MacOSScreenCapturer.displaySources().then { displaySources in + MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in callback(data, displaySources as CFArray, nil) }.catch { error in callback(data, nil, error.localizedDescription as CFString) From 1bbb7dd1260f3dfbad9f7909907627ca371c6381 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Fri, 21 Oct 2022 14:21:45 +0200 Subject: [PATCH 49/63] Leave Zed room when LiveKit room disconnects --- crates/call/src/room.rs | 25 ++++++- crates/collab/src/integration_tests.rs | 70 +++++++++++++++++-- crates/live_kit_client/Cargo.toml | 2 +- .../Sources/LiveKitBridge/LiveKitBridge.swift | 29 +++++++- crates/live_kit_client/src/prod.rs | 46 +++++++++++- crates/live_kit_client/src/test.rs | 63 +++++++++++------ crates/live_kit_server/src/api.rs | 3 +- 7 files changed, 200 insertions(+), 38 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index 98b223281c..ff63e09605 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -8,6 +8,7 @@ use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate}; +use postage::stream::Stream; use project::Project; use std::{mem, os::unix::prelude::OsStrExt, sync::Arc}; use util::{post_inc, ResultExt}; @@ -80,8 +81,26 @@ impl Room { let live_kit_room = if let Some(connection_info) = live_kit_connection_info { let room = live_kit_client::Room::new(); - let mut track_changes = room.remote_video_track_updates(); + let mut status = room.status(); + // Consume the initial status of the room. + let _ = status.try_recv(); let _maintain_room = cx.spawn_weak(|this, mut cx| async move { + while let Some(status) = status.next().await { + let this = if let Some(this) = this.upgrade(&cx) { + this + } else { + break; + }; + + if status == live_kit_client::ConnectionState::Disconnected { + this.update(&mut cx, |this, cx| this.leave(cx).log_err()); + break; + } + } + }); + + let mut track_changes = room.remote_video_track_updates(); + let _maintain_tracks = cx.spawn_weak(|this, mut cx| async move { while let Some(track_change) = track_changes.next().await { let this = if let Some(this) = this.upgrade(&cx) { this @@ -94,14 +113,17 @@ impl Room { }); } }); + cx.foreground() .spawn(room.connect(&connection_info.server_url, &connection_info.token)) .detach_and_log_err(cx); + Some(LiveKitRoom { room, screen_track: ScreenTrack::None, next_publish_id: 0, _maintain_room, + _maintain_tracks, }) } else { None @@ -725,6 +747,7 @@ struct LiveKitRoom { screen_track: ScreenTrack, next_publish_id: usize, _maintain_room: Task<()>, + _maintain_tracks: Task<()>, } pub enum ScreenTrack { diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index 37dfedfe55..e353102c23 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -253,12 +253,13 @@ async fn test_basic_calls( } ); - // User B leaves the room. - active_call_b.update(cx_b, |call, cx| { - call.hang_up(cx).unwrap(); - assert!(call.room().is_none()); - }); - deterministic.run_until_parked(); + // User B gets disconnected from the LiveKit server, which causes them + // to automatically leave the room. + server + .test_live_kit_server + .disconnect_client(client_b.peer_id().unwrap().to_string()) + .await; + active_call_b.update(cx_b, |call, _| assert!(call.room().is_none())); assert_eq!( room_participants(&room_a, cx_a), RoomParticipants { @@ -452,6 +453,63 @@ async fn test_leaving_room_on_disconnection( pending: Default::default() } ); + + // Call user B again from client A. + active_call_a + .update(cx_a, |call, cx| { + call.invite(client_b.user_id().unwrap(), None, cx) + }) + .await + .unwrap(); + let room_a = active_call_a.read_with(cx_a, |call, _| call.room().unwrap().clone()); + + // User B receives the call and joins the room. + let mut incoming_call_b = active_call_b.read_with(cx_b, |call, _| call.incoming()); + incoming_call_b.next().await.unwrap().unwrap(); + active_call_b + .update(cx_b, |call, cx| call.accept_incoming(cx)) + .await + .unwrap(); + let room_b = active_call_b.read_with(cx_b, |call, _| call.room().unwrap().clone()); + deterministic.run_until_parked(); + assert_eq!( + room_participants(&room_a, cx_a), + RoomParticipants { + remote: vec!["user_b".to_string()], + pending: Default::default() + } + ); + assert_eq!( + room_participants(&room_b, cx_b), + RoomParticipants { + remote: vec!["user_a".to_string()], + pending: Default::default() + } + ); + + // User B gets disconnected from the LiveKit server, which causes it + // to automatically leave the room. + server + .test_live_kit_server + .disconnect_client(client_b.peer_id().unwrap().to_string()) + .await; + deterministic.run_until_parked(); + active_call_a.update(cx_a, |call, _| assert!(call.room().is_none())); + active_call_b.update(cx_b, |call, _| assert!(call.room().is_none())); + assert_eq!( + room_participants(&room_a, cx_a), + RoomParticipants { + remote: Default::default(), + pending: Default::default() + } + ); + assert_eq!( + room_participants(&room_b, cx_b), + RoomParticipants { + remote: Default::default(), + pending: Default::default() + } + ); } #[gpui::test(iterations = 10)] diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index ce555ccbc3..d0f54782b9 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -34,6 +34,7 @@ core-graphics = "0.22.3" futures = "0.3" log = { version = "0.4.16", features = ["kv_unstable_serde"] } parking_lot = "0.11.1" +postage = { version = "0.4.1", features = ["futures-traits"] } async-trait = { version = "0.1", optional = true } lazy_static = { version = "1.4", optional = true } @@ -60,7 +61,6 @@ jwt = "0.16" lazy_static = "1.4" objc = "0.2" parking_lot = "0.11.1" -postage = { version = "0.4.1", features = ["futures-traits"] } serde = { version = "1.0", features = ["derive", "rc"] } sha2 = "0.10" simplelog = "0.9" diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index c3f0e64449..657b559c5e 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -5,15 +5,28 @@ import ScreenCaptureKit class LKRoomDelegate: RoomDelegate { var data: UnsafeRawPointer + var onDidDisconnect: @convention(c) (UnsafeRawPointer) -> Void var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void var onDidUnsubscribeFromRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) { + init( + data: UnsafeRawPointer, + onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, + onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, + onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) + { self.data = data + self.onDidDisconnect = onDidDisconnect self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack self.onDidUnsubscribeFromRemoteVideoTrack = onDidUnsubscribeFromRemoteVideoTrack } + func room(_ room: Room, didUpdate connectionState: ConnectionState, oldValue: ConnectionState) { + if connectionState.isDisconnected { + self.onDidDisconnect(self.data) + } + } + func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { if track.kind == .video { self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque()) @@ -62,8 +75,18 @@ class LKVideoRenderer: NSObject, VideoRenderer { } @_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack) +public func LKRoomDelegateCreate( + data: UnsafeRawPointer, + onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, + onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, + onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void +) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate( + data: data, + onDidDisconnect: onDidDisconnect, + onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, + onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack + ) return Unmanaged.passRetained(delegate).toOpaque() } diff --git a/crates/live_kit_client/src/prod.rs b/crates/live_kit_client/src/prod.rs index 35a5705a24..47fd4f0b69 100644 --- a/crates/live_kit_client/src/prod.rs +++ b/crates/live_kit_client/src/prod.rs @@ -11,6 +11,7 @@ use futures::{ pub use media::core_video::CVImageBuffer; use media::core_video::CVImageBufferRef; use parking_lot::Mutex; +use postage::watch; use std::{ ffi::c_void, sync::{Arc, Weak}, @@ -19,6 +20,7 @@ use std::{ extern "C" { fn LKRoomDelegateCreate( callback_data: *mut c_void, + on_did_disconnect: extern "C" fn(callback_data: *mut c_void), on_did_subscribe_to_remote_video_track: extern "C" fn( callback_data: *mut c_void, publisher_id: CFStringRef, @@ -75,8 +77,18 @@ extern "C" { pub type Sid = String; +#[derive(Clone, Eq, PartialEq)] +pub enum ConnectionState { + Disconnected, + Connected { url: String, token: String }, +} + pub struct Room { native_room: *const c_void, + connection: Mutex<( + watch::Sender, + watch::Receiver, + )>, remote_video_track_subscribers: Mutex>>, _delegate: RoomDelegate, } @@ -87,13 +99,18 @@ impl Room { let delegate = RoomDelegate::new(weak_room.clone()); Self { native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + connection: Mutex::new(watch::channel_with(ConnectionState::Disconnected)), remote_video_track_subscribers: Default::default(), _delegate: delegate, } }) } - pub fn connect(&self, url: &str, token: &str) -> impl Future> { + pub fn status(&self) -> watch::Receiver { + self.connection.lock().1.clone() + } + + pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { let url = CFString::new(url); let token = CFString::new(token); let (did_connect, tx, rx) = Self::build_done_callback(); @@ -107,7 +124,23 @@ impl Room { ) } - async { rx.await.unwrap().context("error connecting to room") } + let this = self.clone(); + let url = url.to_string(); + let token = token.to_string(); + async move { + match rx.await.unwrap().context("error connecting to room") { + Ok(()) => { + *this.connection.lock().0.borrow_mut() = + ConnectionState::Connected { url, token }; + Ok(()) + } + Err(err) => Err(err), + } + } + } + + fn did_disconnect(&self) { + *self.connection.lock().0.borrow_mut() = ConnectionState::Disconnected; } pub fn display_sources(self: &Arc) -> impl Future>> { @@ -265,6 +298,7 @@ impl RoomDelegate { let native_delegate = unsafe { LKRoomDelegateCreate( weak_room as *mut c_void, + Self::on_did_disconnect, Self::on_did_subscribe_to_remote_video_track, Self::on_did_unsubscribe_from_remote_video_track, ) @@ -275,6 +309,14 @@ impl RoomDelegate { } } + extern "C" fn on_did_disconnect(room: *mut c_void) { + let room = unsafe { Weak::from_raw(room as *mut Room) }; + if let Some(room) = room.upgrade() { + room.did_disconnect(); + } + let _ = Weak::into_raw(room); + } + extern "C" fn on_did_subscribe_to_remote_video_track( room: *mut c_void, publisher_id: CFStringRef, diff --git a/crates/live_kit_client/src/test.rs b/crates/live_kit_client/src/test.rs index 23076e13a5..329e4e1176 100644 --- a/crates/live_kit_client/src/test.rs +++ b/crates/live_kit_client/src/test.rs @@ -7,7 +7,8 @@ use lazy_static::lazy_static; use live_kit_server::token; use media::core_video::CVImageBuffer; use parking_lot::Mutex; -use std::{future::Future, sync::Arc}; +use postage::watch; +use std::{future::Future, mem, sync::Arc}; lazy_static! { static ref SERVERS: Mutex>> = Default::default(); @@ -145,6 +146,16 @@ impl TestServer { Ok(()) } + pub async fn disconnect_client(&self, client_identity: String) { + self.background.simulate_random_delay().await; + let mut server_rooms = self.rooms.lock(); + for room in server_rooms.values_mut() { + if let Some(room) = room.client_rooms.remove(&client_identity) { + *room.0.lock().connection.0.borrow_mut() = ConnectionState::Disconnected; + } + } + } + async fn publish_video_track(&self, token: String, local_track: LocalVideoTrack) -> Result<()> { self.background.simulate_random_delay().await; let claims = live_kit_server::token::validate(&token, &self.secret_key)?; @@ -227,7 +238,10 @@ impl live_kit_server::api::Client for TestApiClient { pub type Sid = String; struct RoomState { - connection: Option, + connection: ( + watch::Sender, + watch::Receiver, + ), display_sources: Vec, video_track_updates: ( async_broadcast::Sender, @@ -235,9 +249,10 @@ struct RoomState { ), } -struct ConnectionState { - url: String, - token: String, +#[derive(Clone, Eq, PartialEq)] +pub enum ConnectionState { + Disconnected, + Connected { url: String, token: String }, } pub struct Room(Mutex); @@ -245,12 +260,16 @@ pub struct Room(Mutex); impl Room { pub fn new() -> Arc { Arc::new(Self(Mutex::new(RoomState { - connection: None, + connection: watch::channel_with(ConnectionState::Disconnected), display_sources: Default::default(), video_track_updates: async_broadcast::broadcast(128), }))) } + pub fn status(&self) -> watch::Receiver { + self.0.lock().connection.1.clone() + } + pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { let this = self.clone(); let url = url.to_string(); @@ -258,7 +277,7 @@ impl Room { async move { let server = TestServer::get(&url)?; server.join_room(token.clone(), this.clone()).await?; - this.0.lock().connection = Some(ConnectionState { url, token }); + *this.0.lock().connection.0.borrow_mut() = ConnectionState::Connected { url, token }; Ok(()) } } @@ -301,32 +320,30 @@ impl Room { } fn test_server(&self) -> Arc { - let this = self.0.lock(); - let connection = this - .connection - .as_ref() - .expect("must be connected to call this method"); - TestServer::get(&connection.url).unwrap() + match self.0.lock().connection.1.borrow().clone() { + ConnectionState::Disconnected => panic!("must be connected to call this method"), + ConnectionState::Connected { url, .. } => TestServer::get(&url).unwrap(), + } } fn token(&self) -> String { - self.0 - .lock() - .connection - .as_ref() - .expect("must be connected to call this method") - .token - .clone() + match self.0.lock().connection.1.borrow().clone() { + ConnectionState::Disconnected => panic!("must be connected to call this method"), + ConnectionState::Connected { token, .. } => token, + } } } impl Drop for Room { fn drop(&mut self) { - if let Some(connection) = self.0.lock().connection.take() { - if let Ok(server) = TestServer::get(&connection.token) { + if let ConnectionState::Connected { token, .. } = mem::replace( + &mut *self.0.lock().connection.0.borrow_mut(), + ConnectionState::Disconnected, + ) { + if let Ok(server) = TestServer::get(&token) { let background = server.background.clone(); background - .spawn(async move { server.leave_room(connection.token).await.unwrap() }) + .spawn(async move { server.leave_room(token).await.unwrap() }) .detach(); } } diff --git a/crates/live_kit_server/src/api.rs b/crates/live_kit_server/src/api.rs index f78cd8f6fb..417a17bdc9 100644 --- a/crates/live_kit_server/src/api.rs +++ b/crates/live_kit_server/src/api.rs @@ -86,7 +86,7 @@ impl Client for LiveKitClient { } async fn create_room(&self, name: String) -> Result<()> { - let x: proto::Room = self + let _: proto::Room = self .request( "twirp/livekit.RoomService/CreateRoom", token::VideoGrant { @@ -99,7 +99,6 @@ impl Client for LiveKitClient { }, ) .await?; - dbg!(x); Ok(()) } From 476020ae8464605436a6f1728674dfa95ea017c1 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 10:04:08 +0200 Subject: [PATCH 50/63] Show shared screen as a pane item --- Cargo.lock | 16 +-- crates/call/Cargo.toml | 1 + crates/call/src/call.rs | 2 +- crates/call/src/participant.rs | 14 +- crates/call/src/room.rs | 65 ++++----- crates/collab/src/integration_tests.rs | 26 +--- crates/language/Cargo.toml | 2 +- crates/workspace/src/pane_group.rs | 35 +---- crates/workspace/src/shared_screen.rs | 175 +++++++++++++++++++++++++ crates/workspace/src/workspace.rs | 76 ++++++++--- 10 files changed, 286 insertions(+), 126 deletions(-) create mode 100644 crates/workspace/src/shared_screen.rs diff --git a/Cargo.lock b/Cargo.lock index 20f32b79b2..65b562c8ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,17 +170,6 @@ dependencies = [ "rust-embed", ] -[[package]] -name = "async-broadcast" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90622698a1218e0b2fb846c97b5f19a0831f6baddee73d9454156365ccfa473b" -dependencies = [ - "easy-parallel", - "event-listener", - "futures-core", -] - [[package]] name = "async-broadcast" version = "0.4.1" @@ -727,6 +716,7 @@ name = "call" version = "0.1.0" dependencies = [ "anyhow", + "async-broadcast", "client", "collections", "futures 0.3.24", @@ -2983,7 +2973,7 @@ name = "language" version = "0.1.0" dependencies = [ "anyhow", - "async-broadcast 0.3.4", + "async-broadcast", "async-trait", "client", "clock", @@ -3156,7 +3146,7 @@ name = "live_kit_client" version = "0.1.0" dependencies = [ "anyhow", - "async-broadcast 0.4.1", + "async-broadcast", "async-trait", "block", "byteorder", diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index 7556be6f77..a7a3331d20 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -27,6 +27,7 @@ project = { path = "../project" } util = { path = "../util" } anyhow = "1.0.38" +async-broadcast = "0.4" futures = "0.3" postage = { version = "0.4.1", features = ["futures-traits"] } diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 6b06d04375..2fb56d83a5 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -1,4 +1,4 @@ -mod participant; +pub mod participant; pub mod room; use anyhow::{anyhow, Result}; diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs index c045bd77dc..dfa456f734 100644 --- a/crates/call/src/participant.rs +++ b/crates/call/src/participant.rs @@ -1,8 +1,8 @@ use anyhow::{anyhow, Result}; use client::{proto, User}; use collections::HashMap; -use gpui::{Task, WeakModelHandle}; -use live_kit_client::Frame; +use gpui::WeakModelHandle; +pub use live_kit_client::Frame; use project::Project; use std::sync::Arc; @@ -41,18 +41,16 @@ pub struct RemoteParticipant { pub user: Arc, pub projects: Vec, pub location: ParticipantLocation, - pub tracks: HashMap, + pub tracks: HashMap>, } #[derive(Clone)] pub struct RemoteVideoTrack { - pub(crate) frame: Option, - pub(crate) _live_kit_track: Arc, - pub(crate) _maintain_frame: Arc>, + pub(crate) live_kit_track: Arc, } impl RemoteVideoTrack { - pub fn frame(&self) -> Option<&Frame> { - self.frame.as_ref() + pub fn frames(&self) -> async_broadcast::Receiver { + self.live_kit_track.frames() } } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index ff63e09605..f19389c1ca 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -7,7 +7,7 @@ use client::{proto, Client, PeerId, TypedEnvelope, User, UserStore}; use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; -use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate}; +use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate, Sid}; use postage::stream::Stream; use project::Project; use std::{mem, os::unix::prelude::OsStrExt, sync::Arc}; @@ -15,9 +15,16 @@ use util::{post_inc, ResultExt}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum Event { - Frame { + ParticipantLocationChanged { participant_id: PeerId, - track_id: live_kit_client::Sid, + }, + RemoteVideoTrackShared { + participant_id: PeerId, + track_id: Sid, + }, + RemoteVideoTrackUnshared { + peer_id: PeerId, + track_id: Sid, }, RemoteProjectShared { owner: Arc, @@ -356,7 +363,12 @@ impl Room { if let Some(remote_participant) = this.remote_participants.get_mut(&peer_id) { remote_participant.projects = participant.projects; - remote_participant.location = location; + if location != remote_participant.location { + remote_participant.location = location; + cx.emit(Event::ParticipantLocationChanged { + participant_id: peer_id, + }); + } } else { this.remote_participants.insert( peer_id, @@ -430,44 +442,16 @@ impl Room { .remote_participants .get_mut(&peer_id) .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - let mut frames = track.frames(); participant.tracks.insert( track_id.clone(), - RemoteVideoTrack { - frame: None, - _live_kit_track: track, - _maintain_frame: Arc::new(cx.spawn_weak(|this, mut cx| async move { - while let Some(frame) = frames.next().await { - let this = if let Some(this) = this.upgrade(&cx) { - this - } else { - break; - }; - - let done = this.update(&mut cx, |this, cx| { - if let Some(track) = - this.remote_participants.get_mut(&peer_id).and_then( - |participant| participant.tracks.get_mut(&track_id), - ) - { - track.frame = Some(frame); - cx.emit(Event::Frame { - participant_id: peer_id, - track_id: track_id.clone(), - }); - false - } else { - true - } - }); - - if done { - break; - } - } - })), - }, + Arc::new(RemoteVideoTrack { + live_kit_track: track, + }), ); + cx.emit(Event::RemoteVideoTrackShared { + participant_id: peer_id, + track_id, + }); } RemoteVideoTrackUpdate::Unsubscribed { publisher_id, @@ -479,6 +463,7 @@ impl Room { .get_mut(&peer_id) .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; participant.tracks.remove(&track_id); + cx.emit(Event::RemoteVideoTrackUnshared { peer_id, track_id }); } } @@ -750,7 +735,7 @@ struct LiveKitRoom { _maintain_tracks: Task<()>, } -pub enum ScreenTrack { +enum ScreenTrack { None, Pending { publish_id: usize }, Published(LocalTrackPublication), diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index f239a0ce58..b414839110 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -5,10 +5,7 @@ use crate::{ }; use ::rpc::Peer; use anyhow::anyhow; -use call::{ - room::{self, Event}, - ActiveCall, ParticipantLocation, Room, -}; +use call::{room, ActiveCall, ParticipantLocation, Room}; use client::{ self, test::FakeHttpClient, Channel, ChannelDetails, ChannelList, Client, Connection, Credentials, EstablishConnectionError, PeerId, User, UserStore, RECEIVE_TIMEOUT, @@ -33,7 +30,7 @@ use language::{ range_to_lsp, tree_sitter_rust, Diagnostic, DiagnosticEntry, FakeLspAdapter, Language, LanguageConfig, LanguageRegistry, OffsetRangeExt, Point, Rope, }; -use live_kit_client::{Frame, MacOSDisplay}; +use live_kit_client::MacOSDisplay; use lsp::{self, FakeLanguageServer}; use parking_lot::Mutex; use project::{ @@ -202,36 +199,25 @@ async fn test_basic_calls( .await .unwrap(); - let frame = Frame { - width: 800, - height: 600, - label: "a".into(), - }; - display.send_frame(frame.clone()); deterministic.run_until_parked(); assert_eq!(events_b.borrow().len(), 1); let event = events_b.borrow().first().unwrap().clone(); - if let Event::Frame { + if let call::room::Event::RemoteVideoTrackShared { participant_id, track_id, } = event { assert_eq!(participant_id, client_a.peer_id().unwrap()); room_b.read_with(cx_b, |room, _| { - assert_eq!( - room.remote_participants()[&client_a.peer_id().unwrap()].tracks[&track_id].frame(), - Some(&frame) - ); + assert!(room.remote_participants()[&client_a.peer_id().unwrap()] + .tracks + .contains_key(&track_id)); }); } else { panic!("unexpected event") } - display.send_frame(frame.clone()); - deterministic.run_until_parked(); - assert_eq!(events_b.borrow().len(), 2); - // User A leaves the room. active_call_a.update(cx_a, |call, cx| { call.hang_up(cx).unwrap(); diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.toml index 96feadbfbc..9a11b80511 100644 --- a/crates/language/Cargo.toml +++ b/crates/language/Cargo.toml @@ -36,7 +36,7 @@ text = { path = "../text" } theme = { path = "../theme" } util = { path = "../util" } anyhow = "1.0.38" -async-broadcast = "0.3.4" +async-broadcast = "0.4" async-trait = "0.1" futures = "0.3" lazy_static = "1.4" diff --git a/crates/workspace/src/pane_group.rs b/crates/workspace/src/pane_group.rs index c778115d91..6c379ffd2a 100644 --- a/crates/workspace/src/pane_group.rs +++ b/crates/workspace/src/pane_group.rs @@ -2,9 +2,7 @@ use crate::{FollowerStatesByLeader, JoinProject, Pane, Workspace}; use anyhow::{anyhow, Result}; use call::{ActiveCall, ParticipantLocation}; use gpui::{ - elements::*, - geometry::{rect::RectF, vector::vec2f}, - Axis, Border, CursorStyle, ModelHandle, MouseButton, RenderContext, ViewHandle, + elements::*, Axis, Border, CursorStyle, ModelHandle, MouseButton, RenderContext, ViewHandle, }; use project::Project; use serde::Deserialize; @@ -144,30 +142,6 @@ impl Member { Border::default() }; - let content = if leader.as_ref().map_or(false, |(_, leader)| { - leader.location == ParticipantLocation::External && !leader.tracks.is_empty() - }) { - let (_, leader) = leader.unwrap(); - let track = leader.tracks.values().next().unwrap(); - let frame = track.frame().cloned(); - Canvas::new(move |bounds, _, cx| { - if let Some(frame) = frame.clone() { - let size = constrain_size_preserving_aspect_ratio( - bounds.size(), - vec2f(frame.width() as f32, frame.height() as f32), - ); - let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.; - cx.scene.push_surface(gpui::mac::Surface { - bounds: RectF::new(origin, size), - image_buffer: frame.image(), - }); - } - }) - .boxed() - } else { - ChildView::new(pane, cx).boxed() - }; - let prompt = if let Some((_, leader)) = leader { match leader.location { ParticipantLocation::SharedProject { @@ -251,7 +225,12 @@ impl Member { }; Stack::new() - .with_child(Container::new(content).with_border(border).boxed()) + .with_child( + ChildView::new(pane, cx) + .contained() + .with_border(border) + .boxed(), + ) .with_children(prompt) .boxed() } diff --git a/crates/workspace/src/shared_screen.rs b/crates/workspace/src/shared_screen.rs new file mode 100644 index 0000000000..e241542f18 --- /dev/null +++ b/crates/workspace/src/shared_screen.rs @@ -0,0 +1,175 @@ +use crate::{Item, ItemNavHistory}; +use anyhow::{anyhow, Result}; +use call::participant::{Frame, RemoteVideoTrack}; +use client::{PeerId, User}; +use futures::StreamExt; +use gpui::{ + elements::*, + geometry::{rect::RectF, vector::vec2f}, + Entity, ModelHandle, RenderContext, Task, View, ViewContext, +}; +use smallvec::SmallVec; +use std::{ + path::PathBuf, + sync::{Arc, Weak}, +}; + +pub enum Event { + Close, +} + +pub struct SharedScreen { + track: Weak, + frame: Option, + pub peer_id: PeerId, + user: Arc, + nav_history: Option, + _maintain_frame: Task<()>, +} + +impl SharedScreen { + pub fn new( + track: &Arc, + peer_id: PeerId, + user: Arc, + cx: &mut ViewContext, + ) -> Self { + let mut frames = track.frames(); + Self { + track: Arc::downgrade(track), + frame: None, + peer_id, + user, + nav_history: Default::default(), + _maintain_frame: cx.spawn(|this, mut cx| async move { + while let Some(frame) = frames.next().await { + this.update(&mut cx, |this, cx| { + this.frame = Some(frame); + cx.notify(); + }) + } + this.update(&mut cx, |_, cx| cx.emit(Event::Close)); + }), + } + } +} + +impl Entity for SharedScreen { + type Event = Event; +} + +impl View for SharedScreen { + fn ui_name() -> &'static str { + "SharedScreen" + } + + fn render(&mut self, _: &mut RenderContext) -> ElementBox { + let frame = self.frame.clone(); + Canvas::new(move |bounds, _, cx| { + if let Some(frame) = frame.clone() { + let size = constrain_size_preserving_aspect_ratio( + bounds.size(), + vec2f(frame.width() as f32, frame.height() as f32), + ); + let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.; + cx.scene.push_surface(gpui::mac::Surface { + bounds: RectF::new(origin, size), + image_buffer: frame.image(), + }); + } + }) + .boxed() + } +} + +impl Item for SharedScreen { + fn deactivated(&mut self, cx: &mut ViewContext) { + if let Some(nav_history) = self.nav_history.as_ref() { + nav_history.push::<()>(None, cx); + } + } + + fn tab_content( + &self, + _: Option, + style: &theme::Tab, + _: &gpui::AppContext, + ) -> gpui::ElementBox { + Flex::row() + .with_child( + Svg::new("icons/disable_screen_sharing_12.svg") + .with_color(style.label.text.color) + .constrained() + .with_width(style.icon_width) + .aligned() + .contained() + .with_margin_right(style.spacing) + .boxed(), + ) + .with_child( + Label::new( + format!("{}'s screen", self.user.github_login), + style.label.clone(), + ) + .aligned() + .boxed(), + ) + .boxed() + } + + fn project_path(&self, _: &gpui::AppContext) -> Option { + Default::default() + } + + fn project_entry_ids(&self, _: &gpui::AppContext) -> SmallVec<[project::ProjectEntryId; 3]> { + Default::default() + } + + fn is_singleton(&self, _: &gpui::AppContext) -> bool { + false + } + + fn set_nav_history(&mut self, history: ItemNavHistory, _: &mut ViewContext) { + self.nav_history = Some(history); + } + + fn clone_on_split(&self, cx: &mut ViewContext) -> Option { + let track = self.track.upgrade()?; + Some(Self::new(&track, self.peer_id, self.user.clone(), cx)) + } + + fn can_save(&self, _: &gpui::AppContext) -> bool { + false + } + + fn save( + &mut self, + _: ModelHandle, + _: &mut ViewContext, + ) -> Task> { + Task::ready(Err(anyhow!("Item::save called on SharedScreen"))) + } + + fn save_as( + &mut self, + _: ModelHandle, + _: PathBuf, + _: &mut ViewContext, + ) -> Task> { + Task::ready(Err(anyhow!("Item::save_as called on SharedScreen"))) + } + + fn reload( + &mut self, + _: ModelHandle, + _: &mut ViewContext, + ) -> Task> { + Task::ready(Err(anyhow!("Item::reload called on SharedScreen"))) + } + + fn to_item_events(event: &Self::Event) -> Vec { + match event { + Event::Close => vec![crate::ItemEvent::CloseItem], + } + } +} diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index 7f84e58c03..3faec363ab 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -6,6 +6,7 @@ pub mod dock; pub mod pane; pub mod pane_group; pub mod searchable; +mod shared_screen; pub mod sidebar; mod status_bar; mod toolbar; @@ -36,6 +37,7 @@ use project::{Project, ProjectEntryId, ProjectPath, ProjectStore, Worktree, Work use searchable::SearchableItemHandle; use serde::Deserialize; use settings::{Autosave, DockAnchor, Settings}; +use shared_screen::SharedScreen; use sidebar::{Sidebar, SidebarButtons, SidebarSide, ToggleSidebarItem}; use smallvec::SmallVec; use status_bar::StatusBar; @@ -1097,14 +1099,7 @@ impl Workspace { if cx.has_global::>() { let call = cx.global::>().clone(); let mut subscriptions = Vec::new(); - subscriptions.push(cx.observe(&call, |_, _, cx| cx.notify())); - subscriptions.push(cx.subscribe(&call, |this, _, event, cx| { - if let call::room::Event::Frame { participant_id, .. } = event { - if this.follower_states_by_leader.contains_key(&participant_id) { - cx.notify(); - } - } - })); + subscriptions.push(cx.subscribe(&call, Self::on_active_call_event)); active_call = Some((call, subscriptions)); } @@ -2517,13 +2512,43 @@ impl Workspace { } fn leader_updated(&mut self, leader_id: PeerId, cx: &mut ViewContext) -> Option<()> { + cx.notify(); + + let call = self.active_call()?; + let room = call.read(cx).room()?.read(cx); + let participant = room.remote_participants().get(&leader_id)?; + let mut items_to_add = Vec::new(); - for (pane, state) in self.follower_states_by_leader.get(&leader_id)? { - if let Some(FollowerItem::Loaded(item)) = state - .active_view_id - .and_then(|id| state.items_by_leader_view_id.get(&id)) - { - items_to_add.push((pane.clone(), item.boxed_clone())); + match participant.location { + call::ParticipantLocation::SharedProject { project_id } => { + if Some(project_id) == self.project.read(cx).remote_id() { + for (pane, state) in self.follower_states_by_leader.get(&leader_id)? { + if let Some(FollowerItem::Loaded(item)) = state + .active_view_id + .and_then(|id| state.items_by_leader_view_id.get(&id)) + { + items_to_add.push((pane.clone(), item.boxed_clone())); + } + } + } + } + call::ParticipantLocation::UnsharedProject => {} + call::ParticipantLocation::External => { + let track = participant.tracks.values().next()?.clone(); + let user = participant.user.clone(); + + 'outer: for (pane, _) in self.follower_states_by_leader.get(&leader_id)? { + for item in pane.read(cx).items_of_type::() { + if item.read(cx).peer_id == leader_id { + items_to_add.push((pane.clone(), Box::new(item))); + continue 'outer; + } + } + + let shared_screen = + cx.add_view(|cx| SharedScreen::new(&track, leader_id, user.clone(), cx)); + items_to_add.push((pane.clone(), Box::new(shared_screen))); + } } } @@ -2532,8 +2557,8 @@ impl Workspace { if pane == self.active_pane { pane.update(cx, |pane, cx| pane.focus_active_item(cx)); } - cx.notify(); } + None } @@ -2561,6 +2586,27 @@ impl Workspace { fn active_call(&self) -> Option<&ModelHandle> { self.active_call.as_ref().map(|(call, _)| call) } + + fn on_active_call_event( + &mut self, + _: ModelHandle, + event: &call::room::Event, + cx: &mut ViewContext, + ) { + match event { + call::room::Event::ParticipantLocationChanged { + participant_id: peer_id, + } + | call::room::Event::RemoteVideoTrackShared { + participant_id: peer_id, + .. + } + | call::room::Event::RemoteVideoTrackUnshared { peer_id, .. } => { + self.leader_updated(*peer_id, cx); + } + _ => {} + } + } } impl Entity for Workspace { From f99d70500caac62ade052c84af7144e171221722 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 10:47:47 +0200 Subject: [PATCH 51/63] Allow opening shared screen via the contacts popover --- crates/collab_ui/src/contact_list.rs | 139 +++++++++++++++++++++++++-- crates/gpui/src/app.rs | 5 + crates/theme/src/theme.rs | 2 +- crates/workspace/src/workspace.rs | 53 +++++++--- styles/src/styleTree/contactList.ts | 5 + 5 files changed, 184 insertions(+), 20 deletions(-) diff --git a/crates/collab_ui/src/contact_list.rs b/crates/collab_ui/src/contact_list.rs index 7a51cc83ec..d6d87393a6 100644 --- a/crates/collab_ui/src/contact_list.rs +++ b/crates/collab_ui/src/contact_list.rs @@ -17,7 +17,7 @@ use serde::Deserialize; use settings::Settings; use theme::IconButton; use util::ResultExt; -use workspace::JoinProject; +use workspace::{JoinProject, OpenSharedScreen}; impl_actions!(contact_list, [RemoveContact, RespondToContactRequest]); impl_internal_actions!(contact_list, [ToggleExpanded, Call, LeaveCall]); @@ -67,6 +67,10 @@ enum ContactEntry { host_user_id: u64, is_last: bool, }, + ParticipantScreen { + peer_id: PeerId, + is_last: bool, + }, IncomingRequest(Arc), OutgoingRequest(Arc), Contact(Arc), @@ -97,6 +101,16 @@ impl PartialEq for ContactEntry { return project_id_1 == project_id_2; } } + ContactEntry::ParticipantScreen { + peer_id: peer_id_1, .. + } => { + if let ContactEntry::ParticipantScreen { + peer_id: peer_id_2, .. + } = other + { + return peer_id_1 == peer_id_2; + } + } ContactEntry::IncomingRequest(user_1) => { if let ContactEntry::IncomingRequest(user_2) = other { return user_1.id == user_2.id; @@ -216,6 +230,15 @@ impl ContactList { &theme.contact_list, cx, ), + ContactEntry::ParticipantScreen { peer_id, is_last } => { + Self::render_participant_screen( + *peer_id, + *is_last, + is_selected, + &theme.contact_list, + cx, + ) + } ContactEntry::IncomingRequest(user) => Self::render_contact_request( user.clone(), this.user_store.clone(), @@ -347,6 +370,9 @@ impl ContactList { follow_user_id: *host_user_id, }); } + ContactEntry::ParticipantScreen { peer_id, .. } => { + cx.dispatch_action(OpenSharedScreen { peer_id: *peer_id }); + } _ => {} } } @@ -430,11 +456,10 @@ impl ContactList { executor.clone(), )); for mat in matches { - let participant = &room.remote_participants()[&PeerId(mat.candidate_id as u32)]; + let peer_id = PeerId(mat.candidate_id as u32); + let participant = &room.remote_participants()[&peer_id]; participant_entries.push(ContactEntry::CallParticipant { - user: room.remote_participants()[&PeerId(mat.candidate_id as u32)] - .user - .clone(), + user: participant.user.clone(), is_pending: false, }); let mut projects = participant.projects.iter().peekable(); @@ -443,7 +468,13 @@ impl ContactList { project_id: project.id, worktree_root_names: project.worktree_root_names.clone(), host_user_id: participant.user.id, - is_last: projects.peek().is_none(), + is_last: projects.peek().is_none() && participant.tracks.is_empty(), + }); + } + if !participant.tracks.is_empty() { + participant_entries.push(ContactEntry::ParticipantScreen { + peer_id, + is_last: true, }); } } @@ -763,6 +794,102 @@ impl ContactList { .boxed() } + fn render_participant_screen( + peer_id: PeerId, + is_last: bool, + is_selected: bool, + theme: &theme::ContactList, + cx: &mut RenderContext, + ) -> ElementBox { + let font_cache = cx.font_cache(); + let host_avatar_height = theme + .contact_avatar + .width + .or(theme.contact_avatar.height) + .unwrap_or(0.); + let row = &theme.project_row.default; + let tree_branch = theme.tree_branch; + let line_height = row.name.text.line_height(font_cache); + let cap_height = row.name.text.cap_height(font_cache); + let baseline_offset = + row.name.text.baseline_offset(font_cache) + (theme.row_height - line_height) / 2.; + + MouseEventHandler::::new(peer_id.0 as usize, cx, |mouse_state, _| { + let tree_branch = *tree_branch.style_for(mouse_state, is_selected); + let row = theme.project_row.style_for(mouse_state, is_selected); + + Flex::row() + .with_child( + Stack::new() + .with_child( + Canvas::new(move |bounds, _, cx| { + let start_x = bounds.min_x() + (bounds.width() / 2.) + - (tree_branch.width / 2.); + let end_x = bounds.max_x(); + let start_y = bounds.min_y(); + let end_y = bounds.min_y() + baseline_offset - (cap_height / 2.); + + cx.scene.push_quad(gpui::Quad { + bounds: RectF::from_points( + vec2f(start_x, start_y), + vec2f( + start_x + tree_branch.width, + if is_last { end_y } else { bounds.max_y() }, + ), + ), + background: Some(tree_branch.color), + border: gpui::Border::default(), + corner_radius: 0., + }); + cx.scene.push_quad(gpui::Quad { + bounds: RectF::from_points( + vec2f(start_x, end_y), + vec2f(end_x, end_y + tree_branch.width), + ), + background: Some(tree_branch.color), + border: gpui::Border::default(), + corner_radius: 0., + }); + }) + .boxed(), + ) + .constrained() + .with_width(host_avatar_height) + .boxed(), + ) + .with_child( + Svg::new("icons/disable_screen_sharing_12.svg") + .with_color(row.icon.color) + .constrained() + .with_width(row.icon.width) + .aligned() + .left() + .contained() + .with_style(row.icon.container) + .boxed(), + ) + .with_child( + Label::new("Screen Sharing".into(), row.name.text.clone()) + .aligned() + .left() + .contained() + .with_style(row.name.container) + .flex(1., false) + .boxed(), + ) + .constrained() + .with_height(theme.row_height) + .contained() + .with_style(row.container) + .boxed() + }) + .with_cursor_style(CursorStyle::PointingHand) + .on_click(MouseButton::Left, move |_, cx| { + cx.dispatch_action(OpenSharedScreen { peer_id }); + }) + .boxed() + } + fn render_header( section: Section, theme: &theme::ContactList, diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index a9020cf350..5cbc786b72 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -3835,6 +3835,11 @@ impl<'a, T: View> ViewContext<'a, T> { self.app.notify_view(self.window_id, self.view_id); } + pub fn dispatch_action(&mut self, action: impl Action) { + self.app + .dispatch_action_at(self.window_id, self.view_id, action) + } + pub fn dispatch_any_action(&mut self, action: Box) { self.app .dispatch_any_action_at(self.window_id, self.view_id, action) diff --git a/crates/theme/src/theme.rs b/crates/theme/src/theme.rs index 3555f77126..db6609fa82 100644 --- a/crates/theme/src/theme.rs +++ b/crates/theme/src/theme.rs @@ -120,6 +120,7 @@ pub struct ContactList { pub struct ProjectRow { #[serde(flatten)] pub container: ContainerStyle, + pub icon: Icon, pub name: ContainedText, } @@ -381,7 +382,6 @@ pub struct Icon { pub container: ContainerStyle, pub color: Color, pub width: f32, - pub path: String, } #[derive(Deserialize, Clone, Copy, Default)] diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index 3faec363ab..a318852359 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -121,12 +121,18 @@ pub struct JoinProject { pub follow_user_id: u64, } +#[derive(Clone, PartialEq)] +pub struct OpenSharedScreen { + pub peer_id: PeerId, +} + impl_internal_actions!( workspace, [ OpenPaths, ToggleFollow, JoinProject, + OpenSharedScreen, RemoveWorktreeFromProject ] ); @@ -166,6 +172,7 @@ pub fn init(app_state: Arc, cx: &mut MutableAppContext) { cx.add_async_action(Workspace::follow_next_collaborator); cx.add_async_action(Workspace::close); cx.add_async_action(Workspace::save_all); + cx.add_action(Workspace::open_shared_screen); cx.add_action(Workspace::add_folder_to_project); cx.add_action(Workspace::remove_folder_from_project); cx.add_action( @@ -1788,6 +1795,15 @@ impl Workspace { item } + pub fn open_shared_screen(&mut self, action: &OpenSharedScreen, cx: &mut ViewContext) { + if let Some(shared_screen) = + self.shared_screen_for_peer(action.peer_id, &self.active_pane, cx) + { + let pane = self.active_pane.clone(); + Pane::add_item(self, &pane, Box::new(shared_screen), false, true, None, cx); + } + } + pub fn activate_item(&mut self, item: &dyn ItemHandle, cx: &mut ViewContext) -> bool { let result = self.panes.iter().find_map(|pane| { pane.read(cx) @@ -2534,20 +2550,10 @@ impl Workspace { } call::ParticipantLocation::UnsharedProject => {} call::ParticipantLocation::External => { - let track = participant.tracks.values().next()?.clone(); - let user = participant.user.clone(); - - 'outer: for (pane, _) in self.follower_states_by_leader.get(&leader_id)? { - for item in pane.read(cx).items_of_type::() { - if item.read(cx).peer_id == leader_id { - items_to_add.push((pane.clone(), Box::new(item))); - continue 'outer; - } + for (pane, _) in self.follower_states_by_leader.get(&leader_id)? { + if let Some(shared_screen) = self.shared_screen_for_peer(leader_id, pane, cx) { + items_to_add.push((pane.clone(), Box::new(shared_screen))); } - - let shared_screen = - cx.add_view(|cx| SharedScreen::new(&track, leader_id, user.clone(), cx)); - items_to_add.push((pane.clone(), Box::new(shared_screen))); } } } @@ -2562,6 +2568,27 @@ impl Workspace { None } + fn shared_screen_for_peer( + &self, + peer_id: PeerId, + pane: &ViewHandle, + cx: &mut ViewContext, + ) -> Option> { + let call = self.active_call()?; + let room = call.read(cx).room()?.read(cx); + let participant = room.remote_participants().get(&peer_id)?; + let track = participant.tracks.values().next()?.clone(); + let user = participant.user.clone(); + + for item in pane.read(cx).items_of_type::() { + if item.read(cx).peer_id == peer_id { + return Some(item); + } + } + + Some(cx.add_view(|cx| SharedScreen::new(&track, peer_id, user.clone(), cx))) + } + pub fn on_window_activation_changed(&mut self, active: bool, cx: &mut ViewContext) { if !active { for pane in &self.panes { diff --git a/styles/src/styleTree/contactList.ts b/styles/src/styleTree/contactList.ts index a58bf90fd1..5aede5d862 100644 --- a/styles/src/styleTree/contactList.ts +++ b/styles/src/styleTree/contactList.ts @@ -166,6 +166,11 @@ export default function contactsPanel(colorScheme: ColorScheme) { projectRow: { ...projectRow, background: background(layer, "on"), + icon: { + margin: { left: nameMargin }, + color: foreground(layer, "variant"), + width: 12, + }, name: { ...projectRow.name, ...text(layer, "mono", { size: "sm" }), From a8bd234aa438dddffcb34986b3089f5a781aaca7 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 10:53:44 +0200 Subject: [PATCH 52/63] Simplify room events --- crates/call/src/room.rs | 16 ++++++---------- crates/collab/src/integration_tests.rs | 15 +++++++-------- crates/workspace/src/workspace.rs | 12 +++--------- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index f19389c1ca..f6722dd13d 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -7,7 +7,7 @@ use client::{proto, Client, PeerId, TypedEnvelope, User, UserStore}; use collections::{BTreeMap, HashSet}; use futures::StreamExt; use gpui::{AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, Task}; -use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate, Sid}; +use live_kit_client::{LocalTrackPublication, LocalVideoTrack, RemoteVideoTrackUpdate}; use postage::stream::Stream; use project::Project; use std::{mem, os::unix::prelude::OsStrExt, sync::Arc}; @@ -18,13 +18,8 @@ pub enum Event { ParticipantLocationChanged { participant_id: PeerId, }, - RemoteVideoTrackShared { + RemoteVideoTracksChanged { participant_id: PeerId, - track_id: Sid, - }, - RemoteVideoTrackUnshared { - peer_id: PeerId, - track_id: Sid, }, RemoteProjectShared { owner: Arc, @@ -448,9 +443,8 @@ impl Room { live_kit_track: track, }), ); - cx.emit(Event::RemoteVideoTrackShared { + cx.emit(Event::RemoteVideoTracksChanged { participant_id: peer_id, - track_id, }); } RemoteVideoTrackUpdate::Unsubscribed { @@ -463,7 +457,9 @@ impl Room { .get_mut(&peer_id) .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; participant.tracks.remove(&track_id); - cx.emit(Event::RemoteVideoTrackUnshared { peer_id, track_id }); + cx.emit(Event::RemoteVideoTracksChanged { + participant_id: peer_id, + }); } } diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index b414839110..961f057bc5 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -203,16 +203,15 @@ async fn test_basic_calls( assert_eq!(events_b.borrow().len(), 1); let event = events_b.borrow().first().unwrap().clone(); - if let call::room::Event::RemoteVideoTrackShared { - participant_id, - track_id, - } = event - { + if let call::room::Event::RemoteVideoTracksChanged { participant_id } = event { assert_eq!(participant_id, client_a.peer_id().unwrap()); room_b.read_with(cx_b, |room, _| { - assert!(room.remote_participants()[&client_a.peer_id().unwrap()] - .tracks - .contains_key(&track_id)); + assert_eq!( + room.remote_participants()[&client_a.peer_id().unwrap()] + .tracks + .len(), + 1 + ); }); } else { panic!("unexpected event") diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index a318852359..c3e54c525b 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -2621,15 +2621,9 @@ impl Workspace { cx: &mut ViewContext, ) { match event { - call::room::Event::ParticipantLocationChanged { - participant_id: peer_id, - } - | call::room::Event::RemoteVideoTrackShared { - participant_id: peer_id, - .. - } - | call::room::Event::RemoteVideoTrackUnshared { peer_id, .. } => { - self.leader_updated(*peer_id, cx); + call::room::Event::ParticipantLocationChanged { participant_id } + | call::room::Event::RemoteVideoTracksChanged { participant_id } => { + self.leader_updated(*participant_id, cx); } _ => {} } From e135b982c1a6fc80015e14ff53fc5b0953d2d965 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 11:02:10 +0200 Subject: [PATCH 53/63] Focus shared screen item when clicking on it --- crates/workspace/src/shared_screen.rs | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/workspace/src/shared_screen.rs b/crates/workspace/src/shared_screen.rs index e241542f18..4a603ea1b8 100644 --- a/crates/workspace/src/shared_screen.rs +++ b/crates/workspace/src/shared_screen.rs @@ -6,7 +6,7 @@ use futures::StreamExt; use gpui::{ elements::*, geometry::{rect::RectF, vector::vec2f}, - Entity, ModelHandle, RenderContext, Task, View, ViewContext, + Entity, ModelHandle, MouseButton, RenderContext, Task, View, ViewContext, }; use smallvec::SmallVec; use std::{ @@ -63,21 +63,27 @@ impl View for SharedScreen { "SharedScreen" } - fn render(&mut self, _: &mut RenderContext) -> ElementBox { + fn render(&mut self, cx: &mut RenderContext) -> ElementBox { + enum Focus {} + let frame = self.frame.clone(); - Canvas::new(move |bounds, _, cx| { - if let Some(frame) = frame.clone() { - let size = constrain_size_preserving_aspect_ratio( - bounds.size(), - vec2f(frame.width() as f32, frame.height() as f32), - ); - let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.; - cx.scene.push_surface(gpui::mac::Surface { - bounds: RectF::new(origin, size), - image_buffer: frame.image(), - }); - } + MouseEventHandler::::new(0, cx, |_, _| { + Canvas::new(move |bounds, _, cx| { + if let Some(frame) = frame.clone() { + let size = constrain_size_preserving_aspect_ratio( + bounds.size(), + vec2f(frame.width() as f32, frame.height() as f32), + ); + let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.; + cx.scene.push_surface(gpui::mac::Surface { + bounds: RectF::new(origin, size), + image_buffer: frame.image(), + }); + } + }) + .boxed() }) + .on_down(MouseButton::Left, |_, cx| cx.focus_parent_view()) .boxed() } } From 088c5bac1f96855f7ccd4430ee0f5a0c8521b0ee Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 11:02:41 +0200 Subject: [PATCH 54/63] Remove stray log statement --- .../LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 657b559c5e..a0326b24a1 100644 --- a/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -59,7 +59,6 @@ class LKVideoRenderer: NSObject, VideoRenderer { } func setSize(_ size: CGSize) { - print("Called setSize", size); } func renderFrame(_ frame: RTCVideoFrame?) { From 874a3605f84a7756349389340fabfa8abe7c703f Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 14:28:58 +0200 Subject: [PATCH 55/63] Init submodules on CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 866d0acc0e..a973436ced 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: uses: actions/checkout@v2 with: clean: false + submodules: 'recursive' - name: Run tests run: cargo test --workspace --no-fail-fast @@ -75,6 +76,7 @@ jobs: uses: actions/checkout@v2 with: clean: false + submodules: 'recursive' - name: Validate version if: ${{ startsWith(github.ref, 'refs/tags/v') }} From 9860dbbbea84e15c1a1dd194df25cf13f12a5dd0 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 15:03:26 +0200 Subject: [PATCH 56/63] Set location on `ActiveCall` even before there's a room We will automatically call `Room::set_location` once a room has been assigned. --- crates/call/src/call.rs | 32 +++++++--- crates/call/src/room.rs | 2 +- crates/collab/src/integration_tests.rs | 61 +++++++++++++++----- crates/collab_ui/src/collab_titlebar_item.rs | 14 ++--- crates/collab_ui/src/contact_list.rs | 24 ++------ 5 files changed, 80 insertions(+), 53 deletions(-) diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 2fb56d83a5..106006007c 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -5,7 +5,7 @@ use anyhow::{anyhow, Result}; use client::{proto, Client, TypedEnvelope, User, UserStore}; use gpui::{ AppContext, AsyncAppContext, Entity, ModelContext, ModelHandle, MutableAppContext, - Subscription, Task, + Subscription, Task, WeakModelHandle, }; pub use participant::ParticipantLocation; use postage::watch; @@ -27,6 +27,7 @@ pub struct IncomingCall { } pub struct ActiveCall { + location: Option>, room: Option<(ModelHandle, Vec)>, incoming_call: ( watch::Sender>, @@ -49,6 +50,7 @@ impl ActiveCall { ) -> Self { Self { room: None, + location: None, incoming_call: watch::channel(), _subscriptions: vec![ client.add_request_handler(cx.handle(), Self::handle_incoming_call), @@ -132,7 +134,9 @@ impl ActiveCall { Room::create(recipient_user_id, initial_project, client, user_store, cx) }) .await?; - this.update(&mut cx, |this, cx| this.set_room(Some(room), cx)); + + this.update(&mut cx, |this, cx| this.set_room(Some(room.clone()), cx)) + .await?; }; Ok(()) @@ -180,7 +184,8 @@ impl ActiveCall { let join = Room::join(&call, self.client.clone(), self.user_store.clone(), cx); cx.spawn(|this, mut cx| async move { let room = join.await?; - this.update(&mut cx, |this, cx| this.set_room(Some(room.clone()), cx)); + this.update(&mut cx, |this, cx| this.set_room(Some(room.clone()), cx)) + .await?; Ok(()) }) } @@ -223,35 +228,46 @@ impl ActiveCall { project: Option<&ModelHandle>, cx: &mut ModelContext, ) -> Task> { + self.location = project.map(|project| project.downgrade()); if let Some((room, _)) = self.room.as_ref() { room.update(cx, |room, cx| room.set_location(project, cx)) } else { - Task::ready(Err(anyhow!("no active call"))) + Task::ready(Ok(())) } } - fn set_room(&mut self, room: Option>, cx: &mut ModelContext) { + fn set_room( + &mut self, + room: Option>, + cx: &mut ModelContext, + ) -> Task> { if room.as_ref() != self.room.as_ref().map(|room| &room.0) { + cx.notify(); if let Some(room) = room { if room.read(cx).status().is_offline() { self.room = None; + Task::ready(Ok(())) } else { let subscriptions = vec![ cx.observe(&room, |this, room, cx| { if room.read(cx).status().is_offline() { - this.set_room(None, cx); + this.set_room(None, cx).detach_and_log_err(cx); } cx.notify(); }), cx.subscribe(&room, |_, _, event, cx| cx.emit(event.clone())), ]; - self.room = Some((room, subscriptions)); + self.room = Some((room.clone(), subscriptions)); + let location = self.location.and_then(|location| location.upgrade(cx)); + room.update(cx, |room, cx| room.set_location(location.as_ref(), cx)) } } else { self.room = None; + Task::ready(Ok(())) } - cx.notify(); + } else { + Task::ready(Ok(())) } } diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs index f6722dd13d..7d5153950d 100644 --- a/crates/call/src/room.rs +++ b/crates/call/src/room.rs @@ -567,7 +567,7 @@ impl Room { }) } - pub fn set_location( + pub(crate) fn set_location( &mut self, project: Option<&ModelHandle>, cx: &mut ModelContext, diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index 961f057bc5..406a97025e 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -1074,15 +1074,9 @@ async fn test_room_location( client_a.fs.insert_tree("/a", json!({})).await; client_b.fs.insert_tree("/b", json!({})).await; - let (project_a, _) = client_a.build_local_project("/a", cx_a).await; - let (project_b, _) = client_b.build_local_project("/b", cx_b).await; - - server - .create_room(&mut [(&client_a, cx_a), (&client_b, cx_b)]) - .await; - let active_call_a = cx_a.read(ActiveCall::global); - let room_a = active_call_a.read_with(cx_a, |call, _| call.room().unwrap().clone()); + let active_call_b = cx_b.read(ActiveCall::global); + let a_notified = Rc::new(Cell::new(false)); cx_a.update({ let notified = a_notified.clone(); @@ -1092,8 +1086,6 @@ async fn test_room_location( } }); - let active_call_b = cx_b.read(ActiveCall::global); - let room_b = active_call_b.read_with(cx_b, |call, _| call.room().unwrap().clone()); let b_notified = Rc::new(Cell::new(false)); cx_b.update({ let b_notified = b_notified.clone(); @@ -1103,10 +1095,18 @@ async fn test_room_location( } }); - room_a - .update(cx_a, |room, cx| room.set_location(Some(&project_a), cx)) + let (project_a, _) = client_a.build_local_project("/a", cx_a).await; + active_call_a + .update(cx_a, |call, cx| call.set_location(Some(&project_a), cx)) .await .unwrap(); + let (project_b, _) = client_b.build_local_project("/b", cx_b).await; + + server + .create_room(&mut [(&client_a, cx_a), (&client_b, cx_b)]) + .await; + let room_a = active_call_a.read_with(cx_a, |call, _| call.room().unwrap().clone()); + let room_b = active_call_b.read_with(cx_b, |call, _| call.room().unwrap().clone()); deterministic.run_until_parked(); assert!(a_notified.take()); assert_eq!( @@ -1161,8 +1161,8 @@ async fn test_room_location( )] ); - room_b - .update(cx_b, |room, cx| room.set_location(Some(&project_b), cx)) + active_call_b + .update(cx_b, |call, cx| call.set_location(Some(&project_b), cx)) .await .unwrap(); deterministic.run_until_parked(); @@ -1187,8 +1187,8 @@ async fn test_room_location( )] ); - room_b - .update(cx_b, |room, cx| room.set_location(None, cx)) + active_call_b + .update(cx_b, |call, cx| call.set_location(None, cx)) .await .unwrap(); deterministic.run_until_parked(); @@ -5070,6 +5070,7 @@ async fn test_following(cx_a: &mut TestAppContext, cx_b: &mut TestAppContext) { .create_room(&mut [(&client_a, cx_a), (&client_b, cx_b)]) .await; let active_call_a = cx_a.read(ActiveCall::global); + let active_call_b = cx_b.read(ActiveCall::global); client_a .fs @@ -5083,11 +5084,20 @@ async fn test_following(cx_a: &mut TestAppContext, cx_b: &mut TestAppContext) { ) .await; let (project_a, worktree_id) = client_a.build_local_project("/a", cx_a).await; + active_call_a + .update(cx_a, |call, cx| call.set_location(Some(&project_a), cx)) + .await + .unwrap(); + let project_id = active_call_a .update(cx_a, |call, cx| call.share_project(project_a.clone(), cx)) .await .unwrap(); let project_b = client_b.build_remote_project(project_id, cx_b).await; + active_call_b + .update(cx_b, |call, cx| call.set_location(Some(&project_b), cx)) + .await + .unwrap(); // Client A opens some editors. let workspace_a = client_a.build_workspace(&project_a, cx_a); @@ -5281,6 +5291,7 @@ async fn test_peers_following_each_other(cx_a: &mut TestAppContext, cx_b: &mut T .create_room(&mut [(&client_a, cx_a), (&client_b, cx_b)]) .await; let active_call_a = cx_a.read(ActiveCall::global); + let active_call_b = cx_b.read(ActiveCall::global); // Client A shares a project. client_a @@ -5296,6 +5307,10 @@ async fn test_peers_following_each_other(cx_a: &mut TestAppContext, cx_b: &mut T ) .await; let (project_a, worktree_id) = client_a.build_local_project("/a", cx_a).await; + active_call_a + .update(cx_a, |call, cx| call.set_location(Some(&project_a), cx)) + .await + .unwrap(); let project_id = active_call_a .update(cx_a, |call, cx| call.share_project(project_a.clone(), cx)) .await @@ -5303,6 +5318,10 @@ async fn test_peers_following_each_other(cx_a: &mut TestAppContext, cx_b: &mut T // Client B joins the project. let project_b = client_b.build_remote_project(project_id, cx_b).await; + active_call_b + .update(cx_b, |call, cx| call.set_location(Some(&project_b), cx)) + .await + .unwrap(); // Client A opens some editors. let workspace_a = client_a.build_workspace(&project_a, cx_a); @@ -5450,6 +5469,7 @@ async fn test_auto_unfollowing(cx_a: &mut TestAppContext, cx_b: &mut TestAppCont .create_room(&mut [(&client_a, cx_a), (&client_b, cx_b)]) .await; let active_call_a = cx_a.read(ActiveCall::global); + let active_call_b = cx_b.read(ActiveCall::global); // Client A shares a project. client_a @@ -5464,11 +5484,20 @@ async fn test_auto_unfollowing(cx_a: &mut TestAppContext, cx_b: &mut TestAppCont ) .await; let (project_a, worktree_id) = client_a.build_local_project("/a", cx_a).await; + active_call_a + .update(cx_a, |call, cx| call.set_location(Some(&project_a), cx)) + .await + .unwrap(); + let project_id = active_call_a .update(cx_a, |call, cx| call.share_project(project_a.clone(), cx)) .await .unwrap(); let project_b = client_b.build_remote_project(project_id, cx_b).await; + active_call_b + .update(cx_b, |call, cx| call.set_location(Some(&project_b), cx)) + .await + .unwrap(); // Client A opens some editors. let workspace_a = client_a.build_workspace(&project_a, cx_a); diff --git a/crates/collab_ui/src/collab_titlebar_item.rs b/crates/collab_ui/src/collab_titlebar_item.rs index 43d9bc0dbf..9027a0937b 100644 --- a/crates/collab_ui/src/collab_titlebar_item.rs +++ b/crates/collab_ui/src/collab_titlebar_item.rs @@ -120,19 +120,15 @@ impl CollabTitlebarItem { } fn window_activation_changed(&mut self, active: bool, cx: &mut ViewContext) { - let workspace = self.workspace.upgrade(cx); - let room = ActiveCall::global(cx).read(cx).room().cloned(); - if let Some((workspace, room)) = workspace.zip(room) { - let workspace = workspace.read(cx); + if let Some(workspace) = self.workspace.upgrade(cx) { let project = if active { - Some(workspace.project().clone()) + Some(workspace.read(cx).project().clone()) } else { None }; - room.update(cx, |room, cx| { - room.set_location(project.as_ref(), cx) - .detach_and_log_err(cx); - }); + ActiveCall::global(cx) + .update(cx, |call, cx| call.set_location(project.as_ref(), cx)) + .detach_and_log_err(cx); } } diff --git a/crates/collab_ui/src/contact_list.rs b/crates/collab_ui/src/contact_list.rs index d6d87393a6..74cf2012fc 100644 --- a/crates/collab_ui/src/contact_list.rs +++ b/crates/collab_ui/src/contact_list.rs @@ -1162,25 +1162,11 @@ impl ContactList { fn call(&mut self, action: &Call, cx: &mut ViewContext) { let recipient_user_id = action.recipient_user_id; let initial_project = action.initial_project.clone(); - let window_id = cx.window_id(); - - let active_call = ActiveCall::global(cx); - cx.spawn_weak(|_, mut cx| async move { - active_call - .update(&mut cx, |active_call, cx| { - active_call.invite(recipient_user_id, initial_project.clone(), cx) - }) - .await?; - if cx.update(|cx| cx.window_is_active(window_id)) { - active_call - .update(&mut cx, |call, cx| { - call.set_location(initial_project.as_ref(), cx) - }) - .await?; - } - anyhow::Ok(()) - }) - .detach_and_log_err(cx); + ActiveCall::global(cx) + .update(cx, |call, cx| { + call.invite(recipient_user_id, initial_project.clone(), cx) + }) + .detach_and_log_err(cx); } fn leave_call(&mut self, _: &LeaveCall, cx: &mut ViewContext) { From 50c4783333b9c325bad1ffc615a5d7095fd1a162 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 15:17:25 +0200 Subject: [PATCH 57/63] Add test for screen-sharing --- crates/collab/src/integration_tests.rs | 65 ++++++++++++++++++++++++-- crates/workspace/src/workspace.rs | 2 +- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/crates/collab/src/integration_tests.rs b/crates/collab/src/integration_tests.rs index 406a97025e..80ed46c336 100644 --- a/crates/collab/src/integration_tests.rs +++ b/crates/collab/src/integration_tests.rs @@ -55,7 +55,7 @@ use std::{ }; use theme::ThemeRegistry; use unindent::Unindent as _; -use workspace::{Item, SplitDirection, ToggleFollow, Workspace}; +use workspace::{shared_screen::SharedScreen, Item, SplitDirection, ToggleFollow, Workspace}; #[ctor::ctor] fn init_logger() { @@ -5058,7 +5058,11 @@ async fn test_contact_requests( } #[gpui::test(iterations = 10)] -async fn test_following(cx_a: &mut TestAppContext, cx_b: &mut TestAppContext) { +async fn test_following( + deterministic: Arc, + cx_a: &mut TestAppContext, + cx_b: &mut TestAppContext, +) { cx_a.foreground().forbid_parking(); cx_a.update(editor::init); cx_b.update(editor::init); @@ -5239,7 +5243,7 @@ async fn test_following(cx_a: &mut TestAppContext, cx_b: &mut TestAppContext) { workspace_a.update(cx_a, |workspace, cx| { workspace.activate_item(&editor_a2, cx) }); - cx_a.foreground().run_until_parked(); + deterministic.run_until_parked(); assert_eq!( workspace_b.read_with(cx_b, |workspace, cx| workspace .active_item(cx) @@ -5269,9 +5273,62 @@ async fn test_following(cx_a: &mut TestAppContext, cx_b: &mut TestAppContext) { editor_a1.id() ); + // Client B activates an external window, which causes a new screen-sharing item to be added to the pane. + let display = MacOSDisplay::new(); + active_call_b + .update(cx_b, |call, cx| call.set_location(None, cx)) + .await + .unwrap(); + active_call_b + .update(cx_b, |call, cx| { + call.room().unwrap().update(cx, |room, cx| { + room.set_display_sources(vec![display.clone()]); + room.share_screen(cx) + }) + }) + .await + .unwrap(); + deterministic.run_until_parked(); + let shared_screen = workspace_a.read_with(cx_a, |workspace, cx| { + workspace + .active_item(cx) + .unwrap() + .downcast::() + .unwrap() + }); + + // Client B activates Zed again, which causes the previous editor to become focused again. + active_call_b + .update(cx_b, |call, cx| call.set_location(Some(&project_b), cx)) + .await + .unwrap(); + deterministic.run_until_parked(); + assert_eq!( + workspace_a.read_with(cx_a, |workspace, cx| workspace + .active_item(cx) + .unwrap() + .id()), + editor_a1.id() + ); + + // Client B activates an external window again, and the previously-opened screen-sharing item + // gets activated. + active_call_b + .update(cx_b, |call, cx| call.set_location(None, cx)) + .await + .unwrap(); + deterministic.run_until_parked(); + assert_eq!( + workspace_a.read_with(cx_a, |workspace, cx| workspace + .active_item(cx) + .unwrap() + .id()), + shared_screen.id() + ); + // Following interrupts when client B disconnects. client_b.disconnect(&cx_b.to_async()).unwrap(); - cx_a.foreground().run_until_parked(); + deterministic.run_until_parked(); assert_eq!( workspace_a.read_with(cx_a, |workspace, _| workspace.leader_for_pane(&pane_a)), None diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index c3e54c525b..e7752219c5 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -6,7 +6,7 @@ pub mod dock; pub mod pane; pub mod pane_group; pub mod searchable; -mod shared_screen; +pub mod shared_screen; pub mod sidebar; mod status_bar; mod toolbar; From 7e4d582d1efc4c970683f1a85ad600848c2000bf Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 16:50:56 +0200 Subject: [PATCH 58/63] Replace `Screen Sharing` label with `Screen` Co-Authored-By: Nathan Sobo --- crates/collab_ui/src/contact_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/collab_ui/src/contact_list.rs b/crates/collab_ui/src/contact_list.rs index 74cf2012fc..1947d14a16 100644 --- a/crates/collab_ui/src/contact_list.rs +++ b/crates/collab_ui/src/contact_list.rs @@ -869,7 +869,7 @@ impl ContactList { .boxed(), ) .with_child( - Label::new("Screen Sharing".into(), row.name.text.clone()) + Label::new("Screen".into(), row.name.text.clone()) .aligned() .left() .contained() From 484c8f7cbe5c90af056be8e19f90d644210d671f Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 17:03:18 +0200 Subject: [PATCH 59/63] Provide LiveKit environment variables on Kubernetes Co-Authored-By: Nathan Sobo --- crates/collab/k8s/manifest.template.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/collab/k8s/manifest.template.yml b/crates/collab/k8s/manifest.template.yml index 628cf92506..06a0e200ec 100644 --- a/crates/collab/k8s/manifest.template.yml +++ b/crates/collab/k8s/manifest.template.yml @@ -70,6 +70,21 @@ spec: secretKeyRef: name: api key: token + - name: LIVE_KIT_SERVER + valueFrom: + secretKeyRef: + name: livekit + key: server + - name: LIVE_KIT_KEY + valueFrom: + secretKeyRef: + name: livekit + key: key + - name: LIVE_KIT_SECRET + valueFrom: + secretKeyRef: + name: livekit + key: secret - name: INVITE_LINK_PREFIX value: ${INVITE_LINK_PREFIX} - name: RUST_LOG From 2b5ac535b91aa5a8cd56deb76a8fe9b7f8dd67ba Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 17:04:33 +0200 Subject: [PATCH 60/63] Temporarily upload app bundle as CI artifact Co-Authored-By: Nathan Sobo --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a973436ced..c74c2ae94c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,9 +85,8 @@ jobs: - name: Create app bundle run: script/bundle - - name: Upload app bundle to workflow run if main branch + - name: Upload app bundle to workflow run uses: actions/upload-artifact@v2 - if: ${{ github.ref == 'refs/heads/main' }} with: name: Zed.dmg path: target/release/Zed.dmg From dce21900a742f64267477a29cb6cfc26139dee97 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 17:06:40 +0200 Subject: [PATCH 61/63] Bump protocol version Co-Authored-By: Nathan Sobo --- crates/rpc/src/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/src/rpc.rs b/crates/rpc/src/rpc.rs index c11caab108..b6aef64677 100644 --- a/crates/rpc/src/rpc.rs +++ b/crates/rpc/src/rpc.rs @@ -6,4 +6,4 @@ pub use conn::Connection; pub use peer::*; mod macros; -pub const PROTOCOL_VERSION: u32 = 38; +pub const PROTOCOL_VERSION: u32 = 39; From 011085a93ffa8007b2d4c1ae59cb1065e248e07b Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 24 Oct 2022 17:36:19 +0200 Subject: [PATCH 62/63] Revert "Temporarily upload app bundle as CI artifact" This reverts commit 2b5ac535b91aa5a8cd56deb76a8fe9b7f8dd67ba. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c74c2ae94c..a973436ced 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,8 +85,9 @@ jobs: - name: Create app bundle run: script/bundle - - name: Upload app bundle to workflow run + - name: Upload app bundle to workflow run if main branch uses: actions/upload-artifact@v2 + if: ${{ github.ref == 'refs/heads/main' }} with: name: Zed.dmg path: target/release/Zed.dmg From 8c2ff695155c3dc4a19d8e48cb6e54edb8c956ae Mon Sep 17 00:00:00 2001 From: Nathan Sobo Date: Mon, 24 Oct 2022 09:42:59 -0600 Subject: [PATCH 63/63] Render a tooltip on toggle screen sharing button Co-Authored-By: Antonio Scandurra --- crates/collab_ui/src/collab_titlebar_item.rs | 21 ++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/collab_ui/src/collab_titlebar_item.rs b/crates/collab_ui/src/collab_titlebar_item.rs index 9027a0937b..2a8870fe66 100644 --- a/crates/collab_ui/src/collab_titlebar_item.rs +++ b/crates/collab_ui/src/collab_titlebar_item.rs @@ -259,11 +259,17 @@ impl CollabTitlebarItem { ) -> Option { let active_call = ActiveCall::global(cx); let room = active_call.read(cx).room().cloned()?; - let icon = if room.read(cx).is_screen_sharing() { - "icons/disable_screen_sharing_12.svg" + let icon; + let tooltip; + + if room.read(cx).is_screen_sharing() { + icon = "icons/disable_screen_sharing_12.svg"; + tooltip = "Stop Sharing Screen" } else { - "icons/enable_screen_sharing_12.svg" - }; + icon = "icons/enable_screen_sharing_12.svg"; + tooltip = "Share Screen"; + } + let titlebar = &theme.workspace.titlebar; Some( MouseEventHandler::::new(0, cx, |state, _| { @@ -284,6 +290,13 @@ impl CollabTitlebarItem { .on_click(MouseButton::Left, move |_, cx| { cx.dispatch_action(ToggleScreenSharing); }) + .with_tooltip::( + 0, + tooltip.into(), + Some(Box::new(ToggleScreenSharing)), + theme.tooltip.clone(), + cx, + ) .aligned() .boxed(), )