From 79a7a0e0e709a61965c6c15070b14f94f3fe8b60 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 31 Aug 2022 11:02:48 +0200 Subject: [PATCH] Capture screen in BGRA8 and render it in `capture` example app --- Cargo.lock | 3 +- crates/capture/Cargo.toml | 1 - crates/capture/src/main.rs | 5 +- crates/gpui/src/platform/mac/renderer.rs | 144 +++++++++++++---------- crates/gpui/src/scene.rs | 8 +- crates/media/Cargo.toml | 4 + crates/media/build.rs | 29 +++++ crates/media/src/bindings.h | 1 + crates/media/src/bindings.rs | 8 ++ crates/media/src/media.rs | 24 +++- 10 files changed, 156 insertions(+), 71 deletions(-) create mode 100644 crates/media/build.rs create mode 100644 crates/media/src/bindings.h create mode 100644 crates/media/src/bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 46abc68765..b2d886db0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -756,7 +756,6 @@ version = "0.1.0" dependencies = [ "bindgen", "block", - "cc", "cocoa", "core-foundation", "core-graphics", @@ -3034,8 +3033,10 @@ dependencies = [ name = "media" version = "0.1.0" dependencies = [ + "bindgen", "block", "core-foundation", + "foreign-types", "metal", "objc", ] diff --git a/crates/capture/Cargo.toml b/crates/capture/Cargo.toml index f52fde195e..54b1d0990e 100644 --- a/crates/capture/Cargo.toml +++ b/crates/capture/Cargo.toml @@ -26,4 +26,3 @@ simplelog = "0.9" [build-dependencies] bindgen = "0.59.2" -cc = "1.0" \ No newline at end of file diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index ed9f56cd18..af67ed9da4 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -128,6 +128,10 @@ impl ScreenCaptureView { let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)]; let _: () = msg_send![config, setQueueDepth: 6]; let _: () = msg_send![config, setShowsCursor: YES]; + let _: () = msg_send![ + config, + setPixelFormat: media::core_video::kCVPixelFormatType_32BGRA + ]; let stream: id = msg_send![class!(SCStream), alloc]; let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output]; @@ -173,7 +177,6 @@ impl ScreenCaptureView { if let Some(this) = this.upgrade(&cx) { this.update(&mut cx, |this, cx| { this.image_buffer = image_buffer; - println!("NEW SURFACE!"); cx.notify(); }) } else { diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index 1023d5a5ef..b0301e9cb6 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -383,6 +383,13 @@ impl Renderer { drawable_size, command_encoder, ); + self.render_surfaces( + layer.surfaces(), + scale_factor, + offset, + drawable_size, + command_encoder, + ); } command_encoder.end_encoding(); @@ -791,80 +798,87 @@ impl Renderer { return; } + command_encoder.set_render_pipeline_state(&self.image_pipeline_state); + command_encoder.set_vertex_buffer( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, + Some(&self.unit_vertices), + 0, + ); + command_encoder.set_vertex_bytes( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, + mem::size_of::() as u64, + [drawable_size.to_float2()].as_ptr() as *const c_void, + ); + for surface in surfaces { - // let origin = surface.bounds.origin() * scale_factor; - // let target_size = surface.bounds.size() * scale_factor; - // let corner_radius = surface.corner_radius * scale_factor; - // let border_width = surface.border.width * scale_factor; - - let width = surface.image_buffer.width(); - let height = surface.image_buffer.height(); - - // We should add this method, but this return CVPixelFormatType and we need MTLPixelFormat - // I found at least one code example that manually maps them. Not sure what other options we have. - let pixel_format = surface.image_buffer.pixel_format_type(); + let origin = surface.bounds.origin() * scale_factor; + let source_size = vec2i( + surface.image_buffer.width() as i32, + surface.image_buffer.height() as i32, + ); + let target_size = surface.bounds.size(); + let pixel_format = if surface.image_buffer.pixel_format_type() + == core_video::kCVPixelFormatType_32BGRA + { + MTLPixelFormat::BGRA8Unorm + } else { + panic!("unsupported pixel format") + }; let texture = self.cv_texture_cache.create_texture_from_image( surface.image_buffer.as_concrete_TypeRef(), ptr::null(), pixel_format, - width, - height, + source_size.x() as usize, + source_size.y() as usize, 0, ); + + align_offset(offset); + let next_offset = *offset + mem::size_of::(); + assert!( + next_offset <= INSTANCE_BUFFER_SIZE, + "instance buffer exhausted" + ); + + command_encoder.set_vertex_buffer( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, + Some(&self.instances), + *offset as u64, + ); + command_encoder.set_vertex_bytes( + shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, + mem::size_of::() as u64, + [source_size.to_float2()].as_ptr() as *const c_void, + ); + command_encoder.set_fragment_texture( + shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, + Some(texture.as_texture_ref()), + ); + + unsafe { + let buffer_contents = + (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; + std::ptr::write( + buffer_contents, + shaders::GPUIImage { + origin: origin.to_float2(), + target_size: target_size.to_float2(), + source_size: source_size.to_float2(), + atlas_origin: Default::default(), + border_top: Default::default(), + border_right: Default::default(), + border_bottom: Default::default(), + border_left: Default::default(), + border_color: Default::default(), + corner_radius: Default::default(), + }, + ); + } + + command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6); + *offset = next_offset; } - - // command_encoder.set_render_pipeline_state(&self.image_pipeline_state); - // command_encoder.set_vertex_buffer( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64, - // Some(&self.unit_vertices), - // 0, - // ); - // command_encoder.set_vertex_bytes( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64, - // mem::size_of::() as u64, - // [drawable_size.to_float2()].as_ptr() as *const c_void, - // ); - - // for (atlas_id, images) in images_by_atlas { - // align_offset(offset); - // let next_offset = *offset + images.len() * mem::size_of::(); - // assert!( - // next_offset <= INSTANCE_BUFFER_SIZE, - // "instance buffer exhausted" - // ); - - // let texture = self.image_cache.atlas_texture(atlas_id).unwrap(); - // command_encoder.set_vertex_buffer( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64, - // Some(&self.instances), - // *offset as u64, - // ); - // command_encoder.set_vertex_bytes( - // shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64, - // mem::size_of::() as u64, - // [vec2i(texture.width() as i32, texture.height() as i32).to_float2()].as_ptr() - // as *const c_void, - // ); - // command_encoder.set_fragment_texture( - // shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64, - // Some(texture), - // ); - - // unsafe { - // let buffer_contents = - // (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage; - // std::ptr::copy_nonoverlapping(images.as_ptr(), buffer_contents, images.len()); - // } - - // command_encoder.draw_primitives_instanced( - // metal::MTLPrimitiveType::Triangle, - // 0, - // 6, - // images.len() as u64, - // ); - // *offset = next_offset; - // } } fn render_path_sprites( diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index 00b8d3c88b..dbd46da1a0 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -397,14 +397,18 @@ impl Layer { } } + pub fn images(&self) -> &[Image] { + self.images.as_slice() + } + fn push_surface(&mut self, surface: Surface) { if can_draw(surface.bounds) { self.surfaces.push(surface); } } - pub fn images(&self) -> &[Image] { - self.images.as_slice() + pub fn surfaces(&self) -> &[Surface] { + self.surfaces.as_slice() } fn push_shadow(&mut self, shadow: Shadow) { diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 06f73caaf5..c2ca04abfa 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -10,5 +10,9 @@ doctest = false [dependencies] block = "0.1" core-foundation = "0.9.3" +foreign-types = "0.3" metal = "0.21.0" objc = "0.2" + +[build-dependencies] +bindgen = "0.59.2" diff --git a/crates/media/build.rs b/crates/media/build.rs new file mode 100644 index 0000000000..3446311d89 --- /dev/null +++ b/crates/media/build.rs @@ -0,0 +1,29 @@ +use std::{env, path::PathBuf, process::Command}; + +fn main() { + let sdk_path = String::from_utf8( + Command::new("xcrun") + .args(&["--sdk", "macosx", "--show-sdk-path"]) + .output() + .unwrap() + .stdout, + ) + .unwrap(); + let sdk_path = sdk_path.trim_end(); + + println!("cargo:rerun-if-changed=src/bindings.h"); + let bindings = bindgen::Builder::default() + .header("src/bindings.h") + .clang_arg(format!("-isysroot{}", sdk_path)) + .clang_arg("-xobjective-c") + .allowlist_var("kCVPixelFormatType_.*") + .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .layout_tests(false) + .generate() + .expect("unable to generate bindings"); + + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + bindings + .write_to_file(out_path.join("bindings.rs")) + .expect("couldn't write dispatch bindings"); +} diff --git a/crates/media/src/bindings.h b/crates/media/src/bindings.h new file mode 100644 index 0000000000..d5ba4eccd6 --- /dev/null +++ b/crates/media/src/bindings.h @@ -0,0 +1 @@ +#import \ No newline at end of file diff --git a/crates/media/src/bindings.rs b/crates/media/src/bindings.rs new file mode 100644 index 0000000000..a1c0b0da3e --- /dev/null +++ b/crates/media/src/bindings.rs @@ -0,0 +1,8 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(unused)] + +use objc::*; + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 855435359c..5db184f3b2 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -1,6 +1,8 @@ #![allow(non_snake_case)] #![allow(non_camel_case_types)] +mod bindings; + use core_foundation::{ base::{CFTypeID, TCFType}, declare_TCFType, impl_CFTypeDescription, impl_TCFType, @@ -34,11 +36,13 @@ pub mod core_video { use std::ptr; use super::*; + pub use crate::bindings::*; use core_foundation::{ base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, }; + use foreign_types::ForeignTypeRef; use io_surface::{IOSurface, IOSurfaceRef}; - use metal::{MTLDevice, MTLPixelFormat}; + use metal::{MTLDevice, MTLPixelFormat, MTLTexture}; #[repr(C)] pub struct __CVImageBuffer(c_void); @@ -65,13 +69,19 @@ pub mod core_video { pub fn height(&self) -> usize { unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } } + + pub fn pixel_format_type(&self) -> OSType { + unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) } + } } + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVImageBufferGetTypeID() -> CFTypeID; fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; + fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType; } #[repr(C)] @@ -130,6 +140,7 @@ pub mod core_video { } } + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVMetalTextureCacheGetTypeID() -> CFTypeID; fn CVMetalTextureCacheCreate( @@ -160,7 +171,18 @@ pub mod core_video { impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); impl_CFTypeDescription!(CVMetalTexture); + impl CVMetalTexture { + pub fn as_texture_ref(&self) -> &metal::TextureRef { + unsafe { + let texture = CVMetalTextureGetTexture(self.as_concrete_TypeRef()); + &metal::TextureRef::from_ptr(texture as *mut _) + } + } + } + + #[link(name = "CoreVideo", kind = "framework")] extern "C" { fn CVMetalTextureGetTypeID() -> CFTypeID; + fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; } }