Capture screen in BGRA8 and render it in capture example app

Cargo.lock

@@ -756,7 +756,6 @@ version = "0.1.0"
 dependencies = [
  "bindgen",
  "block",
- "cc",
  "cocoa",
  "core-foundation",
  "core-graphics",
@@ -3034,8 +3033,10 @@ dependencies = [
 name = "media"
 version = "0.1.0"
 dependencies = [
+ "bindgen",
  "block",
  "core-foundation",
+ "foreign-types",
  "metal",
  "objc",
 ]

crates/capture/Cargo.toml

@@ -26,4 +26,3 @@ simplelog = "0.9"
 
 [build-dependencies]
 bindgen = "0.59.2"
-cc = "1.0"

crates/capture/src/main.rs

@@ -128,6 +128,10 @@ impl ScreenCaptureView {
                 let _: () = msg_send![config, setMinimumFrameInterval: bindings::CMTimeMake(1, 60)];
                 let _: () = msg_send![config, setQueueDepth: 6];
                 let _: () = msg_send![config, setShowsCursor: YES];
+                let _: () = msg_send![
+                    config,
+                    setPixelFormat: media::core_video::kCVPixelFormatType_32BGRA
+                ];
 
                 let stream: id = msg_send![class!(SCStream), alloc];
                 let stream: id = msg_send![stream, initWithFilter: filter configuration: config delegate: output];
@@ -173,7 +177,6 @@ impl ScreenCaptureView {
                 if let Some(this) = this.upgrade(&cx) {
                     this.update(&mut cx, |this, cx| {
                         this.image_buffer = image_buffer;
-                        println!("NEW SURFACE!");
                         cx.notify();
                     })
                 } else {

crates/gpui/src/platform/mac/renderer.rs

@@ -383,6 +383,13 @@ impl Renderer {
                 drawable_size,
                 command_encoder,
             );
+            self.render_surfaces(
+                layer.surfaces(),
+                scale_factor,
+                offset,
+                drawable_size,
+                command_encoder,
+            );
         }
 
         command_encoder.end_encoding();
@@ -791,80 +798,87 @@ impl Renderer {
             return;
         }
 
+        command_encoder.set_render_pipeline_state(&self.image_pipeline_state);
+        command_encoder.set_vertex_buffer(
+            shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64,
+            Some(&self.unit_vertices),
+            0,
+        );
+        command_encoder.set_vertex_bytes(
+            shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64,
+            mem::size_of::<shaders::vector_float2>() as u64,
+            [drawable_size.to_float2()].as_ptr() as *const c_void,
+        );
+
         for surface in surfaces {
-            // let origin = surface.bounds.origin() * scale_factor;
+            let origin = surface.bounds.origin() * scale_factor;
-            // let target_size = surface.bounds.size() * scale_factor;
+            let source_size = vec2i(
-            // let corner_radius = surface.corner_radius * scale_factor;
+                surface.image_buffer.width() as i32,
-            // let border_width = surface.border.width * scale_factor;
+                surface.image_buffer.height() as i32,
-
+            );
-            let width = surface.image_buffer.width();
+            let target_size = surface.bounds.size();
-            let height = surface.image_buffer.height();
+            let pixel_format = if surface.image_buffer.pixel_format_type()
-
+                == core_video::kCVPixelFormatType_32BGRA
-            // We should add this method, but this return CVPixelFormatType and we need MTLPixelFormat
+            {
-            // I found at least one code example that manually maps them. Not sure what other options we have.
+                MTLPixelFormat::BGRA8Unorm
-            let pixel_format = surface.image_buffer.pixel_format_type();
+            } else {
+                panic!("unsupported pixel format")
+            };
 
             let texture = self.cv_texture_cache.create_texture_from_image(
                 surface.image_buffer.as_concrete_TypeRef(),
                 ptr::null(),
                 pixel_format,
-                width,
+                source_size.x() as usize,
-                height,
+                source_size.y() as usize,
                 0,
             );
+
+            align_offset(offset);
+            let next_offset = *offset + mem::size_of::<shaders::GPUIImage>();
+            assert!(
+                next_offset <= INSTANCE_BUFFER_SIZE,
+                "instance buffer exhausted"
+            );
+
+            command_encoder.set_vertex_buffer(
+                shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64,
+                Some(&self.instances),
+                *offset as u64,
+            );
+            command_encoder.set_vertex_bytes(
+                shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64,
+                mem::size_of::<shaders::vector_float2>() as u64,
+                [source_size.to_float2()].as_ptr() as *const c_void,
+            );
+            command_encoder.set_fragment_texture(
+                shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64,
+                Some(texture.as_texture_ref()),
+            );
+
+            unsafe {
+                let buffer_contents =
+                    (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage;
+                std::ptr::write(
+                    buffer_contents,
+                    shaders::GPUIImage {
+                        origin: origin.to_float2(),
+                        target_size: target_size.to_float2(),
+                        source_size: source_size.to_float2(),
+                        atlas_origin: Default::default(),
+                        border_top: Default::default(),
+                        border_right: Default::default(),
+                        border_bottom: Default::default(),
+                        border_left: Default::default(),
+                        border_color: Default::default(),
+                        corner_radius: Default::default(),
+                    },
+                );
+            }
+
+            command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6);
+            *offset = next_offset;
         }
-
-        // command_encoder.set_render_pipeline_state(&self.image_pipeline_state);
-        // command_encoder.set_vertex_buffer(
-        //     shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64,
-        //     Some(&self.unit_vertices),
-        //     0,
-        // );
-        // command_encoder.set_vertex_bytes(
-        //     shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexViewportSize as u64,
-        //     mem::size_of::<shaders::vector_float2>() as u64,
-        //     [drawable_size.to_float2()].as_ptr() as *const c_void,
-        // );
-
-        // for (atlas_id, images) in images_by_atlas {
-        //     align_offset(offset);
-        //     let next_offset = *offset + images.len() * mem::size_of::<shaders::GPUIImage>();
-        //     assert!(
-        //         next_offset <= INSTANCE_BUFFER_SIZE,
-        //         "instance buffer exhausted"
-        //     );
-
-        //     let texture = self.image_cache.atlas_texture(atlas_id).unwrap();
-        //     command_encoder.set_vertex_buffer(
-        //         shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexImages as u64,
-        //         Some(&self.instances),
-        //         *offset as u64,
-        //     );
-        //     command_encoder.set_vertex_bytes(
-        //         shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexAtlasSize as u64,
-        //         mem::size_of::<shaders::vector_float2>() as u64,
-        //         [vec2i(texture.width() as i32, texture.height() as i32).to_float2()].as_ptr()
-        //             as *const c_void,
-        //     );
-        //     command_encoder.set_fragment_texture(
-        //         shaders::GPUIImageFragmentInputIndex_GPUIImageFragmentInputIndexAtlas as u64,
-        //         Some(texture),
-        //     );
-
-        //     unsafe {
-        //         let buffer_contents =
-        //             (self.instances.contents() as *mut u8).add(*offset) as *mut shaders::GPUIImage;
-        //         std::ptr::copy_nonoverlapping(images.as_ptr(), buffer_contents, images.len());
-        //     }
-
-        //     command_encoder.draw_primitives_instanced(
-        //         metal::MTLPrimitiveType::Triangle,
-        //         0,
-        //         6,
-        //         images.len() as u64,
-        //     );
-        //     *offset = next_offset;
-        // }
     }
 
     fn render_path_sprites(

crates/gpui/src/scene.rs

@@ -397,14 +397,18 @@ impl Layer {
         }
     }
 
+    pub fn images(&self) -> &[Image] {
+        self.images.as_slice()
+    }
+
     fn push_surface(&mut self, surface: Surface) {
         if can_draw(surface.bounds) {
             self.surfaces.push(surface);
         }
     }
 
-    pub fn images(&self) -> &[Image] {
+    pub fn surfaces(&self) -> &[Surface] {
-        self.images.as_slice()
+        self.surfaces.as_slice()
     }
 
     fn push_shadow(&mut self, shadow: Shadow) {

crates/media/Cargo.toml

@@ -10,5 +10,9 @@ doctest = false
 [dependencies]
 block = "0.1"
 core-foundation = "0.9.3"
+foreign-types = "0.3"
 metal = "0.21.0"
 objc = "0.2"
+
+[build-dependencies]
+bindgen = "0.59.2"

crates/media/build.rs

@@ -0,0 +1,29 @@
+use std::{env, path::PathBuf, process::Command};
+
+fn main() {
+    let sdk_path = String::from_utf8(
+        Command::new("xcrun")
+            .args(&["--sdk", "macosx", "--show-sdk-path"])
+            .output()
+            .unwrap()
+            .stdout,
+    )
+    .unwrap();
+    let sdk_path = sdk_path.trim_end();
+
+    println!("cargo:rerun-if-changed=src/bindings.h");
+    let bindings = bindgen::Builder::default()
+        .header("src/bindings.h")
+        .clang_arg(format!("-isysroot{}", sdk_path))
+        .clang_arg("-xobjective-c")
+        .allowlist_var("kCVPixelFormatType_.*")
+        .parse_callbacks(Box::new(bindgen::CargoCallbacks))
+        .layout_tests(false)
+        .generate()
+        .expect("unable to generate bindings");
+
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    bindings
+        .write_to_file(out_path.join("bindings.rs"))
+        .expect("couldn't write dispatch bindings");
+}

crates/media/src/bindings.h

@@ -0,0 +1 @@
+#import <CoreVideo/CVPixelFormatDescription.h>

crates/media/src/bindings.rs

@@ -0,0 +1,8 @@
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(non_snake_case)]
+#![allow(unused)]
+
+use objc::*;
+
+include!(concat!(env!("OUT_DIR"), "/bindings.rs"));

crates/media/src/media.rs

@@ -1,6 +1,8 @@
 #![allow(non_snake_case)]
 #![allow(non_camel_case_types)]
 
+mod bindings;
+
 use core_foundation::{
     base::{CFTypeID, TCFType},
     declare_TCFType, impl_CFTypeDescription, impl_TCFType,
@@ -34,11 +36,13 @@ pub mod core_video {
     use std::ptr;
 
     use super::*;
+    pub use crate::bindings::*;
     use core_foundation::{
         base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef,
     };
+    use foreign_types::ForeignTypeRef;
     use io_surface::{IOSurface, IOSurfaceRef};
-    use metal::{MTLDevice, MTLPixelFormat};
+    use metal::{MTLDevice, MTLPixelFormat, MTLTexture};
 
     #[repr(C)]
     pub struct __CVImageBuffer(c_void);
@@ -65,13 +69,19 @@ pub mod core_video {
         pub fn height(&self) -> usize {
             unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) }
         }
+
+        pub fn pixel_format_type(&self) -> OSType {
+            unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) }
+        }
     }
 
+    #[link(name = "CoreVideo", kind = "framework")]
     extern "C" {
         fn CVImageBufferGetTypeID() -> CFTypeID;
         fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef;
         fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize;
         fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize;
+        fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType;
     }
 
     #[repr(C)]
@@ -130,6 +140,7 @@ pub mod core_video {
         }
     }
 
+    #[link(name = "CoreVideo", kind = "framework")]
     extern "C" {
         fn CVMetalTextureCacheGetTypeID() -> CFTypeID;
         fn CVMetalTextureCacheCreate(
@@ -160,7 +171,18 @@ pub mod core_video {
     impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID);
     impl_CFTypeDescription!(CVMetalTexture);
 
+    impl CVMetalTexture {
+        pub fn as_texture_ref(&self) -> &metal::TextureRef {
+            unsafe {
+                let texture = CVMetalTextureGetTexture(self.as_concrete_TypeRef());
+                &metal::TextureRef::from_ptr(texture as *mut _)
+            }
+        }
+    }
+
+    #[link(name = "CoreVideo", kind = "framework")]
     extern "C" {
         fn CVMetalTextureGetTypeID() -> CFTypeID;
+        fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void;
     }
 }