mirrors/zed
1
0
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2024-12-24 17:28:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Change ABI from pair of u32s to single u64

Browse source
This commit is contained in:
Isaac Clayton 2022-06-09 15:49:55 +02:00
parent 7edcf7c423
commit 5b40734f80
5 changed files with 152 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
crates/plugin/src/lib.rs
View file

@ -1,50 +1,55 @@
pub use bincode;
pub use serde;
#[repr(C)]
// TODO: move the implementation to one place?
pub struct __Buffer {
pub ptr: *const u8,
pub len: usize,
pub ptr: u32, // *const u8,
pub len: u32, // usize,
}
impl __Buffer {
pub fn into_u64(self) -> u64 {
((self.ptr as u64) << 32) | (self.len as u64)
}
pub fn from_u64(packed: u64) -> Self {
__Buffer {
ptr: (packed >> 32) as u32,
len: packed as u32,
}
}
}
/// Allocates a buffer with an exact size.
/// We don't return the size because it has to be passed in anyway.
#[no_mangle]
pub extern "C" fn __alloc_buffer(len: usize) -> *const u8 {
let vec = vec![0; len];
pub extern "C" fn __alloc_buffer(len: u32) -> u32 {
let vec = vec![0; len as usize];
let buffer = unsafe { __Buffer::from_vec(vec) };
return buffer.ptr;
}
// /// Frees a given buffer, requires the size.
// #[no_mangle]
// pub extern "C" fn __free_buffer(ptr: *const u8, len: usize) {
// let buffer = Buffer { ptr, len };
// let vec = unsafe { buffer.to_vec() };
// std::mem::drop(vec);
// }
/// Frees a given buffer, requires the size.
#[no_mangle]
pub extern "C" fn __free_buffer(buffer: u64) {
let vec = unsafe { __Buffer::from_u64(buffer).to_vec() };
std::mem::drop(vec);
}
impl __Buffer {
#[inline(always)]
pub unsafe fn to_vec(&self) -> Vec<u8> {
core::slice::from_raw_parts(self.ptr, self.len).to_vec()
core::slice::from_raw_parts(self.ptr as *const u8, self.len as usize).to_vec()
}
#[inline(always)]
pub unsafe fn from_vec(mut vec: Vec<u8>) -> __Buffer {
vec.shrink_to(0);
let ptr = vec.as_ptr();
let len = vec.len();
let ptr = vec.as_ptr() as u32;
let len = vec.len() as u32;
std::mem::forget(vec);
__Buffer { ptr, len }
}
#[inline(always)]
pub fn leak_to_heap(self) -> *const __Buffer {
let boxed = Box::new(self);
let ptr = Box::<__Buffer>::into_raw(boxed) as *const __Buffer;
return ptr;
}
}
pub mod prelude {

45
crates/plugin_macros/src/lib.rs
View file

@ -59,10 +59,9 @@ pub fn export(args: TokenStream, function: TokenStream) -> TokenStream {
#[no_mangle]
// TODO: switch len from usize to u32?
pub extern "C" fn #outer_fn_name(ptr: *const u8, len: usize) -> *const ::plugin::__Buffer {
pub extern "C" fn #outer_fn_name(packed_buffer: u64) -> u64 {
// setup
let buffer = ::plugin::__Buffer { ptr, len };
let data = unsafe { buffer.to_vec() };
let data = unsafe { ::plugin::__Buffer::from_u64(packed_buffer).to_vec() };
// operation
let data: #ty = match ::plugin::bincode::deserialize(&data) {
@ -74,8 +73,8 @@ pub fn export(args: TokenStream, function: TokenStream) -> TokenStream {
let new_data = new_data.unwrap();
// teardown
let new_buffer = unsafe { ::plugin::__Buffer::from_vec(new_data) };
return new_buffer.leak_to_heap();
let new_buffer = unsafe { ::plugin::__Buffer::from_vec(new_data) }.into_u64();
return new_buffer;
}
})
}
@ -101,8 +100,8 @@ pub fn import(args: TokenStream, function: TokenStream) -> TokenStream {
// block: todo!(),
// };
// let inner_fn_name = format_ident!("{}", inner_fn.sig.ident);
// let outer_fn_name = format_ident!("__{}", inner_fn_name);
let outer_fn_name = format_ident!("{}", fn_declare.sig.ident);
let inner_fn_name = format_ident!("__{}", outer_fn_name);
// let variadic = inner_fn.sig.inputs.len();
// let i = (0..variadic).map(syn::Index::from);
@ -135,32 +134,26 @@ pub fn import(args: TokenStream, function: TokenStream) -> TokenStream {
// TokenStream::from(quote! {
// extern "C" {
// #[no_mangle]
// fn #outer_fn_name(ptr: *const u8, len: usize) -> *const ::plugin::__Buffer;
// fn #inner_fn_name(buffer: u64) -> u64;
// }
// #[no_mangle]
// fn #inner_fn_name #params -> #output {
// println!("executing command: {}", string);
// // serialize data
// let data = #collect_params;
// fn #outer_fn_name #args /* (string: &str) */ -> #return_type /* Option<Vec<u8>> */ {
// dbg!("executing command: {}", string);
// // setup
// let data = #args_collect;
// let data = ::plugin::bincode::serialize(&data).unwrap();
// let buffer = unsafe { ::plugin::__Buffer::from_vec(data) };
// let ptr = buffer.ptr;
// let len = buffer.len;
// // leak data to heap
// buffer.leak_to_heap();
// // call extern function
// let result = unsafe { __command(ptr, len) };
// // get result
// let result = todo!(); // convert into box
// // deserialize data
// let data: Option<String> = match ::plugin::bincode::deserialize(&data) {
// // operation
// let new_buffer = unsafe { #inner_fn_name(buffer.into_u64()) };
// let new_data = unsafe { ::plugin::__Buffer::from_u64(new_buffer).to_vec() };
// // teardown
// match ::plugin::bincode::deserialize(&new_data) {
// Ok(d) => d,
// Err(e) => panic!("Data passed to function not deserializable."),
// };
// return data;
// Err(e) => panic!("Data returned from function not deserializable."),
// }
// }
// })
todo!()

143
crates/plugin_runtime/src/wasi.rs
View file

@ -8,15 +8,34 @@ use serde::{de::DeserializeOwned, Serialize};
use wasi_common::{dir, file};
use wasmtime::{
AsContext, AsContextMut, Caller, Config, Engine, Extern, Instance, Linker, Module, Store,
StoreContext, StoreContextMut, Trap, TypedFunc,
StoreContext, StoreContextMut, Trap, TypedFunc, WasmParams,
};
use wasmtime::{IntoFunc, Memory};
use wasmtime_wasi::{Dir, WasiCtx, WasiCtxBuilder};
pub struct WasiResource(u32);
#[repr(C)]
struct WasiBuffer {
ptr: u32,
len: u32,
}
impl WasiBuffer {
pub fn into_u64(self) -> u64 {
((self.ptr as u64) << 32) | (self.len as u64)
}
pub fn from_u64(packed: u64) -> Self {
WasiBuffer {
ptr: (packed >> 32) as u32,
len: packed as u32,
}
}
}
pub struct WasiFn<A: Serialize, R: DeserializeOwned> {
function: TypedFunc<(u32, u32), u32>,
function: TypedFunc<u64, u64>,
_function_type: PhantomData<fn(A) -> R>,
}
@ -31,37 +50,6 @@ impl<A: Serialize, R: DeserializeOwned> Clone for WasiFn<A, R> {
}
}
// impl<A: Serialize, R: DeserializeOwned> WasiFn<A, R> {
// #[inline(always)]
// pub async fn call(&self, runtime: &mut Wasi, arg: A) -> Result<R, Error> {
// runtime.call(self, arg).await
// }
// }
// type signature derived from:
// https://docs.rs/wasmtime/latest/wasmtime/struct.Linker.html#method.func_wrap2_async
// macro_rules! dynHostFunction {
// () => {
// Box<
// dyn for<'a> Fn(Caller<'a, WasiCtx>, u32, u32)
// -> Box<dyn Future<Output = u32> + Send + 'a>
// + Send
// + Sync
// + 'static
// >
// };
// }
// macro_rules! implHostFunction {
// () => {
// impl for<'a> Fn(Caller<'a, WasiCtx>, u32, u32)
// -> Box<dyn Future<Output = u32> + Send + 'a>
// + Send
// + Sync
// + 'static
// };
// }
pub struct WasiPluginBuilder {
wasi_ctx: WasiCtx,
engine: Engine,
@ -73,7 +61,7 @@ impl WasiPluginBuilder {
let mut config = Config::default();
config.async_support(true);
let engine = Engine::new(&config)?;
let mut linker = Linker::new(&engine);
let linker = Linker::new(&engine);
Ok(WasiPluginBuilder {
// host_functions: HashMap::new(),
@ -96,10 +84,10 @@ impl WasiPluginBuilder {
name: &str,
function: impl Fn(A) -> R + Send + Sync + 'static,
) -> Result<Self, Error> {
self.linker.func_wrap2_async(
self.linker.func_wrap1_async(
"env",
name,
move |mut caller: Caller<'_, WasiCtxAlloc>, ptr: u32, len: u32| {
&format!("__{}", name),
move |mut caller: Caller<'_, WasiCtxAlloc>, packed_buffer: u64| {
// TODO: use try block once avaliable
let result: Result<(Memory, Vec<u8>), Trap> = (|| {
// grab a handle to the memory
@ -109,8 +97,11 @@ impl WasiPluginBuilder {
};
// get the args passed from Guest
let args =
Wasi::deserialize_from_buffer(&mut plugin_memory, &caller, ptr, len)?;
let args = Wasi::deserialize_from_buffer(
&mut plugin_memory,
&caller,
WasiBuffer::from_u64(packed_buffer),
)?;
// Call the Host-side function
let result: R = function(args);
@ -125,8 +116,7 @@ impl WasiPluginBuilder {
Box::new(async move {
let (mut plugin_memory, result) = result?;
// todo!();
let (ptr, len) = Wasi::serialize_to_buffer(
let buffer = Wasi::serialize_to_buffer(
caller.data().alloc_buffer(),
&mut plugin_memory,
&mut caller,
@ -134,7 +124,7 @@ impl WasiPluginBuilder {
)
.await?;
Ok(7u32)
Ok(buffer.into_u64())
})
},
)?;
@ -159,7 +149,7 @@ impl WasiPluginBuilder {
#[derive(Copy, Clone)]
struct WasiAlloc {
alloc_buffer: TypedFunc<u32, u32>,
free_buffer: TypedFunc<u32, u32>,
free_buffer: TypedFunc<u64, ()>,
}
struct WasiCtxAlloc {
@ -174,7 +164,7 @@ impl WasiCtxAlloc {
.alloc_buffer
}
fn free_buffer(&self) -> TypedFunc<u32, u32> {
fn free_buffer(&self) -> TypedFunc<u64, ()> {
self.alloc
.expect("allocator has been not initialized, cannot free buffer!")
.free_buffer
@ -351,48 +341,46 @@ impl Wasi {
plugin_memory: &mut Memory,
mut store: impl AsContextMut<Data = WasiCtxAlloc>,
item: Vec<u8>,
) -> Result<(u32, u32), Error> {
) -> Result<WasiBuffer, Error> {
// allocate a buffer and write the argument to that buffer
let buffer_len = item.len() as u32;
let buffer_ptr = alloc_buffer.call_async(&mut store, buffer_len).await?;
plugin_memory.write(&mut store, buffer_ptr as usize, &item)?;
Ok((buffer_ptr, buffer_len))
let len = item.len() as u32;
let ptr = alloc_buffer.call_async(&mut store, len).await?;
plugin_memory.write(&mut store, ptr as usize, &item)?;
Ok(WasiBuffer { ptr, len })
}
/// Takes `ptr to a `(ptr, len)` pair, and returns `(ptr, len)`.
fn deref_buffer(
plugin_memory: &mut Memory,
store: impl AsContext<Data = WasiCtxAlloc>,
buffer: u32,
) -> Result<(u32, u32), Error> {
// create a buffer to read the (ptr, length) pair into
// this is a total of 4 + 4 = 8 bytes.
let raw_buffer = &mut [0; 8];
plugin_memory.read(store, buffer as usize, raw_buffer)?;
// /// Takes `ptr to a `(ptr, len)` pair, and returns `(ptr, len)`.
// fn deref_buffer(
// plugin_memory: &mut Memory,
// store: impl AsContext<Data = WasiCtxAlloc>,
// buffer: u32,
// ) -> Result<(u32, u32), Error> {
// // create a buffer to read the (ptr, length) pair into
// // this is a total of 4 + 4 = 8 bytes.
// let raw_buffer = &mut [0; 8];
// plugin_memory.read(store, buffer as usize, raw_buffer)?;
// use these bytes (wasm stores things little-endian)
// to get a pointer to the buffer and its length
let b = raw_buffer;
let buffer_ptr = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
let buffer_len = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
// // use these bytes (wasm stores things little-endian)
// // to get a pointer to the buffer and its length
// let b = raw_buffer;
// let buffer_ptr = u32::from_le_bytes([b[0], b[1], b[2], b[3]]);
// let buffer_len = u32::from_le_bytes([b[4], b[5], b[6], b[7]]);
return Ok((buffer_ptr, buffer_len));
}
// return Ok((buffer_ptr, buffer_len));
// }
/// Takes a `(ptr, len)` pair and returns the corresponding deserialized buffer.
fn deserialize_from_buffer<R: DeserializeOwned>(
plugin_memory: &mut Memory,
store: impl AsContext<Data = WasiCtxAlloc>,
buffer_ptr: u32,
buffer_len: u32,
buffer: WasiBuffer,
) -> Result<R, Error> {
let buffer_ptr = buffer_ptr as usize;
let buffer_len = buffer_len as usize;
let buffer_end = buffer_ptr + buffer_len;
let buffer_start = buffer.ptr as usize;
let buffer_end = buffer_start + buffer.len as usize;
// read the buffer at this point into a byte array
// deserialize the byte array into the provided serde type
let result = &plugin_memory.data(store.as_context())[buffer_ptr..buffer_end];
let result = &plugin_memory.data(store.as_context())[buffer_start..buffer_end];
let result = bincode::deserialize(result)?;
// TODO: this is handled wasm-side
@ -402,6 +390,7 @@ impl Wasi {
Ok(result)
}
/// Retrieves the handle to a function of a given type.
pub fn function<A: Serialize, R: DeserializeOwned, T: AsRef<str>>(
&mut self,
name: T,
@ -409,7 +398,7 @@ impl Wasi {
let fun_name = format!("__{}", name.as_ref());
let fun = self
.instance
.get_typed_func::<(u32, u32), u32, _>(&mut self.store, &fun_name)?;
.get_typed_func::<u64, u64, _>(&mut self.store, &fun_name)?;
Ok(WasiFn {
function: fun,
_function_type: PhantomData,
@ -417,6 +406,7 @@ impl Wasi {
}
// TODO: dont' use as for conversions
/// Asynchronously calls a function defined Guest-side.
pub async fn call<A: Serialize, R: DeserializeOwned>(
&mut self,
handle: &WasiFn<A, R>,
@ -444,16 +434,13 @@ impl Wasi {
// this returns a ptr to a (ptr, lentgh) pair
let result_buffer = handle
.function
.call_async(&mut self.store, arg_buffer)
.call_async(&mut self.store, arg_buffer.into_u64())
.await?;
let (result_buffer_ptr, result_buffer_len) =
Self::deref_buffer(&mut plugin_memory, &mut self.store, result_buffer)?;
Self::deserialize_from_buffer(
&mut plugin_memory,
&mut self.store,
result_buffer_ptr,
result_buffer_len,
WasiBuffer::from_u64(result_buffer),
)
}
}

17
crates/zed/src/languages/language_plugin.rs
View file

@ -12,8 +12,14 @@ pub async fn new_json(executor: Arc<Background>) -> Result<PluginLspAdapter> {
let plugin = WasiPluginBuilder::new_with_default_ctx()?
.host_function("command", |command: String| {
// TODO: actual thing
std::process::Command::new(command).output().unwrap();
Some("Hello".to_string())
dbg!(&command);
let mut args = command.split(' ');
let command = args.next().unwrap();
std::process::Command::new(command)
.args(args)
.output()
.log_err()
.map(|output| dbg!(output.stdout))
})?
.init(include_bytes!("../../../../plugins/bin/json_language.wasm"))
.await?;
@ -24,7 +30,7 @@ pub struct PluginLspAdapter {
name: WasiFn<(), String>,
server_args: WasiFn<(), Vec<String>>,
fetch_latest_server_version: WasiFn<(), Option<String>>,
fetch_server_binary: WasiFn<(PathBuf, String), Option<PathBuf>>,
fetch_server_binary: WasiFn<(PathBuf, String), Result<PathBuf, String>>,
cached_server_binary: WasiFn<PathBuf, Option<PathBuf>>,
label_for_completion: WasiFn<String, Option<String>>,
initialization_options: WasiFn<(), String>,
@ -102,9 +108,10 @@ impl LspAdapter for PluginLspAdapter {
async move {
let mut runtime = runtime.lock().await;
let handle = runtime.attach_path(&container_dir)?;
let result: Option<PathBuf> = runtime.call(&function, (container_dir, version)).await?;
let result: Result<PathBuf, String> =
runtime.call(&function, (container_dir, version)).await?;
runtime.remove_resource(handle)?;
result.ok_or_else(|| anyhow!("Could not load cached server binary"))
result.map_err(|e| anyhow!("{}", e))
}
.boxed()
}

49
plugins/json_language/src/lib.rs
View file

@ -5,7 +5,7 @@ use std::fs;
use std::path::PathBuf;
// #[import]
// fn my_command(string: &str) -> Option<String>;
// fn command(string: &str) -> Option<String>;
// #[no_mangle]
// // TODO: switch len from usize to u32?
@ -29,39 +29,34 @@ use std::path::PathBuf;
// }
extern "C" {
fn __command(ptr: *const u8, len: usize) -> *mut ::plugin::__Buffer;
fn __command(buffer: u64) -> u64;
}
#[no_mangle]
fn command(string: &str) -> Option<String> {
fn command(string: &str) -> Option<Vec<u8>> {
dbg!("executing command: {}", string);
// serialize data
// setup
let data = string;
let data = ::plugin::bincode::serialize(&data).unwrap();
let buffer = unsafe { ::plugin::__Buffer::from_vec(data) };
let ptr = buffer.ptr;
let len = buffer.len;
// leak data to heap
buffer.leak_to_heap();
// call extern function
let result = unsafe { __command(ptr, len) };
// get result
let new_buffer = unsafe { Box::from_raw(result) }; // convert into box
let new_data = unsafe { new_buffer.to_vec() };
// deserialize data
let new_data: Option<String> = match ::plugin::bincode::deserialize(&new_data) {
// operation
let new_buffer = unsafe { __command(buffer.into_u64()) };
let new_data = unsafe { ::plugin::__Buffer::from_u64(new_buffer).to_vec() };
let new_data: Option<Vec<u8>> = match ::plugin::bincode::deserialize(&new_data) {
Ok(d) => d,
Err(e) => panic!("Data returned from function not deserializable."),
};
// teardown
return new_data;
}
// TODO: some sort of macro to generate ABI bindings
extern "C" {
pub fn hello(item: u32) -> u32;
pub fn bye(item: u32) -> u32;
}
// extern "C" {
// pub fn hello(item: u32) -> u32;
// pub fn bye(item: u32) -> u32;
// }
// #[bind]
// pub async fn name(u32) -> u32 {
@ -99,11 +94,15 @@ pub fn fetch_latest_server_version() -> Option<String> {
}
// TODO: command returns error code
let output = command("npm info vscode-json-languageserver --json")?;
let output =
command("npm info vscode-json-languageserver --json").expect("could not run command");
// if !output.is_ok() {
// return None;
// }
let output = String::from_utf8(output).unwrap();
dbg!(&output);
let mut info: NpmInfo = serde_json::from_str(&output).ok()?;
info.versions.pop()
}
@ -120,6 +119,8 @@ pub fn fetch_server_binary(container_dir: PathBuf, version: String) -> Result<Pa
"npm install vscode-json-languageserver@{}",
version
));
let output = output.map(String::from_utf8);
dbg!(&output);
if output.is_none() {
return Err("failed to install vscode-json-languageserver".to_string());
}
@ -142,20 +143,28 @@ pub fn fetch_server_binary(container_dir: PathBuf, version: String) -> Result<Pa
#[export]
pub fn cached_server_binary(container_dir: PathBuf) -> Option<PathBuf> {
let mut last_version_dir = None;
println!("reading directory");
let mut entries = fs::read_dir(&container_dir).ok()?;
while let Some(entry) = entries.next() {
println!("looking at entries");
let entry = entry.ok()?;
println!("some more stuff");
if entry.file_type().ok()?.is_dir() {
println!("this is it!");
last_version_dir = Some(entry.path());
}
}
let last_version_dir = last_version_dir?;
println!("here we go");
let bin_path = last_version_dir.join(BIN_PATH);
if bin_path.exists() {
dbg!(&bin_path);
Some(bin_path)
} else {
println!("no binary found");
None
}
}