From 84091e5331e8bd212c7bbc2a9f1ce4f6fb55ade9 Mon Sep 17 00:00:00 2001 From: Chirantan Ekbote Date: Fri, 10 Sep 2021 18:43:17 +0900 Subject: [PATCH] Add vhost-user vsock vmm Add the vmm side for the vhost-user vsock device. BUG=b:179756331 TEST=Connect to vshd inside a VM with a vhost-user vsock device Change-Id: I332adbb6f8d6cfc8dff16375e93d946ecad2e84b Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3153213 Tested-by: kokoro Commit-Queue: Chirantan Ekbote Reviewed-by: Daniel Verkamp Reviewed-by: Keiichi Watanabe --- devices/src/virtio/vhost/user/vmm/mod.rs | 2 + devices/src/virtio/vhost/user/vmm/vsock.rs | 162 +++++++++++++++++++++ src/crosvm.rs | 2 + src/error.rs | 4 + src/linux.rs | 18 ++- src/main.rs | 4 + 6 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 devices/src/virtio/vhost/user/vmm/vsock.rs diff --git a/devices/src/virtio/vhost/user/vmm/mod.rs b/devices/src/virtio/vhost/user/vmm/mod.rs index fd495ea269..50ec9f2b10 100644 --- a/devices/src/virtio/vhost/user/vmm/mod.rs +++ b/devices/src/virtio/vhost/user/vmm/mod.rs @@ -10,6 +10,7 @@ mod gpu; mod handler; mod mac80211_hwsim; mod net; +mod vsock; mod wl; mod worker; @@ -21,6 +22,7 @@ pub use self::gpu::*; pub use self::handler::VhostUserHandler; pub use self::mac80211_hwsim::*; pub use self::net::*; +pub use self::vsock::*; pub use self::wl::*; use remain::sorted; diff --git a/devices/src/virtio/vhost/user/vmm/vsock.rs b/devices/src/virtio/vhost/user/vmm/vsock.rs new file mode 100644 index 0000000000..4a7f287c2a --- /dev/null +++ b/devices/src/virtio/vhost/user/vmm/vsock.rs @@ -0,0 +1,162 @@ +// Copyright 2021 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +use std::{cell::RefCell, os::unix::net::UnixStream, path::Path, thread}; + +use base::{error, Event, RawDescriptor}; +use cros_async::Executor; +use data_model::Le64; +use vm_memory::GuestMemory; +use vmm_vhost::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures}; + +use crate::virtio::{ + vhost::{ + user::vmm::{handler::VhostUserHandler, worker::Worker, Error, Result}, + vsock, + }, + Interrupt, Queue, VirtioDevice, TYPE_VSOCK, VIRTIO_F_VERSION_1, +}; + +pub struct Vsock { + kill_evt: Option, + worker_thread: Option>, + handler: RefCell, + queue_sizes: Vec, +} + +impl Vsock { + pub fn new>(base_features: u64, socket_path: P) -> Result { + let socket = UnixStream::connect(socket_path).map_err(Error::SocketConnect)?; + + let init_features = VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); + let allow_features = init_features + | base_features + | 1 << VIRTIO_F_VERSION_1 + | 1 << virtio_sys::vhost::VIRTIO_RING_F_INDIRECT_DESC + | 1 << virtio_sys::vhost::VIRTIO_RING_F_EVENT_IDX + | 1 << virtio_sys::vhost::VIRTIO_F_NOTIFY_ON_EMPTY + | 1 << virtio_sys::vhost::VHOST_F_LOG_ALL + | 1 << virtio_sys::vhost::VIRTIO_F_ANY_LAYOUT; + let allow_protocol_features = + VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::CONFIG; + + let mut handler = VhostUserHandler::new_from_stream( + socket, + vsock::QUEUE_SIZES.len() as u64, + allow_features, + init_features, + allow_protocol_features, + )?; + let queue_sizes = handler.queue_sizes(vsock::QUEUE_SIZE, vsock::QUEUE_SIZES.len())?; + + Ok(Vsock { + kill_evt: None, + worker_thread: None, + handler: RefCell::new(handler), + queue_sizes, + }) + } +} + +impl Drop for Vsock { + fn drop(&mut self) { + if let Some(kill_evt) = self.kill_evt.take() { + // Ignore the result because there is nothing we can do about it. + let _ = kill_evt.write(1); + } + + if let Some(worker_thread) = self.worker_thread.take() { + let _ = worker_thread.join(); + } + } +} + +impl VirtioDevice for Vsock { + fn keep_rds(&self) -> Vec { + Vec::new() + } + + fn features(&self) -> u64 { + self.handler.borrow().avail_features + } + + fn ack_features(&mut self, features: u64) { + if let Err(e) = self.handler.borrow_mut().ack_features(features) { + error!("failed to enable features 0x{:x}: {}", features, e); + } + } + + fn device_type(&self) -> u32 { + TYPE_VSOCK + } + + fn queue_max_sizes(&self) -> &[u16] { + self.queue_sizes.as_slice() + } + + fn read_config(&self, offset: u64, data: &mut [u8]) { + if let Err(e) = self.handler.borrow_mut().read_config::(offset, data) { + error!("failed to read config: {}", e); + } + } + + fn activate( + &mut self, + mem: GuestMemory, + interrupt: Interrupt, + queues: Vec, + queue_evts: Vec, + ) { + if let Err(e) = self + .handler + .borrow_mut() + .activate(&mem, &interrupt, &queues, &queue_evts) + { + error!("failed to activate queues: {}", e); + return; + } + + let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) { + Ok(v) => v, + Err(e) => { + error!("failed creating kill Event pair: {}", e); + return; + } + }; + self.kill_evt = Some(self_kill_evt); + + let worker_result = thread::Builder::new() + .name("vhost_user_vsock".to_string()) + .spawn(move || { + let ex = Executor::new().expect("failed to create an executor"); + let mut worker = Worker { + queues, + mem, + kill_evt, + }; + if let Err(e) = worker.run(&ex, interrupt) { + error!("failed to start a worker: {}", e); + } + worker + }); + + match worker_result { + Err(e) => { + error!("failed to spawn virtio_vsock worker: {}", e); + } + Ok(join_handle) => { + self.worker_thread = Some(join_handle); + } + } + } + + fn reset(&mut self) -> bool { + if let Err(e) = self.handler.borrow_mut().reset(self.queue_sizes.len()) { + error!("Failed to reset vsock device: {}", e); + false + } else { + true + } + } +} diff --git a/src/crosvm.rs b/src/crosvm.rs index dc6e78de3a..098cfccb68 100644 --- a/src/crosvm.rs +++ b/src/crosvm.rs @@ -369,6 +369,7 @@ pub struct Config { pub vhost_user_gpu: Vec, pub vhost_user_mac80211_hwsim: Option, pub vhost_user_net: Vec, + pub vhost_user_vsock: Vec, pub vhost_user_wl: Vec, #[cfg(feature = "direct")] pub direct_pmio: Option, @@ -458,6 +459,7 @@ impl Default for Config { vhost_user_fs: Vec::new(), vhost_user_mac80211_hwsim: None, vhost_user_net: Vec::new(), + vhost_user_vsock: Vec::new(), vhost_user_wl: Vec::new(), #[cfg(feature = "direct")] direct_pmio: None, diff --git a/src/error.rs b/src/error.rs index a7353a4b5c..e3001f683a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -150,6 +150,7 @@ pub enum Error { VhostUserMac80211HwsimNew(VhostUserVmmError), VhostUserNetDeviceNew(VhostUserVmmError), VhostUserNetWithNetArgs, + VhostUserVsockDeviceNew(VhostUserVmmError), VhostUserWlDeviceNew(VhostUserVmmError), VhostVsockDeviceNew(virtio::vhost::Error), VirtioPciDev(base::Error), @@ -312,6 +313,9 @@ impl Display for Error { f, "vhost-user-net cannot be used with any of --host_ip, --netmask or --mac" ), + VhostUserVsockDeviceNew(e) => { + write!(f, "failed to set up vhost-user vsock device: {}", e) + } VhostUserWlDeviceNew(e) => { write!(f, "failed to set up vhost-user wl device: {}", e) } diff --git a/src/linux.rs b/src/linux.rs index adedad5e60..16788992cf 100644 --- a/src/linux.rs +++ b/src/linux.rs @@ -34,7 +34,8 @@ use devices::vfio::{VfioCommonSetup, VfioCommonTrait}; use devices::virtio::snd::cras_backend::Parameters as CrasSndParameters; use devices::virtio::vhost::user::vmm::{ Block as VhostUserBlock, Console as VhostUserConsole, Fs as VhostUserFs, - Mac80211Hwsim as VhostUserMac80211Hwsim, Net as VhostUserNet, Wl as VhostUserWl, + Mac80211Hwsim as VhostUserMac80211Hwsim, Net as VhostUserNet, Vsock as VhostUserVsock, + Wl as VhostUserWl, }; use devices::virtio::{self, Console, VirtioDevice}; #[cfg(feature = "gpu")] @@ -646,6 +647,17 @@ fn create_vhost_user_net_device(cfg: &Config, opt: &VhostUserOption) -> DeviceRe }) } +fn create_vhost_user_vsock_device(cfg: &Config, opt: &VhostUserOption) -> DeviceResult { + let dev = VhostUserVsock::new(virtio::base_features(cfg.protected_vm), &opt.socket) + .map_err(Error::VhostUserVsockDeviceNew)?; + + Ok(VirtioDeviceStub { + dev: Box::new(dev), + // no sandbox here because virtqueue handling is exported to a different process. + jail: None, + }) +} + fn create_vhost_user_wl_device(cfg: &Config, opt: &VhostUserWlOption) -> DeviceResult { // The crosvm wl device expects us to connect the tube before it will accept a vhost-user // connection. @@ -1353,6 +1365,10 @@ fn create_virtio_devices( devs.push(create_vhost_user_net_device(cfg, net)?); } + for vsock in &cfg.vhost_user_vsock { + devs.push(create_vhost_user_vsock_device(cfg, vsock)?); + } + for opt in &cfg.vhost_user_wl { devs.push(create_vhost_user_wl_device(cfg, opt)?); } diff --git a/src/main.rs b/src/main.rs index dd10f109d1..2fd74fa317 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1861,6 +1861,9 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument:: "vhost-user-net" => cfg.vhost_user_net.push(VhostUserOption { socket: PathBuf::from(value.unwrap()), }), + "vhost-user-vsock" => cfg.vhost_user_vsock.push(VhostUserOption { + socket: PathBuf::from(value.unwrap()), + }), "vhost-user-wl" => { let mut components = value.unwrap().splitn(2, ":"); let socket = components.next().map(PathBuf::from).ok_or_else(|| { @@ -2216,6 +2219,7 @@ iommu=on|off - indicates whether to enable virtio IOMMU for this device"), Argument::value("vhost-user-gpu", "SOCKET_PATH", "Paths to a vhost-user socket for gpu"), Argument::value("vhost-user-mac80211-hwsim", "SOCKET_PATH", "Path to a socket for vhost-user mac80211_hwsim"), Argument::value("vhost-user-net", "SOCKET_PATH", "Path to a socket for vhost-user net"), + Argument::value("vhost-user-vsock", "SOCKET_PATH", "Path to a socket for vhost-user vsock"), Argument::value("vhost-user-wl", "SOCKET_PATH:TUBE_PATH", "Paths to a vhost-user socket for wayland and a Tube socket for additional wayland-specific messages"), Argument::value("vhost-user-fs", "SOCKET_PATH:TAG", "Path to a socket path for vhost-user fs, and tag for the shared dir"),