devices: Support vhost-user-wl

Support running the wl device as an out-of-tree process.  Unlike other
vhost-user devices, this one uses a second socket for additional
wayland-specific messages between the device and VMM.  Since
virtio-wayland is not a standardized device and it's likely it will be
merged into the gpu device, it's not worth the effort to try to
incorporate the wayland-specific extensions into the regular vhost-user
protocol.

BUG=b:179755841
TEST=start a crostini vm with a vhost-user-wl device and play
     gnome-mahjongg

Change-Id: I455d32393426aff2acd392092ff82f6cc970d903
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2919156
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Chirantan Ekbote <chirantan@chromium.org>
Reviewed-by: Keiichi Watanabe <keiichiw@chromium.org>

devices/src/virtio/vhost/user/mod.rs

@@ -6,12 +6,14 @@ mod block;
 mod fs;
 mod handler;
 mod net;
+mod wl;
 mod worker;
 
 pub use self::block::*;
 pub use self::fs::*;
 pub use self::handler::VhostUserHandler;
 pub use self::net::*;
+pub use self::wl::*;
 
 use remain::sorted;
 use thiserror::Error as ThisError;

devices/src/virtio/vhost/user/wl.rs

@@ -0,0 +1,156 @@
+// Copyright 2021 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+use std::cell::RefCell;
+use std::path::Path;
+use std::thread;
+
+use base::{error, Event, RawDescriptor};
+use cros_async::Executor;
+use vm_memory::GuestMemory;
+use vmm_vhost::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
+
+use crate::virtio::vhost::user::worker::Worker;
+use crate::virtio::vhost::user::{Result, VhostUserHandler};
+use crate::virtio::wl::{
+    QUEUE_SIZE, QUEUE_SIZES, VIRTIO_WL_F_SEND_FENCES, VIRTIO_WL_F_TRANS_FLAGS,
+};
+use crate::virtio::{Interrupt, Queue, VirtioDevice, TYPE_WL};
+
+pub struct Wl {
+    kill_evt: Option<Event>,
+    worker_thread: Option<thread::JoinHandle<Worker>>,
+    handler: RefCell<VhostUserHandler>,
+    queue_sizes: Vec<u16>,
+}
+
+impl Wl {
+    pub fn new<P: AsRef<Path>>(base_features: u64, socket_path: P) -> Result<Wl> {
+        let default_queue_size = QUEUE_SIZES.len();
+
+        let allow_features = 1u64 << crate::virtio::VIRTIO_F_VERSION_1
+            | 1 << VIRTIO_WL_F_TRANS_FLAGS
+            | 1 << VIRTIO_WL_F_SEND_FENCES
+            | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+        let init_features = base_features | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
+        let allow_protocol_features =
+            VhostUserProtocolFeatures::MQ | VhostUserProtocolFeatures::CONFIG;
+
+        let mut handler = VhostUserHandler::new_from_path(
+            socket_path,
+            default_queue_size as u64,
+            allow_features,
+            init_features,
+            allow_protocol_features,
+        )?;
+        let queue_sizes = handler.queue_sizes(QUEUE_SIZE, default_queue_size)?;
+
+        Ok(Wl {
+            kill_evt: None,
+            worker_thread: None,
+            handler: RefCell::new(handler),
+            queue_sizes,
+        })
+    }
+}
+
+impl VirtioDevice for Wl {
+    fn keep_rds(&self) -> Vec<RawDescriptor> {
+        Vec::new()
+    }
+
+    fn device_type(&self) -> u32 {
+        TYPE_WL
+    }
+
+    fn queue_max_sizes(&self) -> &[u16] {
+        &self.queue_sizes
+    }
+
+    fn features(&self) -> u64 {
+        self.handler.borrow().avail_features
+    }
+
+    fn ack_features(&mut self, features: u64) {
+        if let Err(e) = self.handler.borrow_mut().ack_features(features) {
+            error!("failed to enable features 0x{:x}: {}", features, e);
+        }
+    }
+
+    fn read_config(&self, _offset: u64, _data: &mut [u8]) {}
+
+    fn activate(
+        &mut self,
+        mem: GuestMemory,
+        interrupt: Interrupt,
+        queues: Vec<Queue>,
+        queue_evts: Vec<Event>,
+    ) {
+        if let Err(e) = self
+            .handler
+            .borrow_mut()
+            .activate(&mem, &interrupt, &queues, &queue_evts)
+        {
+            error!("failed to activate queues: {}", e);
+            return;
+        }
+
+        let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
+            Ok(v) => v,
+            Err(e) => {
+                error!("failed creating kill Event pair: {}", e);
+                return;
+            }
+        };
+        self.kill_evt = Some(self_kill_evt);
+
+        let worker_result = thread::Builder::new()
+            .name("vhost_user_wl".to_string())
+            .spawn(move || {
+                let ex = Executor::new().expect("failed to create an executor");
+                let mut worker = Worker {
+                    queues,
+                    mem,
+                    kill_evt,
+                };
+
+                if let Err(e) = worker.run(&ex, interrupt) {
+                    error!("failed to start a worker: {}", e);
+                }
+                worker
+            });
+
+        match worker_result {
+            Err(e) => {
+                error!("failed to spawn vhost_user_wl worker: {}", e);
+            }
+            Ok(join_handle) => {
+                self.worker_thread = Some(join_handle);
+            }
+        }
+    }
+
+    fn reset(&mut self) -> bool {
+        if let Err(e) = self.handler.borrow_mut().reset(self.queue_sizes.len()) {
+            error!("Failed to reset wl device: {}", e);
+            false
+        } else {
+            true
+        }
+    }
+}
+
+impl Drop for Wl {
+    fn drop(&mut self) {
+        if let Some(kill_evt) = self.kill_evt.take() {
+            if let Some(worker_thread) = self.worker_thread.take() {
+                if let Err(e) = kill_evt.write(1) {
+                    error!("failed to write to kill_evt: {}", e);
+                    return;
+                }
+                let _ = worker_thread.join();
+            }
+        }
+    }
+}

src/crosvm.rs

@@ -67,6 +67,11 @@ pub struct VhostUserFsOption {
     pub tag: String,
 }
 
+pub struct VhostUserWlOption {
+    pub socket: PathBuf,
+    pub vm_tube: PathBuf,
+}
+
 /// A bind mount for directories in the plugin process.
 pub struct BindMount {
     pub src: PathBuf,
@@ -259,6 +264,7 @@ pub struct Config {
     pub vhost_user_blk: Vec<VhostUserOption>,
     pub vhost_user_fs: Vec<VhostUserFsOption>,
     pub vhost_user_net: Vec<VhostUserOption>,
+    pub vhost_user_wl: Vec<VhostUserWlOption>,
     #[cfg(feature = "direct")]
     pub direct_pmio: Option<DirectIoOption>,
     #[cfg(feature = "direct")]
@@ -338,6 +344,7 @@ impl Default for Config {
             vhost_user_blk: Vec::new(),
             vhost_user_fs: Vec::new(),
             vhost_user_net: Vec::new(),
+            vhost_user_wl: Vec::new(),
             #[cfg(feature = "direct")]
             direct_pmio: None,
             #[cfg(feature = "direct")]

src/linux.rs

@@ -32,10 +32,11 @@ use libc::{self, c_int, gid_t, uid_t};
 
 use acpi_tables::sdt::SDT;
 
-use base::net::{UnixSeqpacketListener, UnlinkUnixSeqpacketListener};
+use base::net::{UnixSeqpacket, UnixSeqpacketListener, UnlinkUnixSeqpacketListener};
 use base::*;
 use devices::virtio::vhost::user::{
     Block as VhostUserBlock, Error as VhostUserError, Fs as VhostUserFs, Net as VhostUserNet,
+    Wl as VhostUserWl,
 };
 #[cfg(feature = "gpu")]
 use devices::virtio::EventDevice;
@@ -63,7 +64,7 @@ use vm_memory::{GuestAddress, GuestMemory, MemoryPolicy};
 use crate::gdb::{gdb_thread, GdbStub};
 use crate::{
     Config, DiskOption, Executable, SharedDir, SharedDirKind, TouchDeviceOption, VhostUserFsOption,
-    VhostUserOption,
+    VhostUserOption, VhostUserWlOption,
 };
 use arch::{
     self, LinuxArch, RunnableLinuxVm, SerialHardware, SerialParameters, VcpuAffinity,
@@ -100,6 +101,7 @@ pub enum Error {
     CloneEvent(base::Error),
     CloneVcpu(base::Error),
     ConfigureVcpu(<Arch as LinuxArch>::Error),
+    ConnectTube(io::Error),
     #[cfg(feature = "audio")]
     CreateAc97(devices::PciDeviceError),
     CreateConsole(arch::serial::Error),
@@ -193,6 +195,7 @@ pub enum Error {
     VhostUserFsDeviceNew(VhostUserError),
     VhostUserNetDeviceNew(VhostUserError),
     VhostUserNetWithNetArgs,
+    VhostUserWlDeviceNew(VhostUserError),
     VhostVsockDeviceNew(virtio::vhost::Error),
     VirtioPciDev(base::Error),
     WaitContextAdd(base::Error),
@@ -223,6 +226,7 @@ impl Display for Error {
             CloneEvent(e) => write!(f, "failed to clone event: {}", e),
             CloneVcpu(e) => write!(f, "failed to clone vcpu: {}", e),
             ConfigureVcpu(e) => write!(f, "failed to configure vcpu: {}", e),
+            ConnectTube(e) => write!(f, "failed to connect to tube: {}", e),
             #[cfg(feature = "audio")]
             CreateAc97(e) => write!(f, "failed to create ac97 device: {}", e),
             CreateConsole(e) => write!(f, "failed to create console device: {}", e),
@@ -334,6 +338,9 @@ impl Display for Error {
                 f,
                 "vhost-user-net cannot be used with any of --host_ip, --netmask or --mac"
             ),
+            VhostUserWlDeviceNew(e) => {
+                write!(f, "failed to set up vhost-user wl device: {}", e)
+            }
             VhostVsockDeviceNew(e) => write!(f, "failed to set up virtual socket device: {}", e),
             VirtioPciDev(e) => write!(f, "failed to create virtio pci dev: {}", e),
             WaitContextAdd(e) => write!(f, "failed to add descriptor to wait context: {}", e),
@@ -879,6 +886,19 @@ fn create_vhost_user_net_device(cfg: &Config, opt: &VhostUserOption) -> DeviceRe
     })
 }
 
+fn create_vhost_user_wl_device(cfg: &Config, opt: &VhostUserWlOption) -> DeviceResult {
+    // The crosvm wl device expects us to connect the tube before it will accept a vhost-user
+    // connection.
+    let dev = VhostUserWl::new(virtio::base_features(cfg.protected_vm), &opt.socket)
+        .map_err(Error::VhostUserWlDeviceNew)?;
+
+    Ok(VirtioDeviceStub {
+        dev: Box::new(dev),
+        // no sandbox here because virtqueue handling is exported to a different process.
+        jail: None,
+    })
+}
+
 #[cfg(feature = "gpu")]
 fn create_gpu_device(
     cfg: &Config,
@@ -1512,6 +1532,10 @@ fn create_virtio_devices(
         devs.push(create_vhost_user_net_device(cfg, net)?);
     }
 
+    for opt in &cfg.vhost_user_wl {
+        devs.push(create_vhost_user_wl_device(cfg, opt)?);
+    }
+
     #[cfg_attr(not(feature = "gpu"), allow(unused_mut))]
     let mut resource_bridges = Vec::<Tube>::new();
 
@@ -2478,6 +2502,13 @@ where
         components.gdb = Some((port, gdb_control_tube));
     }
 
+    for wl_cfg in &cfg.vhost_user_wl {
+        let wayland_host_tube = UnixSeqpacket::connect(&wl_cfg.vm_tube)
+            .map(Tube::new)
+            .map_err(Error::ConnectTube)?;
+        control_tubes.push(TaggedControlTube::VmMemory(wayland_host_tube));
+    }
+
     let (wayland_host_tube, wayland_device_tube) = Tube::pair().map_err(Error::CreateTube)?;
     control_tubes.push(TaggedControlTube::VmMemory(wayland_host_tube));
     // Balloon gets a special socket so balloon requests can be forwarded from the main process.

src/main.rs

@@ -26,7 +26,7 @@ use crosvm::DirectIoOption;
 use crosvm::{
     argument::{self, print_help, set_arguments, Argument},
     platform, BindMount, Config, DiskOption, Executable, GidMap, SharedDir, TouchDeviceOption,
-    VhostUserFsOption, VhostUserOption, DISK_ID_LEN,
+    VhostUserFsOption, VhostUserOption, VhostUserWlOption, DISK_ID_LEN,
 };
 #[cfg(feature = "gpu")]
 use devices::virtio::gpu::{GpuMode, GpuParameters};
@@ -1679,6 +1679,23 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
         "vhost-user-net" => cfg.vhost_user_net.push(VhostUserOption {
             socket: PathBuf::from(value.unwrap()),
         }),
+        "vhost-user-wl" => {
+            let mut components = value.unwrap().splitn(2, ":");
+            let socket = components.next().map(PathBuf::from).ok_or_else(|| {
+                argument::Error::InvalidValue {
+                    value: value.unwrap().to_owned(),
+                    expected: String::from("missing socket path"),
+                }
+            })?;
+            let vm_tube = components.next().map(PathBuf::from).ok_or_else(|| {
+                argument::Error::InvalidValue {
+                    value: value.unwrap().to_owned(),
+                    expected: String::from("missing vm tube path"),
+                }
+            })?;
+            cfg.vhost_user_wl
+                .push(VhostUserWlOption { socket, vm_tube });
+        }
         "vhost-user-fs" => {
             // (socket:tag)
             let param = value.unwrap();
@@ -1978,6 +1995,7 @@ writeback=BOOL - Indicates whether the VM can use writeback caching (default: fa
           Argument::value("balloon_bias_mib", "N", "Amount to bias balance of memory between host and guest as the balloon inflates, in MiB."),
           Argument::value("vhost-user-blk", "SOCKET_PATH", "Path to a socket for vhost-user block"),
           Argument::value("vhost-user-net", "SOCKET_PATH", "Path to a socket for vhost-user net"),
+          Argument::value("vhost-user-wl", "SOCKET_PATH:TUBE_PATH", "Paths to a vhost-user socket for wayland and a Tube socket for additional wayland-specific messages"),
           Argument::value("vhost-user-fs", "SOCKET_PATH:TAG",
                           "Path to a socket path for vhost-user fs, and tag for the shared dir"),
           #[cfg(feature = "direct")]