vhost_user: refactor device activation

Move worker thread creation from frontend device into handler. This
removes the need to duplicate the worker thread boilerplate in each
individual device.

BUG=b:201745804
TEST=compiles

Change-Id: Ib6567402a809b9cafe286f575751419e31469d76
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3716338
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Alexandre Courbot <acourbot@chromium.org>
Reviewed-by: Keiichi Watanabe <keiichiw@chromium.org>
Commit-Queue: David Stevens <stevensd@chromium.org>
This commit is contained in:
David Stevens 2022-06-07 16:00:34 +09:00 committed by Chromeos LUCI
parent 908adc6403
commit 48b0b429d6
11 changed files with 122 additions and 358 deletions

View file

@ -13,7 +13,7 @@ use virtio_sys::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
use vm_memory::GuestMemory;
use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, worker::Worker};
use crate::virtio::vhost::user::vmm::handler::VhostUserHandler;
use crate::virtio::{block::common::virtio_blk_config, DeviceType, Interrupt, Queue, VirtioDevice};
const VIRTIO_BLK_F_SEG_MAX: u32 = 2;
@ -27,7 +27,7 @@ const QUEUE_SIZE: u16 = 256;
pub struct Block {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -105,45 +105,17 @@ impl VirtioDevice for Block {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "block")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_virtio_blk".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost-user virtio_blk worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -12,12 +12,12 @@ use vm_memory::GuestMemory;
use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::console::{virtio_console_config, QUEUE_SIZE};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, worker::Worker, Error, Result};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, Error, Result};
use crate::virtio::{DeviceType, Interrupt, Queue, VirtioDevice};
pub struct Console {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -88,44 +88,17 @@ impl VirtioDevice for Console {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "console")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_virtio_console".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost-user virtio_console worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -14,14 +14,14 @@ use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use vmm_vhost::Error as VhostUserError;
use crate::virtio::fs::{virtio_fs_config, FS_MAX_TAG_LEN, QUEUE_SIZE};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, worker::Worker, Error, Result};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, Error, Result};
use crate::virtio::{copy_config, DeviceType};
use crate::virtio::{Interrupt, Queue, VirtioDevice};
pub struct Fs {
cfg: virtio_fs_config,
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -120,45 +120,17 @@ impl VirtioDevice for Fs {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "fs")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_virtio_fs".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost-user virtio_fs worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -12,7 +12,7 @@ use crate::{
pci::{PciBarConfiguration, PciCapability},
virtio::{
gpu::QUEUE_SIZES,
vhost::user::vmm::{worker::Worker, Result, VhostUserHandler},
vhost::user::vmm::{Result, VhostUserHandler},
virtio_gpu_config, DeviceType, Interrupt, PciCapabilityType, Queue, VirtioDevice,
VirtioPciShmCap, GPU_BAR_NUM, GPU_BAR_OFFSET, VIRTIO_GPU_F_CONTEXT_INIT,
VIRTIO_GPU_F_CREATE_GUEST_HANDLE, VIRTIO_GPU_F_RESOURCE_BLOB, VIRTIO_GPU_F_RESOURCE_SYNC,
@ -37,7 +37,7 @@ enum GpuState {
pub struct Gpu {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
state: GpuState,
queue_sizes: Vec<u16>,
@ -149,45 +149,17 @@ impl VirtioDevice for Gpu {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "gpu")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_gpu".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost_user_gpu worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -5,8 +5,9 @@
mod sys;
use std::io::Write;
use std::thread;
use base::{AsRawDescriptor, Event, Tube};
use base::{error, AsRawDescriptor, Event, Tube};
use vm_memory::GuestMemory;
use vmm_vhost::message::{
VhostUserConfigFlags, VhostUserProtocolFeatures, VhostUserVirtioFeatures,
@ -14,6 +15,7 @@ use vmm_vhost::message::{
use vmm_vhost::{VhostBackend, VhostUserMaster, VhostUserMemoryRegionInfo, VringConfigData};
use crate::virtio::vhost::user::vmm::handler::sys::SocketMaster;
use crate::virtio::vhost::user::vmm::worker::Worker;
use crate::virtio::vhost::user::vmm::{Error, Result};
use crate::virtio::{Interrupt, Queue};
@ -212,12 +214,13 @@ impl VhostUserHandler {
/// Activates vrings.
pub fn activate(
&mut self,
mem: &GuestMemory,
interrupt: &Interrupt,
queues: &[Queue],
queue_evts: &[Event],
) -> Result<()> {
self.set_mem_table(mem)?;
mem: GuestMemory,
interrupt: Interrupt,
queues: Vec<Queue>,
queue_evts: Vec<Event>,
label: &str,
) -> Result<(thread::JoinHandle<()>, Event)> {
self.set_mem_table(&mem)?;
let msix_config_opt = interrupt
.get_msix_config()
@ -230,10 +233,29 @@ impl VhostUserHandler {
let irqfd = msix_config
.get_irqfd(queue.vector as usize)
.unwrap_or_else(|| interrupt.get_interrupt_evt());
self.activate_vring(mem, queue_index, queue, queue_evt, irqfd)?;
self.activate_vring(&mem, queue_index, queue, queue_evt, irqfd)?;
}
Ok(())
drop(msix_config);
let label = format!("vhost_user_virtio_{}", label);
let kill_evt = Event::new().map_err(Error::CreateEvent)?;
let self_kill_evt = kill_evt.try_clone().map_err(Error::CreateEvent)?;
thread::Builder::new()
.name(label.clone())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start {} worker: {}", label, e);
}
})
.map(|worker_result| (worker_result, self_kill_evt))
.map_err(Error::SpawnWorker)
}
/// Deactivates all vrings.

View file

@ -8,33 +8,20 @@ use std::path::Path;
use std::thread;
use base::{error, Event, RawDescriptor};
use remain::sorted;
use thiserror::Error as ThisError;
use vm_memory::GuestMemory;
use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, worker::Worker, Error};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, Error};
use crate::virtio::{DeviceType, Interrupt, Queue, VirtioDevice};
use std::result::Result;
#[sorted]
#[derive(ThisError, Debug)]
enum Mac80211HwsimError {
#[error("failed to activate queues: {0}")]
ActivateQueue(Error),
#[error("failed to kill event pair: {0}")]
CreateKillEventPair(base::Error),
#[error("failed to spawn mac80211_hwsim worker: {0}")]
SpawnWorker(std::io::Error),
}
const QUEUE_SIZE: u16 = 256;
const QUEUE_COUNT: usize = 2;
pub struct Mac80211Hwsim {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -63,44 +50,6 @@ impl Mac80211Hwsim {
queue_sizes,
})
}
fn activate_internal(
&mut self,
mem: GuestMemory,
interrupt: Interrupt,
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) -> Result<(), Mac80211HwsimError> {
self.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.map_err(Mac80211HwsimError::ActivateQueue)?;
let (self_kill_evt, kill_evt) = Event::new()
.and_then(|e| Ok((e.try_clone()?, e)))
.map_err(Mac80211HwsimError::CreateKillEventPair)?;
self.kill_evt = Some(self_kill_evt);
let join_handle = thread::Builder::new()
.name("vhost_user_mac80211_hwsim".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
})
.map_err(Mac80211HwsimError::SpawnWorker)?;
self.worker_thread = Some(join_handle);
Ok(())
}
}
impl Drop for Mac80211Hwsim {
@ -147,8 +96,20 @@ impl VirtioDevice for Mac80211Hwsim {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self.activate_internal(mem, interrupt, queues, queue_evts) {
error!("Failed to activate mac80211_hwsim: {}", e);
match self.handler.borrow_mut().activate(
mem,
interrupt,
queues,
queue_evts,
"mac80211_hwsim",
) {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -131,6 +131,9 @@ pub enum Error {
/// Failed to create Master from a UDS path.
#[error("failed to connect to device socket while creating instance: {0}")]
SocketConnectOnMasterCreate(VhostError),
/// Failed to spawn worker thread.
#[error("failed to spawn worker: {0}")]
SpawnWorker(std::io::Error),
/// The tag for the Fs device was too long to fit in the config space.
#[error("tag is too long: {len} > {max}")]
TagTooLong { len: usize, max: usize },

View file

@ -12,7 +12,7 @@ use virtio_sys::virtio_net;
use vm_memory::GuestMemory;
use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, worker::Worker, Error};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, Error};
use crate::virtio::{DeviceType, Interrupt, Queue, VirtioDevice, VirtioNetConfig};
type Result<T> = std::result::Result<T, Error>;
@ -21,7 +21,7 @@ const QUEUE_SIZE: u16 = 1024;
pub struct Net {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -101,44 +101,17 @@ impl VirtioDevice for Net {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "net")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_virtio_net".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn virtio_net worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -12,13 +12,13 @@ use vm_memory::GuestMemory;
use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::snd::layout::virtio_snd_config;
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, worker::Worker, Error, Result};
use crate::virtio::vhost::user::vmm::{handler::VhostUserHandler, Error, Result};
use crate::virtio::{DeviceType, Interrupt, Queue, VirtioDevice};
// A vhost-user snd device.
pub struct Snd {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -97,46 +97,17 @@ impl VirtioDevice for Snd {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "snd")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) =
match Event::new().and_then(|evt| Ok((evt.try_clone()?, evt))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_virtio_snd".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost-user virtio_snd worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -11,7 +11,7 @@ use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::{
vhost::{
user::vmm::{handler::VhostUserHandler, worker::Worker, Error, Result},
user::vmm::{handler::VhostUserHandler, Error, Result},
vsock,
},
DeviceType, Interrupt, Queue, VirtioDevice,
@ -19,7 +19,7 @@ use crate::virtio::{
pub struct Vsock {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -100,44 +100,17 @@ impl VirtioDevice for Vsock {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "vsock")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_vsock".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn virtio_vsock worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}

View file

@ -10,7 +10,7 @@ use base::{error, Event, RawDescriptor};
use vm_memory::GuestMemory;
use vmm_vhost::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::virtio::vhost::user::vmm::{worker::Worker, Result, VhostUserHandler};
use crate::virtio::vhost::user::vmm::{Result, VhostUserHandler};
use crate::virtio::wl::{
QUEUE_SIZE, QUEUE_SIZES, VIRTIO_WL_F_SEND_FENCES, VIRTIO_WL_F_TRANS_FLAGS,
};
@ -18,7 +18,7 @@ use crate::virtio::{DeviceType, Interrupt, Queue, VirtioDevice};
pub struct Wl {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
worker_thread: Option<thread::JoinHandle<()>>,
handler: RefCell<VhostUserHandler>,
queue_sizes: Vec<u16>,
}
@ -85,45 +85,17 @@ impl VirtioDevice for Wl {
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
match self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
.activate(mem, interrupt, queues, queue_evts, "wl")
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_wl".to_string())
.spawn(move || {
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost_user_wl worker: {}", e);
}
Ok(join_handle) => {
Ok((join_handle, kill_evt)) => {
self.worker_thread = Some(join_handle);
self.kill_evt = Some(kill_evt);
}
Err(e) => {
error!("failed to activate queues: {}", e);
}
}
}