mirrors/crosvm
1
0
Fork 0
mirror of https://chromium.googlesource.com/crosvm/crosvm synced 2025-02-05 18:20:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support out-of-tree vhost-user-gpu

Browse source 
BUG=b:179755651
TEST=`glxinfo -B | grep virgl` inside a crostini vm

Change-Id: I37b98fcccfb8e56af4d07a2afe828ca3b5087b1f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2987595
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Gurchetan Singh <gurchetansingh@chromium.org>
Reviewed-by: Keiichi Watanabe <keiichiw@chromium.org>
Commit-Queue: Chirantan Ekbote <chirantan@chromium.org>
This commit is contained in:
Chirantan Ekbote 2021-06-25 18:31:41 +09:00 committed by Commit Bot
parent 084c87baad
commit 44292f58f3
7 changed files with 318 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

218
devices/src/virtio/vhost/user/vmm/gpu.rs Normal file
View file

@ -0,0 +1,218 @@
// Copyright 2021 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use std::{cell::RefCell, path::Path, thread};
use base::{error, Event, RawDescriptor, Tube};
use cros_async::Executor;
use vm_memory::GuestMemory;
use vmm_vhost::vhost_user::message::{VhostUserProtocolFeatures, VhostUserVirtioFeatures};
use crate::{
pci::{PciBarConfiguration, PciCapability},
virtio::{
gpu::QUEUE_SIZES,
vhost::user::vmm::{worker::Worker, Result, VhostUserHandler},
virtio_gpu_config, Interrupt, PciCapabilityType, Queue, VirtioDevice, VirtioPciShmCap,
GPU_BAR_NUM, GPU_BAR_OFFSET, GPU_BAR_SIZE, TYPE_GPU, VIRTIO_GPU_F_CONTEXT_INIT,
VIRTIO_GPU_F_CREATE_GUEST_HANDLE, VIRTIO_GPU_F_RESOURCE_BLOB, VIRTIO_GPU_F_RESOURCE_SYNC,
VIRTIO_GPU_F_RESOURCE_UUID, VIRTIO_GPU_F_VIRGL, VIRTIO_GPU_SHM_ID_HOST_VISIBLE,
},
PciAddress,
};
pub struct Gpu {
kill_evt: Option<Event>,
worker_thread: Option<thread::JoinHandle<Worker>>,
handler: RefCell<VhostUserHandler>,
host_tube: Tube,
queue_sizes: Vec<u16>,
}
impl Gpu {
pub fn new<P: AsRef<Path>>(
base_features: u64,
socket_path: P,
host_tube: Tube,
device_tube: Tube,
) -> Result<Gpu> {
let default_queue_size = QUEUE_SIZES.len();
let allow_features = 1u64 << crate::virtio::VIRTIO_F_VERSION_1
| 1 << VIRTIO_GPU_F_VIRGL
| 1 << VIRTIO_GPU_F_RESOURCE_UUID
| 1 << VIRTIO_GPU_F_RESOURCE_BLOB
| 1 << VIRTIO_GPU_F_CONTEXT_INIT
| 1 << VIRTIO_GPU_F_RESOURCE_SYNC
| 1 << VIRTIO_GPU_F_CREATE_GUEST_HANDLE
| VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
let init_features = base_features | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits();
let allow_protocol_features =
VhostUserProtocolFeatures::CONFIG | VhostUserProtocolFeatures::SLAVE_REQ;
let mut handler = VhostUserHandler::new_from_path(
socket_path,
default_queue_size as u64,
allow_features,
init_features,
allow_protocol_features,
)?;
handler.set_device_request_channel(device_tube)?;
Ok(Gpu {
kill_evt: None,
worker_thread: None,
handler: RefCell::new(handler),
host_tube,
queue_sizes: QUEUE_SIZES[..].to_vec(),
})
}
}
impl VirtioDevice for Gpu {
fn keep_rds(&self) -> Vec<RawDescriptor> {
Vec::new()
}
fn device_type(&self) -> u32 {
TYPE_GPU
}
fn queue_max_sizes(&self) -> &[u16] {
&self.queue_sizes
}
fn features(&self) -> u64 {
self.handler.borrow().avail_features
}
fn ack_features(&mut self, features: u64) {
if let Err(e) = self.handler.borrow_mut().ack_features(features) {
error!("failed to enable features 0x{:x}: {}", features, e);
}
}
fn read_config(&self, offset: u64, data: &mut [u8]) {
if let Err(e) = self
.handler
.borrow_mut()
.read_config::<virtio_gpu_config>(offset, data)
{
error!("failed to read gpu config: {}", e);
}
}
fn write_config(&mut self, offset: u64, data: &[u8]) {
if let Err(e) = self
.handler
.borrow_mut()
.write_config::<virtio_gpu_config>(offset, data)
{
error!("failed to write gpu config: {}", e);
}
}
fn activate(
&mut self,
mem: GuestMemory,
interrupt: Interrupt,
queues: Vec<Queue>,
queue_evts: Vec<Event>,
) {
if let Err(e) = self
.handler
.borrow_mut()
.activate(&mem, &interrupt, &queues, &queue_evts)
{
error!("failed to activate queues: {}", e);
return;
}
let (self_kill_evt, kill_evt) = match Event::new().and_then(|e| Ok((e.try_clone()?, e))) {
Ok(v) => v,
Err(e) => {
error!("failed creating kill Event pair: {}", e);
return;
}
};
self.kill_evt = Some(self_kill_evt);
let worker_result = thread::Builder::new()
.name("vhost_user_gpu".to_string())
.spawn(move || {
let ex = Executor::new().expect("failed to create an executor");
let mut worker = Worker {
queues,
mem,
kill_evt,
};
if let Err(e) = worker.run(&ex, interrupt) {
error!("failed to start a worker: {}", e);
}
worker
});
match worker_result {
Err(e) => {
error!("failed to spawn vhost_user_gpu worker: {}", e);
}
Ok(join_handle) => {
self.worker_thread = Some(join_handle);
}
}
}
fn get_device_bars(&mut self, address: PciAddress) -> Vec<PciBarConfiguration> {
if let Err(e) = self.host_tube.send(&address) {
error!("failed to send `PciAddress` to gpu device: {}", e);
return Vec::new();
}
match self.host_tube.recv() {
Ok(cfg) => cfg,
Err(e) => {
error!(
"failed to receive `PciBarConfiguration` from gpu device: {}",
e
);
Vec::new()
}
}
}
fn get_device_caps(&self) -> Vec<Box<dyn PciCapability>> {
vec![Box::new(VirtioPciShmCap::new(
PciCapabilityType::SharedMemoryConfig,
GPU_BAR_NUM,
GPU_BAR_OFFSET,
GPU_BAR_SIZE,
VIRTIO_GPU_SHM_ID_HOST_VISIBLE,
))]
}
fn reset(&mut self) -> bool {
if let Err(e) = self.handler.borrow_mut().reset(self.queue_sizes.len()) {
error!("Failed to reset gpu device: {}", e);
false
} else {
true
}
}
}
impl Drop for Gpu {
fn drop(&mut self) {
if let Some(kill_evt) = self.kill_evt.take() {
if let Some(worker_thread) = self.worker_thread.take() {
if let Err(e) = kill_evt.write(1) {
error!("failed to write to kill_evt: {}", e);
return;
}
let _ = worker_thread.join();
}
}
}
}

27
devices/src/virtio/vhost/user/vmm/handler.rs
View file

@ -6,7 +6,7 @@ use std::io::Write;
use std::os::unix::net::UnixStream;
use std::path::Path;
use base::{AsRawDescriptor, Event};
use base::{AsRawDescriptor, Event, Tube};
use vm_memory::GuestMemory;
use vmm_vhost::vhost_user::message::{
VhostUserConfigFlags, VhostUserProtocolFeatures, VhostUserVirtioFeatures,
@ -144,6 +144,31 @@ impl VhostUserHandler {
.map_err(Error::CopyConfig)
}
/// Writes `data` into the device configuration space at `offset`.
pub fn write_config<T>(&mut self, offset: u64, data: &[u8]) -> Result<()> {
let config_len = std::mem::size_of::<T>() as u64;
let data_len = data.len() as u64;
offset
.checked_add(data_len)
.and_then(|l| if l <= config_len { Some(()) } else { None })
.ok_or(Error::InvalidConfigOffset {
data_len,
offset,
config_len,
})?;
self.vu
.set_config(offset as u32, VhostUserConfigFlags::empty(), data)
.map_err(Error::SetConfig)
}
/// Sets the channel for device-specific messages.
pub fn set_device_request_channel(&mut self, channel: Tube) -> Result<()> {
self.vu
.set_slave_request_fd(&channel.as_raw_descriptor())
.map_err(Error::SetDeviceRequestChannel)
}
/// Sets the memory map regions so it can translate the vring addresses.
pub fn set_mem_table(&mut self, mem: &GuestMemory) -> Result<()> {
let mut regions: Vec<VhostUserMemoryRegionInfo> = Vec::new();

10
devices/src/virtio/vhost/user/vmm/mod.rs
View file

@ -5,6 +5,8 @@
mod block;
mod console;
mod fs;
#[cfg(feature = "gpu")]
mod gpu;
mod handler;
mod mac80211_hwsim;
mod net;
@ -14,6 +16,8 @@ mod worker;
pub use self::block::*;
pub use self::console::*;
pub use self::fs::*;
#[cfg(feature = "gpu")]
pub use self::gpu::*;
pub use self::handler::VhostUserHandler;
pub use self::mac80211_hwsim::*;
pub use self::net::*;
@ -67,6 +71,12 @@ pub enum Error {
/// Failed to reset owner.
#[error("failed to reset owner: {0}")]
ResetOwner(VhostError),
/// Failed to set config.
#[error("failed to set config: {0}")]
SetConfig(VhostError),
/// Failed to set device request channel.
#[error("failed to set device request channel: {0}")]
SetDeviceRequestChannel(VhostError),
/// Failed to set features.
#[error("failed to set features: {0}")]
SetFeatures(VhostError),

2
src/crosvm.rs
View file

@ -274,6 +274,7 @@ pub struct Config {
pub vhost_user_blk: Vec<VhostUserOption>,
pub vhost_user_console: Vec<VhostUserOption>,
pub vhost_user_fs: Vec<VhostUserFsOption>,
pub vhost_user_gpu: Vec<VhostUserOption>,
pub vhost_user_mac80211_hwsim: Option<VhostUserOption>,
pub vhost_user_net: Vec<VhostUserOption>,
pub vhost_user_wl: Vec<VhostUserWlOption>,
@ -360,6 +361,7 @@ impl Default for Config {
balloon_bias: 0,
vhost_user_blk: Vec::new(),
vhost_user_console: Vec::new(),
vhost_user_gpu: Vec::new(),
vhost_user_fs: Vec::new(),
vhost_user_mac80211_hwsim: None,
vhost_user_net: Vec::new(),

4
src/error.rs
View file

@ -41,6 +41,7 @@ pub enum Error {
BuildVm(<Arch as LinuxArch>::Error),
ChownTpmStorage(base::Error),
CloneEvent(base::Error),
CloneTube(base::TubeError),
CloneVcpu(base::Error),
ConfigureHotPlugDevice(<Arch as LinuxArch>::Error),
ConfigureVcpu(<Arch as LinuxArch>::Error),
@ -143,6 +144,7 @@ pub enum Error {
VhostUserBlockDeviceNew(VhostUserVmmError),
VhostUserConsoleDeviceNew(VhostUserVmmError),
VhostUserFsDeviceNew(VhostUserVmmError),
VhostUserGpuDeviceNew(VhostUserVmmError),
VhostUserMac80211HwsimNew(VhostUserVmmError),
VhostUserNetDeviceNew(VhostUserVmmError),
VhostUserNetWithNetArgs,
@ -175,6 +177,7 @@ impl Display for Error {
BuildVm(e) => write!(f, "The architecture failed to build the vm: {}", e),
ChownTpmStorage(e) => write!(f, "failed to chown tpm storage: {}", e),
CloneEvent(e) => write!(f, "failed to clone event: {}", e),
CloneTube(e) => write!(f, "failed to clone tube: {}", e),
CloneVcpu(e) => write!(f, "failed to clone vcpu: {}", e),
ConfigureHotPlugDevice(e) => write!(f, "Failed to configure pci hotplug device:{}", e),
ConfigureVcpu(e) => write!(f, "failed to configure vcpu: {}", e),
@ -294,6 +297,7 @@ impl Display for Error {
write!(f, "failed to set up vhost-user console device: {}", e)
}
VhostUserFsDeviceNew(e) => write!(f, "failed to set up vhost-user fs device: {}", e),
VhostUserGpuDeviceNew(e) => write!(f, "failed to set up vhost-user gpu device: {}", e),
VhostUserMac80211HwsimNew(e) => {
write!(f, "failed to set up vhost-user mac80211_hwsim device {}", e)
}

58
src/linux.rs
View file

@ -30,17 +30,19 @@ use base::net::{UnixSeqpacket, UnixSeqpacketListener, UnlinkUnixSeqpacketListene
use base::*;
use devices::serial_device::{SerialHardware, SerialParameters};
use devices::vfio::{VfioCommonSetup, VfioCommonTrait};
#[cfg(feature = "gpu")]
use devices::virtio::gpu::{DEFAULT_DISPLAY_HEIGHT, DEFAULT_DISPLAY_WIDTH};
#[cfg(feature = "audio_cras")]
use devices::virtio::snd::cras_backend::Parameters as CrasSndParameters;
use devices::virtio::vhost::user::vmm::{
Block as VhostUserBlock, Console as VhostUserConsole, Fs as VhostUserFs,
Mac80211Hwsim as VhostUserMac80211Hwsim, Net as VhostUserNet, Wl as VhostUserWl,
};
#[cfg(feature = "gpu")]
use devices::virtio::EventDevice;
use devices::virtio::{self, Console, VirtioDevice};
#[cfg(feature = "gpu")]
use devices::virtio::{
gpu::{DEFAULT_DISPLAY_HEIGHT, DEFAULT_DISPLAY_WIDTH},
vhost::user::vmm::Gpu as VhostUserGpu,
EventDevice,
};
#[cfg(feature = "audio")]
use devices::Ac97Dev;
use devices::ProtectionType;
@ -655,6 +657,30 @@ fn create_vhost_user_wl_device(cfg: &Config, opt: &VhostUserWlOption) -> DeviceR
})
}
#[cfg(feature = "gpu")]
fn create_vhost_user_gpu_device(
cfg: &Config,
opt: &VhostUserOption,
host_tube: Tube,
device_tube: Tube,
) -> DeviceResult {
// The crosvm gpu device expects us to connect the tube before it will accept a vhost-user
// connection.
let dev = VhostUserGpu::new(
virtio::base_features(cfg.protected_vm),
&opt.socket,
host_tube,
device_tube,
)
.map_err(Error::VhostUserGpuDeviceNew)?;
Ok(VirtioDeviceStub {
dev: Box::new(dev),
// no sandbox here because virtqueue handling is exported to a different process.
jail: None,
})
}
#[cfg(feature = "gpu")]
fn create_gpu_device(
cfg: &Config,
@ -1210,6 +1236,7 @@ fn create_virtio_devices(
_exit_evt: &Event,
wayland_device_tube: Tube,
gpu_device_tube: Tube,
vhost_user_gpu_tubes: Vec<(Tube, Tube)>,
balloon_device_tube: Tube,
disk_device_tubes: &mut Vec<Tube>,
pmem_device_tubes: &mut Vec<Tube>,
@ -1328,6 +1355,16 @@ fn create_virtio_devices(
devs.push(create_vhost_user_wl_device(cfg, opt)?);
}
#[cfg(feature = "gpu")]
for (opt, (host_tube, device_tube)) in cfg.vhost_user_gpu.iter().zip(vhost_user_gpu_tubes) {
devs.push(create_vhost_user_gpu_device(
cfg,
opt,
host_tube,
device_tube,
)?);
}
#[cfg_attr(not(feature = "gpu"), allow(unused_mut))]
let mut resource_bridges = Vec::<Tube>::new();
@ -1563,6 +1600,7 @@ fn create_devices(
control_tubes: &mut Vec<TaggedControlTube>,
wayland_device_tube: Tube,
gpu_device_tube: Tube,
vhost_user_gpu_tubes: Vec<(Tube, Tube)>,
balloon_device_tube: Tube,
disk_device_tubes: &mut Vec<Tube>,
pmem_device_tubes: &mut Vec<Tube>,
@ -1577,6 +1615,7 @@ fn create_devices(
exit_evt,
wayland_device_tube,
gpu_device_tube,
vhost_user_gpu_tubes,
balloon_device_tube,
disk_device_tubes,
pmem_device_tubes,
@ -2362,6 +2401,16 @@ where
control_tubes.push(TaggedControlTube::VmMemory(wayland_host_tube));
}
let mut vhost_user_gpu_tubes = Vec::with_capacity(cfg.vhost_user_gpu.len());
for _ in 0..cfg.vhost_user_gpu.len() {
let (host_tube, device_tube) = Tube::pair().map_err(Error::CreateTube)?;
vhost_user_gpu_tubes.push((
host_tube.try_clone().map_err(Error::CloneTube)?,
device_tube,
));
control_tubes.push(TaggedControlTube::VmMemory(host_tube));
}
let (wayland_host_tube, wayland_device_tube) = Tube::pair().map_err(Error::CreateTube)?;
control_tubes.push(TaggedControlTube::VmMemory(wayland_host_tube));
// Balloon gets a special socket so balloon requests can be forwarded from the main process.
@ -2463,6 +2512,7 @@ where
&mut control_tubes,
wayland_device_tube,
gpu_device_tube,
vhost_user_gpu_tubes,
balloon_device_tube,
&mut disk_device_tubes,
&mut pmem_device_tubes,

4
src/main.rs
View file

@ -1874,6 +1874,9 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
"vhost-user-console" => cfg.vhost_user_console.push(VhostUserOption {
socket: PathBuf::from(value.unwrap()),
}),
"vhost-user-gpu" => cfg.vhost_user_gpu.push(VhostUserOption {
socket: PathBuf::from(value.unwrap()),
}),
"vhost-user-mac80211-hwsim" => {
cfg.vhost_user_mac80211_hwsim = Some(VhostUserOption {
socket: PathBuf::from(value.unwrap()),
@ -2228,6 +2231,7 @@ iommu=on|off - indicates whether to enable virtio IOMMU for this device"),
Argument::value("balloon_bias_mib", "N", "Amount to bias balance of memory between host and guest as the balloon inflates, in MiB."),
Argument::value("vhost-user-blk", "SOCKET_PATH", "Path to a socket for vhost-user block"),
Argument::value("vhost-user-console", "SOCKET_PATH", "Path to a socket for vhost-user console"),
Argument::value("vhost-user-gpu", "SOCKET_PATH", "Paths to a vhost-user socket for gpu"),
Argument::value("vhost-user-mac80211-hwsim", "SOCKET_PATH", "Path to a socket for vhost-user mac80211_hwsim"),
Argument::value("vhost-user-net", "SOCKET_PATH", "Path to a socket for vhost-user net"),
Argument::value("vhost-user-wl", "SOCKET_PATH:TUBE_PATH", "Paths to a vhost-user socket for wayland and a Tube socket for additional wayland-specific messages"),