vfio: Integrate VFIO device into pci device model

Create VFIO device and VFIO PCI device in create_devices() function, and
intergrate it into PciRootBridge, so guest could see this vfio device.

Add a vfio config parameter, this config point to passthrough or mdev
device sysfs path.

For passthrough case, first user unbind host device from its driver,
then bind host device to vfio-pci. Like:
echo 0000:00:02.0 > /sys/bus/pci/devices/0000:00:02.0/driver/unbind
ech0 8086 1912 > /sys/bus/pci/drivers/vfio-pci/new_id
Finally pass the sysfs to crosvm through
--vfio=/sys/bus/pci/devices/0000:00:02.0

For mdev case, user create a mdev device through
echo $UUID > mdev_type/create, then pass this mdev device to crosvm like
--vfio=/sys/bus/pci/devices/0000:00:02.0/$UUID

BUG=chromium:992270
TEST=none

Change-Id: I0f59d6e93f62f9ab0727ad3a867d204f4ff6ad2d
Signed-off-by: Xiong Zhang <xiong.y.zhang@intel.corp-partner.google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/1581140
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Reviewed-by: Zach Reizner <zachr@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>

devices/src/pci/vfio_pci.rs

@@ -139,8 +139,7 @@ impl PciDevice for VfioPciDevice {
     }
 
     fn keep_fds(&self) -> Vec<RawFd> {
-        let fds = Vec::new();
-        fds
+        self.device.keep_fds()
     }
 
     fn assign_irq(

devices/src/vfio.rs

@@ -111,6 +111,7 @@ impl AsRawFd for VfioContainer {
 
 struct VfioGroup {
     group: File,
+    container: VfioContainer,
 }
 
 impl VfioGroup {
@@ -163,7 +164,10 @@ impl VfioGroup {
 
         Self::kvm_device_add_group(vm, &group_file)?;
 
-        Ok(VfioGroup { group: group_file })
+        Ok(VfioGroup {
+            group: group_file,
+            container,
+        })
     }
 
     fn kvm_device_add_group(vm: &Vm, group: &File) -> Result<File, VfioError> {
@@ -230,6 +234,7 @@ struct VfioRegion {
 /// Vfio device for exposing regions which could be read/write to kernel vfio device.
 pub struct VfioDevice {
     dev: File,
+    group: VfioGroup,
     regions: Vec<VfioRegion>,
 }
 
@@ -254,6 +259,7 @@ impl VfioDevice {
 
         Ok(VfioDevice {
             dev: new_dev,
+            group,
             regions: dev_regions,
         })
     }
@@ -368,6 +374,15 @@ impl VfioDevice {
             );
         }
     }
+
+    /// get vfio device's fds which are passed into minijail process
+    pub fn keep_fds(&self) -> Vec<RawFd> {
+        let mut fds = Vec::new();
+        fds.push(self.as_raw_fd());
+        fds.push(self.group.as_raw_fd());
+        fds.push(self.group.container.as_raw_fd());
+        fds
+    }
 }
 
 impl AsRawFd for VfioDevice {

seccomp/x86_64/vfio_device.policy

@@ -0,0 +1,10 @@
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+@include /usr/share/policy/crosvm/common_device.policy
+
+# VFIO_DEVICE_SET_IRQS, VFIO_IOMMU_MAP/UNMAP_DMA
+ioctl: arg1 == 0x3B6E || arg1 == 0x3B71 || arg1 == 0x3B72
+readlink: 1
+pread64: 1
+pwrite64: 1

src/crosvm.rs

@@ -109,6 +109,7 @@ pub struct Config {
     pub virtio_keyboard: Option<PathBuf>,
     pub virtio_input_evdevs: Vec<PathBuf>,
     pub split_irqchip: bool,
+    pub vfio: Option<PathBuf>,
 }
 
 impl Default for Config {
@@ -153,6 +154,7 @@ impl Default for Config {
             virtio_keyboard: None,
             virtio_input_evdevs: Vec::new(),
             split_irqchip: false,
+            vfio: None,
         }
     }
 }

src/linux.rs

@@ -27,7 +27,10 @@ use libc::{self, c_int, gid_t, uid_t};
 
 use audio_streams::DummyStreamSource;
 use devices::virtio::{self, VirtioDevice};
-use devices::{self, HostBackendDeviceProvider, PciDevice, VirtioPciDevice, XhciController};
+use devices::{
+    self, HostBackendDeviceProvider, PciDevice, VfioDevice, VfioPciDevice, VirtioPciDevice,
+    XhciController,
+};
 use io_jail::{self, Minijail};
 use kvm::*;
 use libcras::CrasClient;
@@ -91,6 +94,7 @@ pub enum Error {
     CreateTimerFd(sys_util::Error),
     CreateTpmStorage(PathBuf, io::Error),
     CreateUsbProvider(devices::usb::host_backend::error::Error),
+    CreateVfioDevice(devices::vfio::VfioError),
     DeviceJail(io_jail::Error),
     DevicePivotRoot(io_jail::Error),
     Disk(io::Error),
@@ -172,6 +176,7 @@ impl Display for Error {
                 write!(f, "failed to create tpm storage dir {}: {}", p.display(), e)
             }
             CreateUsbProvider(e) => write!(f, "failed to create usb provider: {}", e),
+            CreateVfioDevice(e) => write!(f, "Failed to create vfio device {}", e),
             DeviceJail(e) => write!(f, "failed to jail device: {}", e),
             DevicePivotRoot(e) => write!(f, "failed to pivot root device: {}", e),
             Disk(e) => write!(f, "failed to load disk image: {}", e),
@@ -979,6 +984,13 @@ fn create_devices(
     let usb_controller = Box::new(XhciController::new(mem.clone(), usb_provider));
     pci_devices.push((usb_controller, simple_jail(&cfg, "xhci.policy")?));
 
+    if cfg.vfio.is_some() {
+        let vfio_path = cfg.vfio.as_ref().unwrap().as_path();
+        let vfiodevice = Box::new(VfioDevice::new(vfio_path, vm).map_err(Error::CreateVfioDevice)?);
+        let vfiopcidevice = Box::new(VfioPciDevice::new(vfiodevice));
+        pci_devices.push((vfiopcidevice, simple_jail(&cfg, "vfio_device.policy")?));
+    }
+
     Ok(pci_devices)
 }
 

src/main.rs

@@ -685,6 +685,24 @@ fn set_argument(cfg: &mut Config, name: &str, value: Option<&str>) -> argument::
             }
             cfg.executable_path = Some(Executable::Bios(PathBuf::from(value.unwrap().to_owned())));
         }
+        "vfio" => {
+            let vfio_path = PathBuf::from(value.unwrap());
+            if !vfio_path.exists() {
+                return Err(argument::Error::InvalidValue {
+                    value: value.unwrap().to_owned(),
+                    expected: "the vfio path does not exist",
+                });
+            }
+            if !vfio_path.is_dir() {
+                return Err(argument::Error::InvalidValue {
+                    value: value.unwrap().to_owned(),
+                    expected: "the vfio path should be directory",
+                });
+            }
+
+            cfg.vfio = Some(vfio_path);
+        }
+
         "help" => return Err(argument::Error::PrintHelp),
         _ => unreachable!(),
     }
@@ -774,6 +792,7 @@ fn run_vm(args: std::env::Args) -> std::result::Result<(), ()> {
           #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
           Argument::flag("split-irqchip", "(EXPERIMENTAL) enable split-irqchip support"),
           Argument::value("bios", "PATH", "Path to BIOS/firmware ROM"),
+          Argument::value("vfio", "PATH", "Path to sysfs of pass through or mdev device"),
           Argument::short_flag('h', "help", "Print help message.")];
 
     let mut cfg = Config::default();