diff --git a/devices/src/pci/vfio_pci.rs b/devices/src/pci/vfio_pci.rs index ca9dc38c73..f0d09fda85 100644 --- a/devices/src/pci/vfio_pci.rs +++ b/devices/src/pci/vfio_pci.rs @@ -1472,6 +1472,11 @@ impl VfioPciDevice { Ok(ranges) } + /// Return the supported iova max address of the Vfio Pci device + pub fn get_max_iova(&self) -> u64 { + self.device.get_max_addr() + } + #[cfg(feature = "direct")] fn coordinated_pm(sysfs_path: &Path, enter: bool) -> anyhow::Result<()> { let path = Path::new(sysfs_path).join("power/coordinated"); diff --git a/devices/src/vfio.rs b/devices/src/vfio.rs index f99d7440b6..7a88406f50 100644 --- a/devices/src/vfio.rs +++ b/devices/src/vfio.rs @@ -702,7 +702,7 @@ pub struct VfioDevice { // vec for vfio device's regions regions: Vec, - iova_alloc: Option>>, + iova_alloc: Arc>, } impl VfioDevice { @@ -731,6 +731,14 @@ impl VfioDevice { group.lock().add_device_num(); let group_descriptor = group.lock().as_raw_descriptor(); + let iova_ranges = container + .lock() + .vfio_iommu_iova_get_iova_ranges()? + .into_iter() + .map(|r| std::ops::RangeInclusive::new(r.start, r.end)); + let iova_alloc = AddressAllocator::new_from_list(iova_ranges, None, None) + .map_err(VfioError::Resources)?; + Ok(VfioDevice { dev, name, @@ -738,7 +746,7 @@ impl VfioDevice { group_descriptor, group_id, regions, - iova_alloc: None, + iova_alloc: Arc::new(Mutex::new(iova_alloc)), }) } @@ -787,7 +795,7 @@ impl VfioDevice { group_descriptor, group_id, regions, - iova_alloc: Some(Arc::new(Mutex::new(iova_alloc))), + iova_alloc: Arc::new(Mutex::new(iova_alloc)), }) } @@ -1352,13 +1360,14 @@ impl VfioDevice { } pub fn alloc_iova(&self, size: u64, align_size: u64, alloc: Alloc) -> Result { - match &self.iova_alloc { - None => Err(VfioError::NoRescAlloc), - Some(iova_alloc) => iova_alloc - .lock() - .allocate_with_align(size, alloc, "alloc_iova".to_owned(), align_size) - .map_err(VfioError::Resources), - } + self.iova_alloc + .lock() + .allocate_with_align(size, alloc, "alloc_iova".to_owned(), align_size) + .map_err(VfioError::Resources) + } + + pub fn get_max_addr(&self) -> u64 { + self.iova_alloc.lock().get_max_addr() } /// Gets the vfio device backing `File`. diff --git a/devices/src/virtio/iommu.rs b/devices/src/virtio/iommu.rs index 27ede3ebc5..6a36082f4a 100644 --- a/devices/src/virtio/iommu.rs +++ b/devices/src/virtio/iommu.rs @@ -747,7 +747,7 @@ impl Iommu { pub fn new( base_features: u64, endpoints: BTreeMap>>>, - phys_max_addr: u64, + iova_max_addr: u64, hp_endpoints_ranges: Vec>, translate_response_senders: Option>, translate_request_rx: Option, @@ -767,7 +767,7 @@ impl Iommu { let input_range = virtio_iommu_range_64 { start: Le64::from(0), - end: phys_max_addr.into(), + end: iova_max_addr.into(), }; let config = virtio_iommu_config { diff --git a/resources/src/address_allocator.rs b/resources/src/address_allocator.rs index 6b15ffd286..57c5d7798f 100644 --- a/resources/src/address_allocator.rs +++ b/resources/src/address_allocator.rs @@ -277,6 +277,11 @@ impl AddressAllocator { .map(|(&alloc, _)| alloc) } + // Return the max address of the allocated address ranges. + pub fn get_max_addr(&self) -> u64 { + self.regions.iter().fold(0, |x, (_, end)| x.max(*end)) + } + /// Returns allocation associated with `alloc`, or None if no such allocation exists. pub fn get(&self, alloc: &Alloc) -> Option<&(u64, u64, String)> { self.allocs.get(alloc) @@ -764,4 +769,15 @@ mod tests { Err(Error::InvalidAlloc(anon)) ); } + + #[test] + fn get_max_address_of_ranges() { + let ranges = vec![ + RangeInclusive::new(0x1000, 0xFFFF), + RangeInclusive::new(0x20000, 0xFFFFF), + ]; + let pool = AddressAllocator::new_from_list(ranges.into_iter(), None, None).unwrap(); + + assert_eq!(pool.get_max_addr(), 0xFFFFF); + } } diff --git a/src/linux/device_helpers.rs b/src/linux/device_helpers.rs index c40656b70b..87fc514bf7 100644 --- a/src/linux/device_helpers.rs +++ b/src/linux/device_helpers.rs @@ -996,7 +996,7 @@ pub fn create_pmem_device( pub fn create_iommu_device( cfg: &Config, - phys_max_addr: u64, + iova_max_addr: u64, endpoints: BTreeMap>>>, hp_endpoints_ranges: Vec>, translate_response_senders: Option>, @@ -1006,7 +1006,7 @@ pub fn create_iommu_device( let dev = virtio::Iommu::new( virtio::base_features(cfg.protected_vm), endpoints, - phys_max_addr, + iova_max_addr, hp_endpoints_ranges, translate_response_senders, translate_request_rx, diff --git a/src/linux/mod.rs b/src/linux/mod.rs index f73eb3b2e3..1ecfe2880b 100644 --- a/src/linux/mod.rs +++ b/src/linux/mod.rs @@ -483,6 +483,7 @@ fn create_devices( #[cfg(feature = "gpu")] render_server_fd: Option, vvu_proxy_device_tubes: &mut Vec, vvu_proxy_max_sibling_mem_size: u64, + iova_max_addr: &mut Option, ) -> DeviceResult, Option)>> { let mut devices: Vec<(Box, Option)> = Vec::new(); let mut balloon_inflate_tube: Option = None; @@ -508,6 +509,10 @@ fn create_devices( vfio_dev.iommu_dev_type(), )?; + *iova_max_addr = Some(max( + vfio_pci_device.get_max_iova(), + iova_max_addr.unwrap_or(0), + )); devices.push((vfio_pci_device, jail)); } @@ -1297,6 +1302,7 @@ where let mut iommu_attached_endpoints: BTreeMap>>> = BTreeMap::new(); + let mut iova_max_addr: Option = None; let mut devices = create_devices( &cfg, &mut vm, @@ -1320,6 +1326,7 @@ where render_server_fd, &mut vvu_proxy_device_tubes, components.memory_size, + &mut iova_max_addr, )?; let mut hp_endpoints_ranges: Vec> = Vec::new(); @@ -1356,7 +1363,7 @@ where let (iommu_host_tube, iommu_device_tube) = Tube::pair().context("failed to create tube")?; let iommu_dev = create_iommu_device( &cfg, - (1u64 << vm.get_guest_phys_addr_bits()) - 1, + iova_max_addr.unwrap_or(u64::MAX), iommu_attached_endpoints, hp_endpoints_ranges, translate_response_senders,