From 9557588786157dd550ae55cd4abf44dc645cb484 Mon Sep 17 00:00:00 2001 From: Xiong Zhang Date: Sun, 26 Sep 2021 14:55:50 +0800 Subject: [PATCH] Devices: Allocate non pci bar mmio at top of HighMmio Some devices like virtio-pmem, intel host bridge need private mmio, these private mmio isn't exposed to Vm through pci bar, and crosvm use mmio allocation interface to allocate them, so guest OS pci bus driver couldn't detect them, and once guest OS reallocate pci mmio, these private mmmio could be assigned to pci device bar. then resource conflict happens. This patch allocates these non pci bar mmio at the top of HighMmio, since HighMmio is large enough and OS allocates pci mmio from low to high. So these non pci bar mmio won't conflict with pci bar. BUG=b:199442120 BUG=b:185084350 TEST=crosvm run --pmem-device xxxx Change-Id: I615d7901d595dd46ba6362f88b71d96baa9c4c8f Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3184549 Reviewed-by: Daniel Verkamp Tested-by: kokoro Commit-Queue: Daniel Verkamp --- resources/src/address_allocator.rs | 82 ++++++++++++++++++++++-------- src/linux.rs | 2 +- x86_64/src/lib.rs | 2 +- 3 files changed, 64 insertions(+), 22 deletions(-) diff --git a/resources/src/address_allocator.rs b/resources/src/address_allocator.rs index 9d8b28d7ae..04fe63e06a 100644 --- a/resources/src/address_allocator.rs +++ b/resources/src/address_allocator.rs @@ -62,15 +62,13 @@ impl AddressAllocator { }) } - /// Allocates a range of addresses from the managed region with an optional tag - /// and minimal alignment. Returns allocated_address. (allocated_address, size, tag) - /// can be retrieved through the `get` method. - pub fn allocate_with_align( + fn internal_allocate_with_align( &mut self, size: u64, alloc: Alloc, tag: String, alignment: u64, + reverse: bool, ) -> Result { let alignment = cmp::max(self.alignment, alignment); @@ -84,24 +82,42 @@ impl AddressAllocator { return Err(Error::BadAlignment); } - // finds first region matching alignment and size. - match self - .regions - .iter() - .find(|range| { - match range.0 % alignment { - 0 => range.0.checked_add(size - 1), - r => range.0.checked_add(size - 1 + alignment - r), - } - .map_or(false, |end| end <= range.1) - }) - .cloned() - { + let region = if !reverse { + // finds first region matching alignment and size. + self.regions + .iter() + .find(|range| { + match range.0 % alignment { + 0 => range.0.checked_add(size - 1), + r => range.0.checked_add(size - 1 + alignment - r), + } + .map_or(false, |end| end <= range.1) + }) + .cloned() + } else { + // finds last region matching alignment and size. + self.regions + .iter() + .rev() + .find(|range| { + range + .1 + .checked_sub(size - 1) + .map_or(false, |start| start & !(alignment - 1) >= range.0) + }) + .cloned() + }; + + match region { Some(slot) => { self.regions.remove(&slot); - let start = match slot.0 % alignment { - 0 => slot.0, - r => slot.0 + alignment - r, + let start = if !reverse { + match slot.0 % alignment { + 0 => slot.0, + r => slot.0 + alignment - r, + } + } else { + (slot.1 - (size - 1)) & !(alignment - 1) }; let end = start + size - 1; if slot.0 < start { @@ -118,6 +134,32 @@ impl AddressAllocator { } } + /// Allocates a range of addresses from the reverse managed region with an optional tag + /// and minimal alignment. Returns allocated_address. (allocated_address, size, tag) + /// can be retrieved through the `get` method. + pub fn reverse_allocate_with_align( + &mut self, + size: u64, + alloc: Alloc, + tag: String, + alignment: u64, + ) -> Result { + self.internal_allocate_with_align(size, alloc, tag, alignment, true) + } + + /// Allocates a range of addresses from the managed region with an optional tag + /// and minimal alignment. Returns allocated_address. (allocated_address, size, tag) + /// can be retrieved through the `get` method. + pub fn allocate_with_align( + &mut self, + size: u64, + alloc: Alloc, + tag: String, + alignment: u64, + ) -> Result { + self.internal_allocate_with_align(size, alloc, tag, alignment, false) + } + pub fn allocate(&mut self, size: u64, alloc: Alloc, tag: String) -> Result { self.allocate_with_align(size, alloc, tag, self.alignment) } diff --git a/src/linux.rs b/src/linux.rs index f5782e70e4..442d9be311 100644 --- a/src/linux.rs +++ b/src/linux.rs @@ -1157,7 +1157,7 @@ fn create_pmem_device( let mapping_address = resources .mmio_allocator(MmioType::High) - .allocate_with_align( + .reverse_allocate_with_align( arena_size, Alloc::PmemDevice(index), format!("pmem_disk_image_{}", index), diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs index 6a7f2f69fb..cc61eaf497 100644 --- a/x86_64/src/lib.rs +++ b/x86_64/src/lib.rs @@ -362,7 +362,7 @@ impl arch::LinuxArch for X8664arch { SystemAllocator::builder() .add_io_addresses(0xc000, 0x10000) .add_low_mmio_addresses(END_ADDR_BEFORE_32BITS, MMIO_SIZE) - .add_high_mmio_addresses(high_mmio_start, u64::max_value() - high_mmio_start) + .add_high_mmio_addresses(high_mmio_start, Self::get_phys_max_addr() - high_mmio_start) .create_allocator(X86_64_IRQ_BASE) .unwrap() }