Devices: Allocate non pci bar mmio at top of HighMmio

Some devices like virtio-pmem, intel host bridge need private mmio, these private
mmio isn't exposed to Vm through pci bar, and crosvm use mmio allocation interface
to allocate them, so guest OS pci bus driver couldn't detect them, and once guest
OS reallocate pci mmio, these private mmmio could be assigned to pci device bar.
then resource conflict happens.

This patch allocates these non pci bar mmio at the top of HighMmio, since HighMmio
is large enough and OS allocates pci mmio from low to high. So these non pci bar
mmio won't conflict with pci bar.

BUG=b:199442120
BUG=b:185084350
TEST=crosvm run --pmem-device xxxx

Change-Id: I615d7901d595dd46ba6362f88b71d96baa9c4c8f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3184549
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>

resources/src/address_allocator.rs

@@ -62,15 +62,13 @@ impl AddressAllocator {
         })
     }
 
-    /// Allocates a range of addresses from the managed region with an optional tag
-    /// and minimal alignment. Returns allocated_address. (allocated_address, size, tag)
-    /// can be retrieved through the `get` method.
-    pub fn allocate_with_align(
+    fn internal_allocate_with_align(
         &mut self,
         size: u64,
         alloc: Alloc,
         tag: String,
         alignment: u64,
+        reverse: bool,
     ) -> Result<u64> {
         let alignment = cmp::max(self.alignment, alignment);
 
@@ -84,24 +82,42 @@ impl AddressAllocator {
             return Err(Error::BadAlignment);
         }
 
-        // finds first region matching alignment and size.
-        match self
-            .regions
-            .iter()
-            .find(|range| {
-                match range.0 % alignment {
-                    0 => range.0.checked_add(size - 1),
-                    r => range.0.checked_add(size - 1 + alignment - r),
-                }
-                .map_or(false, |end| end <= range.1)
-            })
-            .cloned()
-        {
+        let region = if !reverse {
+            // finds first region matching alignment and size.
+            self.regions
+                .iter()
+                .find(|range| {
+                    match range.0 % alignment {
+                        0 => range.0.checked_add(size - 1),
+                        r => range.0.checked_add(size - 1 + alignment - r),
+                    }
+                    .map_or(false, |end| end <= range.1)
+                })
+                .cloned()
+        } else {
+            // finds last region matching alignment and size.
+            self.regions
+                .iter()
+                .rev()
+                .find(|range| {
+                    range
+                        .1
+                        .checked_sub(size - 1)
+                        .map_or(false, |start| start & !(alignment - 1) >= range.0)
+                })
+                .cloned()
+        };
+
+        match region {
             Some(slot) => {
                 self.regions.remove(&slot);
-                let start = match slot.0 % alignment {
-                    0 => slot.0,
-                    r => slot.0 + alignment - r,
+                let start = if !reverse {
+                    match slot.0 % alignment {
+                        0 => slot.0,
+                        r => slot.0 + alignment - r,
+                    }
+                } else {
+                    (slot.1 - (size - 1)) & !(alignment - 1)
                 };
                 let end = start + size - 1;
                 if slot.0 < start {
@@ -118,6 +134,32 @@ impl AddressAllocator {
         }
     }
 
+    /// Allocates a range of addresses from the reverse managed region with an optional tag
+    /// and minimal alignment. Returns allocated_address. (allocated_address, size, tag)
+    /// can be retrieved through the `get` method.
+    pub fn reverse_allocate_with_align(
+        &mut self,
+        size: u64,
+        alloc: Alloc,
+        tag: String,
+        alignment: u64,
+    ) -> Result<u64> {
+        self.internal_allocate_with_align(size, alloc, tag, alignment, true)
+    }
+
+    /// Allocates a range of addresses from the managed region with an optional tag
+    /// and minimal alignment. Returns allocated_address. (allocated_address, size, tag)
+    /// can be retrieved through the `get` method.
+    pub fn allocate_with_align(
+        &mut self,
+        size: u64,
+        alloc: Alloc,
+        tag: String,
+        alignment: u64,
+    ) -> Result<u64> {
+        self.internal_allocate_with_align(size, alloc, tag, alignment, false)
+    }
+
     pub fn allocate(&mut self, size: u64, alloc: Alloc, tag: String) -> Result<u64> {
         self.allocate_with_align(size, alloc, tag, self.alignment)
     }

src/linux.rs

@@ -1157,7 +1157,7 @@ fn create_pmem_device(
 
     let mapping_address = resources
         .mmio_allocator(MmioType::High)
-        .allocate_with_align(
+        .reverse_allocate_with_align(
             arena_size,
             Alloc::PmemDevice(index),
             format!("pmem_disk_image_{}", index),

x86_64/src/lib.rs

@@ -362,7 +362,7 @@ impl arch::LinuxArch for X8664arch {
         SystemAllocator::builder()
             .add_io_addresses(0xc000, 0x10000)
             .add_low_mmio_addresses(END_ADDR_BEFORE_32BITS, MMIO_SIZE)
-            .add_high_mmio_addresses(high_mmio_start, u64::max_value() - high_mmio_start)
+            .add_high_mmio_addresses(high_mmio_start, Self::get_phys_max_addr() - high_mmio_start)
             .create_allocator(X86_64_IRQ_BASE)
             .unwrap()
     }