devices: proxy - Add support for proxying PciDevices

PCI adds a configuration space to the existing memory mapped IO
supported by BusDevices.

Add the ability to set configuration space as optional to the BusDevice
trait so that ProxyDevice can be shared.

PCI devices can have more than one memory mapped region. Expand the bus
so that it has the ability to pass an absolute address instead of an
offset. This will allow the PCI device to know which BAR is being
written to.

Change-Id: I055cd516c49a74316a9547df471290f05d865b0a
Signed-off-by: Dylan Reid <dgreid@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/1103663
Reviewed-by: Sonny Rao <sonnyrao@chromium.org>

aarch64/src/lib.rs

@@ -227,11 +227,11 @@ impl arch::LinuxArch for AArch64 {
         let serial = Arc::new(Mutex::new(devices::Serial::new_out(
             com_evt_1_3.try_clone()?,
             Box::new(stdout()))));
-        bus.insert(serial.clone(), AARCH64_SERIAL_ADDR, AARCH64_SERIAL_SIZE)
+        bus.insert(serial.clone(), AARCH64_SERIAL_ADDR, AARCH64_SERIAL_SIZE, false)
             .expect("failed to add serial device");
 
         let rtc = Arc::new(Mutex::new(devices::pl030::Pl030::new(rtc_evt)));
-        bus.insert(rtc, AARCH64_RTC_ADDR, AARCH64_RTC_SIZE)
+        bus.insert(rtc, AARCH64_RTC_ADDR, AARCH64_RTC_SIZE, false)
             .expect("failed to add rtc device");
         Ok(())
     }

devices/src/bus.rs

@@ -36,26 +36,48 @@ pub enum Error {
 
 pub type Result<T> = result::Result<T, Error>;
 
+/// Holds a base and length representing the address space occupied by a `BusDevice`.
+///
+/// * base - The address at which the range start.
+/// * len - The length of the range in bytes.
+/// * full_addr - If true, return the full address from `get_device`, otherwise return the offset
+///               from `base`
 #[derive(Debug, Copy, Clone)]
-struct BusRange(u64, u64);
+pub struct BusRange {
+    pub base: u64,
+    pub len: u64,
+    pub full_addr: bool,
+}
+
+impl BusRange {
+    /// Returns true if `addr` is within the range.
+    pub fn contains(&self, addr: u64) -> bool {
+        self.base <= addr && addr < self.base + self.len
+    }
+
+    /// Returns true if there is overlap with the given range.
+    pub fn overlaps(&self, base: u64, len: u64) -> bool {
+        self.base < (base + len) && base < self.base + self.len
+    }
+}
 
 impl Eq for BusRange {}
 
 impl PartialEq for BusRange {
     fn eq(&self, other: &BusRange) -> bool {
-        self.0 == other.0
+        self.base == other.base
     }
 }
 
 impl Ord for BusRange {
     fn cmp(&self, other: &BusRange) -> Ordering {
-        self.0.cmp(&other.0)
+        self.base.cmp(&other.base)
     }
 }
 
 impl PartialOrd for BusRange {
     fn partial_cmp(&self, other: &BusRange) -> Option<Ordering> {
-        self.0.partial_cmp(&other.0)
+        self.base.partial_cmp(&other.base)
     }
 }
 
@@ -75,50 +97,41 @@ impl Bus {
     }
 
     fn first_before(&self, addr: u64) -> Option<(BusRange, &Mutex<BusDevice>)> {
-        // for when we switch to rustc 1.17: self.devices.range(..addr).iter().rev().next()
-        for (range, dev) in self.devices.iter().rev() {
-            if range.0 <= addr {
-                return Some((*range, dev));
-            }
-        }
-        None
+        let(range, dev) =  self.devices.range(..=BusRange {base:addr, len:1, full_addr: false})
+                                       .rev()
+                                       .next()?;
+        Some((*range, dev))
     }
 
     fn get_device(&self, addr: u64) -> Option<(u64, &Mutex<BusDevice>)> {
-        if let Some((BusRange(start, len), dev)) = self.first_before(addr) {
-            let offset = addr - start;
-            if offset < len {
-                return Some((offset, dev));
+        if let Some((range, dev)) = self.first_before(addr) {
+            let offset = addr - range.base;
+            if offset < range.len {
+                if range.full_addr {
+                    return Some((addr, dev));
+                } else {
+                    return Some((offset, dev));
+                }
             }
         }
         None
     }
 
     /// Puts the given device at the given address space.
-    pub fn insert(&mut self, device: Arc<Mutex<BusDevice>>, base: u64, len: u64) -> Result<()> {
+    pub fn insert(&mut self, device: Arc<Mutex<BusDevice>>, base: u64, len: u64, full_addr: bool)
+        -> Result<()>
+    {
         if len == 0 {
             return Err(Error::Overlap);
         }
 
-        // Reject all cases where the new device's base is within an old device's range.
-        if self.get_device(base).is_some() {
+        // Reject all cases where the new device's range overlaps with an existing device.
+        if self.devices.iter().any(|(range, _dev)| range.overlaps(base, len)) {
             return Err(Error::Overlap);
         }
 
-        // The above check will miss an overlap in which the new device's base address is before the
-        // range of another device. To catch that case, we search for a device with a range before
-        // the new device's range's end. If there is no existing device in that range that starts
-        // after the new device, then there will be no overlap.
-        if let Some((BusRange(start, _), _)) = self.first_before(base + len - 1) {
-            // Such a device only conflicts with the new device if it also starts after the new
-            // device because of our initial `get_device` check above.
-            if start >= base {
-                return Err(Error::Overlap);
-            }
-        }
-
         if self.devices
-               .insert(BusRange(base, len), device)
+               .insert(BusRange{base, len, full_addr}, device)
                .is_some() {
             return Err(Error::Overlap);
         }
@@ -177,24 +190,41 @@ mod tests {
     fn bus_insert() {
         let mut bus = Bus::new();
         let dummy = Arc::new(Mutex::new(DummyDevice));
-        assert!(bus.insert(dummy.clone(), 0x10, 0).is_err());
-        assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_ok());
-        assert!(bus.insert(dummy.clone(), 0x0f, 0x10).is_err());
-        assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_err());
-        assert!(bus.insert(dummy.clone(), 0x10, 0x15).is_err());
-        assert!(bus.insert(dummy.clone(), 0x12, 0x15).is_err());
-        assert!(bus.insert(dummy.clone(), 0x12, 0x01).is_err());
-        assert!(bus.insert(dummy.clone(), 0x0, 0x20).is_err());
-        assert!(bus.insert(dummy.clone(), 0x20, 0x05).is_ok());
-        assert!(bus.insert(dummy.clone(), 0x25, 0x05).is_ok());
-        assert!(bus.insert(dummy.clone(), 0x0, 0x10).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x10, 0, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, false).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x0f, 0x10, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x15, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x12, 0x15, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x12, 0x01, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x0, 0x20, false).is_err());
+        assert!(bus.insert(dummy.clone(), 0x20, 0x05, false).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x25, 0x05, false).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x0, 0x10, false).is_ok());
+    }
+
+    #[test]
+    fn bus_insert_full_addr() {
+        let mut bus = Bus::new();
+        let dummy = Arc::new(Mutex::new(DummyDevice));
+        assert!(bus.insert(dummy.clone(), 0x10, 0, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, true).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x0f, 0x10, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x15, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x12, 0x15, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x12, 0x01, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x0, 0x20, true).is_err());
+        assert!(bus.insert(dummy.clone(), 0x20, 0x05, true).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x25, 0x05, true).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x0, 0x10, true).is_ok());
     }
 
     #[test]
     fn bus_read_write() {
         let mut bus = Bus::new();
         let dummy = Arc::new(Mutex::new(DummyDevice));
-        assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, false).is_ok());
         assert!(bus.read(0x10, &mut [0, 0, 0, 0]));
         assert!(bus.write(0x10, &[0, 0, 0, 0]));
         assert!(bus.read(0x11, &mut [0, 0, 0, 0]));
@@ -211,7 +241,7 @@ mod tests {
     fn bus_read_write_values() {
         let mut bus = Bus::new();
         let dummy = Arc::new(Mutex::new(ConstantDevice));
-        assert!(bus.insert(dummy.clone(), 0x10, 0x10).is_ok());
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, false).is_ok());
 
         let mut values = [0, 1, 2, 3];
         assert!(bus.read(0x10, &mut values));
@@ -221,4 +251,42 @@ mod tests {
         assert_eq!(values, [5, 6, 7, 8]);
         assert!(bus.write(0x15, &values));
     }
+
+    #[test]
+    fn bus_read_write_full_addr_values() {
+        let mut bus = Bus::new();
+        let dummy = Arc::new(Mutex::new(ConstantDevice));
+        assert!(bus.insert(dummy.clone(), 0x10, 0x10, true).is_ok());
+
+        let mut values = [0u8; 4];
+        assert!(bus.read(0x10, &mut values));
+        assert_eq!(values, [0x10, 0x11, 0x12, 0x13]);
+        assert!(bus.write(0x10, &values));
+        assert!(bus.read(0x15, &mut values));
+        assert_eq!(values, [0x15, 0x16, 0x17, 0x18]);
+        assert!(bus.write(0x15, &values));
+    }
+
+    #[test]
+    fn bus_range_contains() {
+        let a = BusRange { base: 0x1000, len: 0x400, full_addr: false };
+        assert!(a.contains(0x1000));
+        assert!(a.contains(0x13ff));
+        assert!(!a.contains(0xfff));
+        assert!(!a.contains(0x1400));
+        assert!(a.contains(0x1200));
+    }
+
+    #[test]
+    fn bus_range_overlap() {
+        let a = BusRange { base: 0x1000, len: 0x400, full_addr: false };
+        assert!(a.overlaps(0x1000, 0x400));
+        assert!(a.overlaps(0xf00, 0x400));
+        assert!(a.overlaps(0x1000, 0x01));
+        assert!(a.overlaps(0xfff, 0x02));
+        assert!(a.overlaps(0x1100, 0x100));
+        assert!(a.overlaps(0x13ff, 0x100));
+        assert!(!a.overlaps(0x1400, 0x100));
+        assert!(!a.overlaps(0xf00, 0x100));
+    }
 }

devices/src/lib.rs

@@ -26,7 +26,7 @@ mod serial;
 pub mod pl030;
 pub mod virtio;
 
-pub use self::bus::{Bus, BusDevice};
+pub use self::bus::{Bus, BusDevice, BusRange};
 pub use self::cmos::Cmos;
 pub use self::pl030::Pl030;
 pub use self::i8042::I8042Device;

devices/src/proxy.rs

@@ -12,7 +12,7 @@ use std::os::unix::net::UnixDatagram;
 use std::process;
 use std::time::Duration;
 
-use byteorder::{NativeEndian, ByteOrder};
+use byteorder::{LittleEndian, NativeEndian, ByteOrder};
 
 use BusDevice;
 use io_jail::{self, Minijail};
@@ -40,7 +40,9 @@ const MSG_SIZE: usize = 24;
 enum Command {
     Read = 0,
     Write = 1,
-    Shutdown = 2,
+    ReadConfig = 2,
+    WriteConfig = 3,
+    Shutdown = 4,
 }
 
 fn child_proc(sock: UnixDatagram, device: &mut BusDevice) {
@@ -63,15 +65,31 @@ fn child_proc(sock: UnixDatagram, device: &mut BusDevice) {
         }
 
         let cmd = NativeEndian::read_u32(&buf[0..]);
-        let len = NativeEndian::read_u32(&buf[4..]) as usize;
-        let offset = NativeEndian::read_u64(&buf[8..]);
 
         let res = if cmd == Command::Read as u32 {
+            let len = NativeEndian::read_u32(&buf[4..]) as usize;
+            let offset = NativeEndian::read_u64(&buf[8..]);
             device.read(offset, &mut buf[16..16 + len]);
             handle_eintr!(sock.send(&buf))
         } else if cmd == Command::Write as u32 {
+            let len = NativeEndian::read_u32(&buf[4..]) as usize;
+            let offset = NativeEndian::read_u64(&buf[8..]);
             device.write(offset, &buf[16..16 + len]);
             handle_eintr!(sock.send(&buf))
+        } else if cmd == Command::ReadConfig as u32 {
+            let reg_idx = NativeEndian::read_u32(&buf[4..]) as usize;
+            let val = device.config_register_read(reg_idx);
+            buf[16] = val as u8;
+            buf[17] = (val >> 8) as u8;
+            buf[18] = (val >> 16) as u8;
+            buf[19] = (val >> 24) as u8;
+            handle_eintr!(sock.send(&buf))
+        } else if cmd == Command::WriteConfig as u32 {
+            let reg_idx = NativeEndian::read_u32(&buf[4..]) as usize;
+            let offset = u64::from(NativeEndian::read_u32(&buf[8..]));
+            let len = u64::from(NativeEndian::read_u32(&buf[16..]));
+            device.config_register_write(reg_idx, offset, &buf[20..(20 + len as usize)]);
+            handle_eintr!(sock.send(&buf))
         } else if cmd == Command::Shutdown as u32 {
             running = false;
             handle_eintr!(sock.send(&buf))
@@ -148,6 +166,20 @@ impl ProxyDevice {
         handle_eintr!(self.sock.send(&buf)).map(|_| ()).map_err(Error::Io)
     }
 
+    fn send_config_cmd(&self, cmd: Command, reg_idx: u32, offset: u64, data: &[u8])
+        -> Result<()>
+    {
+        let mut buf = [0; MSG_SIZE];
+        NativeEndian::write_u32(&mut buf[0..], cmd as u32);
+        NativeEndian::write_u32(&mut buf[4..], reg_idx);
+        NativeEndian::write_u64(&mut buf[8..], offset);
+        NativeEndian::write_u32(&mut buf[16..], data.len() as u32);
+        buf[20..20 + data.len()].clone_from_slice(data);
+        handle_eintr!(self.sock.send(&buf))
+            .map(|_| ())
+            .map_err(Error::Io)
+    }
+
     fn recv_resp(&self, data: &mut [u8]) -> Result<()> {
         let mut buf = [0; MSG_SIZE];
         handle_eintr!(self.sock.recv(&mut buf)).map_err(Error::Io)?;
@@ -160,6 +192,26 @@ impl ProxyDevice {
         let mut buf = [0; MSG_SIZE];
         handle_eintr!(self.sock.recv(&mut buf)).map(|_| ()).map_err(Error::Io)
     }
+
+    pub fn config_register_write(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
+        let res = self
+            .send_config_cmd(Command::WriteConfig, reg_idx as u32, offset, data)
+            .and_then(|_| self.wait());
+        if let Err(e) = res {
+            error!("failed write to child device process: {}", e);
+        }
+    }
+
+    pub fn config_register_read(&self, reg_idx: usize) -> u32 {
+        let mut data = [0u8; 4];
+        let res = self
+            .send_config_cmd(Command::ReadConfig, reg_idx as u32, 0, &[])
+            .and_then(|_| self.recv_resp(&mut data));
+        if let Err(e) = res {
+            error!("failed write to child device process: {}", e);
+        }
+        LittleEndian::read_u32(&data)
+    }
 }
 
 impl BusDevice for ProxyDevice {

src/linux.rs

@@ -320,13 +320,15 @@ fn register_mmio(bus: &mut devices::Bus,
         bus
             .insert(Arc::new(Mutex::new(proxy_dev)),
             mmio_base,
-            mmio_len)
+            mmio_len,
+            false)
             .unwrap();
     } else {
         bus
             .insert(Arc::new(Mutex::new(mmio_device)),
             mmio_base,
-            mmio_len)
+            mmio_len,
+            false)
             .unwrap();
     }
 

x86_64/src/lib.rs

@@ -348,37 +348,41 @@ impl arch::LinuxArch for X8664arch {
                                          map_err(|e| Error::CloneEventFd(e))?,
                                          Box::new(stdout()))));
         let nul_device = Arc::new(Mutex::new(NoDevice));
-        io_bus.insert(stdio_serial.clone(), 0x3f8, 0x8).unwrap();
+        io_bus.insert(stdio_serial.clone(), 0x3f8, 0x8, false).unwrap();
         io_bus.insert(Arc::new(Mutex::new(
             devices::Serial::new_sink(com_evt_2_4.try_clone().
                                       map_err(|e| Error::CloneEventFd(e))?))),
                       0x2f8,
-                      0x8)
+                      0x8,
+                      false)
             .unwrap();
         io_bus.insert(Arc::new(Mutex::new(
             devices::Serial::new_sink(com_evt_1_3.try_clone().
                                       map_err(|e| Error::CloneEventFd(e))?))),
                       0x3e8,
-                      0x8)
+                      0x8,
+                      false)
             .unwrap();
         io_bus.insert(Arc::new(Mutex::new(
             devices::Serial::new_sink(com_evt_2_4.try_clone().
                                       map_err(|e| Error::CloneEventFd(e))?))),
                       0x2e8,
-                      0x8)
+                      0x8,
+                      false)
             .unwrap();
-        io_bus.insert(Arc::new(Mutex::new(devices::Cmos::new())), 0x70, 0x2)
+        io_bus.insert(Arc::new(Mutex::new(devices::Cmos::new())), 0x70, 0x2, false)
             .unwrap();
         io_bus.insert(Arc::new(Mutex::new(
             devices::I8042Device::new(exit_evt.try_clone().
                                       map_err(|e| Error::CloneEventFd(e))?))),
                       0x061,
-                      0x4)
+                      0x4,
+                      false)
             .unwrap();
-        io_bus.insert(nul_device.clone(), 0x040, 0x8).unwrap(); // ignore pit
-        io_bus.insert(nul_device.clone(), 0x0ed, 0x1).unwrap(); // most likely this one does nothing
-        io_bus.insert(nul_device.clone(), 0x0f0, 0x2).unwrap(); // ignore fpu
-        io_bus.insert(nul_device.clone(), 0xcf8, 0x8).unwrap(); // ignore pci
+        io_bus.insert(nul_device.clone(), 0x040, 0x8, false).unwrap(); // ignore pit
+        io_bus.insert(nul_device.clone(), 0x0ed, 0x1, false).unwrap(); // most likely this one does nothing
+        io_bus.insert(nul_device.clone(), 0x0f0, 0x2, false).unwrap(); // ignore fpu
+        io_bus.insert(nul_device.clone(), 0xcf8, 0x8, false).unwrap(); // ignore pci
 
         vm.register_irqfd(&com_evt_1_3, 4).map_err(Error::RegisterIrqfd)?;
         vm.register_irqfd(&com_evt_2_4, 3).map_err(Error::RegisterIrqfd)?;