Vfio-pci: Release irq_num at pci device remove

Device gets irq_num from system_allocator when device is added, this
irq_num should return to system_allocator when device is hotplug removed.

this commit adds a ReleseOneIrq interface into VmIrqRequest, it release
one gsi into system_allocator. So vfio-pci device could call it at device
close function.

For msi and msix interrupt vectors, they have irq tube already and could
call ReleaseOneIrq easily.
For legacy INTx, the gsi for vfio-pci's INTx is gotten from host, and this
gsi maybe share with other device, so the commit doesn't release this gsi
at vfio-pci device's remove, otherwise the gsi shared device will be broken.

BUG=b:185084350
TEST=Boot a guest and hotplug vfio pci device repeatedly

Change-Id: Ibcca226b4b5b2092b5bc94bef8219eea82979086
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2955580
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>

devices/src/pci/msix.rs

@@ -506,6 +506,19 @@ impl MsixConfig {
             None => None,
         }
     }
+
+    pub fn destroy(&mut self) {
+        while let Some(irq) = self.irq_vec.pop() {
+            let request = VmIrqRequest::ReleaseOneIrq {
+                gsi: irq.gsi,
+                irqfd: irq.irqfd,
+            };
+            if self.msi_device_socket.send(&request).is_err() {
+                continue;
+            }
+            let _ = self.msi_device_socket.recv::<VmIrqResponse>();
+        }
+    }
 }
 
 impl AsRawDescriptor for MsixConfig {

devices/src/pci/vfio_pci.rs

@@ -230,6 +230,17 @@ impl VfioMsiCap {
     fn get_msi_irqfd(&self) -> Option<&Event> {
         self.irqfd.as_ref()
     }
+
+    fn destroy(&mut self) {
+        if let Some(gsi) = self.gsi {
+            if let Some(irqfd) = self.irqfd.take() {
+                let request = VmIrqRequest::ReleaseOneIrq { gsi, irqfd };
+                if self.vm_socket_irq.send(&request).is_ok() {
+                    let _ = self.vm_socket_irq.recv::<VmIrqResponse>();
+                }
+            }
+        }
+    }
 }
 
 // MSI-X registers in MSI-X capability
@@ -382,6 +393,10 @@ impl VfioMsixCap {
 
         Some(irqfds)
     }
+
+    fn destroy(&mut self) {
+        self.config.destroy()
+    }
 }
 
 struct VfioMsixAllocator {
@@ -825,6 +840,12 @@ impl VfioPciDevice {
     }
 
     fn close(&mut self) {
+        if let Some(msi) = self.msi_cap.as_mut() {
+            msi.destroy();
+        }
+        if let Some(msix) = self.msix_cap.as_mut() {
+            msix.destroy();
+        }
         self.disable_bars_mmap();
         self.device.close();
     }

resources/src/system_allocator.rs

@@ -129,6 +129,11 @@ impl SystemAllocator {
             .ok()
     }
 
+    /// release irq to system irq number pool
+    pub fn release_irq(&mut self, irq: u32) {
+        let _ = self.irq_allocator.release_containing(irq.into());
+    }
+
     /// Reserves the next available system irq number.
     pub fn reserve_irq(&mut self, irq: u32) -> bool {
         let id = self.get_anon_alloc();

src/linux.rs

@@ -3386,6 +3386,7 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
                                                     }
                                                 }
                                                 IrqSetup::Route(route) => irq_chip.route_irq(route),
+                                                IrqSetup::UnRegister(irq, ev) => irq_chip.unregister_irq_event(irq, ev),
                                             },
                                             &mut sys_allocator,
                                         )

vm_control/src/lib.rs

@@ -480,13 +480,20 @@ pub enum VmMemoryResponse {
 #[derive(Serialize, Deserialize, Debug)]
 pub enum VmIrqRequest {
     /// Allocate one gsi, and associate gsi to irqfd with register_irqfd()
-    AllocateOneMsi { irqfd: Event },
+    AllocateOneMsi {
+        irqfd: Event,
+    },
     /// Add one msi route entry into the IRQ chip.
     AddMsiRoute {
         gsi: u32,
         msi_address: u64,
         msi_data: u32,
     },
+    // unregister_irqfs() and release gsi
+    ReleaseOneIrq {
+        gsi: u32,
+        irqfd: Event,
+    },
 }
 
 /// Data to set up an IRQ event or IRQ route on the IRQ chip.
@@ -495,6 +502,7 @@ pub enum VmIrqRequest {
 pub enum IrqSetup<'a> {
     Event(u32, &'a Event),
     Route(IrqRoute),
+    UnRegister(u32, &'a Event),
 }
 
 impl VmIrqRequest {
@@ -539,6 +547,11 @@ impl VmIrqRequest {
                     Err(e) => VmIrqResponse::Err(e),
                 }
             }
+            ReleaseOneIrq { gsi, ref irqfd } => {
+                let _ = set_up_irq(IrqSetup::UnRegister(gsi, irqfd));
+                sys_allocator.release_irq(gsi);
+                VmIrqResponse::Ok
+            }
         }
     }
 }