crosvm: extend suspend commands about generating sleepbtn event

As a part of full guest suspension/resumption support extend crosvm
suspend command about generating sleepbtn press. This allows to emulate
PM1 and inject the vSCI to the guest. When the guest kernel is built
with ACPI button support enabled, it allows to suspend guest with use of
some guest daemon listening on ACPI events e.g. acpid
(acpid detects the ACPI sleepbtn event and triggers the guest
 suspension).

During suspension process (VmRequest::Suspend), before VmResponse::Ok is
advertise, there is need to wait for actual guest suspension.
This is required, e.g. when crosvm suspend command is triggered
automatically by vm_concierge during host suspension process and
prevents suspending host before the guest is suspended.

Above is achieved by taking advantage of KVM_SYSTEM_EVENT_S2IDLE
request sent by hypervisor on behalf of a non-privileged VM. It is used
to wake up blocked thread, which is awaiting non-privileged guest
suspension to finish.

The enhanced suspend process introduced by this patch is triggered
only when the --s2idle flag is passed.

BUG=b:194391015
TEST=Suspend the guest by issuing "crosvm suspend
/run/vm/vm.<hash>/crosvm.sock" and make sure that the guest actually
triggers suspend process and finalizes it.

Change-Id: I38cf5a786275fc4afa4ba7642ef0be779f8f1548
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3602869
Tested-by: kokoro <noreply+kokoro@google.com>
Commit-Queue: Grzegorz Jaszczyk <jaszczyk@google.com>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>

src/linux/mod.rs

@@ -49,7 +49,7 @@ use hypervisor::{HypervisorCap, ProtectionType, Vm, VmCap};
 use minijail::{self, Minijail};
 use resources::{Alloc, SystemAllocator};
 use rutabaga_gfx::RutabagaGralloc;
-use sync::Mutex;
+use sync::{Condvar, Mutex};
 use vm_control::*;
 use vm_memory::{GuestAddress, GuestMemory, MemoryPolicy};
 
@@ -1773,6 +1773,8 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
     #[cfg(target_os = "android")]
     android::set_process_profiles(&cfg.task_profiles)?;
 
+    let guest_suspended_cvar = Arc::new((Mutex::new(false), Condvar::new()));
+
     for (cpu_id, vcpu) in vcpus.into_iter().enumerate() {
         let (to_vcpu_channel, from_main_channel) = mpsc::channel();
         let vcpu_affinity = match linux.vcpu_affinity.clone() {
@@ -1818,6 +1820,7 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
                 ),
             },
             cfg.userspace_msr.clone(),
+            guest_suspended_cvar.clone(),
         )?;
         vcpu_handles.push((handle, to_vcpu_channel));
     }
@@ -1982,6 +1985,7 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
                                             &mut linux.bat_control,
                                             &vcpu_handles,
                                             cfg.force_s2idle,
+                                            guest_suspended_cvar.clone(),
                                         ),
                                     };
 

src/linux/vcpu.rs

@@ -6,6 +6,7 @@ use std::collections::BTreeMap;
 use std::fs::{File, OpenOptions};
 use std::io::prelude::*;
 use std::sync::{mpsc, Arc, Barrier};
+use sync::{Condvar, Mutex};
 
 use std::thread;
 use std::thread::JoinHandle;
@@ -263,17 +264,30 @@ where
 }
 
 #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-fn handle_s2idle_request(_privileged_vm: bool) {}
+fn handle_s2idle_request(
+    _privileged_vm: bool,
+    _guest_suspended_cvar: &Arc<(Mutex<bool>, Condvar)>,
+) {
+}
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-fn handle_s2idle_request(privileged_vm: bool) {
+fn handle_s2idle_request(privileged_vm: bool, guest_suspended_cvar: &Arc<(Mutex<bool>, Condvar)>) {
     const POWER_STATE_FREEZE: &[u8] = b"freeze";
 
-    // For non privileged guests, we silently ignore the suspend request
+    // For non privileged guests, wake up blocked thread on condvar, which is awaiting
+    // non-privileged guest suspension to finish.
     if !privileged_vm {
+        let (lock, cvar) = &**guest_suspended_cvar;
+        let mut guest_suspended = lock.lock();
+        *guest_suspended = true;
+
+        cvar.notify_one();
+        info!("dbg: s2idle notified");
+
         return;
     }
 
+    // For privileged guests, proceed with the suspend request
     let mut power_state = match OpenOptions::new().write(true).open("/sys/power/state") {
         Ok(s) => s,
         Err(err) => {
@@ -307,6 +321,7 @@ fn vcpu_loop<V>(
     >,
     #[cfg(all(target_arch = "x86_64", feature = "gdb"))] guest_mem: GuestMemory,
     msr_handlers: MsrHandlers,
+    guest_suspended_cvar: Arc<(Mutex<bool>, Condvar)>,
 ) -> ExitState
 where
     V: VcpuArch + 'static,
@@ -471,7 +486,7 @@ where
                     return ExitState::Stop;
                 }
                 Ok(VcpuExit::SystemEventS2Idle) => {
-                    handle_s2idle_request(privileged_vm);
+                    handle_s2idle_request(privileged_vm, &guest_suspended_cvar);
                 }
                 #[rustfmt::skip] Ok(VcpuExit::Debug { .. }) => {
                     #[cfg(all(target_arch = "x86_64", feature = "gdb"))]
@@ -548,6 +563,7 @@ pub fn run_vcpu<V>(
     privileged_vm: bool,
     vcpu_cgroup_tasks_file: Option<File>,
     userspace_msr: BTreeMap<u32, MsrConfig>,
+    guest_suspended_cvar: Arc<(Mutex<bool>, Condvar)>,
 ) -> Result<JoinHandle<()>>
 where
     V: VcpuArch + 'static,
@@ -631,6 +647,7 @@ where
                     #[cfg(all(target_arch = "x86_64", feature = "gdb"))]
                     guest_mem,
                     msr_handlers,
+                    guest_suspended_cvar,
                 )
             };
 

vm_control/src/lib.rs

@@ -36,7 +36,7 @@ pub use balloon_control::BalloonStats;
 use balloon_control::{BalloonTubeCommand, BalloonTubeResult};
 
 use base::{
-    error, trace, warn, with_as_descriptor, AsRawDescriptor, Error as SysError, Event,
+    error, info, trace, warn, with_as_descriptor, AsRawDescriptor, Error as SysError, Event,
     ExternalMapping, FromRawDescriptor, IntoRawDescriptor, Killable, MappedRegion,
     MemoryMappingArena, MemoryMappingBuilder, MemoryMappingBuilderUnix, MmapError, Protection,
     Result, SafeDescriptor, SharedMemory, Tube, SIGRTMIN,
@@ -47,7 +47,7 @@ use rutabaga_gfx::{
     DrmFormat, ImageAllocationInfo, RutabagaGralloc, RutabagaGrallocFlags, RutabagaHandle,
     VulkanInfo,
 };
-use sync::Mutex;
+use sync::{Condvar, Mutex};
 use vm_memory::GuestAddress;
 
 /// Struct that describes the offset and stride of a plane located in GPU memory.
@@ -1022,6 +1022,35 @@ fn map_descriptor(
     }
 }
 
+fn generate_sleep_button_event(
+    pm: &mut Option<Arc<Mutex<dyn PmResource>>>,
+    guest_suspended_cvar: &Arc<(Mutex<bool>, Condvar)>,
+) {
+    // During suspend also emulate sleepbtn, which allows to suspend VM (if running e.g. acpid and
+    // reacts on sleep button events)
+    if let Some(pm) = pm {
+        pm.lock().slpbtn_evt();
+    } else {
+        error!("generating sleepbtn during suspend not supported");
+    }
+
+    let (lock, cvar) = &**guest_suspended_cvar;
+    let mut guest_suspended = lock.lock();
+
+    *guest_suspended = false;
+
+    // Wait for notification about guest suspension, if not received after 15sec,
+    // proceed anyway.
+    let result = cvar.wait_timeout(guest_suspended, std::time::Duration::from_secs(15));
+    guest_suspended = result.0;
+
+    if result.1.timed_out() {
+        warn!("Guest suspension timeout - proceeding anyway");
+    } else if *guest_suspended {
+        info!("Guest suspended");
+    }
+}
+
 impl VmRequest {
     /// Executes this request on the given Vm and other mutable state.
     ///
@@ -1039,6 +1068,7 @@ impl VmRequest {
         bat_control: &mut Option<BatControl>,
         vcpu_handles: &[(JoinHandle<()>, mpsc::Sender<VcpuControl>)],
         force_s2idle: bool,
+        guest_suspended_cvar: Arc<(Mutex<bool>, Condvar)>,
     ) -> VmResponse {
         match *self {
             VmRequest::Exit => {
@@ -1064,6 +1094,10 @@ impl VmRequest {
                 }
             }
             VmRequest::Suspend => {
+                if force_s2idle {
+                    generate_sleep_button_event(pm, &guest_suspended_cvar);
+                }
+
                 *run_mode = Some(VmRunMode::Suspending);
                 VmResponse::Ok
             }