qcow: add convert API and export it in qcow_utils

This will be used in vm_concierge's ExportDiskImage function in order to
allow a minimal qcow2 image to be written on the fly (containing only
the required clusters in a tightly-packed image file). It also allows
future flexibility to change the underlying disk image file format while
still exporting qcow2 images (e.g. via `vmc export`).

For testing, add a qcow_img `convert` command, which can convert
between raw and qcow2 as both source and destination.

BUG=None
TEST=Use qcow_img to convert a raw image to qcow2 and back and verify
     its contents are the same as the original.

Change-Id: I74167aca9a9c857d892e24adf5ee17afc0f6e6b5
Signed-off-by: Daniel Verkamp <dverkamp@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/1272060
Commit-Ready: ChromeOS CL Exonerator Bot <chromiumos-cl-exonerator@appspot.gserviceaccount.com>
Reviewed-by: Dylan Reid <dgreid@chromium.org>

qcow/src/qcow.rs

@@ -40,19 +40,27 @@ pub enum Error {
     InvalidRefcountTableOffset,
     NoRefcountClusters,
     OpeningFile(io::Error),
+    ReadingData(io::Error),
     ReadingHeader(io::Error),
     ReadingPointers(io::Error),
     ReadingRefCounts(io::Error),
     ReadingRefCountBlock(refcount::Error),
     SeekingFile(io::Error),
+    SettingFileSize(io::Error),
     SettingRefcountRefcount(io::Error),
     SizeTooSmallForNumberOfClusters,
     WritingHeader(io::Error),
     UnsupportedRefcountOrder,
     UnsupportedVersion(u32),
+    WritingData(io::Error),
 }
 pub type Result<T> = std::result::Result<T, Error>;
 
+pub enum ImageType {
+    Raw,
+    Qcow2,
+}
+
 // QCOW magic constant that starts the header.
 const QCOW_MAGIC: u32 = 0x5146_49fb;
 // Default to a cluster size of 2^DEFAULT_CLUSTER_BITS
@@ -1114,6 +1122,129 @@ fn div_round_up_u32(dividend: u32, divisor: u32) -> u32 {
     (dividend + divisor - 1) / divisor
 }
 
+fn convert_copy<R, W>(reader: &mut R, writer: &mut W, offset: u64, size: u64) -> Result<()>
+where
+    R: Read + Seek,
+    W: Write + Seek,
+{
+    const CHUNK_SIZE: usize = 65536;
+    let mut buf = [0; CHUNK_SIZE];
+    let mut read_count = 0;
+    reader
+        .seek(SeekFrom::Start(offset))
+        .map_err(Error::SeekingFile)?;
+    writer
+        .seek(SeekFrom::Start(offset))
+        .map_err(Error::SeekingFile)?;
+    loop {
+        let this_count = min(CHUNK_SIZE as u64, size - read_count) as usize;
+        let nread = reader
+            .read(&mut buf[..this_count])
+            .map_err(Error::ReadingData)?;
+        writer.write(&buf[..nread]).map_err(Error::WritingData)?;
+        read_count = read_count + nread as u64;
+        if nread == 0 || read_count == size {
+            break;
+        }
+    }
+
+    Ok(())
+}
+
+fn convert_reader_writer<R, W>(reader: &mut R, writer: &mut W, size: u64) -> Result<()>
+where
+    R: Read + Seek + SeekHole,
+    W: Write + Seek,
+{
+    let mut offset = 0;
+    while offset < size {
+        // Find the next range of data.
+        let next_data = match reader.seek_data(offset).map_err(Error::SeekingFile)? {
+            Some(o) => o,
+            None => {
+                // No more data in the file.
+                break;
+            }
+        };
+        let next_hole = match reader.seek_hole(next_data).map_err(Error::SeekingFile)? {
+            Some(o) => o,
+            None => {
+                // This should not happen - there should always be at least one hole
+                // after any data.
+                return Err(Error::SeekingFile(io::Error::from_raw_os_error(EINVAL)));
+            }
+        };
+        let count = next_hole - next_data;
+        convert_copy(reader, writer, next_data, count)?;
+        offset = next_hole;
+    }
+
+    Ok(())
+}
+
+fn convert_reader<R>(reader: &mut R, dst_file: File, dst_type: ImageType) -> Result<()>
+where
+    R: Read + Seek + SeekHole,
+{
+    let src_size = reader.seek(SeekFrom::End(0)).map_err(Error::SeekingFile)?;
+    reader
+        .seek(SeekFrom::Start(0))
+        .map_err(Error::SeekingFile)?;
+
+    // Ensure the destination file is empty before writing to it.
+    dst_file.set_len(0).map_err(Error::SettingFileSize)?;
+
+    match dst_type {
+        ImageType::Qcow2 => {
+            let mut dst_writer = QcowFile::new(dst_file, src_size)?;
+            convert_reader_writer(reader, &mut dst_writer, src_size)
+        }
+        ImageType::Raw => {
+            let mut dst_writer = dst_file;
+            // Set the length of the destination file to convert it into a sparse file
+            // of the desired size.
+            dst_writer
+                .set_len(src_size)
+                .map_err(Error::SettingFileSize)?;
+            convert_reader_writer(reader, &mut dst_writer, src_size)
+        }
+    }
+}
+
+/// Copy the contents of a disk image in `src_file` into `dst_file`.
+/// The type of `src_file` is automatically detected, and the output file type is
+/// determined by `dst_type`.
+pub fn convert(src_file: File, dst_file: File, dst_type: ImageType) -> Result<()> {
+    let src_type = detect_image_type(&src_file)?;
+    match src_type {
+        ImageType::Qcow2 => {
+            let mut src_reader = QcowFile::from(src_file)?;
+            convert_reader(&mut src_reader, dst_file, dst_type)
+        }
+        ImageType::Raw => {
+            // src_file is a raw file.
+            let mut src_reader = src_file;
+            convert_reader(&mut src_reader, dst_file, dst_type)
+        }
+    }
+}
+
+/// Detect the type of an image file by checking for a valid qcow2 header.
+fn detect_image_type(file: &File) -> Result<ImageType> {
+    let mut f = file;
+    let orig_seek = f.seek(SeekFrom::Current(0)).map_err(Error::SeekingFile)?;
+    f.seek(SeekFrom::Start(0)).map_err(Error::SeekingFile)?;
+    let magic = f.read_u32::<BigEndian>().map_err(Error::ReadingHeader)?;
+    let image_type = if magic == QCOW_MAGIC {
+        ImageType::Qcow2
+    } else {
+        ImageType::Raw
+    };
+    f.seek(SeekFrom::Start(orig_seek))
+        .map_err(Error::SeekingFile)?;
+    Ok(image_type)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

qcow_utils/src/qcow_img.rs

@@ -41,6 +41,10 @@ fn show_usage(program_name: &str) {
         "{} dd <file_name> <source_file> - Write bytes from the raw source_file to the file.",
         program_name
     );
+    println!(
+        "{} convert <src_file> <dst_file> - Convert from src_file to dst_file.",
+        program_name
+    );
 }
 
 fn main() -> std::result::Result<(), ()> {
@@ -95,6 +99,14 @@ fn main() -> std::result::Result<(), ()> {
             };
             dd(&matches.free[1], &matches.free[2], count)
         }
+        "convert" => {
+            if matches.free.len() < 2 {
+                println!("Source and destination files are required.");
+                show_usage(&args[0]);
+                return Err(());
+            }
+            convert(&matches.free[1], &matches.free[2])
+        }
         c => {
             println!("invalid subcommand: {:?}", c);
             Err(())
@@ -270,3 +282,45 @@ fn dd(file_path: &str, source_path: &str, count: Option<usize>) -> std::result::
 
     Ok(())
 }
+
+// Reads the file at `src_path` and writes it to `dst_path`.
+// The output format is detected based on the `dst_path` file extension.
+fn convert(src_path: &str, dst_path: &str) -> std::result::Result<(), ()> {
+    let src_file = match OpenOptions::new().read(true).open(src_path) {
+        Ok(f) => f,
+        Err(_) => {
+            println!("Failed to open source file {}", src_path);
+            return Err(());
+        }
+    };
+
+    let dst_file = match OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(dst_path)
+    {
+        Ok(f) => f,
+        Err(_) => {
+            println!("Failed to open destination file {}", dst_path);
+            return Err(());
+        }
+    };
+
+    let dst_type = if dst_path.ends_with("qcow2") {
+        qcow::ImageType::Qcow2
+    } else {
+        qcow::ImageType::Raw
+    };
+
+    match qcow::convert(src_file, dst_file, dst_type) {
+        Ok(_) => {
+            println!("Converted {} to {}", src_path, dst_path);
+            Ok(())
+        }
+        Err(_) => {
+            println!("Failed to copy from {} to {}", src_path, dst_path);
+            Err(())
+        }
+    }
+}

qcow_utils/src/qcow_utils.h

@@ -13,6 +13,14 @@ extern "C" {
 // Create a basic, empty qcow2 file that can grow to `virtual_size` at `path`.
 int create_qcow_with_size(const char *path, uint64_t virtual_size);
 
+// Copy the source disk image from `src_fd` into `dst_fd` as a qcow2 image file.
+// Returns 0 on success or a negated errno value on failure.
+int convert_to_qcow2(int src_fd, int dst_fd);
+
+// Copy the source disk image from `src_fd` into `dst_fd` as a raw image file.
+// Returns 0 on success or a negated errno value on failure.
+int convert_to_raw(int src_fd, int dst_fd);
+
 #ifdef __cplusplus
 };
 #endif

qcow_utils/src/qcow_utils.rs

@@ -7,12 +7,13 @@
 extern crate libc;
 extern crate qcow;
 
-use libc::EINVAL;
+use libc::{EINVAL, EIO};
 use std::ffi::CStr;
-use std::fs::OpenOptions;
+use std::fs::{File, OpenOptions};
 use std::os::raw::{c_char, c_int};
+use std::os::unix::io::FromRawFd;
 
-use qcow::QcowFile;
+use qcow::{ImageType, QcowFile};
 
 #[no_mangle]
 pub unsafe extern "C" fn create_qcow_with_size(path: *const c_char, virtual_size: u64) -> c_int {
@@ -42,3 +43,27 @@ pub unsafe extern "C" fn create_qcow_with_size(path: *const c_char, virtual_size
         Err(_) => -1,
     }
 }
+
+#[no_mangle]
+pub unsafe extern "C" fn convert_to_qcow2(src_fd: c_int, dst_fd: c_int) -> c_int {
+    // The caller is responsible for passing valid file descriptors.
+    let src_file = File::from_raw_fd(src_fd);
+    let dst_file = File::from_raw_fd(dst_fd);
+
+    match qcow::convert(src_file, dst_file, ImageType::Qcow2) {
+        Ok(_) => 0,
+        Err(_) => -EIO,
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn convert_to_raw(src_fd: c_int, dst_fd: c_int) -> c_int {
+    // The caller is responsible for passing valid file descriptors.
+    let src_file = File::from_raw_fd(src_fd);
+    let dst_file = File::from_raw_fd(dst_fd);
+
+    match qcow::convert(src_file, dst_file, ImageType::Raw) {
+        Ok(_) => 0,
+        Err(_) => -EIO,
+    }
+}