add compression/decompression utility

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "avml"
-version = "0.1.1"
+version = "0.1.2"
 license = "MIT"
 description = "A portable volatile memory acquisition tool"
 authors = ["[email protected]"]
@@ -13,18 +13,20 @@ readme = "README.md"
 [features]
 default = ["put", "blobstore"]
 put = ["reqwest"]
-blobstore = ["azure", "retry", "tokio-core", "url"]
+blobstore = ["azure", "retry", "tokio-core", "url", "azure_sdk_core", "azure_sdk_storage_core"]
 
 [dependencies]
 elf = "0.0.10"
 byteorder = "1.3.2"
 clap = {version = "2.33.0", default-features = false}
 snap = "0.2.5"
-url = { version = "1.7.2", optional = true }
+url = { version = "2.1.0", optional = true }
 tokio-core = { version = "0.1.17", optional = true }
 retry = { version = "0.5.1", optional = true }
-reqwest = { version = "0.9.18", default-features = false, features = ["rustls-tls"], optional = true }
-azure = { version = "0.12.0", package = "azure_sdk_for_rust", optional = true}
+reqwest = { version = "0.9.19", default-features = false, features = ["rustls-tls"], optional = true }
+azure = { version = "0.23.1", package = "azure_sdk_storage_blob", optional = true}
+azure_sdk_core = { version = "0.20.3", optional = true }
+azure_sdk_storage_core  = { version = "0.20.4", optional = true }
 
 [profile.release]
 opt-level="z"

src/bin/avml-convert.rs

@@ -0,0 +1,82 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#[macro_use]
+extern crate clap;
+extern crate avml;
+extern crate byteorder;
+extern crate elf;
+extern crate snap;
+
+use clap::{App, Arg};
+use snap::Reader;
+use std::error::Error;
+use std::fs::metadata;
+use std::io::prelude::*;
+use std::io::SeekFrom;
+
+fn convert(src: String, dst: String, compress: bool) -> Result<(), Box<dyn Error>> {
+    let src_len = metadata(&src)?.len();
+    let mut image = avml::image::Image::new(1, &src, &dst)?;
+
+    loop {
+        let current = image.src.seek(SeekFrom::Current(0))?;
+        if current >= src_len {
+            break;
+        }
+
+        let header = avml::image::Header::read(&image.src)?;
+        let mut new_header = header.clone();
+        new_header.version = if compress { 2 } else { 1 };
+
+        match header.version {
+            1 => {
+                avml::image::copy_block(&new_header, &mut image.src, &mut image.dst)?;
+            }
+            2 => {
+                let mut reader = Reader::new(&image.src);
+                avml::image::copy_block(&new_header, &mut reader, &mut image.dst)?;
+                image.src.seek(SeekFrom::Current(8))?;
+            }
+            _ => unimplemented!(),
+        }
+    }
+
+    Ok(())
+}
+
+fn run_app() -> Result<(), Box<dyn Error>> {
+    let args = App::new("avml-convert")
+        .author(crate_authors!())
+        .about("AVML compress/decompress tool")
+        .version(crate_version!())
+        .args(&[
+            Arg::with_name("compress")
+                .long("compress")
+                .help("compress pages via snappy"),
+            Arg::with_name("source")
+                .help("name of the source file to read to on local system")
+                .required(true),
+            Arg::with_name("destination")
+                .help("name of the destination file to write to on local system")
+                .required(true),
+        ])
+        .get_matches();
+
+    let compress = args.is_present("compress");
+    let src = value_t!(args.value_of("source"), String)?;
+    let dst = value_t!(args.value_of("destination"), String)?;
+
+    convert(src, dst, compress)?;
+    Ok(())
+}
+
+fn main() {
+    ::std::process::exit(match run_app() {
+        Ok(_) => 0,
+        Err(err) => {
+            eprintln!("error: {:?}", err);
+            1
+        }
+    });
+}

src/main.rs → src/bin/avml.rs

@@ -3,34 +3,14 @@
 
 #[macro_use]
 extern crate clap;
-extern crate byteorder;
-extern crate elf;
-extern crate snap;
-
-#[cfg(feature = "put")]
-extern crate reqwest;
-
-#[cfg(feature = "blobstore")]
-extern crate azure;
-#[cfg(feature = "blobstore")]
-extern crate retry;
-#[cfg(feature = "blobstore")]
-extern crate tokio_core;
-#[cfg(feature = "blobstore")]
-extern crate url;
+extern crate avml;
 
 use clap::{App, Arg};
 use std::error::Error;
 #[cfg(any(feature = "blobstore", feature = "put"))]
 use std::fs;
 use std::fs::metadata;
-
-#[cfg(feature = "blobstore")]
-mod blobstore;
-mod image;
-mod iomem;
-#[cfg(feature = "put")]
-mod upload;
+use std::ops::Range;
 
 fn kcore(
     ranges: &[std::ops::Range<u64>],
@@ -41,7 +21,7 @@ fn kcore(
         return Err(From::from("locked down kcore"));
     }
 
-    let mut image = image::Image::new(version, "/proc/kcore", filename)?;
+    let mut image = avml::image::Image::new(version, "/proc/kcore", filename)?;
     let mut file = elf::File::open_stream(&mut image.src).expect("unable to analyze /proc/kcore");
     file.phdrs.retain(|&x| x.progtype == elf::types::PT_LOAD);
     file.phdrs.sort_by(|a, b| a.vaddr.cmp(&b.vaddr));
@@ -50,7 +30,7 @@ fn kcore(
     for range in ranges {
         for phdr in &file.phdrs {
             if range.start == phdr.vaddr - start {
-                image.handle_block(phdr.offset, range.start, phdr.memsz)?;
+                image.write_block(phdr.offset, Range{start: range.start, end: phdr.memsz})?;
             }
         }
     }
@@ -63,28 +43,24 @@ fn phys(
     mem: &str,
     version: u32,
 ) -> Result<(), Box<dyn Error>> {
-    let mut image = image::Image::new(version, mem, filename)?;
+    let mut image = avml::image::Image::new(version, mem, filename)?;
     for range in ranges {
-        image.handle_block(range.start, range.start, 1 + range.end - range.start)?;
+        image.write_block(range.start, range.clone())?;
     }
 
     Ok(())
 }
 
 macro_rules! try_method {
-    ($func:expr, $src:expr) => {{
-        eprintln!("trying {}", $src);
-        if let Err(err) = $func {
-            eprintln!("failed {}: {}", $src, err);
-        } else {
-            eprintln!("succeeded {}", $src);
+    ($func:expr) => {{
+        if $func.is_ok() {
             return Ok(());
         }
     }};
 }
 
 fn get_mem(src: Option<&str>, dst: &str, version: u32) -> Result<(), Box<dyn Error>> {
-    let ranges = iomem::parse("/proc/iomem")?;
+    let ranges = avml::iomem::parse("/proc/iomem")?;
 
     if let Some(source) = src {
         let result = match source {
@@ -99,9 +75,9 @@ fn get_mem(src: Option<&str>, dst: &str, version: u32) -> Result<(), Box<dyn Err
         return result;
     }
 
-    try_method!(phys(&ranges, dst, "/dev/crash", version), "/dev/crash");
-    try_method!(kcore(&ranges, dst, version), "/proc/kcore");
-    try_method!(phys(&ranges, dst, "/dev/mem", version), "/dev/mem");
+    try_method!(phys(&ranges, dst, "/dev/crash", version));
+    try_method!(kcore(&ranges, dst, version));
+    try_method!(phys(&ranges, dst, "/dev/mem", version));
 
     Err(From::from("unable to read physical memory"))
 }
@@ -161,7 +137,7 @@ fn run_app() -> Result<(), Box<dyn Error>> {
     {
         let url = args.value_of("url");
         if let Some(url) = url {
-            upload::put(&dst, url)?;
+            avml::upload::put(&dst, url)?;
             if args.is_present("delete") {
                 fs::remove_file(&dst)?;
             }
@@ -178,7 +154,7 @@ fn run_app() -> Result<(), Box<dyn Error>> {
         } * 1024
             * 1024;
         if let Some(sas_url) = sas_url {
-            blobstore::upload_sas(&dst, sas_url, sas_block_size)?;
+            avml::blobstore::upload_sas(&dst, sas_url, sas_block_size)?;
             delete = true;
         }
     }

src/blobstore.rs

@@ -1,9 +1,11 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-use azure::core::errors::AzureError;
 use azure::prelude::*;
-use azure::storage::blob::{BlobBlockType, BlockList};
+use azure_sdk_core::errors::AzureError;
+use azure_sdk_core::prelude::*;
+use azure_sdk_storage_core::prelude::*;
+
 use byteorder::{LittleEndian, WriteBytesExt};
 use retry::{delay::jitter, delay::Exponential, retry, OperationResult};
 use std::cmp;