From b4725823ce8a834fb4bc2f924d181f4efd12cd48 Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:06:28 +0300 Subject: overhauled CLI --- Cargo.toml | 17 ++ README.md | 67 +++++++- build.rs | 3 + src/attribute/mod.rs | 30 ++++ src/bin/mft_dump.rs | 376 ++++++++++++++++++++++++++++++++++++------ tests/fixtures.rs | 27 +++ tests/skeptic.rs | 1 + tests/test_cli.rs | 62 +++++++ tests/test_cli_interactive.rs | 82 +++++++++ 9 files changed, 611 insertions(+), 54 deletions(-) create mode 100644 build.rs create mode 100644 tests/fixtures.rs create mode 100644 tests/skeptic.rs create mode 100644 tests/test_cli.rs create mode 100644 tests/test_cli_interactive.rs diff --git a/Cargo.toml b/Cargo.toml index 3bc1afd..101d28a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,12 +27,29 @@ winstructs = "0.2.0" lru = "0.1.15" itertools = "0.8.0" +# `mft_dump` dependencies +simplelog = "0.5.3" +dialoguer = "0.4.0" +indoc = "0.3" + [dependencies.chrono] version = "0.4.6" features = ["serde"] [dev-dependencies] criterion = "0.2" +skeptic = "0.13" +assert_cmd = "0.11" +predicates = "1" +env_logger = "0.6.1" +tempfile = "3" + +# rexpect relies on unix process semantics, but it's only used for process interaction tests. +[target.'cfg(not(target_os = "windows"))'.dev-dependencies] +rexpect = "0.3" + +[build-dependencies] +skeptic = "0.13" [[bin]] name = "mft_dump" diff --git a/README.md b/README.md index f9df1e0..2a8428e 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,70 @@ [![Build Status](https://dev.azure.com/benamram/DFIR/_apis/build/status/omerbenamram.mft?branchName=master)](https://dev.azure.com/benamram/DFIR/_build/latest?definitionId=5&branchName=master) ![crates.io](https://img.shields.io/crates/v/mft.svg) -A work-in-progress cross platform MFT (master file table) parser. +# MFT + +This is a parser for the MFT (master file table) format. +Supported rust version is latest stable rust (minimum 1.34) or nightly. -Inspired by the work at: https://github.com/forensicmatt/RustyMft, but is not API compatible. +[Documentation](https://docs.rs/mft) + +Python bindings are available as well at https://github.com/omerbenamram/pymft-rs (and at PyPi https://pypi.org/project/mft/) + +## Features + - Implemented using 100% safe rust - and works on all platforms supported by rust (that have stdlib). + - Supports JSON and CSV outputs. + - Supports extracting resident data streams. + +## Installation (associated binary utility): + - Download latest executable release from https://github.com/omerbenamram/mft/releases + - Releases are automatically built for for Windows, macOS, and Linux. (64-bit executables only) + - Build from sources using `cargo install mft` + +# `mft_dump` (Binary utility): +The main binary utility provided with this crate is `mft_dump`, and it provides a quick way to convert mft snapshots to different output formats. + +Some examples + - `mft_dump ` will dump contents of mft entries as JSON. + - `mft_dump -o csv ` will dump contents of mft entries as CSV. + - `mft_dump -e -o json ` will extract all resident streams in MFT to files. + +# Library usage: +```rust +use mft::MftParser; +use mft::attribute::MftAttributeContent; +use std::path::PathBuf; + +fn main() { + // Change this to a path of your .evtx sample. + let fp = PathBuf::from(format!("{}/samples/MFT", std::env::var("CARGO_MANIFEST_DIR").unwrap())); + + let mut parser = MftParser::from_path(fp).unwrap(); + for entry in parser.iter_entries() { + match entry { + Ok(e) => { + for attribute in e.iter_attributes().filter_map(|attr| attr.ok()) { + match attribute.data { + MftAttributeContent::AttrX10(standard_info) => { + println!("\tX10 attribute: {:#?}", standard_info) + }, + MftAttributeContent::AttrX30(filename_attribute) => { + println!("\tX30 attribute: {:#?}", filename_attribute) + }, + _ => { + println!("\tSome other attribute: {:#?}", attribute) + } + } + + } + } + Err(err) => eprintln!("{}", err), + } + } +} +``` + +## Thanks/Resources: + - https://docs.microsoft.com/en-us/windows/desktop/DevNotes/master-file-table + - https://github.com/libyal/libfsntfs/blob/master/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc + - https://github.com/forensicmatt/RustyMft diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..64b479b --- /dev/null +++ b/build.rs @@ -0,0 +1,3 @@ +fn main() { + skeptic::generate_doc_tests(&["README.md"]); +} diff --git a/src/attribute/mod.rs b/src/attribute/mod.rs index c397a2e..e50f21c 100644 --- a/src/attribute/mod.rs +++ b/src/attribute/mod.rs @@ -67,6 +67,36 @@ impl MftAttributeContent { } } + /// Converts the given attributes into a `IndexRootAttr`, consuming the object attribute object. + pub fn into_index_root(self) -> Option { + match self { + MftAttributeContent::AttrX90(content) => Some(content), + _ => None, + } + } + /// Converts the given attributes into a `ObjectIdAttr`, consuming the object attribute object. + pub fn into_object_id(self) -> Option { + match self { + MftAttributeContent::AttrX40(content) => Some(content), + _ => None, + } + } + /// Converts the given attributes into a `StandardInfoAttr`, consuming the object attribute object. + pub fn into_standard_info(self) -> Option { + match self { + MftAttributeContent::AttrX10(content) => Some(content), + _ => None, + } + } + /// Converts the given attributes into a `DataAttr`, consuming the object attribute object. + pub fn into_data(self) -> Option { + match self { + MftAttributeContent::AttrX80(content) => Some(content), + _ => None, + } + } + + /// Converts the given attributes into a `FileNameAttr`, consuming the object attribute object. pub fn into_file_name(self) -> Option { match self { MftAttributeContent::AttrX30(content) => Some(content), diff --git a/src/bin/mft_dump.rs b/src/bin/mft_dump.rs index fdb1f26..aff2cb0 100644 --- a/src/bin/mft_dump.rs +++ b/src/bin/mft_dump.rs @@ -1,17 +1,34 @@ use clap::{App, Arg, ArgMatches}; use env_logger; -use log::info; +use indoc::indoc; +use log::{info, Level}; +use mft::attribute::{MftAttributeContent, MftAttributeType}; use mft::mft::MftParser; use mft::{MftEntry, ReadSeek}; +use dialoguer::Confirmation; use mft::csv::FlatMftEntryWithName; -use std::io; + +use snafu::{Backtrace, ErrorCompat, Snafu}; +use std::fs::File; use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::process::exit; + +use std::{fs, io, path}; + +/// Simple error macro for use inside of internal errors in `EvtxDump` +macro_rules! err { + ($($tt:tt)*) => { Err(Box::::from(format!($($tt)*))) } +} + +type StdErr = Box; +#[derive(Debug, PartialOrd, PartialEq)] enum OutputFormat { JSON, + JSONL, CSV, } @@ -19,6 +36,7 @@ impl OutputFormat { pub fn from_str(s: &str) -> Option { match s { "json" => Some(OutputFormat::JSON), + "jsonl" => Some(OutputFormat::JSONL), "csv" => Some(OutputFormat::CSV), _ => None, } @@ -27,87 +45,295 @@ impl OutputFormat { struct MftDump { filepath: PathBuf, - indent: bool, + // We use an option here to be able to move the output out of mftdump from a mutable reference. + output: Option>, + data_streams_output: Option, + verbosity_level: Option, output_format: OutputFormat, + backtraces: bool, } impl MftDump { - pub fn from_cli_matches(matches: &ArgMatches) -> Self { - MftDump { + pub fn from_cli_matches(matches: &ArgMatches) -> Result { + let output_format = + OutputFormat::from_str(matches.value_of("output-format").unwrap_or_default()) + .expect("Validated with clap default values"); + + let backtraces = matches.is_present("backtraces"); + + let output: Option> = if let Some(path) = matches.value_of("output-target") { + match Self::create_output_file(path, !matches.is_present("no-confirm-overwrite")) { + Ok(f) => Some(Box::new(f)), + Err(e) => { + return err!( + "An error occurred while creating output file at `{}` - `{}`", + path, + e + ); + } + } + } else { + Some(Box::new(io::stdout())) + }; + + let data_streams_output = if let Some(path) = matches.value_of("data-streams-target") { + let path = PathBuf::from(path); + Self::create_output_dir(&path)?; + Some(path) + } else { + None + }; + + let verbosity_level = match matches.occurrences_of("verbose") { + 0 => None, + 1 => Some(Level::Info), + 2 => Some(Level::Debug), + 3 => Some(Level::Trace), + _ => { + eprintln!("using more than -vvv does not affect verbosity level"); + Some(Level::Trace) + } + }; + + Ok(MftDump { filepath: PathBuf::from(matches.value_of("INPUT").expect("Required argument")), - indent: !matches.is_present("no-indent"), - output_format: OutputFormat::from_str( - matches.value_of("output-format").unwrap_or_default(), - ) - .expect("Validated with clap default values"), - } + output, + data_streams_output, + verbosity_level, + output_format, + backtraces, + }) } - pub fn print_json_entry(&self, entry: &MftEntry) { - let json_str = if self.indent { - serde_json::to_string_pretty(&entry).expect("It should be valid UTF-8") + fn create_output_dir(path: impl AsRef) -> Result<(), StdErr> { + let p = path.as_ref(); + + if p.exists() { + if !p.is_dir() { + return err!("There is a file at {}, refusing to overwrite", p.display()); + } + // p exists and is a directory, it's ok to add files. } else { - serde_json::to_string(&entry).expect("It should be valid UTF-8") - }; + fs::create_dir_all(path)? + } - println!("{}", json_str); + Ok(()) } - pub fn print_csv_entry( - &self, - entry: &MftEntry, - parser: &mut MftParser, - writer: &mut csv::Writer, - ) { - let flat_entry = FlatMftEntryWithName::from_entry(&entry, parser); + /// If `prompt` is passed, will display a confirmation prompt before overwriting files. + fn create_output_file( + path: impl AsRef, + prompt: bool, + ) -> Result> { + let p = path.as_ref(); - writer.serialize(flat_entry).expect("Writing to CSV failed"); + if p.is_dir() { + return err!( + "There is a directory at {}, refusing to overwrite", + p.display() + ); + } + + if p.exists() { + if prompt { + match Confirmation::new() + .with_text(&format!( + "Are you sure you want to override output file at {}", + p.display() + )) + .default(false) + .interact() + { + Ok(true) => Ok(File::create(p)?), + Ok(false) => err!("Cancelled"), + Err(e) => err!( + "Failed to write confirmation prompt to term caused by\n{}", + e + ), + } + } else { + Ok(File::create(p)?) + } + } else { + // Ok to assume p is not an existing directory + match p.parent() { + Some(parent) => + // Parent exist + { + if parent.exists() { + Ok(File::create(p)?) + } else { + fs::create_dir_all(parent)?; + Ok(File::create(p)?) + } + } + None => err!("Output file cannot be root."), + } + } } - pub fn parse_file(&self) { - info!("Opening file {:?}", &self.filepath); - let mut mft_handler = match MftParser::from_path(&self.filepath) { - Ok(mft_handler) => mft_handler, - Err(error) => { - eprintln!( - "Failed to parse {:?}, failed with: [{}]", - &self.filepath, error - ); - std::process::exit(-1); + /// Main entry point for `EvtxDump` + pub fn run(&mut self) -> Result<(), StdErr> { + self.try_to_initialize_logging(); + + let mut parser = match MftParser::from_path(&self.filepath) { + Ok(parser) => parser, + Err(e) => { + return err!( + "Failed to open file {}.\n\tcaused by: {}", + self.filepath.display(), + &e + ) } }; + // Since the JSON parser can do away with a &mut Write, but the csv parser needs ownership + // of `Write`, we eagerly create the csv writer here, moving the Box out from + // `Mftdump` and replacing it with None placeholder. let mut csv_writer = match self.output_format { - OutputFormat::CSV => Some(csv::Writer::from_writer(io::stdout())), + OutputFormat::CSV => { + Some(csv::Writer::from_writer(self.output.take().expect( + "There can only be one flow accessing the output at a time", + ))) + } _ => None, }; - let number_of_entries = mft_handler.get_entry_count(); + let number_of_entries = parser.get_entry_count(); for i in 0..number_of_entries { - let entry = mft_handler.get_entry(i); + let entry = parser.get_entry(i); let entry = match entry { Ok(entry) => entry, Err(error) => { - eprintln!("Failed to parse MFT entry {}, failed with: [{}]", i, error); + eprintln!("{}", error); + + if self.backtraces { + if let Some(bt) = error.backtrace() { + eprintln!("{}", bt); + } + } continue; } }; + if let Some(data_streams_dir) = &self.data_streams_output { + if let Ok(Some(path)) = parser.get_full_path_for_entry(&entry) { + let sanitized_path = sanitized(&path.to_string_lossy().to_string()); + + for (i, (name, stream)) in entry + .iter_attributes() + .filter_map(|a| a.ok()) + .filter_map(|a| { + if a.header.type_code == MftAttributeType::DATA { + // resident + let name = a.header.name.clone(); + if let Some(data) = a.data.into_data() { + Some((name, data)) + } else { + None + } + } else { + None + } + }) + .enumerate() + { + let orig_path_component: String = data_streams_dir + .join(&sanitized_path) + .to_string_lossy() + .to_string() + .chars() + .take(150) + .collect(); + + let data_stream_path = format!( + "{parent_name}__{stream_number}_{stream_name}.dontrun", + parent_name = orig_path_component, + stream_number = i, + stream_name = name + ); + + let mut f = File::create(&data_stream_path)?; + f.write_all(stream.data())?; + } + } + } + match self.output_format { - OutputFormat::JSON => self.print_json_entry(&entry), + OutputFormat::JSON | OutputFormat::JSONL => self.print_json_entry(&entry)?, OutputFormat::CSV => self.print_csv_entry( &entry, - &mut mft_handler, + &mut parser, csv_writer .as_mut() .expect("CSV Writer is for OutputFormat::CSV"), - ), + )?, } } + + Ok(()) + } + + fn try_to_initialize_logging(&self) { + if let Some(level) = self.verbosity_level { + match simplelog::WriteLogger::init( + level.to_level_filter(), + simplelog::Config::default(), + io::stderr(), + ) { + Ok(_) => {} + Err(e) => eprintln!("Failed to initialize logging: {:?}", e), + }; + } + } + + pub fn print_json_entry(&mut self, entry: &MftEntry) -> Result<(), Box> { + let out = self + .output + .as_mut() + .expect("CSV Flow cannot occur, so `Mftdump` should still Own `output`"); + + let json_str = if self.output_format == OutputFormat::JSON { + serde_json::to_vec_pretty(&entry).expect("It should be valid UTF-8") + } else { + serde_json::to_vec(&entry).expect("It should be valid UTF-8") + }; + + out.write_all(&json_str)?; + out.write_all(b"\n")?; + + Ok(()) + } + + pub fn print_csv_entry( + &self, + entry: &MftEntry, + parser: &mut MftParser, + writer: &mut csv::Writer, + ) -> Result<(), Box> { + let flat_entry = FlatMftEntryWithName::from_entry(&entry, parser); + + writer.serialize(flat_entry)?; + + Ok(()) } } +// adapter from python version +// https://github.com/pallets/werkzeug/blob/9394af646038abf8b59d6f866a1ea5189f6d46b8/src/werkzeug/utils.py#L414 +pub fn sanitized(component: &str) -> String { + let mut buf = String::with_capacity(component.len()); + for c in component.chars() { + match c { + _sep if path::is_separator(c) => buf.push('_'), + _ => buf.push(c), + } + } + + buf +} + fn main() { env_logger::init(); @@ -116,23 +342,69 @@ fn main() { .author("Omer B. ") .about("Utility for parsing MFT snapshots") .arg(Arg::with_name("INPUT").required(true)) - .arg( - Arg::with_name("no-indent") - .long("--no-indent") - .takes_value(false) - .help("When set, output will not be indented (works only with JSON output)."), - ) .arg( Arg::with_name("output-format") .short("-o") .long("--output-format") .takes_value(true) - .possible_values(&["csv", "json"]) + .possible_values(&["csv", "json", "jsonl"]) .default_value("json") .help("Output format."), ) + .arg( + Arg::with_name("output-target") + .long("--output") + .short("-f") + .takes_value(true) + .help(indoc!("Writes output to the file specified instead of stdout, errors will still be printed to stderr. + Will ask for confirmation before overwriting files, to allow overwriting, pass `--no-confirm-overwrite` + Will create parent directories if needed.")), + ) + .arg( + Arg::with_name("data-streams-target") + .long("--extract-resident-streams") + .short("-e") + .takes_value(true) + .help(indoc!("Writes resident data streams to the given directory, resident streams will be named\ + _/.norun")), + ) + .arg( + Arg::with_name("no-confirm-overwrite") + .long("--no-confirm-overwrite") + .takes_value(false) + .help(indoc!("When set, will not ask for confirmation before overwriting files, useful for automation")), + ) + .arg(Arg::with_name("verbose") + .short("-v") + .multiple(true) + .takes_value(false) + .help(indoc!(r#" + Sets debug prints level for the application: + -v - info + -vv - debug + -vvv - trace + NOTE: trace output is only available in debug builds, as it is extremely verbose."#)) + ) + .arg( + Arg::with_name("backtraces") + .long("--backtraces") + .takes_value(false) + .help("If set, a backtrace will be printed with some errors if available")) .get_matches(); - let app = MftDump::from_cli_matches(&matches); - app.parse_file(); + let mut app = match MftDump::from_cli_matches(&matches) { + Ok(app) => app, + Err(e) => { + eprintln!("An error occurred while setting up the app: {}", &e); + exit(1); + } + }; + + match app.run() { + Ok(()) => {} + Err(e) => { + eprintln!("A runtime error has occurred {}", &e); + exit(1); + } + }; } diff --git a/tests/fixtures.rs b/tests/fixtures.rs new file mode 100644 index 0000000..17dfed6 --- /dev/null +++ b/tests/fixtures.rs @@ -0,0 +1,27 @@ +#![allow(dead_code)] +use std::path::PathBuf; + +use std::sync::{Once, ONCE_INIT}; + +static LOGGER_INIT: Once = ONCE_INIT; + +// Rust runs the tests concurrently, so unless we synchronize logging access +// it will crash when attempting to run `cargo test` with some logging facilities. +pub fn ensure_env_logger_initialized() { + LOGGER_INIT.call_once(env_logger::init); +} + +pub fn samples_dir() -> PathBuf { + PathBuf::from(file!()) + .parent() + .unwrap() + .parent() + .unwrap() + .join("samples") + .canonicalize() + .unwrap() +} + +pub fn mft_sample() -> PathBuf { + samples_dir().join("MFT") +} diff --git a/tests/skeptic.rs b/tests/skeptic.rs new file mode 100644 index 0000000..ff46c9c --- /dev/null +++ b/tests/skeptic.rs @@ -0,0 +1 @@ +include!(concat!(env!("OUT_DIR"), "/skeptic-tests.rs")); diff --git a/tests/test_cli.rs b/tests/test_cli.rs new file mode 100644 index 0000000..890b055 --- /dev/null +++ b/tests/test_cli.rs @@ -0,0 +1,62 @@ +mod fixtures; + +use fixtures::*; + +use assert_cmd::prelude::*; +use std::fs; +use std::fs::File; +use std::io::{Read, Write}; +use std::process::Command; +use tempfile::tempdir; + +#[test] +fn it_respects_directory_output() { + let d = tempdir().unwrap(); + let f = d.as_ref().join("test.out"); + + let sample = mft_sample(); + + let mut cmd = Command::cargo_bin("mft_dump").expect("failed to find binary"); + cmd.args(&["-f", &f.to_string_lossy(), sample.to_str().unwrap()]); + + assert!( + cmd.output().unwrap().stdout.is_empty(), + "Expected output to be printed to file, but was printed to stdout" + ); + + let mut expected = vec![]; + + File::open(&f).unwrap().read_to_end(&mut expected).unwrap(); + assert!( + !expected.is_empty(), + "Expected output to be printed to file" + ) +} + +#[test] +fn test_it_refuses_to_overwrite_directory() { + let d = tempdir().unwrap(); + + let sample = mft_sample(); + let mut cmd = Command::cargo_bin("mft_dump").expect("failed to find binary"); + cmd.args(&["-f", &d.path().to_string_lossy(), sample.to_str().unwrap()]); + + cmd.assert().failure().code(1); +} + +#[test] +fn test_it_exports_resident_streams() { + let d = tempdir().unwrap(); + + let sample = mft_sample(); + let mut cmd = Command::cargo_bin("mft_dump").expect("failed to find binary"); + cmd.args(&[ + "-e", + &d.path().to_string_lossy().to_string(), + &sample.to_string_lossy().to_string(), + ]); + + cmd.assert().success(); + + assert_eq!(fs::read_dir(d.path()).unwrap().count(), 1108) +} diff --git a/tests/test_cli_interactive.rs b/tests/test_cli_interactive.rs new file mode 100644 index 0000000..5f7111f --- /dev/null +++ b/tests/test_cli_interactive.rs @@ -0,0 +1,82 @@ +// The interactive tests are in a separate file, +// since they use `rexpect`, which internally uses quirky fork semantics to open a pty. +// They will fail if tried to be executed concurrently any other CLI test. + +mod fixtures; + +use fixtures::*; + +use std::fs::File; +use std::io::{Read, Write}; +use tempfile::tempdir; + +use assert_cmd::cargo::cargo_bin; +#[cfg(not(target_os = "windows"))] +use rexpect::spawn; + +// It should behave the same on windows, but interactive testing relies on unix pty internals. +#[test] +#[cfg(not(target_os = "windows"))] +fn test_it_confirms_before_overwriting_a_file() { + let d = tempdir().unwrap(); + let f = d.as_ref().join("test.out"); + + let mut file = File::create(&f).unwrap(); + file.write_all(b"I'm a file!").unwrap(); + + let sample = mft_sample(); + + let cmd_string = format!( + "{bin} -f {output_file} {sample}", + bin = cargo_bin("mft_dump").display(), + output_file = f.to_string_lossy(), + sample = sample.to_str().unwrap() + ); + + let mut p = spawn(&cmd_string, Some(10000)).unwrap(); + p.exp_regex(r#"Are you sure you want to override.*"#) + .unwrap(); + p.send_line("y").unwrap(); + p.exp_eof().unwrap(); + + let mut expected = vec![]; + + File::open(&f).unwrap().read_to_end(&mut expected).unwrap(); + assert!( + !expected.len() > 100, + "Expected output to be printed to file" + ) +} + +#[test] +#[cfg(not(target_os = "windows"))] +fn test_it_confirms_before_overwriting_a_file_and_quits() { + let d = tempdir().unwrap(); + let f = d.as_ref().join("test.out"); + + let mut file = File::create(&f).unwrap(); + file.write_all(b"I'm a file!").unwrap(); + + let sample = mft_sample(); + + let cmd_string = format!( + "{bin} -f {output_file} {sample}", + bin = cargo_bin("mft_dump").display(), + output_file = f.to_string_lossy(), + sample = sample.to_str().unwrap() + ); + + let mut p = spawn(&cmd_string, Some(10000)).unwrap(); + p.exp_regex(r#"Are you sure you want to override.*"#) + .unwrap(); + p.send_line("n").unwrap(); + p.exp_eof().unwrap(); + + let mut expected = vec![]; + + File::open(&f).unwrap().read_to_end(&mut expected).unwrap(); + assert!( + !expected.len() > 100, + "Expected output to be printed to file" + ) +} -- cgit v1.2.3 From 2f6c0d02c39e896b39bd9d92b63f505abd93de32 Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:07:56 +0300 Subject: cargo fix --- src/bin/mft_dump.rs | 6 +++--- tests/test_cli.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bin/mft_dump.rs b/src/bin/mft_dump.rs index aff2cb0..7815da1 100644 --- a/src/bin/mft_dump.rs +++ b/src/bin/mft_dump.rs @@ -1,16 +1,16 @@ use clap::{App, Arg, ArgMatches}; use env_logger; use indoc::indoc; -use log::{info, Level}; +use log::Level; -use mft::attribute::{MftAttributeContent, MftAttributeType}; +use mft::attribute::MftAttributeType; use mft::mft::MftParser; use mft::{MftEntry, ReadSeek}; use dialoguer::Confirmation; use mft::csv::FlatMftEntryWithName; -use snafu::{Backtrace, ErrorCompat, Snafu}; +use snafu::ErrorCompat; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; diff --git a/tests/test_cli.rs b/tests/test_cli.rs index 890b055..b4397a1 100644 --- a/tests/test_cli.rs +++ b/tests/test_cli.rs @@ -5,7 +5,7 @@ use fixtures::*; use assert_cmd::prelude::*; use std::fs; use std::fs::File; -use std::io::{Read, Write}; +use std::io::{Read}; use std::process::Command; use tempfile::tempdir; -- cgit v1.2.3 From 24e48139e30a78e7675ae9bea3c1520346c25442 Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:14:31 +0300 Subject: fmt --- src/mft.rs | 3 --- tests/test_cli.rs | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mft.rs b/src/mft.rs index 4f16233..9d78543 100644 --- a/src/mft.rs +++ b/src/mft.rs @@ -5,9 +5,6 @@ use crate::{EntryHeader, ReadSeek}; use log::{debug, trace}; use snafu::ResultExt; - - - use lru::LruCache; use std::fs::{self, File}; use std::io::{BufReader, Cursor, SeekFrom}; diff --git a/tests/test_cli.rs b/tests/test_cli.rs index b4397a1..d167b3e 100644 --- a/tests/test_cli.rs +++ b/tests/test_cli.rs @@ -5,7 +5,7 @@ use fixtures::*; use assert_cmd::prelude::*; use std::fs; use std::fs::File; -use std::io::{Read}; +use std::io::Read; use std::process::Command; use tempfile::tempdir; -- cgit v1.2.3 From 816005abbac3ca1e307dfa0049eb65c43bc16ef8 Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:27:53 +0300 Subject: optimise execution time by 25% --- src/utils.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index fb9b812..25470e7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,11 +1,19 @@ use crate::ReadSeek; use byteorder::ReadBytesExt; use std::char::decode_utf16; +use std::fmt::Write; use std::io; pub fn to_hex_string(bytes: &[u8]) -> String { - let strs: Vec = bytes.iter().map(|b| format!("{:02X}", b)).collect(); - strs.join("") + let len = bytes.len(); + // Each byte is represented by 2 ascii bytes. + let mut s = String::with_capacity(len * 2); + + for byte in bytes { + write!(s, "{:02X}", byte).expect("Writing to an allocated string cannot fail"); + } + + s } /// Reads a utf16 string from the given stream. -- cgit v1.2.3 From c9a6557fc6b65aaa2a807691bace876eb46228ea Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:38:36 +0300 Subject: dont blow up stdout with output in doctests --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a8428e..449cf9a 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Some examples - `mft_dump -e -o json ` will extract all resident streams in MFT to files. # Library usage: -```rust +```rust,no_run use mft::MftParser; use mft::attribute::MftAttributeContent; use std::path::PathBuf; -- cgit v1.2.3 From a4f609d87c7eb29e37c7900d0cb40b7fd79fc84e Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:52:00 +0300 Subject: tiny things --- README.md | 2 +- src/bin/mft_dump.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 449cf9a..b425151 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The main binary utility provided with this crate is `mft_dump`, and it provides Some examples - `mft_dump ` will dump contents of mft entries as JSON. - `mft_dump -o csv ` will dump contents of mft entries as CSV. - - `mft_dump -e -o json ` will extract all resident streams in MFT to files. + - `mft_dump --extract-resident-streams -o json ` will extract all resident streams in MFT to files in . # Library usage: ```rust,no_run diff --git a/src/bin/mft_dump.rs b/src/bin/mft_dump.rs index 7815da1..28829e3 100644 --- a/src/bin/mft_dump.rs +++ b/src/bin/mft_dump.rs @@ -18,7 +18,7 @@ use std::process::exit; use std::{fs, io, path}; -/// Simple error macro for use inside of internal errors in `EvtxDump` +/// Simple error macro for use inside of internal errors in `MftDump` macro_rules! err { ($($tt:tt)*) => { Err(Box::::from(format!($($tt)*))) } } -- cgit v1.2.3 From b3ea99a193d4d7ad9a31a4e188371c867510bb84 Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Mon, 3 Jun 2019 16:57:03 +0300 Subject: use correct assert --- tests/test_cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.rs b/tests/test_cli.rs index d167b3e..0e093b4 100644 --- a/tests/test_cli.rs +++ b/tests/test_cli.rs @@ -58,5 +58,5 @@ fn test_it_exports_resident_streams() { cmd.assert().success(); - assert_eq!(fs::read_dir(d.path()).unwrap().count(), 1108) + assert_eq!(fs::read_dir(d.path()).unwrap().count(), 2146) } -- cgit v1.2.3 From 32cdab569d4d14df589a0e5123b134ea1a16e010 Mon Sep 17 00:00:00 2001 From: Omer Ben-Amram Date: Tue, 4 Jun 2019 00:00:14 +0300 Subject: fix collisions --- .gitignore | 1 + Cargo.toml | 1 + src/bin/mft_dump.rs | 42 ++++++++++++++++++++++++++++++++++-------- tests/test_cli.rs | 2 +- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index cb14a42..4dd8588 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock Cargo.lock +*.rmeta \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 101d28a..30a0494 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ num-derive = "0.2" winstructs = "0.2.0" lru = "0.1.15" itertools = "0.8.0" +rand = "0.6.5" # `mft_dump` dependencies simplelog = "0.5.3" diff --git a/src/bin/mft_dump.rs b/src/bin/mft_dump.rs index 28829e3..d24db4f 100644 --- a/src/bin/mft_dump.rs +++ b/src/bin/mft_dump.rs @@ -12,10 +12,12 @@ use mft::csv::FlatMftEntryWithName; use snafu::ErrorCompat; use std::fs::File; +use std::io::Read; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::exit; +use std::fmt::Write as FmtWrite; use std::{fs, io, path}; /// Simple error macro for use inside of internal errors in `MftDump` @@ -242,18 +244,29 @@ impl MftDump { let orig_path_component: String = data_streams_dir .join(&sanitized_path) .to_string_lossy() - .to_string() - .chars() - .take(150) - .collect(); + .to_string(); + // Add some random bits to prevent collisions + let random: [u8; 6] = rand::random(); + let rando_string: String = to_hex_string(&random); + + let truncated: String = orig_path_component.chars().take(150).collect(); let data_stream_path = format!( - "{parent_name}__{stream_number}_{stream_name}.dontrun", - parent_name = orig_path_component, + "{path}__{random}_{stream_number}_{stream_name}.dontrun", + path = truncated, + random = rando_string, stream_number = i, stream_name = name ); + if PathBuf::from(&data_stream_path).exists() { + return err!( + "Tried to override an existing stream {} already exists!\ + This is a bug, please report to github!", + data_stream_path + ); + } + let mut f = File::create(&data_stream_path)?; f.write_all(stream.data())?; } @@ -320,6 +333,18 @@ impl MftDump { } } +fn to_hex_string(bytes: &[u8]) -> String { + let len = bytes.len(); + // Each byte is represented by 2 ascii bytes. + let mut s = String::with_capacity(len * 2); + + for byte in bytes { + write!(s, "{:02X}", byte).expect("Writing to an allocated string cannot fail"); + } + + s +} + // adapter from python version // https://github.com/pallets/werkzeug/blob/9394af646038abf8b59d6f866a1ea5189f6d46b8/src/werkzeug/utils.py#L414 pub fn sanitized(component: &str) -> String { @@ -365,8 +390,9 @@ fn main() { .long("--extract-resident-streams") .short("-e") .takes_value(true) - .help(indoc!("Writes resident data streams to the given directory, resident streams will be named\ - _/.norun")), + .help(indoc!("Writes resident data streams to the given directory. + Resident streams will be named like - `{path}___{stream_number}_{stream_name}.dontrun` + random is added to prevent collisions.")), ) .arg( Arg::with_name("no-confirm-overwrite") diff --git a/tests/test_cli.rs b/tests/test_cli.rs index 0e093b4..b247ad5 100644 --- a/tests/test_cli.rs +++ b/tests/test_cli.rs @@ -58,5 +58,5 @@ fn test_it_exports_resident_streams() { cmd.assert().success(); - assert_eq!(fs::read_dir(d.path()).unwrap().count(), 2146) + assert_eq!(fs::read_dir(d.path()).unwrap().count(), 2142) } -- cgit v1.2.3