Merge pull request #38 from forensicmatt/attribute-updates

Attribute updates

7 files changed, 262 insertions, 34 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 878e430..dd589d1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.2] - 2020-03-10
+
+### Fixed
+- Attribute list parsing
+
+### Changed
+- Warn on fixup mismatch instead of fail
+
+### Added
+- Additional File Attribute flags
+- `valid_fixup` field to MftEntry
+
 ## [0.5.1] - 2020-02-06
 
 ### Fixed
diff --git a/Cargo.toml b/Cargo.toml
index f5dd314..5fdb27a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,7 +6,7 @@ repository = "https://github.com/omerbenamram/mft"
 license = "MIT/Apache-2.0"
 readme = "README.md"
 
-version = "0.5.2-alpha.0"
+version = "0.5.2"
 authors = ["Omer Ben-Amram <omerbenamram@gmail.com>"]
 edition = "2018"
 
diff --git a/samples/entry_102130_fixup_issue b/samples/entry_102130_fixup_issue
new file mode 100644
index 0000000..60d1b7c
--- /dev/null
+++ b/samples/entry_102130_fixup_issue
diff --git a/src/attribute/mod.rs b/src/attribute/mod.rs
index 8ac11ce..481d69c 100644
--- a/src/attribute/mod.rs
+++ b/src/attribute/mod.rs
@@ -10,6 +10,8 @@ pub mod x90;
 use crate::err::Result;
 use crate::{impl_serialize_for_bitflags, ReadSeek};
 
+use std::io::Cursor;
+
 use bitflags::bitflags;
 
 use crate::attribute::raw::RawAttribute;
@@ -39,9 +41,26 @@ impl MftAttributeContent {
             MftAttributeType::StandardInformation => Ok(MftAttributeContent::AttrX10(
                 StandardInfoAttr::from_reader(stream)?,
             )),
-            MftAttributeType::AttributeList => Ok(MftAttributeContent::AttrX20(
-                AttributeListAttr::from_stream(stream)?,
-            )),
+            MftAttributeType::AttributeList => {
+                // An attribute list is a buffer of attribute entries which are varying sizes if 
+                // the attributes contain names. Thus, we must know when to stop reading. To
+                // do this, we will create a buffer of the attribute, and stop reading attribute
+                // entries when we reach the end of the buffer.
+                let content_size = resident.data_size;
+                
+                let mut attribute_buffer = vec![0; content_size as usize];
+                stream.read_exact(&mut attribute_buffer)?;
+
+                // Create a new stream that the attribute will read from.
+                let mut new_stream = Cursor::new(attribute_buffer);
+
+                let attr_list = AttributeListAttr::from_stream(
+                    &mut new_stream, 
+                    Some(content_size as u64)
+                )?;
+
+                Ok(MftAttributeContent::AttrX20(attr_list))
+            },
             MftAttributeType::FileName => Ok(MftAttributeContent::AttrX30(
                 FileNameAttr::from_stream(stream)?,
             )),
@@ -67,6 +86,14 @@ impl MftAttributeContent {
         }
     }
 
+    /// Converts the given attributes into a 'AttributeListAttr', consuming the object attribute object.
+    pub fn into_attribute_list(self) -> Option<AttributeListAttr> {
+        match self {
+            MftAttributeContent::AttrX20(content) => Some(content),
+            _ => None,
+        }
+    }
+
     /// Converts the given attributes into a `IndexRootAttr`, consuming the object attribute object.
     pub fn into_index_root(self) -> Option<IndexRootAttr> {
         match self {
@@ -109,11 +136,11 @@ impl MftAttributeContent {
 #[serde(untagged)]
 pub enum MftAttributeContent {
     Raw(RawAttribute),
-    AttrX80(DataAttr),
     AttrX10(StandardInfoAttr),
     AttrX20(AttributeListAttr),
     AttrX30(FileNameAttr),
     AttrX40(ObjectIdAttr),
+    AttrX80(DataAttr),
     AttrX90(IndexRootAttr),
     /// Empty - used when data is non resident.
     None,
@@ -158,10 +185,15 @@ pub enum MftAttributeType {
 }
 
 bitflags! {
+    /// Flag sources:
+    /// https://github.com/EricZimmerman/MFT/blob/3bed2626ee85e9a96a6db70a17407d0c3696056a/MFT/Attributes/StandardInfo.cs#L10
+    /// https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-fscc/ca28ec38-f155-4768-81d6-4bfeb8586fc9
+    /// 
     pub struct FileAttributeFlags: u32 {
         const FILE_ATTRIBUTE_READONLY             = 0x0000_0001;
         const FILE_ATTRIBUTE_HIDDEN               = 0x0000_0002;
         const FILE_ATTRIBUTE_SYSTEM               = 0x0000_0004;
+        const FILE_ATTRIBUTE_DIRECTORY            = 0x0000_0010;
         const FILE_ATTRIBUTE_ARCHIVE              = 0x0000_0020;
         const FILE_ATTRIBUTE_DEVICE               = 0x0000_0040;
         const FILE_ATTRIBUTE_NORMAL               = 0x0000_0080;
@@ -172,6 +204,11 @@ bitflags! {
         const FILE_ATTRIBUTE_OFFLINE              = 0x0000_1000;
         const FILE_ATTRIBUTE_NOT_CONTENT_INDEXED  = 0x0000_2000;
         const FILE_ATTRIBUTE_ENCRYPTED            = 0x0000_4000;
+        const FILE_ATTRIBUTE_INTEGRITY_STREAM     = 0x0000_8000;
+        const FILE_ATTRIBUTE_NO_SCRUB_DATA        = 0x0002_0000;
+        const FILE_ATTRIBUTE_HAS_EA               = 0x0004_0000;
+        const FILE_ATTRIBUTE_IS_DIRECTORY         = 0x1000_0000;
+        const FILE_ATTRIBUTE_INDEX_VIEW           = 0x2000_0000;
     }
 }
 
diff --git a/src/attribute/x20.rs b/src/attribute/x20.rs
index 8437823..d196ae9 100644
--- a/src/attribute/x20.rs
+++ b/src/attribute/x20.rs
@@ -1,6 +1,5 @@
 use crate::err::{Error, Result};
 use crate::ReadSeek;
-use log::trace;
 
 use byteorder::{LittleEndian, ReadBytesExt};
 use encoding::all::UTF_16LE;
@@ -11,30 +10,174 @@ use serde::Serialize;
 use std::io::SeekFrom;
 use winstructs::ntfs::mft_reference::MftReference;
 
+
+/// The AttributeListAttr represents the $20 attribute, which contains a list
+/// of attribute entries in child entries.
+/// 
 #[derive(Serialize, Clone, Debug)]
 pub struct AttributeListAttr {
+    /// A list of AttributeListEntry that make up this AttributeListAttr
+    pub entries: Vec<AttributeListEntry>
+}
+impl AttributeListAttr {
+    /// Read AttributeListAttr from stream. Stream should be the size of the attribute's data itself
+    /// if no stream_size is passed in.
+    /// 
+    ///  # Example
+    ///
+    /// Parse a raw buffer.
+    ///
+    /// ```
+    /// use mft::attribute::x20::AttributeListAttr;
+    /// # use std::io::Cursor;
+    /// let attribute_content_buffer: &[u8] = &[
+    /// 	0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x1A,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    /// 	0x23,0x27,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x12,0x07,0x80,0xF8,0xFF,0xFF,
+    /// 	0x30,0x00,0x00,0x00,0x20,0x00,0x00,0x1A,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    /// 	0x23,0x27,0x00,0x00,0x00,0x00,0x01,0x00,0x03,0x00,0x00,0x00,0x69,0x00,0x6E,0x00,
+    /// 	0x30,0x00,0x00,0x00,0x20,0x00,0x00,0x1A,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    /// 	0x0F,0xCF,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x8A,0x0C,0xA0,0xF8,0xFF,0xFF,
+    /// 	0x90,0x00,0x00,0x00,0x28,0x00,0x04,0x1A,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    /// 	0x0F,0xCF,0x01,0x00,0x00,0x00,0x02,0x00,0x01,0x00,0x24,0x00,0x49,0x00,0x33,0x00,
+    /// 	0x30,0x00,0x79,0x00,0x73,0x00,0xAD,0xEF,0xA0,0x00,0x00,0x00,0x28,0x00,0x04,0x1A,
+    /// 	0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0F,0xCF,0x01,0x00,0x00,0x00,0x02,0x00,
+    /// 	0x02,0x00,0x24,0x00,0x49,0x00,0x33,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x78,0x56,
+    /// 	0xB0,0x00,0x00,0x00,0x28,0x00,0x04,0x1A,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    /// 	0x0F,0xCF,0x01,0x00,0x00,0x00,0x02,0x00,0x03,0x00,0x24,0x00,0x49,0x00,0x33,0x00,
+    /// 	0x30,0x00,0x00,0x00,0x00,0x00,0x65,0x00,0x00,0x01,0x00,0x00,0x30,0x00,0x09,0x1A,
+    /// 	0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x23,0x27,0x00,0x00,0x00,0x00,0x01,0x00,
+    /// 	0x08,0x00,0x24,0x00,0x54,0x00,0x58,0x00,0x46,0x00,0x5F,0x00,0x44,0x00,0x41,0x00,
+    /// 	0x54,0x00,0x41,0x00,0x00,0x00,0x00,0x00
+    /// ];
+    ///
+    /// let attribute_list = AttributeListAttr::from_stream(
+    ///     &mut Cursor::new(attribute_content_buffer),
+    ///     None
+    /// ).unwrap();
+    ///
+    /// assert_eq!(attribute_list.entries.len(), 7);
+    /// ```
+    pub fn from_stream<S: ReadSeek>(
+        mut stream: &mut S,
+        stream_size: Option<u64>
+    ) -> Result<AttributeListAttr> {
+        let mut start_offset = stream.tell()?;
+        let end_offset = match stream_size {
+            Some(s) => s,
+            None => {
+                // If no stream size was passed in we seek to the end of the stream,
+                // then tell to get the ending offset, then seek back to the start,
+                // thus, its better to just pass the stream size.
+                stream.seek(
+                    SeekFrom::End(0)
+                )?;
+
+                let offset = stream.tell()?;
+
+                stream.seek(
+                    SeekFrom::Start(0)
+                )?;
+
+                offset
+            }
+        };
+
+        let mut entries: Vec<AttributeListEntry> = Vec::new();
+
+        // iterate attribute content parsing attribute list entries
+        while start_offset < end_offset {
+            // parse the entry from the stream
+            let attr_entry = AttributeListEntry::from_stream(
+                &mut stream
+            )?;
+
+            // update the starting offset
+            start_offset += attr_entry.record_length as u64;
+
+            // add attribute entry to entries vec
+            entries.push(attr_entry);
+
+            // seek the stream to next start offset to avoid padding
+            stream.seek(
+                SeekFrom::Start(start_offset)
+            )?;
+        }
+
+        Ok(Self{
+            entries
+        })
+    }
+}
+
+
+/// An AttributeListAttr is made up off multiple AttributeListEntry structs.
+/// https://docs.microsoft.com/en-us/windows/win32/devnotes/attribute-list-entry
+/// 
+#[derive(Serialize, Clone, Debug)]
+pub struct AttributeListEntry {
+    /// The attribute code
     pub attribute_type: u32,
+    /// This entry length
     pub record_length: u16,
-    pub first_vcn: u64,
-    pub base_reference: MftReference,
-    pub attribute_id: u16,
+    /// Attribute name length (0 means no name)
+    pub name_length: u8,
+    /// Attribute name offset
+    pub name_offset: u8,
+    /// This member is zero unless the attribute requires multiple file record
+    /// segments and unless this entry is a reference to a segment other than the first one.
+    /// In this case, this value is the lowest VCN that is described by the referenced segment.
+    pub lowest_vcn: u64,
+    /// The segments MFT reference
+    pub segment_reference: MftReference,
+    /// The attribute's id
+    pub reserved: u16,
+    /// The attribute's name
     pub name: String,
 }
-
-impl AttributeListAttr {
-    pub fn from_stream<S: ReadSeek>(stream: &mut S) -> Result<AttributeListAttr> {
+impl AttributeListEntry {
+    /// Create AttributeListEntry from a stream.
+    /// 
+    ///  # Example
+    ///
+    /// Parse a raw buffer.
+    ///
+    /// ```
+    /// use mft::attribute::x20::AttributeListEntry;
+    /// # use std::io::Cursor;
+    /// let attribute_buffer: &[u8] = &[
+    /// 	0x10,0x00,0x00,0x00,0x20,0x00,0x00,0x1A,
+    /// 	0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+    ///     0x23,0x27,0x00,0x00,0x00,0x00,0x01,0x00,
+    /// 	0x00,0x00,0x12,0x07,0x80,0xF8,0xFF,0xFF
+    /// ];
+    ///
+    /// let attribute_entry = AttributeListEntry::from_stream(
+    ///     &mut Cursor::new(attribute_buffer)
+    /// ).unwrap();
+    ///
+    /// assert_eq!(attribute_entry.attribute_type, 16);
+    /// assert_eq!(attribute_entry.record_length, 32);
+    /// assert_eq!(attribute_entry.name_length, 0);
+    /// assert_eq!(attribute_entry.name_offset, 26);
+    /// assert_eq!(attribute_entry.lowest_vcn, 0);
+    /// assert_eq!(attribute_entry.segment_reference.entry, 10019);
+    /// assert_eq!(attribute_entry.segment_reference.sequence, 1);
+    /// assert_eq!(attribute_entry.reserved, 0);
+    /// assert_eq!(attribute_entry.name, "".to_string());
+    /// ```
+    pub fn from_stream<S: ReadSeek>(
+        stream: &mut S
+    ) -> Result<AttributeListEntry> {
         let start_offset = stream.tell()?;
-
-        trace!("Offset {}: AttributeListAttr", start_offset);
-
+        
         let attribute_type = stream.read_u32::<LittleEndian>()?;
         let record_length = stream.read_u16::<LittleEndian>()?;
         let name_length = stream.read_u8()?;
         let name_offset = stream.read_u8()?;
-        let first_vcn = stream.read_u64::<LittleEndian>()?;
-        let base_reference =
-            MftReference::from_reader(stream).map_err(Error::failed_to_read_mft_reference)?;
-        let attribute_id = stream.read_u16::<LittleEndian>()?;
+        let lowest_vcn = stream.read_u64::<LittleEndian>()?;
+        let segment_reference = MftReference::from_reader(stream)
+            .map_err(Error::failed_to_read_mft_reference)?;
+        let reserved = stream.read_u16::<LittleEndian>()?;
 
         let name = if name_length > 0 {
             stream.seek(SeekFrom::Start(start_offset + u64::from(name_offset)))?;
@@ -50,12 +193,14 @@ impl AttributeListAttr {
             String::new()
         };
 
-        Ok(AttributeListAttr {
+        Ok(AttributeListEntry {
             attribute_type,
             record_length,
-            first_vcn,
-            base_reference,
-            attribute_id,
+            name_length,
+            name_offset,
+            lowest_vcn,
+            segment_reference,
+            reserved,
             name,
         })
     }
diff --git a/src/entry.rs b/src/entry.rs
index 1ce8f35..0c0bb48 100644
--- a/src/entry.rs
+++ b/src/entry.rs
@@ -1,7 +1,7 @@
 use crate::err::{Error, Result};
 use crate::impl_serialize_for_bitflags;
 
-use log::trace;
+use log::{trace, warn};
 
 use winstructs::ntfs::mft_reference::MftReference;
 
@@ -29,6 +29,10 @@ pub const FILE_HEADER: &[u8; 4] = b"FILE";
 pub struct MftEntry {
     pub header: EntryHeader,
     pub data: Vec<u8>,
+    /// Valid fixup allows you to check if the fixup value in the entry's blocks
+    /// matched the fixup array value. It is optional because in the case of 
+    /// from_buffer_skip_fixup(), no fixup is even checked, thus, valid_fixup is None
+    pub valid_fixup: Option<bool>
 }
 
 impl ser::Serialize for MftEntry {
@@ -36,10 +40,11 @@ impl ser::Serialize for MftEntry {
     where
         S: Serializer,
     {
-        let mut state = serializer.serialize_struct("Color", 2)?;
+        let mut state = serializer.serialize_struct("MftEntry", 2)?;
         let attributes: Vec<MftAttribute> = self.iter_attributes().filter_map(Result::ok).collect();
         state.serialize_field("header", &self.header)?;
         state.serialize_field("attributes", &attributes)?;
+        state.serialize_field("valid_fixup", &self.valid_fixup)?;
         state.end()
     }
 }
@@ -175,13 +180,16 @@ impl MftEntry {
         let entry_header = EntryHeader::from_reader(&mut cursor, entry_number)?;
         trace!("Number of sectors: {:#?}", entry_header);
 
-        if entry_header.is_valid() {
-            Self::apply_fixups(&entry_header, &mut buffer)?;
-        }
+        let valid_fixup = if entry_header.is_valid() {
+            Some(Self::apply_fixups(&entry_header, &mut buffer)?)
+        } else {
+            None
+        };
 
         Ok(MftEntry {
             header: entry_header,
             data: buffer,
+            valid_fixup
         })
     }
 
@@ -203,6 +211,7 @@ impl MftEntry {
         Ok(MftEntry {
             header: entry_header,
             data: buffer,
+            valid_fixup: None
         })
     }
 
@@ -236,7 +245,10 @@ impl MftEntry {
     /// https://docs.microsoft.com/en-us/windows/desktop/devnotes/multi-sector-header
     /// **Note**: The fixup will be written at the end of each 512-byte stride,
     /// even if the device has more (or less) than 512 bytes per sector.
-    fn apply_fixups(header: &EntryHeader, buffer: &mut [u8]) -> Result<()> {
+    /// The returned result is true if all fixup blocks had the fixup array value, or
+    /// false if a block's fixup value did not match the array's value.
+    fn apply_fixups(header: &EntryHeader, buffer: &mut [u8]) -> Result<bool> {
+        let mut valid_fixup = true;
         let number_of_fixups = u32::from(header.usa_size - 1);
         trace!("Number of fixups: {}", number_of_fixups);
 
@@ -262,17 +274,22 @@ impl MftEntry {
                 &mut buffer[end_of_sector_bytes_start_offset..end_of_sector_bytes_end_offset];
 
             if end_of_sector_bytes != update_sequence {
-                return Err(Error::FailedToApplyFixup {
+                // An item in the block did not match the fixup array value
+                warn!(
+                    "[entry: {}] fixup bytes are not equal to update sequence value - stride_number: {}, end_of_sector_bytes: {:?}, fixup_bytes: {:?}",
+                    header.record_number,
                     stride_number,
-                    end_of_sector_bytes: end_of_sector_bytes.to_vec(),
-                    fixup_bytes: fixup_bytes.to_vec(),
-                });
+                    end_of_sector_bytes.to_vec(),
+                    fixup_bytes.to_vec()
+                );
+
+                valid_fixup = false;
             }
 
             end_of_sector_bytes.copy_from_slice(&fixup_bytes);
         }
 
-        Ok(())
+        Ok(valid_fixup)
     }
 
     pub fn is_allocated(&self) -> bool {
diff --git a/tests/test_entry.rs b/tests/test_entry.rs
new file mode 100644
index 0000000..c95f2c7
--- /dev/null
+++ b/tests/test_entry.rs
@@ -0,0 +1,17 @@
+use mft::entry::MftEntry;
+use serde_json;
+
+#[test]
+fn test_entry_invalid_fixup_value() {
+    let mft_entry_buffer = include_bytes!("../samples/entry_102130_fixup_issue");
+
+    let entry = MftEntry::from_buffer(
+        mft_entry_buffer.to_vec(), 
+        102130
+    ).expect("Failed to parse entry");
+
+    assert_eq!(entry.valid_fixup, Some(false));
+
+    let mft_json_value = serde_json::to_value(&entry).expect("Error serializing MftEntry");
+    assert_eq!(mft_json_value["valid_fixup"], serde_json::value::Value::from(false));
+}
\ No newline at end of file