crates/clawhdf5-format/src/fractal_heap.rs

@@ -9,6 +9,7 @@ use byteorder::{ByteOrder, LittleEndian};
 use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records};
 use crate::error::FormatError;
 use crate::filter_pipeline::FilterPipeline;
+use crate::storage::{Storage, Window, len_usize, read_exact_at, require_contiguous};
 
 /// Parsed fractal heap header (signature "FRHP").
 #[derive(Debug, Clone)]
@@ -138,6 +139,36 @@ impl FractalHeapHeader {
         offset_size: u8,
         length_size: u8,
     ) -> Result<FractalHeapHeader, FormatError> {
+        Self::parse_in(&file_data, offset as u64, offset_size, length_size)
+    }
+
+    /// [`Self::parse`] over any [`Storage`]: one read of the header (two
+    /// when it holds an I/O filter pipeline).
+    pub fn parse_in(
+        file: &dyn Storage,
+        offset: u64,
+        offset_size: u8,
+        length_size: u8,
+    ) -> Result<FractalHeapHeader, FormatError> {
+        // Every field up to the checksum, without and with the filter
+        // information; the window holds all of it (or ends at the end of
+        // the file), so its bounds checks are the whole-file ones.
+        let (os, ls) = (usize::from(offset_size), usize::from(length_size));
+        let unfiltered_len = 26 + 12 * ls + 3 * os;
+        let mut w = Window::read(file, offset, unfiltered_len)?;
+        if w.bytes.len() == unfiltered_len {
+            let filter_len = usize::from(u16::from_le_bytes([w.bytes[7], w.bytes[8]]));
+            if filter_len > 0 {
+                w = Window::read(file, offset, unfiltered_len + ls + 4 + filter_len)?;
+            }
+        }
+        let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
+        let read_offset = |_: &[u8], pos: usize, size: u8| {
+            w.ensure(pos, usize::from(size))?;
+            read_offset(&w.bytes, pos, size)
+        };
+        let file_data: &[u8] = &w.bytes;
+        let offset = 0usize;
         ensure_len(file_data, offset, 5)?;
         if &file_data[offset..offset + 4] != b"FRHP" {
             return Err(FormatError::InvalidFractalHeapSignature);
@@ -148,9 +179,6 @@ impl FractalHeapHeader {
             return Err(FormatError::InvalidFractalHeapVersion(version));
         }
 
-        let os = offset_size as usize;
-        let ls = length_size as usize;
-
         let mut pos = offset + 5;
         ensure_len(file_data, pos, 2)?;
         let heap_id_length = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]);
@@ -353,6 +381,18 @@ impl FractalHeapHeader {
         file_data: &[u8],
         id_bytes: &[u8],
         offset_size: u8,
+    ) -> Result<Vec<u8>, FormatError> {
+        self.read_managed_object_in(&file_data, id_bytes, offset_size)
+    }
+
+    /// [`Self::read_managed_object`] over any [`Storage`]. A huge object
+    /// found through the huge-object v2 B-tree still needs the whole file
+    /// in memory ([`FormatError::ContiguousStorageRequired`] otherwise).
+    pub fn read_managed_object_in(
+        &self,
+        file_data: &dyn Storage,
+        id_bytes: &[u8],
+        offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
         let Some(&first) = id_bytes.first() else {
             return Err(FormatError::UnexpectedEof {
@@ -383,7 +423,7 @@ impl FractalHeapHeader {
     }
 
     /// Read a huge object (heap ID type 1).
-    fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result<Vec<u8>, FormatError> {
+    fn read_huge_object(&self, file: &dyn Storage, id: &[u8]) -> Result<Vec<u8>, FormatError> {
         let os = usize::from(self.offset_size);
         let ls = usize::from(self.length_size);
         // (address, stored length, filter mask, decoded length); the last two
@@ -414,18 +454,17 @@ impl FractalHeapHeader {
             let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8);
             ensure_len(id, 1, key_len)?;
             let key = le_uint(&id[1..1 + key_len]);
-            self.find_huge_record(file_data, key)?
+            self.find_huge_record(file, key)?
         };
 
         let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?;
         let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?;
-        ensure_len(file_data, start, len)?;
-        let stored = &file_data[start..start + len];
+        let stored = read_exact_at(file, start as u64, len)?;
         match &self.filter_pipeline {
-            None => Ok(stored.to_vec()),
+            None => Ok(stored.into_owned()),
             Some(pipeline) => {
                 let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?;
-                let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?;
+                let out = crate::filters::decompress_chunk_masked(&stored, pipeline, mem, 1, mask)?;
                 if out.len() != mem {
                     return Err(heap_error("filtered huge object decoded to the wrong size"));
                 }
@@ -438,7 +477,7 @@ impl FractalHeapHeader {
     /// (address, stored length, filter mask, decoded length).
     fn find_huge_record(
         &self,
-        file_data: &[u8],
+        file: &dyn Storage,
         key: u64,
     ) -> Result<(u64, u64, u32, u64), FormatError> {
         if is_undefined(self.huge_btree_address, self.offset_size) {
@@ -446,6 +485,8 @@ impl FractalHeapHeader {
                 "huge object ID but the heap has no huge-object index",
             ));
         }
+        // The v2 B-tree is read from a slice until it is converted.
+        let file_data = require_contiguous(file, "a huge fractal-heap object's B-tree")?;
         let hdr = BTreeV2Header::parse(
             file_data,
             self.huge_btree_address as usize,
@@ -515,7 +556,7 @@ impl FractalHeapHeader {
     /// Read a managed object (heap ID type 0).
     fn read_heap_managed(
         &self,
-        file_data: &[u8],
+        file_data: &dyn Storage,
         id_bytes: &[u8],
         offset_size: u8,
     ) -> Result<Vec<u8>, FormatError> {
@@ -565,7 +606,7 @@ impl FractalHeapHeader {
     /// through its filter pipeline, so the block is decoded first.
     fn read_from_direct_block(
         &self,
-        file_data: &[u8],
+        file: &dyn Storage,
         block: DirectBlock,
         target_offset: u64,
         length: usize,
@@ -581,9 +622,9 @@ impl FractalHeapHeader {
             let stored_len = usize::try_from(block.filtered_size)
                 .map_err(|_| heap_error("direct block size"))?;
             let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?;
-            ensure_len(file_data, block.addr, stored_len)?;
+            let stored = read_exact_at(file, block.addr as u64, stored_len)?;
             let decoded = crate::filters::decompress_chunk_masked(
-                &file_data[block.addr..block.addr + stored_len],
+                &stored,
                 pipeline,
                 size,
                 1,
@@ -597,17 +638,16 @@ impl FractalHeapHeader {
             .checked_add(local_offset)
             .ok_or(FormatError::UnexpectedEof {
                 expected: usize::MAX,
-                available: file_data.len(),
+                available: len_usize(file),
             })?;
-        ensure_len(file_data, pos, length)?;
-        Ok(file_data[pos..pos + length].to_vec())
+        Ok(read_exact_at(file, pos as u64, length)?.into_owned())
     }
 
     /// Read an object by traversing an indirect block to find the right direct block.
     #[allow(clippy::too_many_arguments)]
     fn read_from_indirect_block(
         &self,
-        file_data: &[u8],
+        file: &dyn Storage,
         iblock_addr: usize,
         nrows: u16,
         iblock_heap_offset: u64,
@@ -621,16 +661,8 @@ impl FractalHeapHeader {
                 "fractal heap: maximum recursion depth exceeded".into(),
             ));
         }
-        // Parse indirect block header
-        ensure_len(file_data, iblock_addr, 4)?;
-        if &file_data[iblock_addr..iblock_addr + 4] != b"FHIB" {
-            return Err(FormatError::InvalidFractalHeapSignature);
-        }
-
         let block_offset_bytes = (self.max_heap_size as usize).div_ceil(8);
         let iblock_header = 5 + offset_size as usize + block_offset_bytes;
-        let mut pos = iblock_addr + iblock_header;
-
         let tw = self.table_width as u64;
         let nrows_usize = nrows as usize;
         let mut current_heap_offset = iblock_heap_offset;
@@ -640,6 +672,38 @@ impl FractalHeapHeader {
         let start_indirect = self.max_direct_rows();
         let max_direct_rows = nrows_usize.min(start_indirect);
 
+        // The block up to its last child entry, in one window: every
+        // position read below lies inside it, so its bounds checks are the
+        // whole-file ones.
+        let direct_entry = usize::from(offset_size)
+            + if self.filter_pipeline.is_some() {
+                usize::from(self.length_size) + 4
+            } else {
+                0
+            };
+        let entries =
+            |rows: usize, entry: usize| rows.saturating_mul(tw as usize).saturating_mul(entry);
+        let block_len = iblock_header
+            .saturating_add(entries(max_direct_rows, direct_entry))
+            .saturating_add(entries(
+                nrows_usize.saturating_sub(start_indirect),
+                usize::from(offset_size),
+            ));
+        let w = Window::read(file, iblock_addr as u64, block_len)?;
+        let ensure_len = |_: &[u8], pos: usize, needed: usize| w.ensure(pos, needed);
+        let read_offset = |_: &[u8], pos: usize, size: u8| {
+            w.ensure(pos, usize::from(size))?;
+            read_offset(&w.bytes, pos, size)
+        };
+        let file_data: &[u8] = &w.bytes;
+
+        // Parse indirect block header
+        ensure_len(file_data, 0, 4)?;
+        if &file_data[..4] != b"FHIB" {
+            return Err(FormatError::InvalidFractalHeapSignature);
+        }
+        let mut pos = iblock_header;
+
         for row in 0..max_direct_rows {
             let block_size = self.block_size_for_row(row);
 
@@ -671,7 +735,7 @@ impl FractalHeapHeader {
                     && target_offset < block_end
                 {
                     return self.read_from_direct_block(
-                        file_data,
+                        file,
                         DirectBlock {
                             addr: child_addr as usize,
                             size: block_size,
@@ -704,7 +768,7 @@ impl FractalHeapHeader {
                     && target_offset < block_end
                 {
                     return self.read_from_indirect_block(
-                        file_data,
+                        file,
                         child_addr as usize,
                         child_nrows,
                         current_heap_offset,
@@ -720,7 +784,7 @@ impl FractalHeapHeader {
 
         Err(FormatError::UnexpectedEof {
             expected: target_offset as usize + length,
-            available: file_data.len(),
+            available: len_usize(file),
         })
     }
 
@@ -1024,4 +1088,110 @@ mod tests {
         let id = [0x40u8, 0, 0, 0, 0, 0, 0];
         assert!(hdr.read_managed_object(&file_data, &id, 8).is_err());
     }
+
+    /// Headers, and managed (in a direct root and through an indirect
+    /// root), huge and tiny objects read identically through a
+    /// `read_at`-only storage, for every truncation of the file.
+    #[test]
+    fn storage_reads_match_slice_reads() {
+        use crate::storage::CountingStorage;
+        let (mut file, header_end) = build_simple_heap(8, 8);
+        // An indirect root block at 600: row 0 holds the direct block at
+        // 256, then three undefined blocks.
+        file[600..604].copy_from_slice(b"FHIB");
+        let mut at = 600 + 5 + 8 + 2;
+        for addr in [256u64, u64::MAX, u64::MAX, u64::MAX] {
+            file[at..at + 8].copy_from_slice(&addr.to_le_bytes());
+            at += 8;
+        }
+        file[900..905].copy_from_slice(b"huge!");
+        let managed_id = |offset: u64, len: u64| {
+            let payload = offset | (len << 16);
+            let mut id = vec![0u8];
+            id.extend_from_slice(&payload.to_le_bytes()[..6]);
+            id
+        };
+        let mut huge = vec![0x10u8];
+        huge.extend_from_slice(&900u64.to_le_bytes());
+        huge.extend_from_slice(&5u64.to_le_bytes());
+        let ids = [
+            managed_id(15, 13),
+            managed_id(15, 200),
+            managed_id(130, 4),
+            huge,
+            vec![0x22, b'a', b'b', b'c', 0, 0, 0],
+        ];
+        let mut cuts: Vec<usize> = (0..=header_end + 1).collect();
+        cuts.extend([256, 260, 271, 280, 600, 610, 620, 640, 900, 903, file.len()]);
+        for cut in cuts {
+            let f = &file[..cut];
+            let storage = CountingStorage::new(f.to_vec());
+            let want = FractalHeapHeader::parse(f, 0, 8, 8);
+            let got = FractalHeapHeader::parse_in(&storage, 0, 8, 8);
+            assert_eq!(format!("{got:?}"), format!("{want:?}"), "cut {cut}");
+            let Ok(direct) = want else { continue };
+            let mut indirect = direct.clone();
+            indirect.root_block_address = 600;
+            indirect.current_rows_in_root_indirect_block = 1;
+            let mut huge_ids = direct.clone();
+            huge_ids.heap_id_length = 17;
+            for hdr in [&direct, &indirect, &huge_ids] {
+                for id in &ids {
+                    assert_eq!(
+                        hdr.read_managed_object_in(&storage, id, 8),
+                        hdr.read_managed_object(f, id, 8),
+                        "cut {cut}"
+                    );
+                }
+            }
+        }
+    }
+
+    /// A huge object found through the huge-object B-tree needs the whole
+    /// file in memory until the B-tree reader is converted: a clean error
+    /// on other storage.
+    #[test]
+    fn huge_object_btree_needs_contiguous_storage() {
+        use crate::storage::CountingStorage;
+        let (file, _) = build_simple_heap(8, 8);
+        let mut hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
+        hdr.huge_btree_address = 700;
+        let storage = CountingStorage::new(file);
+        assert_eq!(
+            hdr.read_managed_object_in(&storage, &[0x10, 1, 0, 0, 0, 0, 0], 8),
+            Err(FormatError::ContiguousStorageRequired(
+                "a huge fractal-heap object's B-tree"
+            ))
+        );
+    }
+
+    /// A header with an I/O filter pipeline (read in a second, longer
+    /// window) parses identically through a `read_at`-only storage, for
+    /// every truncation.
+    #[test]
+    fn filtered_header_parses_identically_through_storage() {
+        use crate::storage::CountingStorage;
+        let (simple, header_end) = build_simple_heap(8, 8);
+        let pipeline = [2u8, 1, 1, 0, 0, 0, 1, 0, 6, 0, 0, 0]; // deflate, level 6
+        let mut header = simple[..header_end - 4].to_vec();
+        header[7..9].copy_from_slice(&(pipeline.len() as u16).to_le_bytes());
+        header.extend_from_slice(&100u64.to_le_bytes()); // root block's stored size
+        header.extend_from_slice(&0u32.to_le_bytes()); // its filter mask
+        header.extend_from_slice(&pipeline);
+        let sum = crate::checksum::jenkins_lookup3(&header);
+        header.extend_from_slice(&sum.to_le_bytes());
+        let mut file = header.clone();
+        file.resize(256, 0);
+        let hdr = FractalHeapHeader::parse(&file, 0, 8, 8).unwrap();
+        assert!(hdr.filter_pipeline.is_some());
+        for cut in 0..=file.len() {
+            let f = &file[..cut];
+            let storage = CountingStorage::new(f.to_vec());
+            assert_eq!(
+                format!("{:?}", FractalHeapHeader::parse_in(&storage, 0, 8, 8)),
+                format!("{:?}", FractalHeapHeader::parse(f, 0, 8, 8)),
+                "cut {cut}"
+            );
+        }
+    }
 }

crates/clawhdf5-format/src/storage.rs

@@ -206,11 +206,52 @@ pub fn read_exact_at(
     if bytes.len() < len {
         // The storage shrank or the backend served a short read inside the
         // file: never parse a partial structure.
-        return Err(eof());
+        return Err(short_read());
     }
     Ok(bytes)
 }
 
+fn short_read() -> FormatError {
+    FormatError::Storage(
+        "short read inside the file (the storage shrank or the backend failed)".into(),
+    )
+}
+
+/// A window of the file: up to `max` bytes read at `base`, fewer only at
+/// the end of the file. Its [`Window::ensure`] reports a bounds failure
+/// exactly as the whole-file check `ensure_len(file_data, base + rel, n)`
+/// did — with the absolute position and the file's length — as long as
+/// every position checked lies within the `max` bytes the window was asked
+/// for: then a position past the window is past the end of the file.
+pub(crate) struct Window<'a> {
+    /// The bytes, from `base` on.
+    pub bytes: Cow<'a, [u8]>,
+    base: usize,
+    file_len: usize,
+}
+
+impl<'a> Window<'a> {
+    /// Read up to `max` bytes at `base`.
+    pub fn read(file: &'a dyn Storage, base: u64, max: usize) -> Result<Self, FormatError> {
+        Ok(Window {
+            bytes: read_upto(file, base, max)?,
+            base: usize::try_from(base).unwrap_or(usize::MAX),
+            file_len: len_usize(file),
+        })
+    }
+
+    /// Check that `[rel, rel + needed)` (relative to `base`) is in the file.
+    pub fn ensure(&self, rel: usize, needed: usize) -> Result<(), FormatError> {
+        match rel.checked_add(needed) {
+            Some(end) if end <= self.bytes.len() => Ok(()),
+            _ => Err(FormatError::UnexpectedEof {
+                expected: self.base.saturating_add(rel).saturating_add(needed),
+                available: self.file_len,
+            }),
+        }
+    }
+}
+
 /// Up to `max` bytes from `offset` on: fewer only at the end of the
 /// storage. For structures whose size is only known once their prefix has
 /// been parsed and whose parsers bound-check what they are given.
@@ -224,9 +265,7 @@ pub fn read_upto(
     let len = usize::try_from(avail).map_or(max, |a| a.min(max));
     let bytes = file.read_at(offset, len)?;
     if bytes.len() < len {
-        return Err(FormatError::Storage(
-            "short read inside the file (the storage shrank or the backend failed)".into(),
-        ));
+        return Err(short_read());
     }
     Ok(bytes)
 }