//! HDF5 Local Heap parsing. #[cfg(not(feature = "std"))] use alloc::string::String; use crate::addr::to_usize; use crate::error::FormatError; use crate::storage::{Storage, len_usize, read_exact_at}; /// Parsed HDF5 Local Heap header. #[derive(Debug, Clone)] pub struct LocalHeap { /// Size of the data segment in bytes. pub data_segment_size: u64, /// Offset of the free list head within the data segment. pub free_list_head_offset: u64, /// File address of the data segment. pub data_segment_address: u64, } fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { let s = size as usize; if pos.checked_add(s).is_none_or(|end| end > data.len()) { return Err(FormatError::UnexpectedEof { expected: pos.saturating_add(s), available: data.len(), }); } let slice = &data[pos..pos + s]; Ok(match size { 2 => u16::from_le_bytes([slice[0], slice[1]]) as u64, 4 => u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]) as u64, 8 => u64::from_le_bytes([ slice[0], slice[1], slice[2], slice[3], slice[4], slice[5], slice[6], slice[7], ]), _ => return Err(FormatError::InvalidOffsetSize(size)), }) } /// First read of a name on a backend without the file in memory: most link /// names are shorter than this. const NAME_READ_START: usize = 64; impl LocalHeap { /// Parse a local heap header at the given offset in the file data. pub fn parse( file_data: &[u8], offset: usize, offset_size: u8, length_size: u8, ) -> Result { Self::parse_in(file_data, offset as u64, offset_size, length_size) } /// [`Self::parse`] over any [`Storage`]: one read of the header. pub fn parse_in( file: &S, offset: u64, offset_size: u8, length_size: u8, ) -> Result { // signature(4) + version(1) + reserved(3) = 8, then length_size*2 + offset_size let ls = length_size as usize; let os = offset_size as usize; let total = 8 + ls * 2 + os; let header = read_exact_at(file, offset, total)?; let file_data: &[u8] = &header; let offset = 0usize; if &file_data[offset..offset + 4] != b"HEAP" { return Err(FormatError::InvalidLocalHeapSignature); } let version = file_data[offset + 4]; if version != 0 { return Err(FormatError::InvalidLocalHeapVersion(version)); } let mut pos = offset + 8; let data_segment_size = read_offset(file_data, pos, length_size)?; pos += ls; let free_list_head_offset = read_offset(file_data, pos, length_size)?; pos += ls; let data_segment_address = read_offset(file_data, pos, offset_size)?; Ok(LocalHeap { data_segment_size, free_list_head_offset, data_segment_address, }) } /// Walk the free list the way libhdf5 does when it loads a heap's data /// (`H5HL__fl_deserialize`), rejecting a heap whose free list points /// outside the data segment. libhdf5 refuses such a heap ("bad heap free /// list"), and names read from it would be garbage. /// /// libhdf5 only loads a heap when it needs a name from it (an empty /// group's broken heap goes unnoticed), so call this before the first /// [`Self::read_string`], not on parse. /// /// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the /// undefined address) is accepted as "no free list" too. pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> { self.validate_free_list_in(file_data, length_size) } /// [`Self::validate_free_list`] over any [`Storage`]: two small reads /// per free block. pub fn validate_free_list_in( &self, file: &S, length_size: u8, ) -> Result<(), FormatError> { const FREE_NULL: u64 = 1; let ls = length_size as usize; let undefined = if ls >= 8 { u64::MAX } else { (1u64 << (8 * ls)) - 1 }; let size = self.data_segment_size; let seg = self.data_segment_address; let mut next = self.free_list_head_offset; // Each free block holds two lengths, so a list longer than this // revisits a block: a cycle. let max_blocks = size / (2 * ls as u64) + 1; let mut walked = 0u64; while next != FREE_NULL && next != undefined { if next >= size || walked >= max_blocks { return Err(FormatError::InvalidLocalHeapFreeList); } walked += 1; let at = seg .checked_add(next) .and_then(|a| usize::try_from(a).ok()) .ok_or(FormatError::InvalidLocalHeapFreeList)?; let block_offset = next; next = read_offset(&read_exact_at(file, at as u64, ls)?, 0, length_size)?; if next == 0 { return Err(FormatError::InvalidLocalHeapFreeList); } let block_size = read_offset(&read_exact_at(file, (at + ls) as u64, ls)?, 0, length_size)?; if block_offset .checked_add(block_size) .is_none_or(|end| end > size) { return Err(FormatError::InvalidLocalHeapFreeList); } } Ok(()) } /// Read a null-terminated string from the heap's data segment at the given byte offset. pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result { self.read_string_in(file_data, string_offset) } /// [`Self::read_string`] over any [`Storage`]: one read of up to 64 /// bytes for a short name, more (each four times the last) up to the end /// of the data segment for a longer one. pub fn read_string_in( &self, file: &S, string_offset: u64, ) -> Result { let file_len = len_usize(file); let seg_addr = to_usize(self.data_segment_address)?; let str_start = seg_addr .checked_add(to_usize(string_offset)?) .ok_or(FormatError::Overflow( "local heap seg_addr + string_offset overflow".into(), ))?; let seg_end = seg_addr .checked_add(to_usize(self.data_segment_size)?) .ok_or(FormatError::Overflow( "local heap seg_addr + data_segment_size overflow".into(), ))?; if str_start >= file_len || str_start >= seg_end { return Err(FormatError::UnexpectedEof { expected: str_start + 1, available: file_len, }); } // Find the null terminator, which lies before the end of the data // segment (or of the file). In memory that is one borrowed slice; // otherwise the bytes are read in growing pieces, so a name costs a // read of about its own length, not of the rest of the segment // (whose size is an untrusted header field). let search_end = seg_end.min(file_len); let total = search_end - str_start; let mut want = if file.as_contiguous().is_some() { total } else { total.min(NAME_READ_START) }; loop { let rest = read_exact_at(file, str_start as u64, want)?; if let Some(len) = rest.iter().position(|&b| b == 0) { let s = core::str::from_utf8(&rest[..len]) .map_err(|_| FormatError::InvalidLocalHeapSignature)?; return Ok(String::from(s)); } if want == total { return Err(FormatError::UnexpectedEof { expected: search_end + 1, available: search_end, }); } want = want.saturating_mul(4).min(total); } } } #[cfg(test)] mod tests { use super::*; fn build_heap_file( heap_offset: usize, data_seg_offset: usize, strings: &[&str], offset_size: u8, length_size: u8, ) -> Vec { // Build data segment let mut data_seg = Vec::new(); for s in strings { data_seg.extend_from_slice(s.as_bytes()); data_seg.push(0); // null terminator } let data_seg_size = data_seg.len(); let total_size = data_seg_offset + data_seg_size + 64; let mut file = vec![0u8; total_size]; // Write heap header at heap_offset let mut pos = heap_offset; file[pos..pos + 4].copy_from_slice(b"HEAP"); pos += 4; file[pos] = 0; // version pos += 1; // reserved 3 pos += 3; // data_segment_size write_val(&mut file, pos, data_seg_size as u64, length_size); pos += length_size as usize; // free_list_head_offset: H5HL_FREE_NULL (no free space) write_val(&mut file, pos, 1, length_size); pos += length_size as usize; // data_segment_address write_val(&mut file, pos, data_seg_offset as u64, offset_size); // Write data segment file[data_seg_offset..data_seg_offset + data_seg_size].copy_from_slice(&data_seg); file } fn write_val(buf: &mut [u8], pos: usize, val: u64, size: u8) { match size { 4 => buf[pos..pos + 4].copy_from_slice(&(val as u32).to_le_bytes()), 8 => buf[pos..pos + 8].copy_from_slice(&val.to_le_bytes()), _ => panic!("test"), } } #[test] fn parse_rejects_near_usize_max_offset_without_panicking() { // Found by fuzzing: `offset + total` overflowed for a crafted // near-usize::MAX offset. let file = build_heap_file(0, 100, &["hello"], 8, 8); let result = LocalHeap::parse(&file, usize::MAX - 4, 8, 8); assert!( matches!(result, Err(FormatError::UnexpectedEof { .. })), "expected a clean UnexpectedEof, got {result:?}" ); } #[test] fn parse_heap_header() { let file = build_heap_file(0, 100, &["hello", "world"], 8, 8); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); assert_eq!(heap.data_segment_address, 100); assert_eq!(heap.data_segment_size, 12); // "hello\0world\0" } #[test] fn read_string_at_offset_0() { let file = build_heap_file(0, 100, &["hello", "world"], 8, 8); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); let s = heap.read_string(&file, 0).unwrap(); assert_eq!(s, "hello"); } #[test] fn read_string_at_offset_6() { let file = build_heap_file(0, 100, &["hello", "world"], 8, 8); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); let s = heap.read_string(&file, 6).unwrap(); assert_eq!(s, "world"); } #[test] fn invalid_signature() { let mut file = build_heap_file(0, 100, &["x"], 8, 8); file[0] = b'X'; let err = LocalHeap::parse(&file, 0, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLocalHeapSignature); } #[test] fn read_string_past_segment() { let file = build_heap_file(0, 100, &["hi"], 8, 8); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); let err = heap.read_string(&file, 100).unwrap_err(); assert!(matches!(err, FormatError::UnexpectedEof { .. })); } #[test] fn parse_heap_4byte_offsets() { let file = build_heap_file(0, 80, &["test"], 4, 4); let heap = LocalHeap::parse(&file, 0, 4, 4).unwrap(); assert_eq!(heap.data_segment_address, 80); let s = heap.read_string(&file, 0).unwrap(); assert_eq!(s, "test"); } /// Heap with data segment `[a, b, c, 0-padding]` whose free list starts /// at `head` and has one block `(next, size)` at offset 8. fn heap_with_free_block(head: u64, next: u64, size: u64) -> Vec { let mut file = build_heap_file(0, 100, &["abcdefg"], 8, 8); file.resize(200, 0); write_val(&mut file, 8, 32, 8); // data segment size write_val(&mut file, 16, head, 8); write_val(&mut file, 108, next, 8); write_val(&mut file, 116, size, 8); file } #[test] fn free_list_inside_the_segment_is_accepted() { let file = heap_with_free_block(8, 1, 24); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); heap.validate_free_list(&file, 8).unwrap(); assert_eq!(heap.read_string(&file, 0).unwrap(), "abcdefg"); // An all-ones head is "no free list" too. let file = heap_with_free_block(u64::MAX, 0, 0); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); assert!(heap.validate_free_list(&file, 8).is_ok()); } #[test] fn bad_free_list_is_rejected_like_libhdf5() { for (head, next, size, why) in [ (40, 1, 8, "head past the segment"), (8, 1, 25, "block runs past the segment"), (8, 0, 8, "next offset of zero"), (8, 8, 8, "cycle"), (8, 999, 8, "next past the segment"), ] { let file = heap_with_free_block(head, next, size); // The header itself parses; the free list is checked on use. let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); assert_eq!( heap.validate_free_list(&file, 8).unwrap_err(), FormatError::InvalidLocalHeapFreeList, "{why}" ); } } #[test] fn invalid_version() { let mut file = build_heap_file(0, 100, &["x"], 8, 8); file[4] = 1; // bad version let err = LocalHeap::parse(&file, 0, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidLocalHeapVersion(1)); } /// Header, free list and strings read identically through a /// `read_at`-only storage, for every truncation of the file. #[test] fn storage_reads_match_slice_reads() { use crate::storage::CountingStorage; let plain = build_heap_file(0, 64, &["", "alpha", "beta"], 8, 8); // A free block of 16 bytes at segment offset 12, ending the list. let mut free = build_heap_file(0, 64, &["", "alpha", "beta", &"x".repeat(20)], 8, 8); free[16..24].copy_from_slice(&12u64.to_le_bytes()); free[64 + 12..64 + 20].copy_from_slice(&1u64.to_le_bytes()); free[64 + 20..64 + 28].copy_from_slice(&16u64.to_le_bytes()); let mut bad_free = free.clone(); bad_free[64 + 20..64 + 28].copy_from_slice(&99u64.to_le_bytes()); for full in [plain, free, bad_free] { for cut in 0..=full.len() { let f = &full[..cut]; let storage = CountingStorage::new(f.to_vec()); let want = LocalHeap::parse(f, 0, 8, 8); let got = LocalHeap::parse_in(&storage, 0, 8, 8); assert_eq!(format!("{got:?}"), format!("{want:?}")); let Ok(heap) = want else { continue }; assert_eq!( heap.validate_free_list_in(&storage, 8), heap.validate_free_list(f, 8) ); for off in [0u64, 1, 2, 6, 7, 11, 100] { assert_eq!(heap.read_string_in(&storage, off), heap.read_string(f, off)); } } } } /// Names of every length around the first read's size, and one with no /// terminator, read identically through a `read_at`-only storage; a /// short name in a heap whose header claims a huge data segment costs /// one small read, not a read of the rest of the file. #[test] fn long_names_and_hostile_segment_sizes() { use crate::storage::CountingStorage; let names: Vec = [0usize, 1, 63, 64, 65, 255, 256, 257, 1000, 5000] .iter() .map(|&n| "n".repeat(n)) .collect(); let refs: Vec<&str> = names.iter().map(String::as_str).collect(); let mut file = build_heap_file(0, 64, &refs, 8, 8); let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); let storage = CountingStorage::new(file.clone()); let mut off = 0u64; for name in &names { let got = heap.read_string_in(&storage, off); assert_eq!(got, heap.read_string(&file, off)); assert_eq!(got.unwrap(), *name); off += name.len() as u64 + 1; } // The last name loses its terminator: both report the same error. let seg_end = 64 + heap.data_segment_size as usize; file[seg_end - 1] = b'n'; let storage = CountingStorage::new(file.clone()); let last = off - names[names.len() - 1].len() as u64 - 1; let want = heap.read_string(&file, last); assert!(want.is_err()); assert_eq!(heap.read_string_in(&storage, last), want); // A 64 MiB file whose heap claims a data segment reaching its end. let mut big = build_heap_file(0, 64, &["short", "names"], 8, 8); big.resize(64 << 20, 0); big[8..16].copy_from_slice(&((64u64 << 20) - 64).to_le_bytes()); let heap = LocalHeap::parse(&big, 0, 8, 8).unwrap(); let storage = CountingStorage::new(big.clone()); assert_eq!(heap.read_string_in(&storage, 6).unwrap(), "names"); assert_eq!((storage.reads(), storage.bytes_read()), (1, 64)); } }