quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'feat/p3-storage-trait' into feat/p3-range-zfp-edit

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/attribute.rs
#	crates/clawhdf5-format/src/btree_v1.rs
#	crates/clawhdf5-format/src/data_layout.rs
#	crates/clawhdf5-format/src/extensible_array.rs
#	crates/clawhdf5-format/src/fixed_array.rs
#	crates/clawhdf5-format/src/fractal_heap.rs
#	crates/clawhdf5-format/src/local_heap.rs
#	crates/clawhdf5-format/src/shared_message.rs
This commit is contained in:
osobh
2026-09-26 14:51:51 -05:00
parent 234dd3e36c 895c79a2fe
commit 437e81cfff
26 changed files with 3480 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/local_heap.rs
+144 -36
View File
@@ -5,6 +5,7 @@ use alloc::string::String;
use crate::addr::to_usize;
use crate::error::FormatError;
use crate::storage::{Storage, len_usize, read_exact_at};
/// Parsed HDF5 Local Heap header.
#[derive(Debug, Clone)]
@@ -17,21 +18,6 @@ pub struct LocalHeap {
pub data_segment_address: u64,
}
/// Checks that `[offset, offset + needed)` fits within `data`, guarding the
/// addition against `usize` overflow from a crafted near-`usize::MAX` offset.
fn ensure_len(data: &[u8], offset: usize, needed: usize) -> Result<(), FormatError> {
if offset
.checked_add(needed)
.is_none_or(|end| end > data.len())
{
return Err(FormatError::UnexpectedEof {
expected: offset.saturating_add(needed),
available: data.len(),
});
}
Ok(())
}
fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
let s = size as usize;
if pos.checked_add(s).is_none_or(|end| end > data.len()) {
@@ -51,6 +37,10 @@ fn read_offset(data: &[u8], pos: usize, size: u8) -> Result<u64, FormatError> {
})
}
/// First read of a name on a backend without the file in memory: most link
/// names are shorter than this.
const NAME_READ_START: usize = 64;
impl LocalHeap {
/// Parse a local heap header at the given offset in the file data.
pub fn parse(
@@ -58,12 +48,24 @@ impl LocalHeap {
offset: usize,
offset_size: u8,
length_size: u8,
) -> Result<LocalHeap, FormatError> {
Self::parse_in(file_data, offset as u64, offset_size, length_size)
}
/// [`Self::parse`] over any [`Storage`]: one read of the header.
pub fn parse_in<S: Storage + ?Sized>(
file: &S,
offset: u64,
offset_size: u8,
length_size: u8,
) -> Result<LocalHeap, FormatError> {
// signature(4) + version(1) + reserved(3) = 8, then length_size*2 + offset_size
let ls = length_size as usize;
let os = offset_size as usize;
let total = 8 + ls * 2 + os;
ensure_len(file_data, offset, total)?;
let header = read_exact_at(file, offset, total)?;
let file_data: &[u8] = &header;
let offset = 0usize;
if &file_data[offset..offset + 4] != b"HEAP" {
return Err(FormatError::InvalidLocalHeapSignature);
@@ -100,6 +102,16 @@ impl LocalHeap {
/// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the
/// undefined address) is accepted as "no free list" too.
pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> {
self.validate_free_list_in(file_data, length_size)
}
/// [`Self::validate_free_list`] over any [`Storage`]: two small reads
/// per free block.
pub fn validate_free_list_in<S: Storage + ?Sized>(
&self,
file: &S,
length_size: u8,
) -> Result<(), FormatError> {
const FREE_NULL: u64 = 1;
let ls = length_size as usize;
let undefined = if ls >= 8 {
@@ -124,11 +136,12 @@ impl LocalHeap {
.and_then(|a| usize::try_from(a).ok())
.ok_or(FormatError::InvalidLocalHeapFreeList)?;
let block_offset = next;
next = read_offset(file_data, at, length_size)?;
next = read_offset(&read_exact_at(file, at as u64, ls)?, 0, length_size)?;
if next == 0 {
return Err(FormatError::InvalidLocalHeapFreeList);
}
let block_size = read_offset(file_data, at + ls, length_size)?;
let block_size =
read_offset(&read_exact_at(file, (at + ls) as u64, ls)?, 0, length_size)?;
if block_offset
.checked_add(block_size)
.is_none_or(|end| end > size)
@@ -141,6 +154,18 @@ impl LocalHeap {
/// Read a null-terminated string from the heap's data segment at the given byte offset.
pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result<String, FormatError> {
self.read_string_in(file_data, string_offset)
}
/// [`Self::read_string`] over any [`Storage`]: one read of up to 64
/// bytes for a short name, more (each four times the last) up to the end
/// of the data segment for a longer one.
pub fn read_string_in<S: Storage + ?Sized>(
&self,
file: &S,
string_offset: u64,
) -> Result<String, FormatError> {
let file_len = len_usize(file);
let seg_addr = to_usize(self.data_segment_address)?;
let str_start =
seg_addr
@@ -154,30 +179,40 @@ impl LocalHeap {
"local heap seg_addr + data_segment_size overflow".into(),
))?;
if str_start >= file_data.len() || str_start >= seg_end {
if str_start >= file_len || str_start >= seg_end {
return Err(FormatError::UnexpectedEof {
expected: str_start + 1,
available: file_data.len(),
available: file_len,
});
}
// Find null terminator
let search_end = seg_end.min(file_data.len());
let mut end = str_start;
while end < search_end && file_data[end] != 0 {
end += 1;
// Find the null terminator, which lies before the end of the data
// segment (or of the file). In memory that is one borrowed slice;
// otherwise the bytes are read in growing pieces, so a name costs a
// read of about its own length, not of the rest of the segment
// (whose size is an untrusted header field).
let search_end = seg_end.min(file_len);
let total = search_end - str_start;
let mut want = if file.as_contiguous().is_some() {
total
} else {
total.min(NAME_READ_START)
};
loop {
let rest = read_exact_at(file, str_start as u64, want)?;
if let Some(len) = rest.iter().position(|&b| b == 0) {
let s = core::str::from_utf8(&rest[..len])
.map_err(|_| FormatError::InvalidLocalHeapSignature)?;
return Ok(String::from(s));
}
if want == total {
return Err(FormatError::UnexpectedEof {
expected: search_end + 1,
available: search_end,
});
}
want = want.saturating_mul(4).min(total);
}
if end >= search_end {
return Err(FormatError::UnexpectedEof {
expected: end + 1,
available: search_end,
});
}
let s = core::str::from_utf8(&file_data[str_start..end])
.map_err(|_| FormatError::InvalidLocalHeapSignature)?;
Ok(String::from(s))
}
}
@@ -346,4 +381,77 @@ mod tests {
let err = LocalHeap::parse(&file, 0, 8, 8).unwrap_err();
assert_eq!(err, FormatError::InvalidLocalHeapVersion(1));
}
/// Header, free list and strings read identically through a
/// `read_at`-only storage, for every truncation of the file.
#[test]
fn storage_reads_match_slice_reads() {
use crate::storage::CountingStorage;
let plain = build_heap_file(0, 64, &["", "alpha", "beta"], 8, 8);
// A free block of 16 bytes at segment offset 12, ending the list.
let mut free = build_heap_file(0, 64, &["", "alpha", "beta", &"x".repeat(20)], 8, 8);
free[16..24].copy_from_slice(&12u64.to_le_bytes());
free[64 + 12..64 + 20].copy_from_slice(&1u64.to_le_bytes());
free[64 + 20..64 + 28].copy_from_slice(&16u64.to_le_bytes());
let mut bad_free = free.clone();
bad_free[64 + 20..64 + 28].copy_from_slice(&99u64.to_le_bytes());
for full in [plain, free, bad_free] {
for cut in 0..=full.len() {
let f = &full[..cut];
let storage = CountingStorage::new(f.to_vec());
let want = LocalHeap::parse(f, 0, 8, 8);
let got = LocalHeap::parse_in(&storage, 0, 8, 8);
assert_eq!(format!("{got:?}"), format!("{want:?}"));
let Ok(heap) = want else { continue };
assert_eq!(
heap.validate_free_list_in(&storage, 8),
heap.validate_free_list(f, 8)
);
for off in [0u64, 1, 2, 6, 7, 11, 100] {
assert_eq!(heap.read_string_in(&storage, off), heap.read_string(f, off));
}
}
}
}
/// Names of every length around the first read's size, and one with no
/// terminator, read identically through a `read_at`-only storage; a
/// short name in a heap whose header claims a huge data segment costs
/// one small read, not a read of the rest of the file.
#[test]
fn long_names_and_hostile_segment_sizes() {
use crate::storage::CountingStorage;
let names: Vec<String> = [0usize, 1, 63, 64, 65, 255, 256, 257, 1000, 5000]
.iter()
.map(|&n| "n".repeat(n))
.collect();
let refs: Vec<&str> = names.iter().map(String::as_str).collect();
let mut file = build_heap_file(0, 64, &refs, 8, 8);
let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap();
let storage = CountingStorage::new(file.clone());
let mut off = 0u64;
for name in &names {
let got = heap.read_string_in(&storage, off);
assert_eq!(got, heap.read_string(&file, off));
assert_eq!(got.unwrap(), *name);
off += name.len() as u64 + 1;
}
// The last name loses its terminator: both report the same error.
let seg_end = 64 + heap.data_segment_size as usize;
file[seg_end - 1] = b'n';
let storage = CountingStorage::new(file.clone());
let last = off - names[names.len() - 1].len() as u64 - 1;
let want = heap.read_string(&file, last);
assert!(want.is_err());
assert_eq!(heap.read_string_in(&storage, last), want);
// A 64 MiB file whose heap claims a data segment reaching its end.
let mut big = build_heap_file(0, 64, &["short", "names"], 8, 8);
big.resize(64 << 20, 0);
big[8..16].copy_from_slice(&((64u64 << 20) - 64).to_le_bytes());
let heap = LocalHeap::parse(&big, 0, 8, 8).unwrap();
let storage = CountingStorage::new(big.clone());
assert_eq!(heap.read_string_in(&storage, 6).unwrap(), "names");
assert_eq!((storage.reads(), storage.bytes_read()), (1, 64));
}
}