quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'fix/p1-vds' into fix/p1-read-gaps

Browse Source 
# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/data_read.rs
This commit is contained in:
osobh
2026-09-25 22:42:04 -05:00
parent 61f36516d7 b4a44a2e66
commit f2e704abf3
23 changed files with 2321 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-format/src/data_layout.rs
+120 -12
View File
@@ -74,21 +74,33 @@ pub enum DataLayout {
},
}
/// Version-1 VDS mapping flag: the source file name is stored by an earlier
/// entry, whose index follows in place of the name.
const VDS_SOURCE_FILE_SHARED: u8 = 0x01;
/// Version-1 VDS mapping flag: likewise for the source dataset name.
const VDS_SOURCE_DSET_SHARED: u8 = 0x02;
/// Version-1 VDS mapping flag: the source is in the virtual file itself
/// (`"."`); no file name is stored.
const VDS_SOURCE_SAME_FILE: u8 = 0x04;
const VDS_ALL_FLAGS: u8 = VDS_SOURCE_FILE_SHARED | VDS_SOURCE_DSET_SHARED | VDS_SOURCE_SAME_FILE;
/// Parse VDS mappings from global-heap object data.
///
/// The global-heap block holding a VDS mapping list is laid out as
/// (reverse-engineered and validated against HDF5 2.0):
/// (`H5D__virtual_store_layout` / `H5D__virtual_load_layout` in libhdf5):
///
/// ```text
/// version(1) · nused(length_size, LE) · entry[nused] · checksum(4)
/// ```
///
/// Each entry is:
/// - source file name — a null-terminated string in **block version 0**; in
/// **block version 1** a same-file reference is encoded as a single `0x04`
/// marker byte (the source file is the virtual file itself) in place of the
/// name;
/// - source dataset name (null-terminated string);
/// - **block version 1 only:** a flags byte. `0x04`: the source is in the
/// virtual file itself and no file name is stored; `0x01`/`0x02`: the
/// source file/dataset name is that of an earlier entry, whose index
/// (`length_size` bytes) is stored instead of the name. libhdf5 2.0 writes
/// version 1 when the file's low version bound is 2.0 and it saves space;
/// - source file name (null-terminated string, unless flagged above);
/// - source dataset name (null-terminated string, unless flagged above);
/// - source selection (serialized `H5S` dataspace selection — self-describing
/// in length);
/// - virtual selection (serialized `H5S` dataspace selection).
@@ -114,7 +126,7 @@ pub fn parse_vds_mappings(
// `nused` is untrusted; don't pre-allocate from it. Each entry consumes at
// least a few bytes, so the loop is naturally bounded by the heap data and
// a bogus `nused` simply errors out on the first short read.
let mut mappings = Vec::new();
let mut mappings: Vec<VdsMapping> = Vec::new();
// Reads one self-describing selection at `pos`, returning its raw bytes and
// advancing past it — bounds-checked so a corrupt selection can't overrun.
let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result<Vec<u8>, FormatError> {
@@ -134,17 +146,57 @@ pub fn parse_vds_mappings(
Ok(bytes)
};
for _ in 0..nused {
// Source file name (with the version-1 same-file marker handled).
let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) {
if version > 1 {
return Err(FormatError::ChunkedReadError(
"unsupported VDS mapping block version".into(),
));
}
for i in 0..nused {
// Version 1 prefixes each entry with a flags byte; a name may then be
// omitted (same file) or replaced by the index of an earlier entry
// holding the same name (`H5D__virtual_load_layout`).
let flags = if version >= 1 {
let f = *heap_data.get(pos).ok_or(FormatError::UnexpectedEof {
expected: pos + 1,
available: heap_data.len(),
})?;
pos += 1;
if f & !VDS_ALL_FLAGS != 0 {
return Err(FormatError::ChunkedReadError(
"unknown VDS mapping flags".into(),
));
}
f
} else {
0
};
// Index of an earlier entry, for a shared name.
let earlier = |pos: &mut usize| -> Result<usize, FormatError> {
let idx = read_length(heap_data, *pos, length_size)?;
*pos += ls;
if idx >= i {
return Err(FormatError::ChunkedReadError(
"VDS mapping shares a name with a later entry".into(),
));
}
Ok(idx as usize)
};
let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 {
String::from(".")
} else if flags & VDS_SOURCE_FILE_SHARED != 0 {
let idx = earlier(&mut pos)?;
mappings[idx].source_file.clone()
} else {
read_null_terminated_string(heap_data, &mut pos)?
};
// Source dataset name.
let source_dataset = read_null_terminated_string(heap_data, &mut pos)?;
let source_dataset = if flags & VDS_SOURCE_DSET_SHARED != 0 {
let idx = earlier(&mut pos)?;
mappings[idx].source_dataset.clone()
} else {
read_null_terminated_string(heap_data, &mut pos)?
};
// Source selection, then virtual selection (both self-describing length).
let source_selection = read_selection(heap_data, &mut pos)?;
@@ -1035,6 +1087,62 @@ mod tests {
assert_eq!(v1.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]);
}
#[test]
fn parse_vds_mappings_v1_shared_names() {
// Written by HDF5 2.0 (h5py, libver=("v200", "v200")) for three
// mappings from `a_rather_long_source_file.h5:a_rather_long_dataset_name`
// and one from the same file: the entries carry flags 0x00, 0x03, 0x03
// and 0x06, so names after the first are stored as entry indices.
let blob: &[u8] = &[
0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x5f, 0x72, 0x61,
0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72,
0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x68, 0x35, 0x00, 0x61, 0x5f, 0x72,
0x61, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74,
0x61, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x00, 0x02, 0x00, 0x00, 0x00,
0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02,
0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00,
0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03,
0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x01,
0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02,
0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00,
0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x01, 0x00, 0x01, 0x00,
0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00,
0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00,
0x01, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02,
0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00,
0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00,
0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01,
0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x8e, 0xa7, 0xea, 0x7a,
];
let mappings = parse_vds_mappings(blob, 8).unwrap();
let names: Vec<(&str, &str)> = mappings
.iter()
.map(|m| (m.source_file.as_str(), m.source_dataset.as_str()))
.collect();
let (file, dset) = ("a_rather_long_source_file.h5", "a_rather_long_dataset_name");
assert_eq!(
names,
vec![(file, dset), (file, dset), (file, dset), (".", dset)]
);
}
#[test]
fn parse_vds_mappings_v1_forward_reference_is_error() {
// Entry 0 claiming to share entry 0's file name must not index past
// the entries decoded so far.
let mut blob = vec![0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x01];
blob.extend_from_slice(&[0u8; 8]);
blob.extend_from_slice(b"d\0");
assert!(parse_vds_mappings(&blob, 8).is_err());
// Unknown flag bits are refused.
let blob = [0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x08, b'd', 0];
assert!(parse_vds_mappings(&blob, 8).is_err());
}
#[test]
fn parse_vds_mappings_external_v0() {
// Block version 0 with an explicit (external) source file name.