Merge branch 'fix/p1-vds' into fix/p1-read-gaps

# Conflicts:
#	CHANGELOG.md
#	crates/clawhdf5-format/src/data_read.rs
This commit is contained in:
osobh
2026-09-25 22:42:04 -05:00
23 changed files with 2321 additions and 287 deletions
@@ -83,6 +83,45 @@ fn read_chunked_dataset(file_data: &[u8], dataset_path: &str) -> (Vec<u8>, Datat
(raw, datatype, dataspace)
}
/// Helper: read a virtual dataset with `vds::read_virtual_dataset`, giving it
/// the dataset's own fill value (same-file sources only).
fn read_virtual_fixture(file_data: &[u8], path: &str) -> (Vec<u8>, Datatype) {
let sig = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, sig).unwrap();
let addr = resolve_path_any(file_data, &sb, path).unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
let layout = DataLayout::parse(
&msg(MessageType::DataLayout).data,
sb.offset_size,
sb.length_size,
)
.unwrap();
let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
file_data,
&hdr.messages,
sb.offset_size,
sb.length_size,
)
.unwrap();
let v = clawhdf5_format::vds::read_virtual_dataset(
file_data,
&layout,
&ds,
&dt,
fill.as_deref(),
sb.offset_size,
sb.length_size,
None,
)
.unwrap();
assert_eq!(v.dims, ds.dimensions);
(v.data, dt)
}
/// Helper: read any dataset (contiguous or chunked) as f64.
fn read_dataset_f64_any(bytes: &[u8], path: &str) -> Vec<f64> {
let sig = find_signature(bytes).unwrap();
@@ -672,7 +711,7 @@ fn v4_virtual_dataset_same_file_read() {
// virt[4:8] <- (unmapped) => fill 0
// virt[8:12] <- src_b[0:4] (ALL) => 20,21,22,23
let file_data = include_bytes!("fixtures/vds_same_file.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "virt");
let (raw, datatype) = read_virtual_fixture(file_data, "virt");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(
values,
@@ -681,6 +720,38 @@ fn v4_virtual_dataset_same_file_read() {
);
}
#[test]
fn v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value() {
// The raw read API has no fill value message, so a virtual dataset with an
// unmapped region is an error there instead of zeros that may be wrong.
let file_data = include_bytes!("fixtures/vds_same_file.h5");
let sig = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, sig).unwrap();
let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
let layout = DataLayout::parse(
&msg(MessageType::DataLayout).data,
sb.offset_size,
sb.length_size,
)
.unwrap();
let err = read_raw_data_full(
file_data,
&layout,
&ds,
&dt,
None,
sb.offset_size,
sb.length_size,
)
.unwrap_err();
assert!(err.to_string().contains("fill value"), "{err}");
}
#[test]
fn v4_virtual_dataset_2d_same_file_read() {
// A 4x4 virtual dataset assembled from two 2x2 same-file sources placed as
@@ -689,7 +760,7 @@ fn v4_virtual_dataset_2d_same_file_read() {
// virt[2:4,2:4] <- src_b = [[5,6],[7,8]]
// everything else -> fill 0
let file_data = include_bytes!("fixtures/vds_2d_same_file.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "virt");
let (raw, datatype) = read_virtual_fixture(file_data, "virt");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(
values,