Merge branch 'fix/p1-vds' into fix/p1-read-gaps
# Conflicts: # CHANGELOG.md # crates/clawhdf5-format/src/data_read.rs
This commit is contained in:
@@ -83,6 +83,45 @@ fn read_chunked_dataset(file_data: &[u8], dataset_path: &str) -> (Vec<u8>, Datat
|
||||
(raw, datatype, dataspace)
|
||||
}
|
||||
|
||||
/// Helper: read a virtual dataset with `vds::read_virtual_dataset`, giving it
|
||||
/// the dataset's own fill value (same-file sources only).
|
||||
fn read_virtual_fixture(file_data: &[u8], path: &str) -> (Vec<u8>, Datatype) {
|
||||
let sig = find_signature(file_data).unwrap();
|
||||
let sb = Superblock::parse(file_data, sig).unwrap();
|
||||
let addr = resolve_path_any(file_data, &sb, path).unwrap();
|
||||
let hdr =
|
||||
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
||||
let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
|
||||
let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
|
||||
let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
|
||||
let layout = DataLayout::parse(
|
||||
&msg(MessageType::DataLayout).data,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)
|
||||
.unwrap();
|
||||
let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
|
||||
file_data,
|
||||
&hdr.messages,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)
|
||||
.unwrap();
|
||||
let v = clawhdf5_format::vds::read_virtual_dataset(
|
||||
file_data,
|
||||
&layout,
|
||||
&ds,
|
||||
&dt,
|
||||
fill.as_deref(),
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(v.dims, ds.dimensions);
|
||||
(v.data, dt)
|
||||
}
|
||||
|
||||
/// Helper: read any dataset (contiguous or chunked) as f64.
|
||||
fn read_dataset_f64_any(bytes: &[u8], path: &str) -> Vec<f64> {
|
||||
let sig = find_signature(bytes).unwrap();
|
||||
@@ -672,7 +711,7 @@ fn v4_virtual_dataset_same_file_read() {
|
||||
// virt[4:8] <- (unmapped) => fill 0
|
||||
// virt[8:12] <- src_b[0:4] (ALL) => 20,21,22,23
|
||||
let file_data = include_bytes!("fixtures/vds_same_file.h5");
|
||||
let (raw, datatype, _) = read_chunked_dataset(file_data, "virt");
|
||||
let (raw, datatype) = read_virtual_fixture(file_data, "virt");
|
||||
let values = read_as_i32(&raw, &datatype).unwrap();
|
||||
assert_eq!(
|
||||
values,
|
||||
@@ -681,6 +720,38 @@ fn v4_virtual_dataset_same_file_read() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value() {
|
||||
// The raw read API has no fill value message, so a virtual dataset with an
|
||||
// unmapped region is an error there instead of zeros that may be wrong.
|
||||
let file_data = include_bytes!("fixtures/vds_same_file.h5");
|
||||
let sig = find_signature(file_data).unwrap();
|
||||
let sb = Superblock::parse(file_data, sig).unwrap();
|
||||
let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
|
||||
let hdr =
|
||||
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
|
||||
let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
|
||||
let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
|
||||
let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
|
||||
let layout = DataLayout::parse(
|
||||
&msg(MessageType::DataLayout).data,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)
|
||||
.unwrap();
|
||||
let err = read_raw_data_full(
|
||||
file_data,
|
||||
&layout,
|
||||
&ds,
|
||||
&dt,
|
||||
None,
|
||||
sb.offset_size,
|
||||
sb.length_size,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("fill value"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v4_virtual_dataset_2d_same_file_read() {
|
||||
// A 4x4 virtual dataset assembled from two 2x2 same-file sources placed as
|
||||
@@ -689,7 +760,7 @@ fn v4_virtual_dataset_2d_same_file_read() {
|
||||
// virt[2:4,2:4] <- src_b = [[5,6],[7,8]]
|
||||
// everything else -> fill 0
|
||||
let file_data = include_bytes!("fixtures/vds_2d_same_file.h5");
|
||||
let (raw, datatype, _) = read_chunked_dataset(file_data, "virt");
|
||||
let (raw, datatype) = read_virtual_fixture(file_data, "virt");
|
||||
let values = read_as_i32(&raw, &datatype).unwrap();
|
||||
assert_eq!(
|
||||
values,
|
||||
|
||||
Reference in New Issue
Block a user