quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases 1 Wiki Activity
Files
63648c70001fdb4a9af8a6a7eeac9b9ebe1d32d8
clawhdf5/crates/clawhdf5-format/tests/integration_test.rs
T
osobhandClaude Opus 5.5 e94a52a88b fix(format): read unmapped VDS elements as the virtual dataset's fill value 
Elements of a virtual dataset that no mapping supplies (unmapped regions,
a missing source file, a missing source dataset) read as 0 instead of the
fill value libhdf5 returns — silent wrong data for any VDS created with a
non-zero fillvalue (read-matrix cases 0471/0472: -1 and 7 read as 0). A
missing source dataset was an error; libhdf5 reads it as fill.

Move VDS assembly into a new vds module following H5Dvirtual.c:
vds::read_virtual_dataset takes the dataset's fill value and a
VdsFileResolver that can refuse a name, and reports how many elements were
unmapped. Sources are read with their own fill value, and a source whose
datatype differs from the virtual dataset's is an error (libhdf5 converts).
File passes the dataset's fill value, resolves source names against the
virtual file's directory, and refuses names that leave it with an error
instead of reading them as fill. read_selection on a VDS goes through the
same fill-aware path.

The raw-read API (read_raw_data_full*) has no fill value, so it now errors
for a VDS with unmapped elements instead of guessing zeros.

Tests: vds_interop::vds_unmapped_regions_read_as_fill_value (external,
same-file, missing file/dataset, sparse source with its own fill, int
fill; earliest and latest format) and
vds_source_outside_directory_is_an_error_not_fill, both against h5py;
integration_test::v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value.

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-25 22:03:47 -05:00

1961 lines
68 KiB
Rust
Raw Blame History

use clawhdf5_format::attribute::{extract_attributes, extract_attributes_full, find_attribute};
use clawhdf5_format::data_layout::DataLayout;
use clawhdf5_format::data_read::{
read_as_f32, read_as_f64, read_as_i32, read_as_i64, read_raw_data, read_raw_data_full,
};
use clawhdf5_format::dataspace::Dataspace;
use clawhdf5_format::datatype::Datatype;
use clawhdf5_format::file_writer::{AttrValue, FileWriter};
use clawhdf5_format::filter_pipeline::{
FILTER_DEFLATE, FILTER_FLETCHER32, FILTER_SHUFFLE, FilterPipeline,
};
use clawhdf5_format::group_v2::resolve_path_any;
use clawhdf5_format::message_type::MessageType;
use clawhdf5_format::object_header::ObjectHeader;
use clawhdf5_format::signature::find_signature;
use clawhdf5_format::superblock::Superblock;
// ============================================================
// Helpers
// ============================================================
/// Helper: navigate to dataset and find the FilterPipeline message.
fn parse_filter_pipeline_from_fixture(file_data: &[u8], dataset_path: &str) -> FilterPipeline {
let offset = find_signature(file_data).expect("signature not found");
let sb = Superblock::parse(file_data, offset).expect("failed to parse superblock");
let addr = resolve_path_any(file_data, &sb, dataset_path).expect("dataset not found");
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)
.expect("failed to parse object header");
let fp_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::FilterPipeline)
.expect("no FilterPipeline message found");
FilterPipeline::parse(&fp_msg.data).expect("failed to parse filter pipeline")
}
/// Helper to read a chunked dataset from a fixture file.
fn read_chunked_dataset(file_data: &[u8], dataset_path: &str) -> (Vec<u8>, Datatype, Dataspace) {
let offset = find_signature(file_data).expect("signature not found");
let sb = Superblock::parse(file_data, offset).expect("failed to parse superblock");
let addr = resolve_path_any(file_data, &sb, dataset_path).expect("dataset not found");
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)
.expect("failed to parse object header");
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let pipeline = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::FilterPipeline)
.map(|m| FilterPipeline::parse(&m.data).unwrap());
let raw = read_raw_data_full(
file_data,
&layout,
&dataspace,
&datatype,
pipeline.as_ref(),
sb.offset_size,
sb.length_size,
)
.unwrap();
(raw, datatype, dataspace)
}
/// Helper: read a virtual dataset with `vds::read_virtual_dataset`, giving it
/// the dataset's own fill value (same-file sources only).
fn read_virtual_fixture(file_data: &[u8], path: &str) -> (Vec<u8>, Datatype) {
let sig = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, sig).unwrap();
let addr = resolve_path_any(file_data, &sb, path).unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
let layout = DataLayout::parse(
&msg(MessageType::DataLayout).data,
sb.offset_size,
sb.length_size,
)
.unwrap();
let fill = clawhdf5_format::fill_value::dataset_fill_value_in(
file_data,
&hdr.messages,
sb.offset_size,
sb.length_size,
)
.unwrap();
let v = clawhdf5_format::vds::read_virtual_dataset(
file_data,
&layout,
&ds,
&dt,
fill.as_deref(),
sb.offset_size,
sb.length_size,
None,
)
.unwrap();
assert_eq!(v.dims, ds.dimensions);
(v.data, dt)
}
/// Helper: read any dataset (contiguous or chunked) as f64.
fn read_dataset_f64_any(bytes: &[u8], path: &str) -> Vec<f64> {
let sig = find_signature(bytes).unwrap();
let sb = Superblock::parse(bytes, sig).unwrap();
let addr = resolve_path_any(bytes, &sb, path).unwrap();
let hdr = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let dt_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data;
let ds_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data;
let dl_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data;
let (dt, _) = Datatype::parse(dt_data).unwrap();
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
let dl = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
match &dl {
DataLayout::Chunked { .. } => {
let pipeline = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::FilterPipeline)
.map(|m| FilterPipeline::parse(&m.data).unwrap());
let raw = clawhdf5_format::chunked_read::read_chunked_data(
bytes,
&dl,
&ds,
&dt,
pipeline.as_ref(),
sb.offset_size,
sb.length_size,
)
.unwrap();
read_as_f64(&raw, &dt).unwrap()
}
_ => {
let raw = read_raw_data(bytes, &dl, &ds, &dt).unwrap();
read_as_f64(&raw, &dt).unwrap()
}
}
}
/// Helper: read any dataset as i32.
fn read_dataset_i32_any(bytes: &[u8], path: &str) -> Vec<i32> {
let sig = find_signature(bytes).unwrap();
let sb = Superblock::parse(bytes, sig).unwrap();
let addr = resolve_path_any(bytes, &sb, path).unwrap();
let hdr = ObjectHeader::parse(bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let dt_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data;
let ds_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data;
let dl_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data;
let (dt, _) = Datatype::parse(dt_data).unwrap();
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
let dl = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(bytes, &dl, &ds, &dt).unwrap();
read_as_i32(&raw, &dt).unwrap()
}
// ============================================================
// Superblock & Object Header Parsing
// ============================================================
#[test]
fn parse_minimal_v2_fixture() {
let data = include_bytes!("fixtures/minimal_v2.h5");
let offset = find_signature(data).expect("signature not found in minimal_v2.h5");
assert_eq!(offset, 0);
let sb = Superblock::parse(data, offset).expect("failed to parse superblock");
assert!(sb.version <= 3);
assert_eq!(sb.offset_size, 8);
assert_eq!(sb.length_size, 8);
assert_eq!(sb.base_address, 0);
assert!(sb.eof_address > 0);
assert!(sb.root_group_address > 0);
}
#[test]
fn parse_minimal_v2_object_header() {
let data = include_bytes!("fixtures/minimal_v2.h5");
let offset = find_signature(data).expect("signature not found");
let sb = Superblock::parse(data, offset).expect("failed to parse superblock");
let root_addr = sb.root_group_address as usize;
let hdr = ObjectHeader::parse(data, root_addr, sb.offset_size, sb.length_size)
.expect("failed to parse root group object header");
assert!(hdr.version == 1 || hdr.version == 2);
assert!(
!hdr.messages.is_empty(),
"root group object header has no messages"
);
}
#[test]
fn parse_simple_dataset_object_header() {
let data = include_bytes!("fixtures/simple_dataset.h5");
let offset = find_signature(data).expect("signature not found");
let sb = Superblock::parse(data, offset).expect("failed to parse superblock");
let root_addr = sb.root_group_address as usize;
let hdr = ObjectHeader::parse(data, root_addr, sb.offset_size, sb.length_size)
.expect("failed to parse root group object header");
assert!(hdr.version == 1 || hdr.version == 2);
assert!(
!hdr.messages.is_empty(),
"root group object header has no messages"
);
}
#[test]
fn parse_simple_dataset_fixture() {
let data = include_bytes!("fixtures/simple_dataset.h5");
let offset = find_signature(data).expect("signature not found in simple_dataset.h5");
assert_eq!(offset, 0);
let sb = Superblock::parse(data, offset).expect("failed to parse superblock");
assert!(sb.version <= 3);
assert_eq!(sb.offset_size, 8);
assert!(sb.eof_address > 0);
assert!(sb.root_group_address > 0);
}
// ============================================================
// Simple Dataset Reading
// ============================================================
#[test]
fn read_simple_dataset_values() {
let file_data = include_bytes!("fixtures/simple_dataset.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let dataset_offset = 0x0320usize;
let hdr = ObjectHeader::parse(file_data, dataset_offset, sb.offset_size, sb.length_size)
.expect("failed to parse dataset object header");
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
assert_eq!(dataspace.dimensions, vec![3]);
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
assert_eq!(datatype.type_size(), 8);
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
match &layout {
DataLayout::Contiguous { address, size } => {
assert_eq!(*address, Some(0x0800));
assert_eq!(*size, 24);
}
other => panic!("expected Contiguous, got {other:?}"),
}
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values, vec![1.0, 2.0, 3.0]);
}
// ============================================================
// Attribute integration tests
// ============================================================
#[test]
fn attrs_h5_dataset_description() {
let file_data = include_bytes!("fixtures/attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let desc = find_attribute(&attrs, "description").expect("description attr not found");
let s = desc.read_as_string().unwrap();
assert_eq!(s, "test dataset");
}
#[test]
fn attrs_h5_dataset_version() {
let file_data = include_bytes!("fixtures/attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let version_attr = find_attribute(&attrs, "version").expect("version attr not found");
let vals = version_attr.read_as_i64().unwrap();
assert_eq!(vals, vec![42]);
}
#[test]
fn attrs_h5_dataset_scale() {
let file_data = include_bytes!("fixtures/attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let scale_attr = find_attribute(&attrs, "scale").expect("scale attr not found");
let vals = scale_attr.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
// 3.14 here is the literal value baked into the binary fixture (fixtures/attrs.h5),
// not an arbitrary sample value, so it cannot be swapped for another constant.
#[allow(clippy::approx_constant)]
let expected = 3.14;
assert!((vals[0] - expected).abs() < 1e-10);
}
#[test]
fn attrs_h5_root_file_type() {
let file_data = include_bytes!("fixtures/attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let root_addr = sb.root_group_address as usize;
let hdr = ObjectHeader::parse(file_data, root_addr, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let ft = find_attribute(&attrs, "file_type").expect("file_type attr not found");
let s = ft.read_as_string().unwrap();
assert_eq!(s, "experiment");
}
#[test]
fn mixed_attrs_h5_temperatures() {
let file_data = include_bytes!("fixtures/mixed_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let exp_addr = resolve_path_any(file_data, &sb, "experiment").unwrap();
let hdr =
ObjectHeader::parse(file_data, exp_addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let temp = find_attribute(&attrs, "temperatures").expect("temperatures attr not found");
let vals = temp.read_as_f64().unwrap();
assert_eq!(vals.len(), 3);
assert!((vals[0] - 22.5).abs() < 1e-10);
assert!((vals[1] - 23.1).abs() < 1e-10);
assert!((vals[2] - 21.8).abs() < 1e-10);
}
#[test]
fn mixed_attrs_h5_name() {
let file_data = include_bytes!("fixtures/mixed_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let exp_addr = resolve_path_any(file_data, &sb, "experiment").unwrap();
let hdr =
ObjectHeader::parse(file_data, exp_addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let name_attr = find_attribute(&attrs, "name").expect("name attr not found");
let s = name_attr.read_as_string().unwrap();
assert_eq!(s, "run_001");
}
#[test]
fn mixed_attrs_h5_iterations() {
let file_data = include_bytes!("fixtures/mixed_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let exp_addr = resolve_path_any(file_data, &sb, "experiment").unwrap();
let hdr =
ObjectHeader::parse(file_data, exp_addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let iter_attr = find_attribute(&attrs, "iterations").expect("iterations attr not found");
let vals = iter_attr.read_as_i64().unwrap();
assert_eq!(vals, vec![1000]);
}
// ============================================================
// Variable-Length Strings
// ============================================================
#[test]
fn vl_strings_h5_names_dataset() {
let file_data = include_bytes!("fixtures/vl_strings.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let names_addr = resolve_path_any(file_data, &sb, "names").unwrap();
let hdr = ObjectHeader::parse(
file_data,
names_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
assert_eq!(dataspace.num_elements(), 3);
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
match &datatype {
Datatype::VariableLength { is_string, .. } => assert!(is_string),
other => panic!("expected VL string type, got {other:?}"),
}
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let strings = clawhdf5_format::vl_data::read_vl_strings(
file_data,
&raw,
dataspace.num_elements(),
sb.offset_size,
sb.length_size,
)
.unwrap();
assert_eq!(strings, vec!["Alice", "Bob", "Charlie"]);
}
#[test]
fn vl_strings_h5_root_vl_attr() {
let file_data = include_bytes!("fixtures/vl_strings.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let root_addr = sb.root_group_address as usize;
let hdr = ObjectHeader::parse(file_data, root_addr, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let vl_attr = find_attribute(&attrs, "vl_attr").expect("vl_attr not found");
let strings = vl_attr
.read_vl_strings(file_data, sb.offset_size, sb.length_size)
.unwrap();
assert_eq!(strings.len(), 1);
assert_eq!(strings[0], "hello variable length");
}
// ============================================================
// Filter Pipeline Integration Tests
// ============================================================
#[test]
fn fixture_chunked_deflate_filter_pipeline() {
let file_data = include_bytes!("fixtures/chunked_deflate.h5");
let fp = parse_filter_pipeline_from_fixture(file_data, "data");
assert!(fp.filters.iter().any(|f| f.filter_id == FILTER_DEFLATE));
let deflate = fp
.filters
.iter()
.find(|f| f.filter_id == FILTER_DEFLATE)
.unwrap();
assert_eq!(deflate.client_data, vec![6]);
}
#[test]
fn fixture_chunked_shuffle_deflate_filter_pipeline() {
let file_data = include_bytes!("fixtures/chunked_shuffle_deflate.h5");
let fp = parse_filter_pipeline_from_fixture(file_data, "data");
assert_eq!(fp.filters.len(), 2);
assert_eq!(fp.filters[0].filter_id, FILTER_SHUFFLE);
assert_eq!(fp.filters[0].client_data, vec![8]);
assert_eq!(fp.filters[1].filter_id, FILTER_DEFLATE);
assert_eq!(fp.filters[1].client_data, vec![6]);
}
#[test]
fn fixture_chunked_fletcher32_filter_pipeline() {
let file_data = include_bytes!("fixtures/chunked_fletcher32.h5");
let fp = parse_filter_pipeline_from_fixture(file_data, "data");
assert_eq!(fp.filters.len(), 1);
assert_eq!(fp.filters[0].filter_id, FILTER_FLETCHER32);
}
#[test]
fn fixture_chunked_deflate_has_chunked_layout() {
let file_data = include_bytes!("fixtures/chunked_deflate.h5");
let offset = find_signature(file_data).expect("signature not found");
let sb = Superblock::parse(file_data, offset).expect("failed to parse superblock");
let addr = resolve_path_any(file_data, &sb, "data").expect("dataset not found");
let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)
.expect("failed to parse object header");
let layout_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.expect("no DataLayout message found");
let layout = DataLayout::parse(&layout_msg.data, sb.offset_size, sb.length_size)
.expect("failed to parse data layout");
match layout {
DataLayout::Chunked { btree_address, .. } => {
assert!(btree_address.is_some(), "btree_address should be set");
}
other => panic!("expected Chunked layout, got {:?}", other),
}
}
// ============================================================
// Chunked Dataset Integration Tests
// ============================================================
#[test]
fn chunked_deflate_read_values() {
let file_data = include_bytes!("fixtures/chunked_deflate.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
#[test]
fn chunked_shuffle_deflate_read_values() {
let file_data = include_bytes!("fixtures/chunked_shuffle_deflate.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
#[test]
fn chunked_fletcher32_read_values() {
let file_data = include_bytes!("fixtures/chunked_fletcher32.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
#[test]
fn chunked_2d_read_values() {
let file_data = include_bytes!("fixtures/chunked_2d.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix");
let values = read_as_f32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 60);
for (i, &v) in values.iter().enumerate() {
assert!(
(v - i as f32).abs() < 1e-6,
"mismatch at index {i}: got {v}"
);
}
}
#[test]
fn chunked_large_read_values() {
let file_data = include_bytes!("fixtures/chunked_large.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "big");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 1000);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as i32, "mismatch at index {i}");
}
}
#[test]
fn chunked_nofilter_read_values() {
let file_data = include_bytes!("fixtures/chunked_nofilter.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "raw");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 50);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
// ============================================================
// V4 Layout Index Type Tests
// ============================================================
#[test]
fn v4_single_chunk_read() {
let file_data = include_bytes!("fixtures/v4_single_chunk.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "small");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values, vec![1.0, 2.0, 3.0]);
}
#[test]
fn v4_single_chunk_deflate_read() {
let file_data = include_bytes!("fixtures/v4_single_chunk_deflate.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "small");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values, vec![1.0, 2.0, 3.0]);
}
#[test]
fn v4_implicit_read() {
let file_data = include_bytes!("fixtures/v4_implicit.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
#[test]
fn v4_fixed_array_read() {
let file_data = include_bytes!("fixtures/v4_fixed_array.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "data");
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values.len(), 100);
for (i, &v) in values.iter().enumerate() {
assert_eq!(v, i as f64, "mismatch at index {i}");
}
}
#[test]
fn v4_virtual_dataset_same_file_read() {
// A 1-D virtual dataset assembled from two same-file sources:
// virt[0:4] <- src_a[1:5] (partial source hyperslab) => 11,12,13,14
// virt[4:8] <- (unmapped) => fill 0
// virt[8:12] <- src_b[0:4] (ALL) => 20,21,22,23
let file_data = include_bytes!("fixtures/vds_same_file.h5");
let (raw, datatype) = read_virtual_fixture(file_data, "virt");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(
values,
vec![11, 12, 13, 14, 0, 0, 0, 0, 20, 21, 22, 23],
"VDS assembly (partial source slice + fill gap) mismatch"
);
}
#[test]
fn v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value() {
// The raw read API has no fill value message, so a virtual dataset with an
// unmapped region is an error there instead of zeros that may be wrong.
let file_data = include_bytes!("fixtures/vds_same_file.h5");
let sig = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, sig).unwrap();
let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap();
let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap();
let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap();
let layout = DataLayout::parse(
&msg(MessageType::DataLayout).data,
sb.offset_size,
sb.length_size,
)
.unwrap();
let err = read_raw_data_full(
file_data,
&layout,
&ds,
&dt,
None,
sb.offset_size,
sb.length_size,
)
.unwrap_err();
assert!(err.to_string().contains("fill value"), "{err}");
}
#[test]
fn v4_virtual_dataset_2d_same_file_read() {
// A 4x4 virtual dataset assembled from two 2x2 same-file sources placed as
// non-contiguous blocks (exercises N-dimensional row-major scatter):
// virt[0:2,0:2] <- src_a = [[1,2],[3,4]]
// virt[2:4,2:4] <- src_b = [[5,6],[7,8]]
// everything else -> fill 0
let file_data = include_bytes!("fixtures/vds_2d_same_file.h5");
let (raw, datatype) = read_virtual_fixture(file_data, "virt");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(
values,
vec![1, 2, 0, 0, 3, 4, 0, 0, 0, 0, 5, 6, 0, 0, 7, 8],
"2-D VDS block scatter mismatch"
);
}
#[test]
fn scaleoffset_float_escale_reads_as_raw() {
// The scale-offset filter's floating-point *E-scale* mode (cd_values[0] = 1)
// is not actually implemented by the HDF5 library: when asked for it, HDF5
// stores the chunk raw (no minbits/minval header) and sets the chunk filter
// mask to skip the filter. So such a dataset must read back verbatim purely
// by honoring the per-chunk filter mask — no E-scale decoder is needed.
let file_data = include_bytes!("fixtures/scaleoffset_float_escale.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "x");
let values = read_as_f64(&raw, &datatype).unwrap();
let expect: Vec<f64> = (0..20).map(|i| i as f64 * 0.25).collect();
assert_eq!(
values, expect,
"E-scale (raw + masked filter) must read verbatim"
);
}
#[test]
fn v4_virtual_dataset_cycle_errors_not_overflow() {
// virt -> virt2 -> virt (both virtual, same file). The reader must reject
// the nested virtual source rather than recurse into a stack overflow.
let file_data = include_bytes!("fixtures/vds_cyclic.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "virt").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds = Dataspace::parse(
&hdr.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data,
sb.length_size,
)
.unwrap();
let (dt, _) = Datatype::parse(
&hdr.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data,
)
.unwrap();
let layout = DataLayout::parse(
&hdr.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data,
sb.offset_size,
sb.length_size,
)
.unwrap();
let r = read_raw_data_full(
file_data,
&layout,
&ds,
&dt,
None,
sb.offset_size,
sb.length_size,
);
assert!(
r.is_err(),
"cyclic virtual dataset must error, not overflow"
);
}
#[test]
fn v4_virtual_dataset_external_file_read() {
use clawhdf5_format::data_read::read_raw_data_full_with_resolver;
// The virtual file maps virt[0:8] <- (external) ext_src.h5:/data = [10..17].
let virt = include_bytes!("fixtures/vds_external_virt.h5");
let src = include_bytes!("fixtures/vds_external_src.h5").to_vec();
let sig = find_signature(virt).unwrap();
let sb = Superblock::parse(virt, sig).unwrap();
let addr = resolve_path_any(virt, &sb, "virt").unwrap();
let hdr = ObjectHeader::parse(virt, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds = Dataspace::parse(
&hdr.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data,
sb.length_size,
)
.unwrap();
let (dt, _) = Datatype::parse(
&hdr.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data,
)
.unwrap();
let layout = DataLayout::parse(
&hdr.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data,
sb.offset_size,
sb.length_size,
)
.unwrap();
// Resolver supplies the external source file's bytes by its stored name.
let resolver = |name: &str| -> Option<Vec<u8>> {
if name == "ext_src.h5" {
Some(src.clone())
} else {
None
}
};
let raw = read_raw_data_full_with_resolver(
virt,
&layout,
&ds,
&dt,
None,
sb.offset_size,
sb.length_size,
Some(&resolver),
)
.unwrap();
let values = read_as_i32(&raw, &dt).unwrap();
assert_eq!(values, vec![10, 11, 12, 13, 14, 15, 16, 17]);
// With no resolver, an external source is a clean error (not wrong data).
let no_resolver = read_raw_data_full_with_resolver(
virt,
&layout,
&ds,
&dt,
None,
sb.offset_size,
sb.length_size,
None,
);
assert!(no_resolver.is_err());
}
#[test]
fn v4_paged_fixed_array_read() {
// 1025 chunks of 16 int32s, gzip-filtered => Fixed Array index whose data
// block is *paged* (page holds 1024 elements). Page 0 is full, page 1 holds
// the single trailing chunk. Chunk k stores value k at its first element.
let file_data = include_bytes!("fixtures/v4_fixed_array_paged.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "big");
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 1025 * 16);
for k in 0..1025usize {
assert_eq!(
values[k * 16],
k as i32,
"chunk-start mismatch at chunk {k}"
);
for j in 1..16 {
assert_eq!(
values[k * 16 + j],
0,
"non-start element nonzero at {}",
k * 16 + j
);
}
}
}
#[test]
fn v4_2d_fixed_array_read() {
let file_data = include_bytes!("fixtures/v4_2d.h5");
let (raw, datatype, _) = read_chunked_dataset(file_data, "matrix");
let values = read_as_f32(&raw, &datatype).unwrap();
assert_eq!(values.len(), 60);
for (i, &v) in values.iter().enumerate() {
assert!(
(v - i as f32).abs() < 1e-6,
"mismatch at index {i}: got {v}"
);
}
}
// ============================================================
// V1 Group Navigation Tests
// ============================================================
#[test]
fn v1_two_groups_resolve_group1() {
let file_data = include_bytes!("fixtures/two_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "group1");
assert!(addr.is_ok(), "should resolve group1 in two_groups.h5");
}
#[test]
fn v1_two_groups_resolve_dataset_in_group() {
let file_data = include_bytes!("fixtures/two_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "group1/values");
assert!(addr.is_ok(), "should resolve group1/values");
}
#[test]
fn v1_two_groups_read_group1_values() {
let file_data = include_bytes!("fixtures/two_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "group1/values").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(values, vec![10, 20, 30]);
}
#[test]
fn v1_two_groups_read_group2_temps() {
let file_data = include_bytes!("fixtures/two_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "group2/temps").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_f32(&raw, &datatype).unwrap();
assert!((values[0] - 98.6).abs() < 0.01);
assert!((values[1] - 37.0).abs() < 0.01);
}
#[test]
fn v1_nested_groups_deep_path() {
let file_data = include_bytes!("fixtures/nested_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "a/b/c/deep");
assert!(
addr.is_ok(),
"should resolve a/b/c/deep in nested_groups.h5"
);
}
#[test]
fn v1_nested_groups_read_deep_dataset() {
let file_data = include_bytes!("fixtures/nested_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "a/b/c/deep").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values, vec![42.0]);
}
// ============================================================
// V2 Group Navigation Tests
// ============================================================
#[test]
fn v2_groups_resolve_sensor_temperature() {
let file_data = include_bytes!("fixtures/v2_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "sensors/temperature");
assert!(
addr.is_ok(),
"should resolve sensors/temperature in v2_groups.h5"
);
}
#[test]
fn v2_groups_read_temperature_values() {
let file_data = include_bytes!("fixtures/v2_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "sensors/temperature").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values, vec![22.5, 23.1, 21.8]);
}
#[test]
fn v2_groups_read_humidity_values() {
let file_data = include_bytes!("fixtures/v2_groups.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "sensors/humidity").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_i32(&raw, &datatype).unwrap();
assert_eq!(values, vec![45, 50, 55]);
}
#[test]
fn v2_many_links_resolve_dataset() {
let file_data = include_bytes!("fixtures/v2_many_links.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let addr = resolve_path_any(file_data, &sb, "dataset_015").unwrap();
let hdr =
ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let (datatype, _) = Datatype::parse(&dt_msg.data).unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &layout, &dataspace, &datatype).unwrap();
let values = read_as_f64(&raw, &datatype).unwrap();
assert_eq!(values, vec![15.0]);
}
// ============================================================
// Write Round-Trip: All Datatype Classes
// ============================================================
#[test]
fn write_roundtrip_f64_dataset() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0, 2.0, 3.0, 4.0, 5.0])
.with_shape(&[5]);
let bytes = fw.finish().unwrap();
let values = read_dataset_f64_any(&bytes, "data");
assert_eq!(values, vec![1.0, 2.0, 3.0, 4.0, 5.0]);
}
#[test]
fn write_roundtrip_f32_dataset() {
let mut fw = FileWriter::new();
let data: Vec<f32> = vec![1.5, 2.5, 3.5];
fw.create_dataset("data")
.with_f32_data(&data)
.with_shape(&[3]);
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let dt_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data;
let ds_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data;
let dl_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data;
let (dt, _) = Datatype::parse(dt_data).unwrap();
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
let dl = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(&bytes, &dl, &ds, &dt).unwrap();
let values = read_as_f32(&raw, &dt).unwrap();
assert_eq!(values, vec![1.5, 2.5, 3.5]);
}
#[test]
fn write_roundtrip_i32_dataset() {
let mut fw = FileWriter::new();
fw.create_dataset("ints").with_i32_data(&[10, 20, 30, 40]);
let bytes = fw.finish().unwrap();
let values = read_dataset_i32_any(&bytes, "ints");
assert_eq!(values, vec![10, 20, 30, 40]);
}
#[test]
fn write_roundtrip_i64_dataset() {
let mut fw = FileWriter::new();
fw.create_dataset("longs").with_i64_data(&[100, 200, 300]);
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "longs").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let dt_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data;
let ds_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data;
let dl_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data;
let (dt, _) = Datatype::parse(dt_data).unwrap();
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
let dl = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(&bytes, &dl, &ds, &dt).unwrap();
let values = read_as_i64(&raw, &dt).unwrap();
assert_eq!(values, vec![100, 200, 300]);
}
#[test]
fn write_roundtrip_u8_dataset() {
let mut fw = FileWriter::new();
fw.create_dataset("bytes").with_u8_data(&[0, 127, 255]);
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "bytes").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let ds_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data;
let dl_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data;
let dt_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data;
let (dt, _) = Datatype::parse(dt_data).unwrap();
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
let dl = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(&bytes, &dl, &ds, &dt).unwrap();
assert_eq!(raw, vec![0, 127, 255]);
}
// ============================================================
// Write Round-Trip: Attributes
// ============================================================
#[test]
fn write_roundtrip_scalar_f64_attr() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0])
.set_attr("scale", AttrValue::F64(3.25));
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let scale = find_attribute(&attrs, "scale").expect("scale attr not found");
let vals = scale.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
assert!((vals[0] - 3.25).abs() < 1e-10);
}
#[test]
fn write_roundtrip_string_attr() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0])
.set_attr("name", AttrValue::String("test_dataset".into()));
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let name = find_attribute(&attrs, "name").expect("name attr not found");
let s = name.read_as_string().unwrap();
assert_eq!(s, "test_dataset");
}
#[test]
fn write_roundtrip_i64_attr() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0])
.set_attr("version", AttrValue::I64(42));
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let ver = find_attribute(&attrs, "version").expect("version attr not found");
let vals = ver.read_as_i64().unwrap();
assert_eq!(vals, vec![42]);
}
#[test]
fn write_roundtrip_f64_array_attr() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0])
.set_attr("temps", AttrValue::F64Array(vec![22.5, 23.1, 21.8]));
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let temps = find_attribute(&attrs, "temps").expect("temps attr not found");
let vals = temps.read_as_f64().unwrap();
assert_eq!(vals.len(), 3);
assert!((vals[0] - 22.5).abs() < 1e-10);
assert!((vals[1] - 23.1).abs() < 1e-10);
assert!((vals[2] - 21.8).abs() < 1e-10);
}
#[test]
fn write_roundtrip_root_attr() {
let mut fw = FileWriter::new();
fw.set_root_attr("file_format", AttrValue::String("HDF5".into()));
fw.create_dataset("dummy").with_f64_data(&[0.0]);
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let hdr = ObjectHeader::parse(
&bytes,
sb.root_group_address as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let fmt = find_attribute(&attrs, "file_format").expect("file_format attr not found");
let s = fmt.read_as_string().unwrap();
assert_eq!(s, "HDF5");
}
// ============================================================
// Write Round-Trip: Chunked + Filters
// ============================================================
#[test]
fn write_roundtrip_chunked_no_filter() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..50).map(|i| i as f64).collect();
fw.create_dataset("data")
.with_f64_data(&data)
.with_shape(&[50])
.with_chunks(&[10]);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "data");
assert_eq!(result, data);
}
#[test]
fn write_roundtrip_chunked_deflate() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
fw.create_dataset("data")
.with_f64_data(&data)
.with_shape(&[100])
.with_chunks(&[25])
.with_deflate(6);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "data");
assert_eq!(result, data);
}
#[test]
fn write_roundtrip_chunked_shuffle_deflate() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
fw.create_dataset("data")
.with_f64_data(&data)
.with_shape(&[100])
.with_chunks(&[50])
.with_shuffle()
.with_deflate(6);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "data");
assert_eq!(result, data);
}
#[test]
fn write_roundtrip_chunked_fletcher32() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..100).map(|i| i as f64).collect();
fw.create_dataset("data")
.with_f64_data(&data)
.with_shape(&[100])
.with_chunks(&[100])
.with_fletcher32();
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "data");
assert_eq!(result, data);
}
// ============================================================
// Write Round-Trip: Resizable Datasets (Extensible Array)
// ============================================================
#[test]
fn write_roundtrip_resizable_dataset() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..30).map(|i| i as f64).collect();
fw.create_dataset("data")
.with_f64_data(&data)
.with_shape(&[30])
.with_chunks(&[10])
.with_maxshape(&[u64::MAX]);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "data");
assert_eq!(result, data);
}
// ============================================================
// Write Round-Trip: Groups
// ============================================================
#[test]
fn write_roundtrip_group_with_dataset() {
let mut fw = FileWriter::new();
let mut grp = fw.create_group("mygroup");
grp.create_dataset("vals")
.with_f64_data(&[10.0, 20.0, 30.0])
.with_shape(&[3]);
let g = grp.finish();
fw.add_group(g);
let bytes = fw.finish().unwrap();
let values = read_dataset_f64_any(&bytes, "mygroup/vals");
assert_eq!(values, vec![10.0, 20.0, 30.0]);
}
#[test]
fn write_roundtrip_multiple_datasets() {
let mut fw = FileWriter::new();
fw.create_dataset("a").with_f64_data(&[1.0, 2.0]);
fw.create_dataset("b").with_f64_data(&[3.0, 4.0]);
fw.create_dataset("c").with_f64_data(&[5.0, 6.0]);
let bytes = fw.finish().unwrap();
assert_eq!(read_dataset_f64_any(&bytes, "a"), vec![1.0, 2.0]);
assert_eq!(read_dataset_f64_any(&bytes, "b"), vec![3.0, 4.0]);
assert_eq!(read_dataset_f64_any(&bytes, "c"), vec![5.0, 6.0]);
}
// ============================================================
// Write Round-Trip: 2D datasets
// ============================================================
#[test]
fn write_roundtrip_2d_contiguous() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..12).map(|i| i as f64).collect();
fw.create_dataset("matrix")
.with_f64_data(&data)
.with_shape(&[3, 4]);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "matrix");
assert_eq!(result, data);
}
#[test]
fn write_roundtrip_2d_chunked() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..24).map(|i| i as f64).collect();
fw.create_dataset("matrix")
.with_f64_data(&data)
.with_shape(&[4, 6])
.with_chunks(&[2, 3]);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "matrix");
assert_eq!(result, data);
}
// ============================================================
// Edge Cases
// ============================================================
#[test]
fn write_roundtrip_single_element_dataset() {
let mut fw = FileWriter::new();
fw.create_dataset("scalar")
.with_f64_data(&[42.0])
.with_shape(&[1]);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "scalar");
assert_eq!(result, vec![42.0]);
}
#[test]
fn write_roundtrip_large_dataset() {
let mut fw = FileWriter::new();
let data: Vec<f64> = (0..10_000).map(|i| i as f64).collect();
fw.create_dataset("big")
.with_f64_data(&data)
.with_shape(&[10_000]);
let bytes = fw.finish().unwrap();
let result = read_dataset_f64_any(&bytes, "big");
assert_eq!(result.len(), 10_000);
assert_eq!(result[0], 0.0);
assert_eq!(result[9999], 9999.0);
}
#[test]
fn write_roundtrip_group_with_attrs() {
let mut fw = FileWriter::new();
let mut grp = fw.create_group("experiment");
grp.set_attr("name", AttrValue::String("run_001".into()));
grp.create_dataset("data")
.with_f64_data(&[1.0, 2.0])
.with_shape(&[2]);
let g = grp.finish();
fw.add_group(g);
let bytes = fw.finish().unwrap();
// Verify dataset
let result = read_dataset_f64_any(&bytes, "experiment/data");
assert_eq!(result, vec![1.0, 2.0]);
// Verify group attribute
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let grp_addr = resolve_path_any(&bytes, &sb, "experiment").unwrap();
let hdr =
ObjectHeader::parse(&bytes, grp_addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let name = find_attribute(&attrs, "name").expect("name attr not found");
let s = name.read_as_string().unwrap();
assert_eq!(s, "run_001");
}
#[test]
fn write_roundtrip_dataset_with_multiple_attrs() {
let mut fw = FileWriter::new();
fw.create_dataset("data")
.with_f64_data(&[1.0, 2.0, 3.0])
.with_shape(&[3])
.set_attr("description", AttrValue::String("test data".into()))
.set_attr("version", AttrValue::I64(2))
.set_attr("scale", AttrValue::F64(0.5));
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let attrs = extract_attributes(&hdr, sb.length_size).unwrap();
let desc = find_attribute(&attrs, "description").expect("description not found");
assert_eq!(desc.read_as_string().unwrap(), "test data");
let ver = find_attribute(&attrs, "version").expect("version not found");
assert_eq!(ver.read_as_i64().unwrap(), vec![2]);
let scale = find_attribute(&attrs, "scale").expect("scale not found");
let vals = scale.read_as_f64().unwrap();
assert!((vals[0] - 0.5).abs() < 1e-10);
}
#[test]
fn superblock_v3_written_files() {
let mut fw = FileWriter::new();
fw.create_dataset("data").with_f64_data(&[1.0]);
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
assert_eq!(sb.version, 3, "FileWriter should produce v3 superblocks");
assert_eq!(sb.offset_size, 8);
assert_eq!(sb.length_size, 8);
}
#[test]
fn write_roundtrip_large_chunked_i32() {
let mut fw = FileWriter::new();
let data: Vec<i32> = (0..5000).collect();
fw.create_dataset("big")
.with_i32_data(&data)
.with_shape(&[5000])
.with_chunks(&[500]);
let bytes = fw.finish().unwrap();
let sig = find_signature(&bytes).unwrap();
let sb = Superblock::parse(&bytes, sig).unwrap();
let addr = resolve_path_any(&bytes, &sb, "big").unwrap();
let hdr = ObjectHeader::parse(&bytes, addr as usize, sb.offset_size, sb.length_size).unwrap();
let dt_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap()
.data;
let ds_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap()
.data;
let dl_data = &hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap()
.data;
let (dt, _) = Datatype::parse(dt_data).unwrap();
let ds = Dataspace::parse(ds_data, sb.length_size).unwrap();
let dl = DataLayout::parse(dl_data, sb.offset_size, sb.length_size).unwrap();
let pipeline = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::FilterPipeline)
.map(|m| FilterPipeline::parse(&m.data).unwrap());
let raw = read_raw_data_full(
&bytes,
&dl,
&ds,
&dt,
pipeline.as_ref(),
sb.offset_size,
sb.length_size,
)
.unwrap();
let values = read_as_i32(&raw, &dt).unwrap();
assert_eq!(values.len(), 5000);
assert_eq!(values[0], 0);
assert_eq!(values[4999], 4999);
}
// ============================================================
// Dense Attribute Tests (AttributeInfo + fractal heap + B-tree v2)
// ============================================================
#[test]
fn dense_attrs_dataset_count() {
let file_data = include_bytes!("fixtures/dense_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
// Should have an AttributeInfo message since >8 attrs triggers dense storage
let has_attr_info = hdr
.messages
.iter()
.any(|m| m.msg_type == MessageType::AttributeInfo);
assert!(
has_attr_info,
"expected AttributeInfo message for dense attributes"
);
let attrs = extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
assert_eq!(attrs.len(), 50, "expected 50 dense attributes");
}
#[test]
fn dense_attrs_dataset_first_attr() {
let file_data = include_bytes!("fixtures/dense_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
let attr_000 = find_attribute(&attrs, "attr_000").expect("attr_000 not found");
let vals = attr_000.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
assert!(
(vals[0] - 0.0).abs() < 1e-10,
"attr_000 should be 0.0, got {}",
vals[0]
);
}
#[test]
fn dense_attrs_dataset_middle_attr() {
let file_data = include_bytes!("fixtures/dense_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
let attr_025 = find_attribute(&attrs, "attr_025").expect("attr_025 not found");
let vals = attr_025.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
assert!(
(vals[0] - 37.5).abs() < 1e-10,
"attr_025 should be 37.5, got {}",
vals[0]
);
}
#[test]
fn dense_attrs_dataset_last_attr() {
let file_data = include_bytes!("fixtures/dense_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
let attr_049 = find_attribute(&attrs, "attr_049").expect("attr_049 not found");
let vals = attr_049.read_as_f64().unwrap();
assert_eq!(vals.len(), 1);
assert!(
(vals[0] - 73.5).abs() < 1e-10,
"attr_049 should be 73.5, got {}",
vals[0]
);
}
#[test]
fn dense_attrs_dataset_all_values_correct() {
let file_data = include_bytes!("fixtures/dense_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
for i in 0..50 {
let name = format!("attr_{i:03}");
let attr = find_attribute(&attrs, &name).unwrap_or_else(|| panic!("{name} not found"));
let vals = attr.read_as_f64().unwrap();
let expected = i as f64 * 1.5;
assert!(
(vals[0] - expected).abs() < 1e-10,
"{name}: expected {expected}, got {}",
vals[0]
);
}
}
#[test]
fn dense_attrs_root_group() {
let file_data = include_bytes!("fixtures/dense_attrs_root.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let hdr = ObjectHeader::parse(
file_data,
sb.root_group_address as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let attrs = extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
assert_eq!(attrs.len(), 20, "expected 20 root group dense attributes");
let attr_00 = find_attribute(&attrs, "root_attr_00").expect("root_attr_00 not found");
let vals = attr_00.read_as_f64().unwrap();
assert!((vals[0] - 0.0).abs() < 1e-10);
let attr_19 = find_attribute(&attrs, "root_attr_19").expect("root_attr_19 not found");
let vals = attr_19.read_as_f64().unwrap();
assert!((vals[0] - 38.0).abs() < 1e-10);
}
#[test]
fn dense_attrs_compact_still_works() {
// Ensure extract_attributes_full also works for compact (non-dense) attributes
let file_data = include_bytes!("fixtures/attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
// extract_attributes_full should return the same results as extract_attributes for compact
let attrs_compact = extract_attributes(&hdr, sb.length_size).unwrap();
let attrs_full =
extract_attributes_full(file_data, &hdr, sb.offset_size, sb.length_size).unwrap();
assert_eq!(attrs_compact.len(), attrs_full.len());
let desc = find_attribute(&attrs_full, "description").expect("description not found");
assert_eq!(desc.read_as_string().unwrap(), "test dataset");
}
#[test]
fn dense_attrs_dataset_data_still_readable() {
// Ensure the dataset data is still readable alongside dense attrs
let file_data = include_bytes!("fixtures/dense_attrs.h5");
let offset = find_signature(file_data).unwrap();
let sb = Superblock::parse(file_data, offset).unwrap();
let data_addr = resolve_path_any(file_data, &sb, "data").unwrap();
let hdr = ObjectHeader::parse(
file_data,
data_addr as usize,
sb.offset_size,
sb.length_size,
)
.unwrap();
let dt_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Datatype)
.unwrap();
let ds_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::Dataspace)
.unwrap();
let dl_msg = hdr
.messages
.iter()
.find(|m| m.msg_type == MessageType::DataLayout)
.unwrap();
let (dt, _) = Datatype::parse(&dt_msg.data).unwrap();
let ds = Dataspace::parse(&ds_msg.data, sb.length_size).unwrap();
let dl = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size).unwrap();
let raw = read_raw_data(file_data, &dl, &ds, &dt).unwrap();
let values = read_as_f64(&raw, &dt).unwrap();
assert_eq!(values, vec![1.0, 2.0, 3.0]);
}
Reference in New Issue View Git Blame Copy Permalink