From 5705866d40d64c9e64081cae008b9eff5e854544 Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 26 Sep 2026 13:09:32 -0500 Subject: [PATCH] format: equivalence harness for the Storage migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/storage_equivalence.rs walks real files and runs every metadata parser converted to Storage twice per object — over the file as a slice and over a read_at-only CountingStorage — and requires identical results, values and errors alike; the only allowed difference is the clean ContiguousStorageRequired from the structures still indexed by a v2 B-tree (dense attributes, a SOHM B-tree index), which is counted. It covers superblock and extension, cache image, SOHM table/list/B-tree, object headers, attributes, fill values, shared messages, symbol-table groups (local heap, B-tree, nodes, names), fractal heaps and their objects, VDS mappings, and fixed/extensible array chunk indexes. Inputs: every fixture; files h5py writes for what the fixtures lack (extensible arrays with super blocks and paged data blocks, paged fixed arrays, a 400-group v1 file with a user block, a 300-link dense group, dense, shared and committed-type attributes, SOHM list and B-tree indexes; honours CLAWHDF5_PYTHON / CLAWHDF5_REQUIRE_INTEROP); and, with CLAWHDF5_STORAGE_CORPUS set, a corpus such as conformance/.cache/corpus. Milestones M2/M3 add their parsers to check_object. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/storage_equivalence.rs | 610 ++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 crates/clawhdf5-format/tests/storage_equivalence.rs diff --git a/crates/clawhdf5-format/tests/storage_equivalence.rs b/crates/clawhdf5-format/tests/storage_equivalence.rs new file mode 100644 index 0000000..a046ffb --- /dev/null +++ b/crates/clawhdf5-format/tests/storage_equivalence.rs @@ -0,0 +1,610 @@ +//! Equivalence harness for the range-read migration +//! (`docs/design/range-reads.md`, milestone M1). +//! +//! Every metadata parser converted to [`Storage`] must give exactly what its +//! `&[u8]` form gives. This walks real files — the fixtures, files h5py +//! writes to exercise the less common structures, and optionally the +//! conformance corpus — and, for every object, runs each converted parser +//! twice: over the file as a slice, and over a [`CountingStorage`] that +//! serves the same bytes through `read_at` only (`as_contiguous()` is +//! `None`, so no parser can fall back to the whole slice). The results must +//! be identical, value for value and error for error. +//! +//! The one allowed difference is [`FormatError::ContiguousStorageRequired`] +//! from the storage path: the structures still indexed by a v2 B-tree (dense +//! attributes, a SOHM B-tree index, huge fractal-heap objects), which fail +//! cleanly instead of reading the whole file. Those are counted. +//! +//! Milestones M2/M3 extend `check_object` with the raw-data and group +//! parsers as they are converted. +//! +//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/ +//! `.nc`/`.h5ad` file under those directories (the conformance corpus is +//! `conformance/.cache/corpus`); `CLAWHDF5_STORAGE_REPORT=1` prints the +//! per-file read counts. +//! - The h5py-written files honour `CLAWHDF5_PYTHON` and +//! `CLAWHDF5_REQUIRE_INTEROP` like the facade's interop tests. + +use std::collections::{HashSet, VecDeque}; +use std::fmt::Debug; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use clawhdf5_format::attribute::{ + extract_attributes_full, extract_attributes_full_in, extract_attributes_tolerant, + extract_attributes_tolerant_in, +}; +use clawhdf5_format::attribute_info::AttributeInfoMessage; +use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in}; +use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records}; +use clawhdf5_format::data_layout::DataLayout; +use clawhdf5_format::dataspace::Dataspace; +use clawhdf5_format::datatype::Datatype; +use clawhdf5_format::error::FormatError; +use clawhdf5_format::extensible_array::{ + ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in, +}; +use clawhdf5_format::fill_value::dataset_fill_value_in; +use clawhdf5_format::fixed_array::{ + FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in, +}; +use clawhdf5_format::fractal_heap::FractalHeapHeader; +use clawhdf5_format::link_info::LinkInfoMessage; +use clawhdf5_format::local_heap::LocalHeap; +use clawhdf5_format::message_type::MessageType; +use clawhdf5_format::object_header::ObjectHeader; +use clawhdf5_format::shared_message::{ + self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in, + parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in, +}; +use clawhdf5_format::signature::split_user_block; +use clawhdf5_format::storage::{CountingStorage, Storage}; +use clawhdf5_format::superblock::Superblock; +use clawhdf5_format::superblock_ext::{ + cache_image_state, cache_image_state_in, read_superblock_extension, + read_superblock_extension_in, +}; +use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode}; + +/// Objects visited per file, heap objects read per heap: enough to cover +/// every structure kind while keeping a 35 000-group file fast. +const MAX_OBJECTS: usize = 1500; +const MAX_HEAP_IDS: usize = 200; + +#[derive(Default, Debug)] +struct Tally { + files: usize, + objects: usize, + checks: usize, + contiguous_required: usize, + reads: u64, + bytes: u64, +} + +struct Walk<'a> { + slice: &'a [u8], + storage: &'a CountingStorage, + name: String, + tally: &'a mut Tally, +} + +impl Walk<'_> { + /// The storage result must equal the slice result, or be the clean + /// "needs the whole file" error. + fn same( + &mut self, + what: &str, + want: &Result, + got: &Result, + ) { + self.tally.checks += 1; + if let Err(FormatError::ContiguousStorageRequired(_)) = got { + self.tally.contiguous_required += 1; + return; + } + let (w, g) = (format!("{want:?}"), format!("{got:?}")); + assert!( + w == g, + "{}: {what} differs\n slice: {}\n storage: {}", + self.name, + &w[..w.len().min(600)], + &g[..g.len().min(600)] + ); + } + + fn st(&self) -> &dyn Storage { + self.storage + } + + fn run(&mut self) { + let slice = self.slice; + let sb = Superblock::parse(slice, 0); + self.same("superblock", &sb, &Superblock::parse_in(self.st(), 0)); + let Ok(sb) = sb else { return }; + let (os, ls) = (sb.offset_size, sb.length_size); + + let want = read_superblock_extension(slice, &sb); + self.same( + "superblock extension", + &want, + &read_superblock_extension_in(self.st(), &sb), + ); + let want = cache_image_state(slice, &sb); + self.same("cache image", &want, &cache_image_state_in(self.st(), &sb)); + let table = load_sohm_table(slice, os, ls); + self.same("SOHM table", &table, &load_sohm_table_in(self.st(), os, ls)); + if let Ok(Some(table)) = &table { + for idx in &table.indexes { + if idx.index_type == 0 { + let want = + parse_sohm_list(slice, idx.index_addr as usize, idx.num_messages, os); + let got = parse_sohm_list_in(self.st(), idx.index_addr, idx.num_messages, os); + self.same("SOHM list", &want, &got); + } else { + let want = parse_sohm_btree_entries(slice, idx.index_addr as usize, os, ls); + let got = parse_sohm_btree_entries_in(self.st(), idx.index_addr, os, ls); + self.same("SOHM B-tree", &want, &got); + } + } + } + + let mut seen = HashSet::new(); + let mut queue = VecDeque::from([sb.root_group_address]); + while let Some(addr) = queue.pop_front() { + if seen.len() >= MAX_OBJECTS || !seen.insert(addr) { + continue; + } + self.tally.objects += 1; + self.check_object(&sb, addr); + // Traversal only (group lookups are milestone M0/M3 work). + if let Ok(children) = + clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr) + { + queue.extend(children.iter().map(|c| c.object_header_address)); + } + } + } + + fn check_object(&mut self, sb: &Superblock, addr: u64) { + let slice = self.slice; + let (os, ls) = (sb.offset_size, sb.length_size); + let header = ObjectHeader::parse(slice, addr as usize, os, ls); + self.same( + "object header", + &header, + &ObjectHeader::parse_in(self.st(), addr, os, ls), + ); + let Ok(header) = header else { return }; + + let want = extract_attributes_full(slice, &header, os, ls); + self.same( + "attributes", + &want, + &extract_attributes_full_in(self.st(), &header, os, ls), + ); + let want = extract_attributes_tolerant(slice, &header, os, ls); + let got = extract_attributes_tolerant_in(self.st(), &header, os, ls); + self.same("attributes (tolerant)", &want, &got); + let want = dataset_fill_value_in(slice, &header.messages, os, ls); + self.same( + "fill value", + &want, + &dataset_fill_value_in(self.storage, &header.messages, os, ls), + ); + + for msg in &header.messages { + if shared_message::is_shared(msg.flags) { + let want = message_data_with_sohm(slice, msg, os, ls); + let got = message_data_with_sohm_in(self.st(), msg, os, ls); + self.same("shared message", &want, &got); + } + match msg.msg_type { + MessageType::SymbolTable => { + if let Ok(stm) = SymbolTableMessage::parse(&msg.data, os) { + self.check_v1_group(&stm, os, ls); + } + } + MessageType::LinkInfo => { + if let Ok(li) = LinkInfoMessage::parse(&msg.data, os) { + self.check_heap( + li.fractal_heap_address, + li.btree_name_index_address, + 4, + os, + ls, + ); + } + } + MessageType::AttributeInfo => { + if let Ok(ai) = AttributeInfoMessage::parse(&msg.data, os) { + self.check_heap( + ai.fractal_heap_address, + ai.btree_name_index_address, + 0, + os, + ls, + ); + } + } + _ => {} + } + } + self.check_layout(&header, os, ls); + } + + /// A symbol-table group: its local heap, B-tree, nodes and names. + fn check_v1_group(&mut self, stm: &SymbolTableMessage, os: u8, ls: u8) { + let slice = self.slice; + let heap = LocalHeap::parse(slice, stm.local_heap_address as usize, os, ls); + self.same( + "local heap", + &heap, + &LocalHeap::parse_in(self.st(), stm.local_heap_address, os, ls), + ); + let nodes = collect_symbol_table_nodes(slice, stm.btree_address, os, ls); + let got = collect_symbol_table_nodes_in(self.st(), stm.btree_address, os, ls); + self.same("group B-tree", &nodes, &got); + let Ok(heap) = heap else { return }; + let want = heap.validate_free_list(slice, ls); + self.same( + "local heap free list", + &want, + &heap.validate_free_list_in(self.st(), ls), + ); + let Ok(nodes) = nodes else { return }; + for &node in nodes.iter().take(MAX_HEAP_IDS) { + let snod = SymbolTableNode::parse(slice, node as usize, os); + self.same( + "symbol table node", + &snod, + &SymbolTableNode::parse_in(self.st(), node, os), + ); + let Ok(snod) = snod else { continue }; + for e in &snod.entries { + let want = heap.read_string(slice, e.link_name_offset); + self.same( + "link name", + &want, + &heap.read_string_in(self.st(), e.link_name_offset), + ); + } + } + } + + /// A dense group's or dense attributes' fractal heap: the header, and + /// the objects its name index points at. `id_at` is where the heap ID + /// starts in a name-index record (after the hash for links). + fn check_heap(&mut self, heap: Option, index: Option, id_at: usize, os: u8, ls: u8) { + let slice = self.slice; + let Some(heap_addr) = heap else { return }; + let fh = FractalHeapHeader::parse(slice, heap_addr as usize, os, ls); + self.same( + "fractal heap", + &fh, + &FractalHeapHeader::parse_in(self.st(), heap_addr, os, ls), + ); + let (Ok(fh), Some(index)) = (fh, index) else { + return; + }; + let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else { + return; + }; + let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else { + return; + }; + let id_len = fh.heap_id_length as usize; + for rec in records.iter().take(MAX_HEAP_IDS) { + let Some(id) = rec.data.get(id_at..id_at + id_len) else { + continue; + }; + let want = fh.read_managed_object(slice, id, os); + self.same( + "heap object", + &want, + &fh.read_managed_object_in(self.st(), id, os), + ); + } + } + + /// A dataset's layout: VDS mappings, and fixed/extensible array chunk + /// indexes. + fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) { + let slice = self.slice; + let find = |t: MessageType| { + header + .messages + .iter() + .find(|m| m.msg_type == t) + .and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok()) + }; + let Some(layout) = find(MessageType::DataLayout) else { + return; + }; + let Ok(layout) = DataLayout::parse(&layout, os, ls) else { + return; + }; + match &layout { + DataLayout::Virtual { .. } => { + let (mut want, mut got) = (layout.clone(), layout.clone()); + let w = want.resolve_vds_mappings(slice, ls).map(|()| want); + let g = got.resolve_vds_mappings_in(self.st(), ls).map(|()| got); + self.same("VDS mappings", &w, &g); + } + DataLayout::Chunked { + chunk_dimensions, + btree_address: Some(addr), + version: 4, + chunk_index_type: Some(kind @ (3 | 4)), + .. + } => { + let (Some(ds), Some(dt)) = + (find(MessageType::Dataspace), find(MessageType::Datatype)) + else { + return; + }; + let (Ok(ds), Ok((dt, _))) = (Dataspace::parse(&ds, ls), Datatype::parse(&dt)) + else { + return; + }; + let rank = ds.dimensions.len(); + if chunk_dimensions.len() < rank { + return; + } + let dims = &chunk_dimensions[..rank]; + let max = ds.max_dimensions.as_deref(); + let es = dt.type_size(); + if *kind == 3 { + let h = FixedArrayHeader::parse(slice, *addr as usize, os, ls); + self.same( + "fixed array header", + &h, + &FixedArrayHeader::parse_in(self.st(), *addr, os, ls), + ); + let Ok(h) = h else { return }; + let want = + read_fixed_array_chunks(slice, &h, &ds.dimensions, max, dims, es, os, ls); + let got = read_fixed_array_chunks_in( + self.st(), + &h, + &ds.dimensions, + max, + dims, + es, + os, + ls, + ); + self.same("fixed array chunks", &want, &got); + } else { + let h = ExtensibleArrayHeader::parse(slice, *addr as usize, os, ls); + let got = ExtensibleArrayHeader::parse_in(self.st(), *addr, os, ls); + self.same("extensible array header", &h, &got); + let Ok(h) = h else { return }; + let want = read_extensible_array_chunks( + slice, + &h, + &ds.dimensions, + max, + dims, + es, + os, + ls, + ); + let got = read_extensible_array_chunks_in( + self.st(), + &h, + &ds.dimensions, + max, + dims, + es, + os, + ls, + ); + self.same("extensible array chunks", &want, &got); + } + } + _ => {} + } + } +} + +fn check_file(path: &Path, tally: &mut Tally) { + let Ok(bytes) = std::fs::read(path) else { + return; + }; + let Ok((_, hdf5)) = split_user_block(&bytes) else { + return; + }; + let storage = CountingStorage::new(hdf5.to_vec()); + let before = (tally.checks, tally.objects); + let mut walk = Walk { + slice: hdf5, + storage: &storage, + name: path.display().to_string(), + tally, + }; + walk.run(); + tally.files += 1; + tally.reads += storage.reads(); + tally.bytes += storage.bytes_read(); + if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") { + eprintln!( + "{:>6} objects {:>7} checks {:>8} reads {:>12} bytes {}", + tally.objects - before.1, + tally.checks - before.0, + storage.reads(), + storage.bytes_read(), + path.display() + ); + } +} + +fn hdf5_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + hdf5_files(&p, out); + } else if p + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf")) + { + out.push(p); + } + } +} + +#[test] +fn fixtures_parse_identically_through_storage() { + let mut files = Vec::new(); + hdf5_files( + &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"), + &mut files, + ); + files.sort(); + assert!(files.len() >= 40, "{} fixtures", files.len()); + let mut tally = Tally::default(); + for f in &files { + check_file(f, &mut tally); + } + eprintln!("fixtures: {tally:?}"); + assert!(tally.objects >= 150 && tally.checks >= 1000, "{tally:?}"); + // Reads really went through read_at. + assert!(tally.reads > tally.objects as u64); +} + +#[test] +fn corpus_parses_identically_through_storage() { + let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else { + eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus"); + return; + }; + let mut files = Vec::new(); + for d in std::env::split_paths(&dirs) { + hdf5_files(&d, &mut files); + } + files.sort(); + let mut tally = Tally::default(); + for f in &files { + check_file(f, &mut tally); + } + eprintln!("corpus: {tally:?}"); + assert!(tally.files > 0); +} + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +/// Files h5py writes to cover what the fixtures do not: extensible arrays +/// deep enough for super blocks and paged data blocks, paged fixed arrays, +/// big symbol-table and dense groups, dense and shared attributes, a SOHM +/// list and a SOHM B-tree, committed datatypes, and a user block. +const GENERATE: &str = r#" +import ctypes, glob, os, sys, h5py, numpy as np +out = sys.argv[1] +def p(n): return os.path.join(out, n) + +with h5py.File(p('ea.h5'), 'w', libver='latest') as f: + # One unlimited dimension: extensible array. 3000 chunks reach super + # blocks; 2-element chunks of int8 keep data small. + d = f.create_dataset('ea', shape=(6000,), maxshape=(None,), chunks=(2,), dtype='i1') + d[:] = np.arange(6000) % 100 + d2 = f.create_dataset('ea_deflate', shape=(4000, 3), maxshape=(None, 3), chunks=(2, 3), + dtype='f4', compression='gzip') + d2[:] = np.random.default_rng(1).random((4000, 3)) + d3 = f.create_dataset('ea_sparse', shape=(100000,), maxshape=(None,), chunks=(4,), dtype='i2') + d3[0:8] = 1; d3[50000:50004] = 2; d3[99996:] = 3 + fa = f.create_dataset('fa_paged', shape=(5000,), chunks=(1,), dtype='u1') + fa[::3] = 7 + fa2 = f.create_dataset('fa', shape=(40, 40), chunks=(8, 8), dtype='f8', compression='gzip') + fa2[:] = 1.5 + for i in range(30): + f.attrs[f'a{i}'] = np.arange(i + 1) + g = f.create_group('dense') + for i in range(300): + g.create_group(f'child{i:04d}').attrs['i'] = i + f['committed'] = np.dtype([('x', 'i4'), ('y', 'f8')]) + f.create_dataset('uses_committed', shape=(3,), dtype=f['committed']) + f['uses_committed'].attrs.create('ta', data=np.zeros(2, dtype=f['committed'].dtype), dtype=f['committed']) + f.attrs['vl'] = ['alpha', 'beta', 'gamma'] + +with h5py.File(p('v1_groups.h5'), 'w', libver='earliest', userblock_size=512) as f: + for i in range(400): + g = f.create_group(f'g{i:04d}') + g.attrs['n'] = i + f.create_dataset('x', data=np.arange(10)) + +libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*')) +if libs: + lib = ctypes.CDLL(libs[0]) + lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint] + lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint] + lib.H5Pset_shared_mesg_phase_change.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint] + for name, list_max in [('sohm_list.h5', 50), ('sohm_btree.h5', 0)]: + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE) + assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0 + # datatype, dataspace, fill value, filter pipeline, attribute + assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, 0x02 | 0x04 | 0x08 | 0x10 | 0x20, 1) >= 0 + assert lib.H5Pset_shared_mesg_phase_change(fcpl.id, list_max, 0) >= 0 + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS) + fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST) + fid = h5py.h5f.create(p(name).encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl) + with h5py.File(fid) as f: + for i in range(20): + d = f.create_dataset(f'd{i}', shape=(10, i + 1), dtype='f4', fillvalue=-1.0, + chunks=(5, 1), compression='gzip') + d.attrs['units'] = 'metres per second, a long enough string to share' + d.attrs['scale'] = np.arange(20, dtype='f8') +print('ok') +"#; + +#[test] +fn h5py_files_parse_identically_through_storage() { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("storage_equivalence"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let out = Command::new(python()) + .args(["-c", GENERATE, dir.to_str().unwrap()]) + .output(); + match out { + Ok(o) if o.status.success() => {} + Ok(o) if interop_required() => panic!( + "h5py generation failed:\n{}\n{}", + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ), + Err(e) if interop_required() => panic!("python not available: {e}"), + _ => { + eprintln!("python3 with h5py unavailable; skipping"); + return; + } + } + let mut files = Vec::new(); + hdf5_files(&dir, &mut files); + files.sort(); + let names: Vec<_> = files + .iter() + .map(|f| f.file_name().unwrap().to_string_lossy().into_owned()) + .collect(); + for want in ["ea.h5", "v1_groups.h5"] { + assert!(names.iter().any(|n| n == want), "{names:?}"); + } + if interop_required() { + assert!(names.iter().any(|n| n == "sohm_btree.h5"), "{names:?}"); + } + let mut tally = Tally::default(); + for f in &files { + check_file(f, &mut tally); + } + eprintln!("h5py files: {tally:?}"); + assert!(tally.objects >= 700, "{tally:?}"); + // Dense attributes and the SOHM B-tree are the known clean errors. + assert!(tally.contiguous_required > 0, "{tally:?}"); +}