crates/clawhdf5-format/tests/storage_equivalence.rs

@@ -0,0 +1,610 @@
+//! Equivalence harness for the range-read migration
+//! (`docs/design/range-reads.md`, milestone M1).
+//!
+//! Every metadata parser converted to [`Storage`] must give exactly what its
+//! `&[u8]` form gives. This walks real files — the fixtures, files h5py
+//! writes to exercise the less common structures, and optionally the
+//! conformance corpus — and, for every object, runs each converted parser
+//! twice: over the file as a slice, and over a [`CountingStorage`] that
+//! serves the same bytes through `read_at` only (`as_contiguous()` is
+//! `None`, so no parser can fall back to the whole slice). The results must
+//! be identical, value for value and error for error.
+//!
+//! The one allowed difference is [`FormatError::ContiguousStorageRequired`]
+//! from the storage path: the structures still indexed by a v2 B-tree (dense
+//! attributes, a SOHM B-tree index, huge fractal-heap objects), which fail
+//! cleanly instead of reading the whole file. Those are counted.
+//!
+//! Milestones M2/M3 extend `check_object` with the raw-data and group
+//! parsers as they are converted.
+//!
+//! - `CLAWHDF5_STORAGE_CORPUS=dir[:dir...]` adds every `.h5`/`.hdf5`/`.he5`/
+//!   `.nc`/`.h5ad` file under those directories (the conformance corpus is
+//!   `conformance/.cache/corpus`); `CLAWHDF5_STORAGE_REPORT=1` prints the
+//!   per-file read counts.
+//! - The h5py-written files honour `CLAWHDF5_PYTHON` and
+//!   `CLAWHDF5_REQUIRE_INTEROP` like the facade's interop tests.
+
+use std::collections::{HashSet, VecDeque};
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+use clawhdf5_format::attribute::{
+    extract_attributes_full, extract_attributes_full_in, extract_attributes_tolerant,
+    extract_attributes_tolerant_in,
+};
+use clawhdf5_format::attribute_info::AttributeInfoMessage;
+use clawhdf5_format::btree_v1::{collect_symbol_table_nodes, collect_symbol_table_nodes_in};
+use clawhdf5_format::btree_v2::{BTreeV2Header, collect_btree_v2_records};
+use clawhdf5_format::data_layout::DataLayout;
+use clawhdf5_format::dataspace::Dataspace;
+use clawhdf5_format::datatype::Datatype;
+use clawhdf5_format::error::FormatError;
+use clawhdf5_format::extensible_array::{
+    ExtensibleArrayHeader, read_extensible_array_chunks, read_extensible_array_chunks_in,
+};
+use clawhdf5_format::fill_value::dataset_fill_value_in;
+use clawhdf5_format::fixed_array::{
+    FixedArrayHeader, read_fixed_array_chunks, read_fixed_array_chunks_in,
+};
+use clawhdf5_format::fractal_heap::FractalHeapHeader;
+use clawhdf5_format::link_info::LinkInfoMessage;
+use clawhdf5_format::local_heap::LocalHeap;
+use clawhdf5_format::message_type::MessageType;
+use clawhdf5_format::object_header::ObjectHeader;
+use clawhdf5_format::shared_message::{
+    self, load_sohm_table, load_sohm_table_in, message_data_with_sohm, message_data_with_sohm_in,
+    parse_sohm_btree_entries, parse_sohm_btree_entries_in, parse_sohm_list, parse_sohm_list_in,
+};
+use clawhdf5_format::signature::split_user_block;
+use clawhdf5_format::storage::{CountingStorage, Storage};
+use clawhdf5_format::superblock::Superblock;
+use clawhdf5_format::superblock_ext::{
+    cache_image_state, cache_image_state_in, read_superblock_extension,
+    read_superblock_extension_in,
+};
+use clawhdf5_format::symbol_table::{SymbolTableMessage, SymbolTableNode};
+
+/// Objects visited per file, heap objects read per heap: enough to cover
+/// every structure kind while keeping a 35 000-group file fast.
+const MAX_OBJECTS: usize = 1500;
+const MAX_HEAP_IDS: usize = 200;
+
+#[derive(Default, Debug)]
+struct Tally {
+    files: usize,
+    objects: usize,
+    checks: usize,
+    contiguous_required: usize,
+    reads: u64,
+    bytes: u64,
+}
+
+struct Walk<'a> {
+    slice: &'a [u8],
+    storage: &'a CountingStorage,
+    name: String,
+    tally: &'a mut Tally,
+}
+
+impl Walk<'_> {
+    /// The storage result must equal the slice result, or be the clean
+    /// "needs the whole file" error.
+    fn same<T: Debug>(
+        &mut self,
+        what: &str,
+        want: &Result<T, FormatError>,
+        got: &Result<T, FormatError>,
+    ) {
+        self.tally.checks += 1;
+        if let Err(FormatError::ContiguousStorageRequired(_)) = got {
+            self.tally.contiguous_required += 1;
+            return;
+        }
+        let (w, g) = (format!("{want:?}"), format!("{got:?}"));
+        assert!(
+            w == g,
+            "{}: {what} differs\n slice:   {}\n storage: {}",
+            self.name,
+            &w[..w.len().min(600)],
+            &g[..g.len().min(600)]
+        );
+    }
+
+    fn st(&self) -> &dyn Storage {
+        self.storage
+    }
+
+    fn run(&mut self) {
+        let slice = self.slice;
+        let sb = Superblock::parse(slice, 0);
+        self.same("superblock", &sb, &Superblock::parse_in(self.st(), 0));
+        let Ok(sb) = sb else { return };
+        let (os, ls) = (sb.offset_size, sb.length_size);
+
+        let want = read_superblock_extension(slice, &sb);
+        self.same(
+            "superblock extension",
+            &want,
+            &read_superblock_extension_in(self.st(), &sb),
+        );
+        let want = cache_image_state(slice, &sb);
+        self.same("cache image", &want, &cache_image_state_in(self.st(), &sb));
+        let table = load_sohm_table(slice, os, ls);
+        self.same("SOHM table", &table, &load_sohm_table_in(self.st(), os, ls));
+        if let Ok(Some(table)) = &table {
+            for idx in &table.indexes {
+                if idx.index_type == 0 {
+                    let want =
+                        parse_sohm_list(slice, idx.index_addr as usize, idx.num_messages, os);
+                    let got = parse_sohm_list_in(self.st(), idx.index_addr, idx.num_messages, os);
+                    self.same("SOHM list", &want, &got);
+                } else {
+                    let want = parse_sohm_btree_entries(slice, idx.index_addr as usize, os, ls);
+                    let got = parse_sohm_btree_entries_in(self.st(), idx.index_addr, os, ls);
+                    self.same("SOHM B-tree", &want, &got);
+                }
+            }
+        }
+
+        let mut seen = HashSet::new();
+        let mut queue = VecDeque::from([sb.root_group_address]);
+        while let Some(addr) = queue.pop_front() {
+            if seen.len() >= MAX_OBJECTS || !seen.insert(addr) {
+                continue;
+            }
+            self.tally.objects += 1;
+            self.check_object(&sb, addr);
+            // Traversal only (group lookups are milestone M0/M3 work).
+            if let Ok(children) =
+                clawhdf5_format::group_v2::resolve_group_children(slice, &sb, addr)
+            {
+                queue.extend(children.iter().map(|c| c.object_header_address));
+            }
+        }
+    }
+
+    fn check_object(&mut self, sb: &Superblock, addr: u64) {
+        let slice = self.slice;
+        let (os, ls) = (sb.offset_size, sb.length_size);
+        let header = ObjectHeader::parse(slice, addr as usize, os, ls);
+        self.same(
+            "object header",
+            &header,
+            &ObjectHeader::parse_in(self.st(), addr, os, ls),
+        );
+        let Ok(header) = header else { return };
+
+        let want = extract_attributes_full(slice, &header, os, ls);
+        self.same(
+            "attributes",
+            &want,
+            &extract_attributes_full_in(self.st(), &header, os, ls),
+        );
+        let want = extract_attributes_tolerant(slice, &header, os, ls);
+        let got = extract_attributes_tolerant_in(self.st(), &header, os, ls);
+        self.same("attributes (tolerant)", &want, &got);
+        let want = dataset_fill_value_in(slice, &header.messages, os, ls);
+        self.same(
+            "fill value",
+            &want,
+            &dataset_fill_value_in(self.storage, &header.messages, os, ls),
+        );
+
+        for msg in &header.messages {
+            if shared_message::is_shared(msg.flags) {
+                let want = message_data_with_sohm(slice, msg, os, ls);
+                let got = message_data_with_sohm_in(self.st(), msg, os, ls);
+                self.same("shared message", &want, &got);
+            }
+            match msg.msg_type {
+                MessageType::SymbolTable => {
+                    if let Ok(stm) = SymbolTableMessage::parse(&msg.data, os) {
+                        self.check_v1_group(&stm, os, ls);
+                    }
+                }
+                MessageType::LinkInfo => {
+                    if let Ok(li) = LinkInfoMessage::parse(&msg.data, os) {
+                        self.check_heap(
+                            li.fractal_heap_address,
+                            li.btree_name_index_address,
+                            4,
+                            os,
+                            ls,
+                        );
+                    }
+                }
+                MessageType::AttributeInfo => {
+                    if let Ok(ai) = AttributeInfoMessage::parse(&msg.data, os) {
+                        self.check_heap(
+                            ai.fractal_heap_address,
+                            ai.btree_name_index_address,
+                            0,
+                            os,
+                            ls,
+                        );
+                    }
+                }
+                _ => {}
+            }
+        }
+        self.check_layout(&header, os, ls);
+    }
+
+    /// A symbol-table group: its local heap, B-tree, nodes and names.
+    fn check_v1_group(&mut self, stm: &SymbolTableMessage, os: u8, ls: u8) {
+        let slice = self.slice;
+        let heap = LocalHeap::parse(slice, stm.local_heap_address as usize, os, ls);
+        self.same(
+            "local heap",
+            &heap,
+            &LocalHeap::parse_in(self.st(), stm.local_heap_address, os, ls),
+        );
+        let nodes = collect_symbol_table_nodes(slice, stm.btree_address, os, ls);
+        let got = collect_symbol_table_nodes_in(self.st(), stm.btree_address, os, ls);
+        self.same("group B-tree", &nodes, &got);
+        let Ok(heap) = heap else { return };
+        let want = heap.validate_free_list(slice, ls);
+        self.same(
+            "local heap free list",
+            &want,
+            &heap.validate_free_list_in(self.st(), ls),
+        );
+        let Ok(nodes) = nodes else { return };
+        for &node in nodes.iter().take(MAX_HEAP_IDS) {
+            let snod = SymbolTableNode::parse(slice, node as usize, os);
+            self.same(
+                "symbol table node",
+                &snod,
+                &SymbolTableNode::parse_in(self.st(), node, os),
+            );
+            let Ok(snod) = snod else { continue };
+            for e in &snod.entries {
+                let want = heap.read_string(slice, e.link_name_offset);
+                self.same(
+                    "link name",
+                    &want,
+                    &heap.read_string_in(self.st(), e.link_name_offset),
+                );
+            }
+        }
+    }
+
+    /// A dense group's or dense attributes' fractal heap: the header, and
+    /// the objects its name index points at. `id_at` is where the heap ID
+    /// starts in a name-index record (after the hash for links).
+    fn check_heap(&mut self, heap: Option<u64>, index: Option<u64>, id_at: usize, os: u8, ls: u8) {
+        let slice = self.slice;
+        let Some(heap_addr) = heap else { return };
+        let fh = FractalHeapHeader::parse(slice, heap_addr as usize, os, ls);
+        self.same(
+            "fractal heap",
+            &fh,
+            &FractalHeapHeader::parse_in(self.st(), heap_addr, os, ls),
+        );
+        let (Ok(fh), Some(index)) = (fh, index) else {
+            return;
+        };
+        let Ok(bt) = BTreeV2Header::parse(slice, index as usize, os, ls) else {
+            return;
+        };
+        let Ok(records) = collect_btree_v2_records(slice, &bt, os, ls) else {
+            return;
+        };
+        let id_len = fh.heap_id_length as usize;
+        for rec in records.iter().take(MAX_HEAP_IDS) {
+            let Some(id) = rec.data.get(id_at..id_at + id_len) else {
+                continue;
+            };
+            let want = fh.read_managed_object(slice, id, os);
+            self.same(
+                "heap object",
+                &want,
+                &fh.read_managed_object_in(self.st(), id, os),
+            );
+        }
+    }
+
+    /// A dataset's layout: VDS mappings, and fixed/extensible array chunk
+    /// indexes.
+    fn check_layout(&mut self, header: &ObjectHeader, os: u8, ls: u8) {
+        let slice = self.slice;
+        let find = |t: MessageType| {
+            header
+                .messages
+                .iter()
+                .find(|m| m.msg_type == t)
+                .and_then(|m| shared_message::message_data_with_sohm(slice, m, os, ls).ok())
+        };
+        let Some(layout) = find(MessageType::DataLayout) else {
+            return;
+        };
+        let Ok(layout) = DataLayout::parse(&layout, os, ls) else {
+            return;
+        };
+        match &layout {
+            DataLayout::Virtual { .. } => {
+                let (mut want, mut got) = (layout.clone(), layout.clone());
+                let w = want.resolve_vds_mappings(slice, ls).map(|()| want);
+                let g = got.resolve_vds_mappings_in(self.st(), ls).map(|()| got);
+                self.same("VDS mappings", &w, &g);
+            }
+            DataLayout::Chunked {
+                chunk_dimensions,
+                btree_address: Some(addr),
+                version: 4,
+                chunk_index_type: Some(kind @ (3 | 4)),
+                ..
+            } => {
+                let (Some(ds), Some(dt)) =
+                    (find(MessageType::Dataspace), find(MessageType::Datatype))
+                else {
+                    return;
+                };
+                let (Ok(ds), Ok((dt, _))) = (Dataspace::parse(&ds, ls), Datatype::parse(&dt))
+                else {
+                    return;
+                };
+                let rank = ds.dimensions.len();
+                if chunk_dimensions.len() < rank {
+                    return;
+                }
+                let dims = &chunk_dimensions[..rank];
+                let max = ds.max_dimensions.as_deref();
+                let es = dt.type_size();
+                if *kind == 3 {
+                    let h = FixedArrayHeader::parse(slice, *addr as usize, os, ls);
+                    self.same(
+                        "fixed array header",
+                        &h,
+                        &FixedArrayHeader::parse_in(self.st(), *addr, os, ls),
+                    );
+                    let Ok(h) = h else { return };
+                    let want =
+                        read_fixed_array_chunks(slice, &h, &ds.dimensions, max, dims, es, os, ls);
+                    let got = read_fixed_array_chunks_in(
+                        self.st(),
+                        &h,
+                        &ds.dimensions,
+                        max,
+                        dims,
+                        es,
+                        os,
+                        ls,
+                    );
+                    self.same("fixed array chunks", &want, &got);
+                } else {
+                    let h = ExtensibleArrayHeader::parse(slice, *addr as usize, os, ls);
+                    let got = ExtensibleArrayHeader::parse_in(self.st(), *addr, os, ls);
+                    self.same("extensible array header", &h, &got);
+                    let Ok(h) = h else { return };
+                    let want = read_extensible_array_chunks(
+                        slice,
+                        &h,
+                        &ds.dimensions,
+                        max,
+                        dims,
+                        es,
+                        os,
+                        ls,
+                    );
+                    let got = read_extensible_array_chunks_in(
+                        self.st(),
+                        &h,
+                        &ds.dimensions,
+                        max,
+                        dims,
+                        es,
+                        os,
+                        ls,
+                    );
+                    self.same("extensible array chunks", &want, &got);
+                }
+            }
+            _ => {}
+        }
+    }
+}
+
+fn check_file(path: &Path, tally: &mut Tally) {
+    let Ok(bytes) = std::fs::read(path) else {
+        return;
+    };
+    let Ok((_, hdf5)) = split_user_block(&bytes) else {
+        return;
+    };
+    let storage = CountingStorage::new(hdf5.to_vec());
+    let before = (tally.checks, tally.objects);
+    let mut walk = Walk {
+        slice: hdf5,
+        storage: &storage,
+        name: path.display().to_string(),
+        tally,
+    };
+    walk.run();
+    tally.files += 1;
+    tally.reads += storage.reads();
+    tally.bytes += storage.bytes_read();
+    if std::env::var("CLAWHDF5_STORAGE_REPORT").is_ok_and(|v| v == "1") {
+        eprintln!(
+            "{:>6} objects {:>7} checks {:>8} reads {:>12} bytes  {}",
+            tally.objects - before.1,
+            tally.checks - before.0,
+            storage.reads(),
+            storage.bytes_read(),
+            path.display()
+        );
+    }
+}
+
+fn hdf5_files(dir: &Path, out: &mut Vec<PathBuf>) {
+    let Ok(entries) = std::fs::read_dir(dir) else {
+        return;
+    };
+    for e in entries.flatten() {
+        let p = e.path();
+        if p.is_dir() {
+            hdf5_files(&p, out);
+        } else if p
+            .extension()
+            .and_then(|x| x.to_str())
+            .is_some_and(|x| matches!(x, "h5" | "hdf5" | "he5" | "nc" | "h5ad" | "hdf"))
+        {
+            out.push(p);
+        }
+    }
+}
+
+#[test]
+fn fixtures_parse_identically_through_storage() {
+    let mut files = Vec::new();
+    hdf5_files(
+        &Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures"),
+        &mut files,
+    );
+    files.sort();
+    assert!(files.len() >= 40, "{} fixtures", files.len());
+    let mut tally = Tally::default();
+    for f in &files {
+        check_file(f, &mut tally);
+    }
+    eprintln!("fixtures: {tally:?}");
+    assert!(tally.objects >= 150 && tally.checks >= 1000, "{tally:?}");
+    // Reads really went through read_at.
+    assert!(tally.reads > tally.objects as u64);
+}
+
+#[test]
+fn corpus_parses_identically_through_storage() {
+    let Ok(dirs) = std::env::var("CLAWHDF5_STORAGE_CORPUS") else {
+        eprintln!("CLAWHDF5_STORAGE_CORPUS not set; skipping the corpus");
+        return;
+    };
+    let mut files = Vec::new();
+    for d in std::env::split_paths(&dirs) {
+        hdf5_files(&d, &mut files);
+    }
+    files.sort();
+    let mut tally = Tally::default();
+    for f in &files {
+        check_file(f, &mut tally);
+    }
+    eprintln!("corpus: {tally:?}");
+    assert!(tally.files > 0);
+}
+
+fn python() -> String {
+    std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
+}
+
+fn interop_required() -> bool {
+    std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
+}
+
+/// Files h5py writes to cover what the fixtures do not: extensible arrays
+/// deep enough for super blocks and paged data blocks, paged fixed arrays,
+/// big symbol-table and dense groups, dense and shared attributes, a SOHM
+/// list and a SOHM B-tree, committed datatypes, and a user block.
+const GENERATE: &str = r#"
+import ctypes, glob, os, sys, h5py, numpy as np
+out = sys.argv[1]
+def p(n): return os.path.join(out, n)
+
+with h5py.File(p('ea.h5'), 'w', libver='latest') as f:
+    # One unlimited dimension: extensible array. 3000 chunks reach super
+    # blocks; 2-element chunks of int8 keep data small.
+    d = f.create_dataset('ea', shape=(6000,), maxshape=(None,), chunks=(2,), dtype='i1')
+    d[:] = np.arange(6000) % 100
+    d2 = f.create_dataset('ea_deflate', shape=(4000, 3), maxshape=(None, 3), chunks=(2, 3),
+                          dtype='f4', compression='gzip')
+    d2[:] = np.random.default_rng(1).random((4000, 3))
+    d3 = f.create_dataset('ea_sparse', shape=(100000,), maxshape=(None,), chunks=(4,), dtype='i2')
+    d3[0:8] = 1; d3[50000:50004] = 2; d3[99996:] = 3
+    fa = f.create_dataset('fa_paged', shape=(5000,), chunks=(1,), dtype='u1')
+    fa[::3] = 7
+    fa2 = f.create_dataset('fa', shape=(40, 40), chunks=(8, 8), dtype='f8', compression='gzip')
+    fa2[:] = 1.5
+    for i in range(30):
+        f.attrs[f'a{i}'] = np.arange(i + 1)
+    g = f.create_group('dense')
+    for i in range(300):
+        g.create_group(f'child{i:04d}').attrs['i'] = i
+    f['committed'] = np.dtype([('x', 'i4'), ('y', 'f8')])
+    f.create_dataset('uses_committed', shape=(3,), dtype=f['committed'])
+    f['uses_committed'].attrs.create('ta', data=np.zeros(2, dtype=f['committed'].dtype), dtype=f['committed'])
+    f.attrs['vl'] = ['alpha', 'beta', 'gamma']
+
+with h5py.File(p('v1_groups.h5'), 'w', libver='earliest', userblock_size=512) as f:
+    for i in range(400):
+        g = f.create_group(f'g{i:04d}')
+        g.attrs['n'] = i
+    f.create_dataset('x', data=np.arange(10))
+
+libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))
+if libs:
+    lib = ctypes.CDLL(libs[0])
+    lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]
+    lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]
+    lib.H5Pset_shared_mesg_phase_change.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint]
+    for name, list_max in [('sohm_list.h5', 50), ('sohm_btree.h5', 0)]:
+        fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)
+        assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0
+        # datatype, dataspace, fill value, filter pipeline, attribute
+        assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, 0x02 | 0x04 | 0x08 | 0x10 | 0x20, 1) >= 0
+        assert lib.H5Pset_shared_mesg_phase_change(fcpl.id, list_max, 0) >= 0
+        fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)
+        fapl.set_libver_bounds(h5py.h5f.LIBVER_LATEST, h5py.h5f.LIBVER_LATEST)
+        fid = h5py.h5f.create(p(name).encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)
+        with h5py.File(fid) as f:
+            for i in range(20):
+                d = f.create_dataset(f'd{i}', shape=(10, i + 1), dtype='f4', fillvalue=-1.0,
+                                     chunks=(5, 1), compression='gzip')
+                d.attrs['units'] = 'metres per second, a long enough string to share'
+                d.attrs['scale'] = np.arange(20, dtype='f8')
+print('ok')
+"#;
+
+#[test]
+fn h5py_files_parse_identically_through_storage() {
+    let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("storage_equivalence");
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    let out = Command::new(python())
+        .args(["-c", GENERATE, dir.to_str().unwrap()])
+        .output();
+    match out {
+        Ok(o) if o.status.success() => {}
+        Ok(o) if interop_required() => panic!(
+            "h5py generation failed:\n{}\n{}",
+            String::from_utf8_lossy(&o.stdout),
+            String::from_utf8_lossy(&o.stderr)
+        ),
+        Err(e) if interop_required() => panic!("python not available: {e}"),
+        _ => {
+            eprintln!("python3 with h5py unavailable; skipping");
+            return;
+        }
+    }
+    let mut files = Vec::new();
+    hdf5_files(&dir, &mut files);
+    files.sort();
+    let names: Vec<_> = files
+        .iter()
+        .map(|f| f.file_name().unwrap().to_string_lossy().into_owned())
+        .collect();
+    for want in ["ea.h5", "v1_groups.h5"] {
+        assert!(names.iter().any(|n| n == want), "{names:?}");
+    }
+    if interop_required() {
+        assert!(names.iter().any(|n| n == "sohm_btree.h5"), "{names:?}");
+    }
+    let mut tally = Tally::default();
+    for f in &files {
+        check_file(f, &mut tally);
+    }
+    eprintln!("h5py files: {tally:?}");
+    assert!(tally.objects >= 700, "{tally:?}");
+    // Dense attributes and the SOHM B-tree are the known clean errors.
+    assert!(tally.contiguous_required > 0, "{tally:?}");
+}