From 8ebd488d9e5da0860793817334b722ac80e9667c Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:52:07 -0500 Subject: [PATCH 01/24] fix(format): size fractal heap child indirect blocks by their row's span A child indirect block in row r of a fractal heap's doubling table spans that row's block size of heap space, so it has log2(size) - log2(start_block_size * width) + 1 rows (libhdf5's H5HF__dtable_size_to_rows). The reader used row - first_indirect_row + 1, which undercounts, so every object stored past the root block's direct rows (512 KiB with libhdf5's defaults) was unreachable: dense groups with a few thousand long link names, or ~20 000 short ones, could not be listed. Regression test: dense_group_whose_heap_outgrows_the_root_direct_rows (h5py writes 2 500 links with 248-byte names; listing compared with h5py). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 + crates/clawhdf5-format/src/fractal_heap.rs | 65 ++++---- .../clawhdf5/tests/dense_storage_interop.rs | 143 ++++++++++++++++++ docs/known-issues.md | 2 + 4 files changed, 183 insertions(+), 33 deletions(-) create mode 100644 crates/clawhdf5/tests/dense_storage_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b87415..a7272ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -299,6 +299,12 @@ flag; Fletcher32 ahead of deflate (NetCDF-4's order). Unknown-message flags follow libhdf5 (`tbogus.h5`): "fail if unknown" is refused, "fail if unknown and writing" is ignored by a reader. +- `clawhdf5-format` reader — dense groups and attributes (links or + attributes kept in a fractal heap indexed by a v2 B-tree): + - A link heap larger than the root indirect block's direct rows (512 KiB + with libhdf5's defaults: a few thousand long link names, or ~20 000 short + ones) could not be listed: child indirect blocks were given the wrong + number of rows, so every link stored in one was unreachable. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index ef442de..c71a41f 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -418,35 +418,35 @@ impl FractalHeapHeader { } // If we have indirect block rows + // A child indirect block in row r spans exactly that row's block size + // of heap space, so it has as many rows as a table of that total size + // needs (not `row - start_indirect + 1`, which undercounts and makes + // every object past the root's direct rows unreachable). for row in start_indirect..nrows_usize { - let _block_size = self.block_size_for_row(row); - let child_nrows = row - start_indirect + 1; + let child_space = self.block_size_for_row(row); + let child_nrows = self.rows_for_size(child_space); for _col in 0..tw { let child_addr = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; - if !is_undefined(child_addr, offset_size) { - // Calculate total heap space covered by this indirect block child - let total_child_space = self.indirect_block_heap_size(child_nrows); - let block_end = current_heap_offset + total_child_space; - if target_offset >= current_heap_offset && target_offset < block_end { - return self.read_from_indirect_block( - file_data, - child_addr as usize, - child_nrows as u16, - current_heap_offset, - target_offset, - length, - offset_size, - depth_remaining - 1, - ); - } - current_heap_offset += total_child_space; - } else { - let total_child_space = self.indirect_block_heap_size(child_nrows); - current_heap_offset += total_child_space; + let block_end = current_heap_offset.saturating_add(child_space); + if !is_undefined(child_addr, offset_size) + && target_offset >= current_heap_offset + && target_offset < block_end + { + return self.read_from_indirect_block( + file_data, + child_addr as usize, + child_nrows, + current_heap_offset, + target_offset, + length, + offset_size, + depth_remaining - 1, + ); } + current_heap_offset = block_end; } } @@ -475,25 +475,24 @@ impl FractalHeapHeader { log2 + 2 } + /// Rows an indirect block needs to span `size` bytes of heap space: + /// `log2(size) - log2(starting_block_size * table_width) + 1`, as + /// libhdf5's `H5HF__dtable_size_to_rows`. + fn rows_for_size(&self, size: u64) -> u16 { + let log2 = |v: u64| 63u32.saturating_sub(v.max(1).leading_zeros()); + let first_row_bits = log2(self.starting_block_size) + log2(u64::from(self.table_width)); + (log2(size).saturating_sub(first_row_bits) + 1) as u16 + } + /// Get block size for a given row in the doubling table. fn block_size_for_row(&self, row: usize) -> u64 { let sbs = self.starting_block_size; if row <= 1 { sbs } else { - sbs * (1u64 << (row - 1)) + sbs.saturating_mul(1u64.checked_shl((row - 1) as u32).unwrap_or(u64::MAX)) } } - - /// Total heap space covered by an indirect block with the given number of rows. - fn indirect_block_heap_size(&self, nrows: usize) -> u64 { - let tw = self.table_width as u64; - let mut total = 0u64; - for row in 0..nrows { - total += self.block_size_for_row(row) * tw; - } - total - } } #[cfg(test)] diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs new file mode 100644 index 0000000..e808d85 --- /dev/null +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -0,0 +1,143 @@ +//! Dense ("new-style") link and attribute storage written by libhdf5 (via +//! h5py): groups whose links live in a fractal heap indexed by a v2 B-tree, +//! and objects whose attributes do. Every listing and value is compared with +//! what h5py itself reports for the same file. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Have h5py write a file with `body` (which sees `f`, `h5py` and `np`), +/// returning the temp dir holding it and its path. +fn h5py_file(body: &str) -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dense.h5").display().to_string(); + let script = format!( + "import h5py, numpy as np\n\ + with h5py.File(r'{path}', 'w', libver='latest') as f:\n{}", + indent(body) + ); + run_python(&script); + (dir, path) +} + +fn indent(body: &str) -> String { + body.lines() + .map(|l| format!(" {l}\n")) + .collect::() +} + +/// The datasets and groups h5py lists in `group`, sorted: links h5py can +/// resolve (hard and soft), without dangling soft links or external links. +fn h5py_listing(path: &str, group: &str) -> (Vec, Vec) { + let out = run_python(&format!( + "import h5py\n\ + ds, gs = [], []\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 g = f[{group:?}]\n\ + \x20 for k in g.keys():\n\ + \x20 if isinstance(g.get(k, getlink=True), h5py.ExternalLink):\n\ + \x20 continue\n\ + \x20 try:\n\ + \x20 o = g[k]\n\ + \x20 except Exception:\n\ + \x20 continue\n\ + \x20 (ds if isinstance(o, h5py.Dataset) else gs).append(k)\n\ + print('\\x1f'.join(sorted(ds)))\n\ + print('\\x1f'.join(sorted(gs)))\n" + )); + let mut lines = out.lines(); + let split = |l: Option<&str>| -> Vec { + l.unwrap_or("") + .split('\x1f') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect() + }; + let ds = split(lines.next()); + let gs = split(lines.next()); + (ds, gs) +} + +fn our_listing(path: &str, group: &str) -> (Vec, Vec) { + let f = File::open(path).unwrap(); + let g = f.group(group).unwrap(); + let mut ds = g.datasets().unwrap(); + let mut gs = g.groups().unwrap(); + ds.sort(); + gs.sort(); + (ds, gs) +} + +fn assert_same_listing(path: &str, group: &str) { + let ours = our_listing(path, group); + let theirs = h5py_listing(path, group); + assert_eq!(ours.0.len(), theirs.0.len(), "dataset count in {group}"); + assert_eq!(ours, theirs, "listing of {group}"); +} + +#[test] +fn dense_group_whose_heap_outgrows_the_root_direct_rows() { + skip_if_no_python!(); + // Long link names make the link heap larger than the root indirect + // block's direct rows can hold (512 KiB with h5py's defaults), so links + // live in child indirect blocks. Those were sized from the wrong row + // count, and every link past the direct rows was unreachable. + let (_dir, path) = h5py_file( + "t = f.create_dataset('t', data=[1.0])\n\ + g = f.create_group('g')\n\ + for i in range(2500):\n\ + \x20 g['n%05d_' % i + 'x' * 240] = t\n", + ); + assert_same_listing(&path, "g"); + let f = File::open(&path).unwrap(); + let last = format!("g/n02499_{}", "x".repeat(240)); + assert_eq!(f.dataset(&last).unwrap().read_f64().unwrap(), vec![1.0]); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 6dfa369..59064cb 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -77,6 +77,8 @@ the VDS item, which is marked. - **Groups and links:** - Groups with a user-defined link type (e.g. 187) cannot be listed. - Dense groups with more than about 22 000 links cannot be listed. + *Partly fixed 2026-09-25:* a link heap past its root block's direct rows + (child indirect blocks) is now read. - Soft links are left out of `datasets()`. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF From 8196fab72a078b57118910b6a5ba42577d276d85 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:53:00 -0500 Subject: [PATCH 02/24] fix(format): read v2 B-tree internal nodes with libhdf5's pointer widths An internal node's child pointer is an address, the child's record count and (below the first internal level) the child subtree's total record count. libhdf5 (H5B2__hdr_init) encodes the record count in the width of a leaf's maximum and the subtree total in the width of cum_max_nrec for that depth, computed level by level from the node size. The reader guessed 2 * leaf_max and leaf_max^depth, which agree at depth 2 but not at depth 3: a 24 000-link group's name index has depth 3, its root's pointers were read 3 bytes wide instead of 2, and listing failed with a garbage heap offset. Regression tests: dense_group_with_a_three_level_name_index (h5py writes 24 000 links; listing compared with h5py) and subtree_capacity_matches_libhdf5. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 + crates/clawhdf5-format/src/btree_v2.rs | 104 +++++++++++------- .../clawhdf5/tests/dense_storage_interop.rs | 22 ++++ docs/known-issues.md | 5 +- 4 files changed, 94 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7272ba..9d94b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -305,6 +305,11 @@ with libhdf5's defaults: a few thousand long link names, or ~20 000 short ones) could not be listed: child indirect blocks were given the wrong number of rows, so every link stored in one was unreachable. + - v2 B-trees of depth 3 or more (a dense group of ~22 000+ links) were + misparsed: internal-node child pointers were read with widths from an + estimate instead of libhdf5's per-depth record capacities, and the + listing failed. The same B-tree code indexes dense attributes, shared + messages and chunks. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/btree_v2.rs b/crates/clawhdf5-format/src/btree_v2.rs index 43fd7b0..0b70981 100644 --- a/crates/clawhdf5-format/src/btree_v2.rs +++ b/crates/clawhdf5-format/src/btree_v2.rs @@ -323,39 +323,21 @@ fn collect_internal_records( let records_start = pos; pos += records_total; - // Compute sizes for child pointers - // max_records at child depth - for variable-width nrec encoding + // Child pointer layout, as libhdf5 computes it (H5B2__hdr_init): the + // child's record count is always encoded in the width needed for a + // *leaf's* maximum, and — below the first internal level — the child + // subtree's total record count in the width needed for the most records + // a subtree of that depth can hold. let child_depth = depth - 1; - let max_nrec_child = if child_depth == 0 { - max_leaf_nrec - } else { - // For internal nodes at child_depth, the true max_nrec depends on the - // node size, record size, and the recursive width of child pointer - // entries (which themselves depend on max_nrec at deeper levels). - // Computing the exact value requires iterating from the leaf level - // upward, as described in the HDF5 spec (III.A.2 "Computing the Size - // of B-tree Nodes"). - // - // We use `max_leaf_nrec * 2` as a conservative upper bound. This - // over-estimates the nrec encoding width, which means we may read - // slightly more bytes per child pointer than strictly necessary, but - // never fewer. The over-read bytes are harmless because we only - // decode `num_records` entries (the actual count from the node header). - // - // Known limitation: for very deep trees (depth > 3) with small record - // sizes, the true max could exceed this estimate, causing us to - // under-allocate the nrec encoding width and misparse child pointers. - // In practice, HDF5 B-tree v2 depths rarely exceed 2-3. - max_leaf_nrec * 2 - }; - let nrec_width = bytes_for_max_records(max_nrec_child); - - // Total records in subtree width (only if depth > 1) + let nrec_width = bytes_for_max_records(max_leaf_nrec); let total_nrec_width = if depth > 1 { - // Width to hold total records in a subtree - // We compute max possible total records at this subtree depth - let max_total = header_max_total_records(max_leaf_nrec, depth - 1); - bytes_for_max_records(max_total) + bytes_for_max_records(cum_max_records( + node_size, + record_size, + offset_size, + max_leaf_nrec, + child_depth, + )) } else { 0 }; @@ -435,14 +417,36 @@ fn collect_internal_records( Ok(()) } -/// Estimate maximum total records at a given depth (for variable-width encoding). -fn header_max_total_records(max_leaf_nrec: u64, depth: u16) -> u64 { - // Conservative: branching factor * max_leaf at each level - let mut total = max_leaf_nrec; - for _ in 0..depth { - total = total.saturating_mul(max_leaf_nrec.max(2)); +/// Most records a subtree whose root is at `depth` can hold (libhdf5's +/// `cum_max_nrec`): a leaf holds `max_leaf_nrec`; an internal node at depth +/// `d` holds `max_nrec(d)` records and `max_nrec(d) + 1` subtrees of depth +/// `d - 1`, where `max_nrec(d)` is what fits in a node once each record is +/// paired with a child pointer of the width depth `d` needs. +fn cum_max_records( + node_size: u32, + record_size: u16, + offset_size: u8, + max_leaf_nrec: u64, + depth: u16, +) -> u64 { + // Internal node overhead: signature(4) + version(1) + type(1) + checksum(4). + const PREFIX: u64 = 10; + let nrec_width = bytes_for_max_records(max_leaf_nrec) as u64; + let mut cum = max_leaf_nrec; + let mut cum_width = 0u64; + for d in 1..=depth { + let ptr = u64::from(offset_size) + nrec_width + if d > 1 { cum_width } else { 0 }; + let max_nrec = u64::from(node_size) + .saturating_sub(PREFIX) + .saturating_sub(ptr) + / (u64::from(record_size) + ptr).max(1); + cum = max_nrec + .saturating_add(1) + .saturating_mul(cum) + .saturating_add(max_nrec); + cum_width = bytes_for_max_records(cum) as u64; } - total + cum } #[cfg(test)] @@ -512,9 +516,15 @@ mod tests { child_nrec: u64, ) -> Vec { let max_leaf = max_records_leaf(node_size, record_size); - let nrec_width = bytes_for_max_records(if depth == 1 { max_leaf } else { max_leaf * 2 }); + let nrec_width = bytes_for_max_records(max_leaf); let total_width = if depth > 1 { - bytes_for_max_records(header_max_total_records(max_leaf, depth - 1)) + bytes_for_max_records(cum_max_records( + node_size, + record_size, + 8, + max_leaf, + depth - 1, + )) } else { 0 }; @@ -673,4 +683,18 @@ mod tests { let records = collect_btree_v2_records(&header, &hdr, 8, 8).unwrap(); assert!(records.is_empty()); } + + #[test] + fn subtree_capacity_matches_libhdf5() { + // A link-name index (11-byte records, 512-byte nodes, 8-byte + // addresses): libhdf5's H5B2__hdr_init gives 45 records per leaf, + // then cum_max_nrec 1 149 at depth 1 and 26 449 at depth 2 — two + // bytes of subtree count in a depth-3 root's child pointers, where + // leaf_max^3 = 91 125 would need three. + let leaf = max_records_leaf(512, 11); + assert_eq!(leaf, 45); + assert_eq!(cum_max_records(512, 11, 8, leaf, 0), 45); + assert_eq!(cum_max_records(512, 11, 8, leaf, 1), 1_149); + assert_eq!(cum_max_records(512, 11, 8, leaf, 2), 26_449); + } } diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index e808d85..ecfff7a 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -141,3 +141,25 @@ fn dense_group_whose_heap_outgrows_the_root_direct_rows() { let last = format!("g/n02499_{}", "x".repeat(240)); assert_eq!(f.dataset(&last).unwrap().read_f64().unwrap(), vec![1.0]); } + +#[test] +fn dense_group_with_a_three_level_name_index() { + skip_if_no_python!(); + // 24 000 links give the link-name v2 B-tree a depth of 3. Internal-node + // child pointers carry the subtree's total record count in a width that + // depends on the most records a subtree can hold; the reader estimated + // that as leaf_max^depth, read the root's pointers 3 bytes wide instead + // of 2, and decoded garbage heap IDs. + let (_dir, path) = h5py_file( + "t = f.create_dataset('t', data=[1.0])\n\ + g = f.create_group('g')\n\ + for i in range(24000):\n\ + \x20 g['l%06d' % i] = t\n", + ); + assert_same_listing(&path, "g"); + let f = File::open(&path).unwrap(); + assert_eq!( + f.dataset("g/l023999").unwrap().read_f64().unwrap(), + vec![1.0] + ); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 59064cb..39cdcf2 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -77,8 +77,9 @@ the VDS item, which is marked. - **Groups and links:** - Groups with a user-defined link type (e.g. 187) cannot be listed. - Dense groups with more than about 22 000 links cannot be listed. - *Partly fixed 2026-09-25:* a link heap past its root block's direct rows - (child indirect blocks) is now read. + **Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had + the wrong row count, and v2 B-tree internal nodes at depth 3+ were read + with the wrong pointer widths. - Soft links are left out of `datasets()`. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF From 85eb7f5ce2e0c237f07697af3b209f691782e9e2 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:52:19 -0500 Subject: [PATCH 03/24] feat(format): read Data Layout message versions 1 and 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HDF5 1.4/1.6-era files store the layout as version 1 or 2: version, dimensionality, class, 5 reserved bytes, an address (contiguous and chunked only), dimensionality 32-bit sizes (with the trailing element-size dimension) and, for compact storage, a 32-bit size and the raw data. They failed with InvalidLayoutVersion — 84 of the 686 files in the audit sweep, 205 datasets. Map them onto the existing variants: chunked uses the same version-1 B-tree chunk index as version 3 and is reported as version 3, so every chunked read path (filters, selections, caches) applies unchanged. Contiguous size is the product of the stored dimensions, which is what libhdf5 computes from the dataspace; a disagreement fails the reader's size check instead of returning wrong data. Fixtures are HDF5's own deflate.h5 (v1, chunked + deflate) and h5ex_g_iterate.h5 (v2, contiguous, one unallocated dataset); the new interop test compares every dataset byte for byte against h5py. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 + crates/clawhdf5-format/src/data_layout.rs | 190 +++++++++++++++++- .../tests/fixtures/legacy/README.md | 11 + .../tests/fixtures/legacy/deflate.h5 | Bin 0 -> 6240 bytes .../tests/fixtures/legacy/h5ex_g_iterate.h5 | Bin 0 -> 2928 bytes .../clawhdf5/tests/legacy_format_interop.rs | 121 +++++++++++ docs/known-issues.md | 2 + 7 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/README.md create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/deflate.h5 create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5 create mode 100644 crates/clawhdf5/tests/legacy_format_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b87415..cf1b36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -236,6 +236,12 @@ - A pipeline with Fletcher32 ahead of the compressor (h5py `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate: output exceeds size limit". +- `clawhdf5-format`: **HDF5 1.4/1.6-era files are readable.** Data Layout + message versions 1 and 2 (compact, contiguous, and chunked through the + version-1 B-tree) failed with `InvalidLayoutVersion` — 84 of the 686 files in + the 2026-09-25 audit sweep, 205 datasets. They now read as libhdf5 does; + checked byte for byte against h5py on HDF5's own test files + (`tests/legacy_format_interop.rs`). ### Storage - `clawhdf5-format`: **half-precision datasets.** diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 8429c5d..ece868c 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -1,7 +1,7 @@ //! HDF5 Data Layout message parsing (message type 0x0008). #[cfg(not(feature = "std"))] -use alloc::{string::String, vec::Vec}; +use alloc::{format, string::String, vec::Vec}; #[cfg(feature = "std")] use std::string::String; @@ -45,7 +45,9 @@ pub enum DataLayout { chunk_dimensions: Vec, /// B-tree address, or `None` if undefined. btree_address: Option, - /// Layout version (3 or 4). + /// Layout version (3 or 4). Version 1/2 messages (HDF5 1.4/1.6-era) + /// use the same version-1 B-tree chunk index as version 3 and are + /// reported as 3. version: u8, /// Chunk index type (v4 only). chunk_index_type: Option, @@ -261,6 +263,7 @@ impl DataLayout { let layout_class = data[1]; match version { + 1 | 2 => Self::parse_v1_v2(data, offset_size), 3 => Self::parse_v3(data, layout_class, offset_size, length_size), // v5 (emitted by HDF5 1.14+/2.0 with `libver=latest`) uses the same // message structure as v4 — only the version number was bumped. @@ -269,6 +272,87 @@ impl DataLayout { } } + /// Layout message versions 1 and 2 (HDF5 before 1.6.3): + /// + /// ```text + /// version(1) · dimensionality(1) · layout class(1) · reserved(5) + /// · address(offset_size) — contiguous and chunked only + /// · dimension sizes(4 × dimensionality) + /// · compact data size(4) · compact raw data — compact only + /// ``` + /// + /// The dimension sizes are the dataset's (contiguous/compact) or the + /// chunk's (chunked) extent plus a trailing element-size dimension, as in + /// version 3's chunked form. libhdf5 ignores them for contiguous storage + /// and sizes the data from the dataspace; the product of the stored + /// dimensions is that same size, and a disagreement (a dimension that was + /// truncated to 32 bits) is caught by the reader's size check rather than + /// returning wrong data. + fn parse_v1_v2(data: &[u8], offset_size: u8) -> Result { + ensure_len(data, 0, 8)?; + let dimensionality = data[1] as usize; + let layout_class = data[2]; + // H5O_LAYOUT_NDIMS: 32 dataspace dimensions + the element-size one. + if dimensionality > 33 { + return Err(FormatError::Overflow(format!( + "data layout dimensionality {dimensionality} exceeds 33" + ))); + } + let mut p = 8; + let os = offset_size as usize; + let address = match layout_class { + 1 | 2 => { + ensure_len(data, p, os)?; + let a = if is_undefined(data, p, offset_size) { + None + } else { + Some(read_offset(data, p, offset_size)?) + }; + p += os; + a + } + 0 => None, + _ => return Err(FormatError::InvalidLayoutClass(layout_class)), + }; + ensure_len(data, p, dimensionality * 4)?; + let dims: Vec = data[p..p + dimensionality * 4] + .as_chunks::<4>() + .0 + .iter() + .map(|c| u32::from_le_bytes(*c)) + .collect(); + p += dimensionality * 4; + match layout_class { + 0 => { + ensure_len(data, p, 4)?; + let size = + u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]) as usize; + ensure_len(data, p + 4, size)?; + Ok(DataLayout::Compact { + data: data[p + 4..p + 4 + size].to_vec(), + }) + } + 1 => { + let size = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d as u64)) + .ok_or_else(|| { + FormatError::Overflow(format!("contiguous layout size {dims:?}")) + })?; + Ok(DataLayout::Contiguous { address, size }) + } + _ => Ok(DataLayout::Chunked { + chunk_dimensions: dims, + btree_address: address, + version: 3, + chunk_index_type: None, + single_chunk_filtered_size: None, + single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, + }), + } + } + fn parse_v3( data: &[u8], layout_class: u8, @@ -546,6 +630,108 @@ impl DataLayout { mod tests { use super::*; + /// Version 1/2 header: version, dimensionality, class, reserved(5). + fn v1v2_header(version: u8, ndims: u8, class: u8) -> Vec { + vec![version, ndims, class, 0, 0, 0, 0, 0] + } + + #[test] + fn v2_compact() { + let mut buf = v1v2_header(2, 2, 0); + // dims (3 elements of 2 bytes) — no address for compact + buf.extend_from_slice(&3u32.to_le_bytes()); + buf.extend_from_slice(&2u32.to_le_bytes()); + buf.extend_from_slice(&6u32.to_le_bytes()); // compact size (u32 in v1/v2) + buf.extend_from_slice(&[1, 0, 2, 0, 3, 0]); + assert_eq!( + DataLayout::parse(&buf, 8, 8).unwrap(), + DataLayout::Compact { + data: vec![1, 0, 2, 0, 3, 0] + } + ); + } + + #[test] + fn v1_contiguous_size_from_dimensions() { + let mut buf = v1v2_header(1, 3, 1); + buf.extend_from_slice(&0x800u32.to_le_bytes()); // 4-byte address + for d in [10u32, 20, 4] { + buf.extend_from_slice(&d.to_le_bytes()); + } + assert_eq!( + DataLayout::parse(&buf, 4, 4).unwrap(), + DataLayout::Contiguous { + address: Some(0x800), + size: 800, + } + ); + } + + #[test] + fn v1_contiguous_undefined_address() { + let mut buf = v1v2_header(1, 2, 1); + buf.extend_from_slice(&[0xFF; 8]); + buf.extend_from_slice(&5u32.to_le_bytes()); + buf.extend_from_slice(&8u32.to_le_bytes()); + assert_eq!( + DataLayout::parse(&buf, 8, 8).unwrap(), + DataLayout::Contiguous { + address: None, + size: 40, + } + ); + } + + #[test] + fn v1_chunked_maps_to_btree_v1_index() { + let mut buf = v1v2_header(1, 3, 2); + buf.extend_from_slice(&0x1234u64.to_le_bytes()); + for d in [50u32, 50, 4] { + buf.extend_from_slice(&d.to_le_bytes()); + } + assert_eq!( + DataLayout::parse(&buf, 8, 8).unwrap(), + DataLayout::Chunked { + chunk_dimensions: vec![50, 50, 4], + btree_address: Some(0x1234), + version: 3, + chunk_index_type: None, + single_chunk_filtered_size: None, + single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, + } + ); + } + + #[test] + fn v1v2_rejects_bad_class_dimensionality_and_truncation() { + assert_eq!( + DataLayout::parse(&v1v2_header(1, 1, 3), 8, 8).unwrap_err(), + FormatError::InvalidLayoutClass(3) + ); + assert!(matches!( + DataLayout::parse(&v1v2_header(2, 34, 1), 8, 8).unwrap_err(), + FormatError::Overflow(_) + )); + // Chunked, dims cut short. + let mut buf = v1v2_header(1, 2, 2); + buf.extend_from_slice(&0x10u64.to_le_bytes()); + buf.extend_from_slice(&7u32.to_le_bytes()); + assert!(matches!( + DataLayout::parse(&buf, 8, 8).unwrap_err(), + FormatError::UnexpectedEof { .. } + )); + // Compact, raw data shorter than its declared size. + let mut buf = v1v2_header(2, 1, 0); + buf.extend_from_slice(&4u32.to_le_bytes()); + buf.extend_from_slice(&100u32.to_le_bytes()); + buf.extend_from_slice(&[0; 4]); + assert!(matches!( + DataLayout::parse(&buf, 8, 8).unwrap_err(), + FormatError::UnexpectedEof { .. } + )); + } + #[test] fn v3_compact() { let mut buf = vec![3u8, 0]; // version=3, class=0 (compact) diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/README.md b/crates/clawhdf5-format/tests/fixtures/legacy/README.md new file mode 100644 index 0000000..b7ee62d --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/legacy/README.md @@ -0,0 +1,11 @@ +# Legacy (HDF5 1.4/1.6-era) fixtures + +Unmodified copies of the HDF Group's own test files from +https://github.com/HDFGroup/hdf5 at a3cf1ea82cc7a66e50029a688121e1b105a7ce88 +(BSD-style license, see that repository's `LICENSE`). Current libraries cannot +write these structures, so they are kept as files. + +| File | Upstream path | Exercises | +|---|---|---| +| `deflate.h5` | `test/testfiles/deflate.h5` | Data Layout message v1, chunked + deflate (v1 B-tree index) | +| `h5ex_g_iterate.h5` | `HDF5Examples/C/H5G/h5ex_g_iterate.h5` | Data Layout message v2, contiguous; an unallocated dataset | diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/deflate.h5 b/crates/clawhdf5-format/tests/fixtures/legacy/deflate.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2f62e2599eee6a832a07f6048b0daba59bd8cf08 GIT binary patch literal 6240 zcmeD5aB<`1lHy_j0S*oZ76t(ZW-tdr{D*=B2~<8z$pWZiMyNmol#u}Cd$>9VfSFKn zs4)x;PhoLXWC!F@5o#|Z*jz@2l+?7G#FA77PN+IQp!pzRWME)qXlQ6= zWNd6?3e43UF#XIB3pCgu8jL_{ff$<1fh-S*1eM5OKYtfSpvz(T=K^x!MkPB&f-#_S z3KWX4@(D&;6YzWjBsnmks{_S3GMJ4+9V{Kf)Lz4(ZW>Ghlok|(Fktqg+XqwbgF_v< z`gR=Z(A{?khdOlk{e`N7xdUbnOdTRWz*LOVqaiRF0;3@?8UmvsFd71*Auu#TpyJls z6DK(t3J5PjYa!3siJs1TLQv~YqT?ZOr)L~IU_l9D4tqJbMwQgRt2N{&HEfdo6A zc?6 File { + File::open(format!("{FIXTURES}/{name}")).unwrap() +} + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Layout v1, chunked (50x50 chunks of a 100x200 dataset), deflate: every +/// read path goes through the version-1 B-tree chunk index. +#[test] +fn layout_v1_chunked_deflate() { + let file = open("deflate.h5"); + let ds = file.dataset("Dataset1").unwrap(); + assert_eq!(ds.shape().unwrap(), [100, 200]); + let expected: Vec = (0..100).flat_map(|_| (0..200).map(|j| j % 5)).collect(); + assert_eq!(ds.read_i32().unwrap(), expected); + + // A hyperslab that straddles four chunks. + let slab = Selection::Hyperslab { + start: vec![48, 48], + stride: vec![1, 1], + count: vec![4, 4], + block: vec![1, 1], + }; + let raw = ds.read_selection(&slab).unwrap(); + let got: Vec = raw + .as_chunks::<4>() + .0 + .iter() + .map(|b| i32::from_le_bytes(*b)) + .collect(); + assert_eq!(got, [3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1]); +} + +/// Layout v2, contiguous: one dataset with storage, one never written (reads +/// as its fill value, 0). +#[test] +fn layout_v2_contiguous() { + let file = open("h5ex_g_iterate.h5"); + assert_eq!(file.dataset("G1/DS2").unwrap().read_i32().unwrap(), [1]); + assert_eq!(file.dataset("DS1").unwrap().read_i32().unwrap(), [0]); +} + +/// Every dataset in every fixture, byte for byte against h5py. +#[test] +fn legacy_fixtures_match_h5py() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + for (name, datasets) in [ + ("deflate.h5", &["Dataset1"][..]), + ("h5ex_g_iterate.h5", &["DS1", "G1/DS2"][..]), + ] { + let path = format!("{FIXTURES}/{name}"); + let script = format!( + r#" +import h5py, numpy as np +f = h5py.File({path:?}, "r") +for n in {datasets:?}: + print(n, np.ascontiguousarray(f[n][()]).tobytes().hex()) +"# + ); + let out = Command::new(python()) + .args(["-c", &script]) + .output() + .unwrap(); + assert!( + out.status.success(), + "h5py: {}", + String::from_utf8_lossy(&out.stderr) + ); + let file = File::open(&path).unwrap(); + for line in String::from_utf8(out.stdout).unwrap().lines() { + let (ds, hex) = line.split_once(' ').unwrap(); + let ours = file + .dataset(ds) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + let ours: String = ours.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(ours, hex, "{name}:{ds}"); + } + } +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 6dfa369..135ff49 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -68,6 +68,8 @@ the VDS item, which is marked. - **Layout message versions 1 and 2** (HDF5 1.6-era files): 84 of the 686 sweep files, `InvalidLayoutVersion`. This is the largest single gap. + **Fixed 2026-09-25:** versions 1 and 2 are parsed (compact, contiguous, + chunked via the v1 B-tree). - **Virtual datasets:** - **Wrong data:** unmapped regions read as 0 instead of the fill value. - `%b` printf-style source names are not expanded. From 36356ba8a10628448f8581a141a4ce2b115ea499 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:53:11 -0500 Subject: [PATCH 04/24] fix(format): keep the array dimensions of compound v1 members Compound datatype version 1 carries, per member, a dimensionality and four dimension sizes (HDF5 before 1.4 had no array class). The parser skipped those 28 bytes, so a member such as `f: f32[4]` came back as a single f32 at the member's offset: the compound's size was right but its members were wrong. libhdf5 wraps such a member in an array type of the first `dimensionality` sizes and ignores the permutation; do the same, and reject a dimensionality above 4 as libhdf5 does. Only files old enough to also use layout message v1 have these, so this became reachable with the previous commit (tarrold.h5, tcompound.h5). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 ++ crates/clawhdf5-format/src/datatype.rs | 85 +++++++++++++++++- .../tests/fixtures/legacy/README.md | 1 + .../tests/fixtures/legacy/tarrold.h5 | Bin 0 -> 6032 bytes .../clawhdf5/tests/legacy_format_interop.rs | 26 +++++- docs/known-issues.md | 4 + 6 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/tarrold.h5 diff --git a/CHANGELOG.md b/CHANGELOG.md index cf1b36d..bb55b44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -300,6 +300,11 @@ - Two threads reading two chunked datasets through one `File` could get each other's chunks (the shared chunk cache was switched between datasets across separate lock acquisitions). The cache is now keyed by dataset. + - Compound datatype version 1 members with legacy array dimensions (HDF5 + before 1.4, which had no array class) were read as a single scalar at + the member's offset; they are now array members, as in libhdf5 + (`tarrold.h5`, `tcompound.h5`). Only reachable once layout versions 1/2 + were readable, since the files that use it are that old. - `clawhdf5-format` reader — errors on valid files: enum and bool datasets through the numeric readers; the "don't filter partial edge chunks" layout flag; Fletcher32 ahead of deflate (NetCDF-4's order). Unknown-message flags diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 436a6cf..6bcb810 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -423,13 +423,36 @@ impl Datatype { ensure_len(data, pos, 4)?; let byte_offset = LittleEndian::read_u32(&data[pos..pos + 4]) as u64; pos += 4; + // v1 members can be fixed-size arrays of their + // datatype (HDF5 before 1.4 had no array class): + // libhdf5 wraps such a member in an array type of the + // first `ndims` of the four stored dimensions and + // ignores the permutation. + let mut legacy_dims = Vec::new(); if version == 1 { ensure_len(data, pos, 28)?; + let ndims = data[pos] as usize; + if ndims > 4 { + return Err(FormatError::InvalidDatatypeVersion { + class: class_id, + version, + }); + } + for i in 0..ndims { + let at = pos + 12 + 4 * i; + legacy_dims.push(LittleEndian::read_u32(&data[at..at + 4])); + } pos += 28; } - let (member_dt, consumed) = + let (mut member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?; pos += consumed; + if !legacy_dims.is_empty() { + member_dt = Datatype::Array { + base_type: Box::new(member_dt), + dimensions: legacy_dims, + }; + } members.push(CompoundMember { name, byte_offset, @@ -1318,6 +1341,66 @@ mod tests { assert_xyid_compound(dt); } + /// A v1 compound member with legacy array dimensions (HDF5 before 1.4, + /// e.g. `tarrold.h5`): `{ i: i16, f: f32[2][3] }`. The member must become + /// an array type, not a scalar at the member's offset. + #[test] + fn test_compound_v1_legacy_array_member() { + let i16le: [u8; 12] = [ + 0x10, 0x08, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + ]; + let f32le: [u8; 20] = [ + 0x11, 0x20, 0x1f, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x17, 0x08, + 0x00, 0x17, 0x7f, 0x00, 0x00, 0x00, + ]; + let mut b = vec![0x16, 0x02, 0x00, 0x00, 28, 0x00, 0x00, 0x00]; + for (name, offset, ndims, dims, dt) in [ + (&b"i"[..], 0u32, 0u8, [0u32; 4], &i16le[..]), + (&b"f"[..], 4, 2, [2, 3, 0, 0], &f32le[..]), + ] { + let mut padded = name.to_vec(); + padded.resize((name.len() + 1 + 7) & !7, 0); + b.extend_from_slice(&padded); + b.extend_from_slice(&offset.to_le_bytes()); + b.extend_from_slice(&[ndims, 0, 0, 0]); + b.extend_from_slice(&[0, 1, 2, 3]); // dimension permutation + b.extend_from_slice(&[0; 4]); + for d in dims { + b.extend_from_slice(&d.to_le_bytes()); + } + b.extend_from_slice(dt); + } + let (dt, consumed) = Datatype::parse(&b).unwrap(); + assert_eq!(consumed, b.len()); + let Datatype::Compound { size, members } = dt else { + panic!("expected Compound, got {dt:?}"); + }; + assert_eq!(size, 28); + assert!(matches!( + members[0].datatype, + Datatype::FixedPoint { size: 2, .. } + )); + match &members[1].datatype { + Datatype::Array { + base_type, + dimensions, + } => { + assert_eq!(dimensions, &[2, 3]); + assert!(matches!( + **base_type, + Datatype::FloatingPoint { size: 4, .. } + )); + } + other => panic!("expected an array member, got {other:?}"), + } + assert_eq!(members[1].datatype.type_size(), 24); + + // More than four legacy dimensions is not a valid message. + let mut bad = b.clone(); + bad[8 + 8 + 4] = 5; // first member's dimensionality + assert!(Datatype::parse(&bad).is_err()); + } + #[test] fn test_compound_v2_padded_names_no_array_fields() { // v2 = v1 without the 28 bytes of per-member array fields; names are diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/README.md b/crates/clawhdf5-format/tests/fixtures/legacy/README.md index b7ee62d..a574759 100644 --- a/crates/clawhdf5-format/tests/fixtures/legacy/README.md +++ b/crates/clawhdf5-format/tests/fixtures/legacy/README.md @@ -9,3 +9,4 @@ write these structures, so they are kept as files. |---|---|---| | `deflate.h5` | `test/testfiles/deflate.h5` | Data Layout message v1, chunked + deflate (v1 B-tree index) | | `h5ex_g_iterate.h5` | `HDF5Examples/C/H5G/h5ex_g_iterate.h5` | Data Layout message v2, contiguous; an unallocated dataset | +| `tarrold.h5` | `test/testfiles/tarrold.h5` | Compound datatype v1 members with legacy array dimensions | diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/tarrold.h5 b/crates/clawhdf5-format/tests/fixtures/legacy/tarrold.h5 new file mode 100644 index 0000000000000000000000000000000000000000..7747ce463fe03cb7ed6d0fa71b5ded8c93e94cb9 GIT binary patch literal 6032 zcmeHLy-veG4E8k@4dq7{2{nj?42%p(DFYKhYEUOYz`#f)Ktf`pJFILS8F_?`JW?N_ zTe0tK1eMYvA*71h!+rL}cb9y%U7cS#?c=rjRvk#f5UOAyaE2eoEdBVqEiUgBuNj_r zWxQtW6h~*IrfiF!ZSX`1T%H#NfB`vQP~5L-UfYxj#f4(PKn0@%AmOi$FmB17j6Z&i z&z9VCKLEz~z^M%k_EmJc7snZBL%@?sEZnhgm9Y$>WE=B}B!M`D_zATKXJg7SmI0k7 zYstrek9fm*C+nAa1B>rPl5>9L&Z0p)KPOaC29$w6V&LNBv`sZivTn%vT6UcLkG;u) zGRFm*H&F8>n~FP%)VF7@ZOK>Ni>tXOPf0B zi!x`Z0WoJkh`x?8pbRJj%78MU3@8K2fHLs!7-)BU-N)NrCLhLjQ*6PX-n&NJRF-=e z_1s7T7`ppe1DjlQ7yE4=j<17m@5#R0v_9Mhu9&v=Yai~$?GCWRSO!sY$oKyoS}V{R JBL}nkcOTQoMppm; literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/legacy_format_interop.rs b/crates/clawhdf5/tests/legacy_format_interop.rs index 5e1767a..e174c61 100644 --- a/crates/clawhdf5/tests/legacy_format_interop.rs +++ b/crates/clawhdf5/tests/legacy_format_interop.rs @@ -9,7 +9,7 @@ use std::process::Command; -use clawhdf5::File; +use clawhdf5::{DType, File}; use clawhdf5_format::selection::Selection; const FIXTURES: &str = concat!( @@ -73,6 +73,29 @@ fn layout_v2_contiguous() { assert_eq!(file.dataset("DS1").unwrap().read_i32().unwrap(), [0]); } +/// Compound datatype version 1 members carrying legacy array dimensions +/// (HDF5 before 1.4 had no array class). h5py: `[('i', ' Date: Fri, 25 Sep 2026 21:54:20 -0500 Subject: [PATCH 05/24] fix(format): locate the address in version-1 shared messages A version-1 shared message reference is version, type, six reserved bytes and then an old-style symbol table entry: link-name offset (length size), object header address, cache type, reserved, scratch. We read the address straight after the reserved bytes, i.e. the link-name offset, and the committed datatype lookup failed with InvalidObjectHeaderVersion (the bytes checked in tcompound.h5: name offset 0x10, then 0x590 = /type1). Datasets of 1.4/1.6-era files that use a committed datatype were unreadable. Skip the name offset. parse_shared_ref has no length size, so add parse_shared_ref_sized and use it in every internal caller; parse_shared_ref keeps its signature and assumes length size == offset size. The old parse_v1_ref unit test encoded the wrong layout and now uses the real bytes. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 ++ crates/clawhdf5-format/src/attribute.rs | 5 +- crates/clawhdf5-format/src/shared_message.rs | 49 +++++++++--- .../tests/fixtures/legacy/README.md | 1 + .../tests/fixtures/legacy/tcompound.h5 | Bin 0 -> 8192 bytes .../clawhdf5/tests/legacy_format_interop.rs | 75 ++++++++++++++++++ docs/known-issues.md | 2 + 7 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/tcompound.h5 diff --git a/CHANGELOG.md b/CHANGELOG.md index bb55b44..2229e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -305,6 +305,14 @@ the member's offset; they are now array members, as in libhdf5 (`tarrold.h5`, `tcompound.h5`). Only reachable once layout versions 1/2 were readable, since the files that use it are that old. +- `clawhdf5-format` reader — errors on valid files: a version-1 shared + message (a committed datatype in HDF5 1.4/1.6-era files) was read as if the + object header address followed the reserved bytes; it follows a link-name + offset (the reference is an old-style symbol table entry), so the reader + followed the name offset and failed with `InvalidObjectHeaderVersion` + (`tcompound.h5`). New `shared_message::parse_shared_ref_sized` takes the + superblock's length size; `parse_shared_ref` assumes it equals the offset + size. - `clawhdf5-format` reader — errors on valid files: enum and bool datasets through the numeric readers; the "don't filter partial edge chunks" layout flag; Fletcher32 ahead of deflate (NetCDF-4's order). Unknown-message flags diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index bb96bd5..36ded53 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -97,7 +97,7 @@ impl AttributeMessage { return Ok(Cow::Borrowed(bytes)); } let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?; - let shared_ref = shared_message::parse_shared_ref(bytes, offset_size)?; + let shared_ref = shared_message::parse_shared_ref_sized(bytes, offset_size, length_size)?; shared_message::resolve_shared_message( file_data, &shared_ref, @@ -407,7 +407,8 @@ pub fn extract_attributes_full( if msg.msg_type == MessageType::Attribute { if shared_message::is_shared(msg.flags) { // Shared attribute: resolve the reference to get actual attribute data - let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?; + let shared_ref = + shared_message::parse_shared_ref_sized(&msg.data, offset_size, length_size)?; let resolved_data = shared_message::resolve_shared_message( file_data, &shared_ref, diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 33334fa..9ab5d0f 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -154,13 +154,29 @@ pub fn is_shared(msg_flags: u8) -> bool { /// /// When the shared flag is set on a message, the data contains a reference /// instead of the actual message content. +/// +/// Assumes the file's length size equals its offset size, which only matters +/// for version-1 references; use [`parse_shared_ref_sized`] when the +/// superblock's length size is known. pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result { + parse_shared_ref_sized(data, offset_size, offset_size) +} + +/// [`parse_shared_ref`] with the superblock's length size, which locates the +/// object header address in a version-1 reference. +pub fn parse_shared_ref_sized( + data: &[u8], + offset_size: u8, + length_size: u8, +) -> Result { ensure_len(data, 0, 2)?; let version = data[0]; let ref_type = data[1]; // Layouts (HDF5 spec IV.A.2 "Shared Message", and libhdf5's decoder): - // v1: version, type, reserved(6), address — always "committed" + // v1: version, type, reserved(6), then an old-style symbol table + // entry: link-name offset(length_size), object header address, + // cache type(4), reserved(4), scratch(16) — always "committed" // v2: version, type, address — always "committed" // v3: version, type, then a fractal-heap ID if type == SOHM, otherwise // an address @@ -177,7 +193,7 @@ pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result address_at(2 + 6), + 1 => address_at(2 + 6 + length_size as usize), 2 => address_at(2), 3 if ref_type == SHARE_TYPE_SOHM => { ensure_len(data, 2, FHEAP_ID_LEN)?; @@ -434,7 +450,7 @@ pub fn message_data_with_sohm<'a>( if !is_shared(msg.flags) { return Ok(Cow::Borrowed(&msg.data)); } - let shared_ref = parse_shared_ref(&msg.data, offset_size)?; + let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?; let table = if shared_ref.heap_id.is_some() { load_sohm_table(file_data, offset_size, length_size)? } else { @@ -514,7 +530,7 @@ pub fn message_data<'a>( if !is_shared(msg.flags) { return Ok(Cow::Borrowed(&msg.data)); } - let shared_ref = parse_shared_ref(&msg.data, offset_size)?; + let shared_ref = parse_shared_ref_sized(&msg.data, offset_size, length_size)?; resolve_shared_message( file_data, &shared_ref, @@ -649,15 +665,26 @@ mod tests { #[test] fn parse_v1_ref() { - let mut data = Vec::new(); - data.push(1); // version - data.push(0); // type - data.extend_from_slice(&[0u8; 6]); // reserved - data.extend_from_slice(&0x5678u64.to_le_bytes()); + // Datatype message of `/group1/dset2` in HDF5's `tcompound.h5` + // (written in 2000): version 1, six reserved bytes, then an old-style + // symbol table entry — link-name offset 0x10, object header address + // 0x590 (the committed datatype `/type1`), cache type, reserved and + // scratch. + let mut data = vec![1, 0, 0, 0, 0, 0, 0, 0]; + data.extend_from_slice(&0x10u64.to_le_bytes()); + data.extend_from_slice(&0x590u64.to_le_bytes()); + data.extend_from_slice(&[0; 24]); - let shared = parse_shared_ref(&data, 8).unwrap(); + let shared = parse_shared_ref_sized(&data, 8, 8).unwrap(); assert_eq!(shared.version, 1); - assert_eq!(shared.object_header_address, Some(0x5678)); + assert_eq!(shared.object_header_address, Some(0x590)); + + // The name offset is a length: 4 bytes here, then an 8-byte address. + let mut data = vec![1, 0, 0, 0, 0, 0, 0, 0]; + data.extend_from_slice(&0x10u32.to_le_bytes()); + data.extend_from_slice(&0x590u64.to_le_bytes()); + let shared = parse_shared_ref_sized(&data, 8, 4).unwrap(); + assert_eq!(shared.object_header_address, Some(0x590)); } #[test] diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/README.md b/crates/clawhdf5-format/tests/fixtures/legacy/README.md index a574759..0eda271 100644 --- a/crates/clawhdf5-format/tests/fixtures/legacy/README.md +++ b/crates/clawhdf5-format/tests/fixtures/legacy/README.md @@ -10,3 +10,4 @@ write these structures, so they are kept as files. | `deflate.h5` | `test/testfiles/deflate.h5` | Data Layout message v1, chunked + deflate (v1 B-tree index) | | `h5ex_g_iterate.h5` | `HDF5Examples/C/H5G/h5ex_g_iterate.h5` | Data Layout message v2, contiguous; an unallocated dataset | | `tarrold.h5` | `test/testfiles/tarrold.h5` | Compound datatype v1 members with legacy array dimensions | +| `tcompound.h5` | `tools/test/testfiles/tcompound.h5` | Version-1 shared messages (committed datatypes); compound v1 array members with data | diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/tcompound.h5 b/crates/clawhdf5-format/tests/fixtures/legacy/tcompound.h5 new file mode 100644 index 0000000000000000000000000000000000000000..d1ec6504cafee27eeda2e6ea97f95bd9bfc8c97d GIT binary patch literal 8192 zcmeHMJ8u&~5T5g0f(Z{f1xa`x9i>SEazlJ45tIT!#G}Rr!b5^23PFMjmmpE31giXu zl(a}eN`pk{G9@J)d~-8%an84P3`B{*Bkj)Y?CkB_?9A-m-rJcgSC0&x7$SyZkpe1_ zpERWUsX$?-tuku`B^+pGI-cdOn)a6!ubooDfo|WNo+k3h<~MBOGl5W{G5YwwvVcbg zcn6tV(lGp%+wav1HN}QJ8ch17BKY`PLXN=MOAxBxov%NeGif(29VEmELrC{@jJl$8 z(D1pl>6pKf|~<^&kLfp<3gC+`%!7v6!RJ~tmrstwb!AtHYMY=3+yq+gJ-ho zBGtpzc~;kXCDkuXsZK;T|C&9U`aIXzZuxf=all~fD6M||zgWPPf3xvx_Tb!*#I6Rg zPvzvCVe!1v`1Lfuc{@K6@@3NqlO`Yn%>)U#+R?iKc`@7BLy@o!PkzYfK(eevs=!!jX{=do{e{;iXD%GgbUmEkw>`79BEh*0r4 zB3ecHghD8&w(*+zyqIMhD61%+P?|){<8w$JFAmYk9}TJla1f!|&H1u=#Ub=7W6~BK zKo8{U@H_A4ny=M1aVR>(5oYG2t{ho8+d(t zK>&|)4xqqi8i3C^1rTtqwEGvNwFJygvuBmzK;7 Vec<(i32, f32)> { + file.dataset(name) + .unwrap() + .read_selection(&Selection::All) + .unwrap() + .as_chunks::<8>() + .0 + .iter() + .map(|b| { + ( + i32::from_be_bytes(b[..4].try_into().unwrap()), + f32::from_be_bytes(b[4..].try_into().unwrap()), + ) + }) + .collect() + }; + assert_eq!( + be_pairs("group1/dset2"), + [(0, 0.0), (1, 1.1), (2, 2.2), (3, 3.3), (4, 4.4)] + ); + assert_eq!( + be_pairs("group2/dset5"), + [(0, 0.0), (1, 0.1), (2, 0.2), (3, 0.3), (4, 0.4)] + ); + + // `/type2`: { int_array: i32[4], float_array: f32[5][6] }, whose array + // members are compound v1 legacy dimensions. + let dset3 = file.dataset("group1/dset3").unwrap(); + assert_eq!( + dset3.dtype().unwrap(), + DType::Compound(vec![ + ( + "int_array".into(), + DType::Array(Box::new(DType::I32), vec![4]) + ), + ( + "float_array".into(), + DType::Array(Box::new(DType::F32), vec![5, 6]) + ), + ]) + ); + let raw = dset3.read_selection(&Selection::All).unwrap(); + assert_eq!(raw.len(), 3 * 6 * (16 + 120)); + assert_eq!( + &raw[..16], + &[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3] + ); + let first: Vec = raw[16..16 + 120] + .as_chunks::<4>() + .0 + .iter() + .map(|b| f32::from_be_bytes(*b)) + .collect(); + let expected: Vec = (0..5) + .flat_map(|i| (0..6).map(move |j| (1 + i + j) as f32)) + .collect(); + assert_eq!(first, expected); +} + /// Every dataset in every fixture, byte for byte against h5py. #[test] fn legacy_fixtures_match_h5py() { @@ -111,6 +176,16 @@ fn legacy_fixtures_match_h5py() { ("deflate.h5", &["Dataset1"][..]), ("h5ex_g_iterate.h5", &["DS1", "G1/DS2"][..]), ("tarrold.h5", &["Dataset1", "Dataset2"][..]), + ( + "tcompound.h5", + &[ + "dset1", + "group1/dset2", + "group1/dset3", + "group1/dset4", + "group2/dset5", + ][..], + ), ] { let path = format!("{FIXTURES}/{name}"); let script = format!( diff --git a/docs/known-issues.md b/docs/known-issues.md index da70ab8..e0a1cae 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -80,6 +80,8 @@ the VDS item, which is marked. - Hyperslab selection versions 1 and 2 are refused. - **Files with a user block:** the base address is not applied. - **Old-style shared messages (version 1)** read the wrong address. + **Fixed 2026-09-25:** the address follows the link-name offset of the + embedded symbol table entry. - **Groups and links:** - Groups with a user-defined link type (e.g. 187) cannot be listed. - Dense groups with more than about 22 000 links cannot be listed. From 1c85986079651d82f0d16a5ae3f9af59fc8311e8 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:55:51 -0500 Subject: [PATCH 06/24] fix(format): read huge, tiny and filtered fractal heap objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A heap ID's type is in bits 4-5 of its first byte (H5HF_ID_TYPE_MASK 0x30); bits 6-7 are the ID version. The reader took the type from bits 6-7, so every huge object ID (0x10) was decoded as a managed one and failed — and since dense attributes are read all at once, one attribute over the heap's 4 KiB managed limit made every attribute on its object unreadable (netcdf4-python's issue671.nc / issue672.nc). - Huge objects (type 1): located directly from the ID when address and length fit in it, otherwise through the huge-object v2 B-tree (record types 1 and 2); filtered huge objects are decoded with the heap's pipeline and their filter mask. - Tiny objects (type 2): read from the ID itself. - Filtered heaps: the header's pipeline is parsed (it was skipped short, so the header checksum was read from the wrong place), indirect-block entries for direct blocks carry their filtered size and mask, and direct blocks are decoded before objects are read from them. - An unknown ID version is an error. FractalHeapHeader gains huge_btree_address, filter_pipeline, root_direct_block_filtered_size, root_direct_block_filter_mask, offset_size and length_size; read_managed_object now accepts any ID type. Regression tests (h5py-written, compared with h5py): dense_attribute_stored_as_a_huge_heap_object, dense_group_with_a_huge_link, dense_group_with_a_filtered_link_heap; unit tests tiny_object_is_read_from_the_id, huge_object_with_a_direct_id, unknown_heap_id_version_is_refused. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 + crates/clawhdf5-format/src/file_writer.rs | 6 + crates/clawhdf5-format/src/fractal_heap.rs | 459 +++++++++++++++--- .../clawhdf5/tests/dense_storage_interop.rs | 112 ++++- docs/known-issues.md | 3 +- 5 files changed, 518 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d94b66..462672c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -310,6 +310,18 @@ estimate instead of libhdf5's per-depth record capacities, and the listing failed. The same B-tree code indexes dense attributes, shared messages and chunks. + - Fractal-heap "huge" objects (larger than the heap's managed-object + limit, 4 KiB by default — e.g. an 8 KiB dense attribute or a link with a + very long name) and "tiny" objects are now read; the ID type was taken + from the wrong bits (6-7, the version, instead of 4-5), so a huge object + failed and took every attribute on its object down with it (NetCDF-4 + files such as netcdf4-python's `issue671.nc`). Huge objects are found + directly from the ID or through the huge-object v2 B-tree, filtered or + not. + - Heaps with an I/O filter pipeline (a group created with a filter on its + creation property list compresses its link heap) are now read: the + header's pipeline was skipped with the wrong size, so its checksum was + looked for in the wrong place, and filtered direct blocks were read raw. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/file_writer.rs b/crates/clawhdf5-format/src/file_writer.rs index 95d5834..dbf59f2 100644 --- a/crates/clawhdf5-format/src/file_writer.rs +++ b/crates/clawhdf5-format/src/file_writer.rs @@ -1947,6 +1947,12 @@ mod tests { root_block_address: 0, current_rows_in_root_indirect_block: 0, managed_objects_count: 0, + huge_btree_address: u64::MAX, + filter_pipeline: None, + root_direct_block_filtered_size: 0, + root_direct_block_filter_mask: 0, + offset_size: 8, + length_size: 8, }; let (off, len) = fh.decode_managed_id(&id).unwrap(); assert_eq!(off, 100); diff --git a/crates/clawhdf5-format/src/fractal_heap.rs b/crates/clawhdf5-format/src/fractal_heap.rs index c71a41f..9bc4ba4 100644 --- a/crates/clawhdf5-format/src/fractal_heap.rs +++ b/crates/clawhdf5-format/src/fractal_heap.rs @@ -1,12 +1,14 @@ //! HDF5 Fractal Heap parsing for v2 group link storage. #[cfg(not(feature = "std"))] -use alloc::vec::Vec; +use alloc::{format, vec::Vec}; #[cfg(feature = "checksum")] use byteorder::{ByteOrder, LittleEndian}; +use crate::btree_v2::{BTreeV2Header, collect_btree_v2_records}; use crate::error::FormatError; +use crate::filter_pipeline::FilterPipeline; /// Parsed fractal heap header (signature "FRHP"). #[derive(Debug, Clone)] @@ -33,6 +35,23 @@ pub struct FractalHeapHeader { pub current_rows_in_root_indirect_block: u16, /// Total number of managed objects. pub managed_objects_count: u64, + /// Address of the v2 B-tree indexing "huge" objects (undefined address + /// when the heap has none). Huge objects are those larger than + /// `max_managed_object_size`; they live outside the heap's blocks. + pub huge_btree_address: u64, + /// The heap's I/O filter pipeline, if it has one. It applies to managed + /// direct blocks and to huge objects. + pub filter_pipeline: Option, + /// Stored (filtered) size of the root direct block; meaningful only when + /// the heap is filtered and its root is a direct block. + pub root_direct_block_filtered_size: u64, + /// Filter mask of the root direct block (bit *i* set = filter *i* + /// skipped); meaningful only when the heap is filtered. + pub root_direct_block_filter_mask: u32, + /// Size of addresses in the file ("Size of Offsets"). + pub offset_size: u8, + /// Size of lengths in the file ("Size of Lengths"). + pub length_size: u8, } fn read_offset(data: &[u8], pos: usize, size: u8) -> Result { @@ -79,6 +98,38 @@ fn is_undefined(val: u64, offset_size: u8) -> bool { } } +/// Little-endian unsigned integer of up to 8 bytes. +fn le_uint(bytes: &[u8]) -> u64 { + bytes + .iter() + .take(8) + .enumerate() + .fold(0u64, |acc, (i, &b)| acc | (u64::from(b) << (i * 8))) +} + +fn heap_error(msg: &str) -> FormatError { + FormatError::ChunkedReadError(format!("fractal heap: {msg}")) +} + +/// Heap ID type, from bits 4-5 of an ID's first byte (libhdf5's +/// `H5HF_ID_TYPE_MASK`, 0x30); bits 6-7 are the ID version, which must be 0. +const HEAP_ID_MANAGED: u8 = 0; +const HEAP_ID_HUGE: u8 = 1; +const HEAP_ID_TINY: u8 = 2; + +/// The type (0 managed, 1 huge, 2 tiny) of a heap ID from its first byte, +/// refusing an ID version other than 0. +fn heap_id_type(first: u8) -> Result { + if first >> 6 != 0 { + return Err(heap_error("unsupported heap ID version")); + } + Ok((first >> 4) & 0x03) +} + +/// v2 B-tree record types indexing a heap's huge objects. +const BTREE_HUGE_INDIRECT: u8 = 1; +const BTREE_HUGE_INDIRECT_FILTERED: u8 = 2; + impl FractalHeapHeader { /// Parse a fractal heap header at the given offset. pub fn parse( @@ -122,11 +173,17 @@ impl FractalHeapHeader { ]); pos += 4; - // Skip several fixed fields: next_huge_object_id(ls), btree_huge_objects_address(os), - // free_space_managed_blocks(ls), managed_block_free_space_manager_address(os), + // next_huge_object_id (length_size) + ensure_len(file_data, pos, ls)?; + pos += ls; + // btree_huge_objects_address (offset_size) + let huge_btree_address = read_offset(file_data, pos, offset_size)?; + pos += os; + + // Skip: free_space_managed_blocks(ls), managed_block_free_space_manager_address(os), // managed_space_in_heap(ls), allocated_managed_space_in_heap(ls), // direct_block_allocation_iterator_offset(ls) - let skip_size = 5 * ls + 2 * os; + let skip_size = 4 * ls + os; ensure_len(file_data, pos, skip_size)?; pos += skip_size; @@ -134,14 +191,9 @@ impl FractalHeapHeader { let managed_objects_count = read_offset(file_data, pos, length_size)?; pos += ls; - // huge_objects_size (length_size) - pos += ls; - // huge_objects_count (length_size) - pos += ls; - // tiny_objects_size (length_size) - pos += ls; - // tiny_objects_count (length_size) - pos += ls; + // huge_objects_size, huge_objects_count, tiny_objects_size, + // tiny_objects_count (length_size each) + pos += 4 * ls; // table_width (2) ensure_len(file_data, pos, 2)?; @@ -175,16 +227,28 @@ impl FractalHeapHeader { ensure_len(file_data, pos, 2)?; let current_rows_in_root_indirect_block = u16::from_le_bytes([file_data[pos], file_data[pos + 1]]); - #[allow(unused_variables, unused_mut, unused_assignments)] - let mut pos = pos + 2; + pos += 2; - // Skip IO filter encoded info if present + // With I/O filters: root direct block's filtered size (length_size), + // its filter mask (4), then the encoded filter pipeline message. + let mut filter_pipeline = None; + let mut root_direct_block_filtered_size = 0; + let mut root_direct_block_filter_mask = 0; if io_filter_encoded_length > 0 { - // root_block_filter_info_size (length_size) + filter_mask (4) - #[allow(unused_assignments)] - { - pos += ls + 4; - } + root_direct_block_filtered_size = read_offset(file_data, pos, length_size)?; + pos += ls; + ensure_len(file_data, pos, 4)?; + root_direct_block_filter_mask = u32::from_le_bytes([ + file_data[pos], + file_data[pos + 1], + file_data[pos + 2], + file_data[pos + 3], + ]); + pos += 4; + let n = io_filter_encoded_length as usize; + ensure_len(file_data, pos, n)?; + filter_pipeline = Some(FilterPipeline::parse(&file_data[pos..pos + n])?); + pos += n; } // Validate header checksum @@ -200,6 +264,8 @@ impl FractalHeapHeader { }); } } + #[cfg(not(feature = "checksum"))] + let _ = pos; Ok(FractalHeapHeader { heap_id_length, @@ -213,13 +279,19 @@ impl FractalHeapHeader { root_block_address, current_rows_in_root_indirect_block, managed_objects_count, + huge_btree_address, + filter_pipeline, + root_direct_block_filtered_size, + root_direct_block_filter_mask, + offset_size, + length_size, }) } /// Decode a managed heap ID into (offset_in_heap, object_length). /// /// The heap ID layout for managed objects (type 0): - /// - Byte 0: bits 6-7 = type (0), bits 4-5 = version (0), bits 0-3 = reserved + /// - Byte 0: bits 6-7 = version (0), bits 4-5 = type (0), bits 0-3 = reserved /// - Bytes 1+: offset (max_heap_size bits, LE) then length (remaining bits, LE) pub fn decode_managed_id(&self, id_bytes: &[u8]) -> Result<(u64, u64), FormatError> { if id_bytes.is_empty() { @@ -229,8 +301,8 @@ impl FractalHeapHeader { }); } - let id_type = (id_bytes[0] >> 6) & 0x03; - if id_type != 0 { + let id_type = heap_id_type(id_bytes[0])?; + if id_type != HEAP_ID_MANAGED { return Err(FormatError::InvalidHeapIdType(id_type)); } @@ -269,12 +341,183 @@ impl FractalHeapHeader { Ok((heap_offset, length_val)) } - /// Read a managed object from the heap given its raw heap ID bytes. + /// Read any object from the heap given its raw heap ID bytes: managed + /// (stored in the heap's blocks), huge (stored outside them, found + /// directly from the ID or through the huge-object v2 B-tree, optionally + /// filtered) or tiny (stored in the ID itself). + /// + /// Despite its name this accepts every ID type; `offset_size` must match + /// the one the header was parsed with. pub fn read_managed_object( &self, file_data: &[u8], id_bytes: &[u8], offset_size: u8, + ) -> Result, FormatError> { + let Some(&first) = id_bytes.first() else { + return Err(FormatError::UnexpectedEof { + expected: 1, + available: 0, + }); + }; + match heap_id_type(first)? { + HEAP_ID_MANAGED => self.read_heap_managed(file_data, id_bytes, offset_size), + HEAP_ID_HUGE => self.read_huge_object(file_data, id_bytes), + HEAP_ID_TINY => self.read_tiny_object(id_bytes), + other => Err(FormatError::InvalidHeapIdType(other)), + } + } + + /// Whether a huge object's ID holds its address and length directly + /// (libhdf5 does this when they fit in the ID), rather than a key into + /// the huge-object B-tree. + fn huge_ids_direct(&self) -> bool { + let room = usize::from(self.heap_id_length).saturating_sub(1); + let os = usize::from(self.offset_size); + let ls = usize::from(self.length_size); + if self.filter_pipeline.is_some() { + room >= os + ls + 4 + ls + } else { + room >= os + ls + } + } + + /// Read a huge object (heap ID type 1). + fn read_huge_object(&self, file_data: &[u8], id: &[u8]) -> Result, FormatError> { + let os = usize::from(self.offset_size); + let ls = usize::from(self.length_size); + // (address, stored length, filter mask, decoded length); the last two + // only matter for a filtered heap. + let (addr, stored_len, mask, mem_len) = if self.huge_ids_direct() { + let body = &id[1..]; + let need = if self.filter_pipeline.is_some() { + os + ls + 4 + ls + } else { + os + ls + }; + ensure_len(body, 0, need)?; + let addr = le_uint(&body[..os]); + let len = le_uint(&body[os..os + ls]); + if self.filter_pipeline.is_some() { + let mask = u32::from_le_bytes([ + body[os + ls], + body[os + ls + 1], + body[os + ls + 2], + body[os + ls + 3], + ]); + let mem = le_uint(&body[os + ls + 4..os + ls + 4 + ls]); + (addr, len, mask, mem) + } else { + (addr, len, 0, len) + } + } else { + let key_len = (usize::from(self.heap_id_length).saturating_sub(1)).min(8); + ensure_len(id, 1, key_len)?; + let key = le_uint(&id[1..1 + key_len]); + self.find_huge_record(file_data, key)? + }; + + let start = usize::try_from(addr).map_err(|_| heap_error("huge object address"))?; + let len = usize::try_from(stored_len).map_err(|_| heap_error("huge object length"))?; + ensure_len(file_data, start, len)?; + let stored = &file_data[start..start + len]; + match &self.filter_pipeline { + None => Ok(stored.to_vec()), + Some(pipeline) => { + let mem = usize::try_from(mem_len).map_err(|_| heap_error("huge object size"))?; + let out = crate::filters::decompress_chunk_masked(stored, pipeline, mem, 1, mask)?; + if out.len() != mem { + return Err(heap_error("filtered huge object decoded to the wrong size")); + } + Ok(out) + } + } + } + + /// Look up huge object `key` in the huge-object v2 B-tree, returning + /// (address, stored length, filter mask, decoded length). + fn find_huge_record( + &self, + file_data: &[u8], + key: u64, + ) -> Result<(u64, u64, u32, u64), FormatError> { + if is_undefined(self.huge_btree_address, self.offset_size) { + return Err(heap_error( + "huge object ID but the heap has no huge-object index", + )); + } + let hdr = BTreeV2Header::parse( + file_data, + self.huge_btree_address as usize, + self.offset_size, + self.length_size, + )?; + let os = usize::from(self.offset_size); + let ls = usize::from(self.length_size); + let filtered = self.filter_pipeline.is_some(); + let (expected_type, rec_len) = if filtered { + (BTREE_HUGE_INDIRECT_FILTERED, os + ls + 4 + ls + ls) + } else { + (BTREE_HUGE_INDIRECT, os + ls + ls) + }; + if hdr.tree_type != expected_type || usize::from(hdr.record_size) < rec_len { + return Err(heap_error("unexpected huge-object B-tree record type")); + } + let records = + collect_btree_v2_records(file_data, &hdr, self.offset_size, self.length_size)?; + for rec in &records { + let d = &rec.data; + if d.len() < rec_len { + continue; + } + let addr = le_uint(&d[..os]); + let len = le_uint(&d[os..os + ls]); + if filtered { + let mask = u32::from_le_bytes([ + d[os + ls], + d[os + ls + 1], + d[os + ls + 2], + d[os + ls + 3], + ]); + let mem = le_uint(&d[os + ls + 4..os + 2 * ls + 4]); + let id = le_uint(&d[os + 2 * ls + 4..os + 3 * ls + 4]); + if id == key { + return Ok((addr, len, mask, mem)); + } + } else { + let id = le_uint(&d[os + ls..os + 2 * ls]); + if id == key { + return Ok((addr, len, 0, len)); + } + } + } + Err(heap_error("huge object not found in its B-tree")) + } + + /// Read a tiny object (heap ID type 2), stored in the ID itself. + fn read_tiny_object(&self, id: &[u8]) -> Result, FormatError> { + // libhdf5 uses a one-byte length (low 4 bits of byte 0) unless the ID + // is long enough to need 12 bits, which then borrow byte 1. + let extended = usize::from(self.heap_id_length).saturating_sub(1) > 17; + let (len, start) = if extended { + ensure_len(id, 0, 2)?; + ( + ((usize::from(id[0] & 0x0F)) << 8 | usize::from(id[1])) + 1, + 2, + ) + } else { + (usize::from(id[0] & 0x0F) + 1, 1) + }; + ensure_len(id, start, len)?; + Ok(id[start..start + len].to_vec()) + } + + /// Read a managed object (heap ID type 0). + fn read_heap_managed( + &self, + file_data: &[u8], + id_bytes: &[u8], + offset_size: u8, ) -> Result, FormatError> { let (heap_offset, obj_len) = self.decode_managed_id(id_bytes)?; @@ -289,12 +532,15 @@ impl FractalHeapHeader { // Root is a direct block self.read_from_direct_block( file_data, - self.root_block_address as usize, - self.starting_block_size, - 0, // block offset in heap = 0 for root + DirectBlock { + addr: self.root_block_address as usize, + size: self.starting_block_size, + heap_offset: 0, + filtered_size: self.root_direct_block_filtered_size, + filter_mask: self.root_direct_block_filter_mask, + }, heap_offset, obj_len as usize, - offset_size, ) } else { // Root is an indirect block — limit recursion to 64 levels @@ -313,27 +559,41 @@ impl FractalHeapHeader { /// Read an object from a direct block. /// - /// The heap offset is relative to the start of the block (including its header), - /// so we just add it to the block address minus the block's heap offset. - #[allow(clippy::too_many_arguments)] + /// The heap offset is relative to the start of the block (including its + /// header), so we just add it to the block address minus the block's heap + /// offset. A filtered heap stores each direct block (header included) + /// through its filter pipeline, so the block is decoded first. fn read_from_direct_block( &self, file_data: &[u8], - block_addr: usize, - _block_size: u64, - block_heap_offset: u64, + block: DirectBlock, target_offset: u64, length: usize, - _offset_size: u8, ) -> Result, FormatError> { - if target_offset < block_heap_offset { + if target_offset < block.heap_offset { return Err(FormatError::UnexpectedEof { - expected: block_heap_offset as usize, + expected: block.heap_offset as usize, available: target_offset as usize, }); } - let local_offset = (target_offset - block_heap_offset) as usize; - let pos = block_addr + let local_offset = (target_offset - block.heap_offset) as usize; + if let Some(pipeline) = &self.filter_pipeline { + let stored_len = usize::try_from(block.filtered_size) + .map_err(|_| heap_error("direct block size"))?; + let size = usize::try_from(block.size).map_err(|_| heap_error("direct block size"))?; + ensure_len(file_data, block.addr, stored_len)?; + let decoded = crate::filters::decompress_chunk_masked( + &file_data[block.addr..block.addr + stored_len], + pipeline, + size, + 1, + block.filter_mask, + )?; + ensure_len(&decoded, local_offset, length)?; + return Ok(decoded[local_offset..local_offset + length].to_vec()); + } + let pos = block + .addr .checked_add(local_offset) .ok_or(FormatError::UnexpectedEof { expected: usize::MAX, @@ -371,19 +631,13 @@ impl FractalHeapHeader { let iblock_header = 5 + offset_size as usize + block_offset_bytes; let mut pos = iblock_addr + iblock_header; - // Compute block sizes for each row using the doubling table let tw = self.table_width as u64; - let nrows_usize = nrows as usize; - - // Build table of (block_size, heap_offset) for each child entry let mut current_heap_offset = iblock_heap_offset; // Rows below max_direct_rows hold direct blocks; rows at/above hold // child indirect blocks. (NOT the FRHP "starting rows" field.) let start_indirect = self.max_direct_rows(); - - // Read child addresses for direct block rows let max_direct_rows = nrows_usize.min(start_indirect); for row in 0..max_direct_rows { @@ -393,35 +647,49 @@ impl FractalHeapHeader { let child_addr = read_offset(file_data, pos, offset_size)?; pos += offset_size as usize; - if self.io_filter_encoded_length > 0 { - // filtered_size(length_size) + filter_mask(4) - // Skip for now - we don't handle filtered direct blocks in fractal heaps - pos += 4; // filter_mask - simplified - } + // A filtered heap stores each direct block's filtered size + // (length_size) and filter mask (4) after its address. + let (filtered_size, filter_mask) = if self.filter_pipeline.is_some() { + let size = read_offset(file_data, pos, self.length_size)?; + pos += usize::from(self.length_size); + ensure_len(file_data, pos, 4)?; + let mask = u32::from_le_bytes([ + file_data[pos], + file_data[pos + 1], + file_data[pos + 2], + file_data[pos + 3], + ]); + pos += 4; + (size, mask) + } else { + (0, 0) + }; - if !is_undefined(child_addr, offset_size) { - let block_end = current_heap_offset + block_size; - if target_offset >= current_heap_offset && target_offset < block_end { - return self.read_from_direct_block( - file_data, - child_addr as usize, - block_size, - current_heap_offset, - target_offset, - length, - offset_size, - ); - } + let block_end = current_heap_offset.saturating_add(block_size); + if !is_undefined(child_addr, offset_size) + && target_offset >= current_heap_offset + && target_offset < block_end + { + return self.read_from_direct_block( + file_data, + DirectBlock { + addr: child_addr as usize, + size: block_size, + heap_offset: current_heap_offset, + filtered_size, + filter_mask, + }, + target_offset, + length, + ); } - current_heap_offset += block_size; + current_heap_offset = block_end; } } - // If we have indirect block rows - // A child indirect block in row r spans exactly that row's block size - // of heap space, so it has as many rows as a table of that total size - // needs (not `row - start_indirect + 1`, which undercounts and makes - // every object past the root's direct rows unreachable). + // Rows at and above `start_indirect` hold child indirect blocks. A + // child in row r spans exactly that row's block size of heap space, + // so it has as many rows as a table of that total size needs. for row in start_indirect..nrows_usize { let child_space = self.block_size_for_row(row); let child_nrows = self.rows_for_size(child_space); @@ -495,6 +763,16 @@ impl FractalHeapHeader { } } +/// A managed direct block's location, extent and (for a filtered heap) its +/// stored size and filter mask. +struct DirectBlock { + addr: usize, + size: u64, + heap_offset: u64, + filtered_size: u64, + filter_mask: u32, +} + #[cfg(test)] mod tests { use super::*; @@ -640,7 +918,7 @@ mod tests { let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); // Build a managed heap ID: - // byte 0: type=0 (bits 6-7 = 00), version=0 (bits 4-5), reserved (bits 0-3) + // byte 0: version=0 (bits 6-7), type=0 (bits 4-5), reserved (bits 0-3) // bytes 1-6: offset (max_heap_size=16 bits) then length (remaining bits) // For offset=0, length=13: // payload = offset | (length << 16) = 0 | (13 << 16) = 0x000D0000 @@ -704,9 +982,46 @@ mod tests { fn invalid_heap_id_type() { let (file_data, _) = build_simple_heap(8, 8); let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); - // Type = 1 (tiny) in bits 6-7 - let id = vec![0x40u8, 0, 0, 0, 0, 0, 0]; // bit 6 set = type 1 + // Type = 1 (huge) in bits 4-5 is not a managed ID + let id = vec![0x10u8, 0, 0, 0, 0, 0, 0]; let err = hdr.decode_managed_id(&id).unwrap_err(); assert_eq!(err, FormatError::InvalidHeapIdType(1)); } + + #[test] + fn tiny_object_is_read_from_the_id() { + let (file_data, _) = build_simple_heap(8, 8); + let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); + // Type 2 (0x20), length - 1 in the low 4 bits, data after. + let id = [0x20 | 2, b'a', b'b', b'c', 0, 0, 0]; + assert_eq!(hdr.read_managed_object(&file_data, &id, 8).unwrap(), b"abc"); + // A length running past the ID is an error, not a short read. + let id = [0x20 | 9, b'a', b'b', b'c', 0, 0, 0]; + assert!(hdr.read_managed_object(&file_data, &id, 8).is_err()); + } + + #[test] + fn huge_object_with_a_direct_id() { + // With IDs long enough for an address and a length, libhdf5 stores + // huge objects' location in the ID instead of the huge-object B-tree. + let (mut file_data, _) = build_simple_heap(8, 8); + let mut hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); + hdr.heap_id_length = 17; + file_data[900..905].copy_from_slice(b"huge!"); + let mut id = vec![0x10u8]; + id.extend_from_slice(&900u64.to_le_bytes()); + id.extend_from_slice(&5u64.to_le_bytes()); + assert_eq!( + hdr.read_managed_object(&file_data, &id, 8).unwrap(), + b"huge!" + ); + } + + #[test] + fn unknown_heap_id_version_is_refused() { + let (file_data, _) = build_simple_heap(8, 8); + let hdr = FractalHeapHeader::parse(&file_data, 0, 8, 8).unwrap(); + let id = [0x40u8, 0, 0, 0, 0, 0, 0]; + assert!(hdr.read_managed_object(&file_data, &id, 8).is_err()); + } } diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index ecfff7a..7f8206a 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -8,7 +8,7 @@ use std::process::Command; -use clawhdf5::File; +use clawhdf5::{AttrValue, File}; fn python() -> String { std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) @@ -163,3 +163,113 @@ fn dense_group_with_a_three_level_name_index() { vec![1.0] ); } + +/// The attribute names h5py reports for `obj`, sorted. +fn h5py_attr_names(path: &str, obj: &str) -> Vec { + let out = run_python(&format!( + "import h5py\n\ + with h5py.File(r'{path}', 'r') as f:\n\ + \x20 print('\\x1f'.join(sorted(f[{obj:?}].attrs.keys())))\n" + )); + out.split('\x1f') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect() +} + +#[test] +fn dense_attribute_stored_as_a_huge_heap_object() { + skip_if_no_python!(); + // More than 8 attributes puts them in dense storage; one larger than the + // heap's 4 KiB managed-object limit is stored as a "huge" object, outside + // the heap blocks and found through the huge-object v2 B-tree. Its heap ID + // (type bits 4-5 = 1) was misread as a managed ID, and the error made + // every attribute on the object unreadable. NetCDF-4 files hit this + // (netcdf-c's issue671.nc / issue672.nc). + let (_dir, path) = h5py_file( + "d = f.create_dataset('d', data=[1.0])\n\ + for i in range(10):\n\ + \x20 d.attrs['a%d' % i] = i\n\ + d.attrs['big'] = np.arange(1024, dtype='f8')\n\ + d.attrs['bigger'] = np.arange(20000, dtype='i8') * 3\n", + ); + let f = File::open(&path).unwrap(); + let attrs = f.dataset("d").unwrap().attrs().unwrap(); + let mut names: Vec = attrs.keys().cloned().collect(); + names.sort(); + assert_eq!(names, h5py_attr_names(&path, "d")); + for i in 0..10 { + assert!( + matches!(attrs[&format!("a{i}")], AttrValue::I64(v) if v == i), + "a{i}: {:?}", + attrs[&format!("a{i}")] + ); + } + let big: Vec = (0..1024).map(f64::from).collect(); + assert!(matches!(&attrs["big"], AttrValue::F64Array(v) if *v == big)); + let bigger: Vec = (0..20000).map(|v| v * 3).collect(); + assert!(matches!(&attrs["bigger"], AttrValue::I64Array(v) if *v == bigger)); +} + +/// A group whose link heap has a deflate I/O filter (set on the group +/// creation property list), with 3 000 links and one link whose message is +/// larger than the heap's managed-object limit, so it is a huge object. +fn huge_link_group(filtered: bool) -> (tempfile::TempDir, String) { + let filter = if filtered { + "import ctypes, glob, os\n\ + lib = ctypes.CDLL(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))[0])\n\ + lib.H5Pset_deflate.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\ + assert lib.H5Pset_deflate(gcpl.id, 6) >= 0\n" + } else { + "" + }; + h5py_file(&format!( + "t = f.create_dataset('t', data=[1.0])\n\ + gcpl = h5py.h5p.create(h5py.h5p.GROUP_CREATE)\n\ + {filter}\ + h5py.h5g.create(f.id, b'g', gcpl=gcpl)\n\ + g = f['g']\n\ + for i in range(3000):\n\ + \x20 g['l%05d' % i] = t\n\ + g['L' * 5000] = t\n" + )) +} + +fn check_huge_link_group(filtered: bool) { + let (_dir, path) = huge_link_group(filtered); + assert_same_listing(&path, "g"); + let f = File::open(&path).unwrap(); + let huge = format!("g/{}", "L".repeat(5000)); + assert_eq!(f.dataset(&huge).unwrap().read_f64().unwrap(), vec![1.0]); + assert_eq!( + f.dataset("g/l02999").unwrap().read_f64().unwrap(), + vec![1.0] + ); +} + +#[test] +fn dense_group_with_a_huge_link() { + skip_if_no_python!(); + check_huge_link_group(false); +} + +#[test] +fn dense_group_with_a_filtered_link_heap() { + skip_if_no_python!(); + // libhdf5 applies a group's filter pipeline to its link heap: direct + // blocks and huge objects are stored deflated, and the heap header + // carries the pipeline. The header's checksum was looked for in the + // wrong place, and filtered blocks were read raw. + if run_python( + "import h5py, glob, os\nprint(len(glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))))", + ) == "0" + { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but h5py's bundled libhdf5 was not found" + ); + eprintln!("SKIP: h5py's bundled libhdf5 not found (needed to set the filter)"); + return; + } + check_huge_link_group(true); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 39cdcf2..774c70e 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -83,7 +83,8 @@ the VDS item, which is marked. - Soft links are left out of `datasets()`. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF - files (`issue671.nc`). + files (`issue671.nc`). **Fixed 2026-09-25:** huge and tiny heap objects, + and filtered heaps, are read. - **Other readers:** - VL-string datasets are not readable through `File`. - Metadata cache images are not supported. From 38d0d4de02f663b55a1191c31af10f3133df3abe Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:57:18 -0500 Subject: [PATCH 07/24] fix(format): skip user-defined links instead of failing the group Link types 65-255 are user-defined: their target is only meaningful to the application that registered the link class. LinkMessage::parse rejects them with InvalidLinkType, and group traversal propagated that, so one such link made the whole group unlistable and every path through it unresolvable (libhdf5's tall.h5 and tudlink.h5, class 187). Group traversal (compact and dense) now leaves user-defined links out, the way h5py leaves out links it cannot open; reserved types (2-63) are still an error. Regression test: user_defined_links_do_not_break_the_listing, on libhdf5's own tools/test/testfiles tall.h5 and tudlink.h5 (BSD-style HDF5 licence, 10 KB and 1 KB), committed as fixtures. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++ crates/clawhdf5-format/src/group_v2.rs | 30 ++++++++++++++++-- .../clawhdf5/tests/dense_storage_interop.rs | 25 +++++++++++++++ crates/clawhdf5/tests/fixtures/tall.h5 | Bin 0 -> 9968 bytes crates/clawhdf5/tests/fixtures/tudlink.h5 | Bin 0 -> 904 bytes docs/known-issues.md | 2 ++ 6 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 crates/clawhdf5/tests/fixtures/tall.h5 create mode 100644 crates/clawhdf5/tests/fixtures/tudlink.h5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 462672c..95da497 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -322,6 +322,11 @@ creation property list compresses its link heap) are now read: the header's pipeline was skipped with the wrong size, so its checksum was looked for in the wrong place, and filtered direct blocks were read raw. + - A user-defined link (link class 65-255, e.g. 187 in libhdf5's + `tall.h5`/`tudlink.h5`) made its whole group unlistable. Such links + cannot be followed without the application that registered the class, so + they are now left out of `datasets()`/`groups()` and path lookup, as h5py + leaves out links it cannot open; reserved link types are still an error. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index 903c7d6..bcfa1be 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -38,6 +38,24 @@ pub fn resolve_v2_group_entries( } } +/// First user-defined link type (HDF5 reserves 2-63; 64 is external). +const FIRST_USER_DEFINED_LINK_TYPE: u8 = 65; + +/// Parse a Link message, or `None` for a user-defined link (type 65-255). +/// +/// A user-defined link's target is only meaningful to the application that +/// registered its class, so, like libhdf5 without that class, we cannot +/// follow it. Leaving it out lets the rest of the group be listed and +/// resolved instead of one such link failing the whole group; reserved +/// types (2-63) are still an error. +fn parse_link(data: &[u8], offset_size: u8) -> Result, FormatError> { + match LinkMessage::parse(data, offset_size) { + Ok(link) => Ok(Some(link)), + Err(FormatError::InvalidLinkType(t)) if t >= FIRST_USER_DEFINED_LINK_TYPE => Ok(None), + Err(e) => Err(e), + } +} + /// Extract link entries from Link messages directly in the object header (compact storage). fn resolve_compact_entries( object_header: &ObjectHeader, @@ -46,7 +64,9 @@ fn resolve_compact_entries( let mut entries = Vec::new(); for msg in &object_header.messages { if msg.msg_type == MessageType::Link { - let link = LinkMessage::parse(&msg.data, offset_size)?; + let Some(link) = parse_link(&msg.data, offset_size)? else { + continue; + }; if let LinkTarget::Hard { object_header_address, } = link.link_target @@ -98,7 +118,9 @@ fn for_each_dense_link( // Read managed object from fractal heap let link_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; - visit(LinkMessage::parse(&link_data, offset_size)?); + if let Some(link) = parse_link(&link_data, offset_size)? { + visit(link); + } } Ok(()) } @@ -178,7 +200,9 @@ fn find_symbolic_link( } else { for msg in &object_header.messages { if msg.msg_type == MessageType::Link { - let link = LinkMessage::parse(&msg.data, offset_size)?; + let Some(link) = parse_link(&msg.data, offset_size)? else { + continue; + }; if link.name == name && is_symbolic(&link.link_target) { found = Some(link.link_target); } diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index 7f8206a..92af6ee 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -273,3 +273,28 @@ fn dense_group_with_a_filtered_link_heap() { } check_huge_link_group(true); } + +fn fixture(name: &str) -> String { + format!("{}/tests/fixtures/{name}", env!("CARGO_MANIFEST_DIR")) +} + +/// `tall.h5` and `tudlink.h5` are libhdf5's own tool test files +/// (`tools/test/testfiles`, BSD-style HDF5 licence). Each has user-defined +/// links of class 187, which h5py lists by name but cannot open, and h5dump +/// prints as `USERDEFINED_LINK`. One such link made the whole group +/// unlistable (`InvalidLinkType(187)`); it is now left out of the listing +/// like any other link that cannot be followed. +#[test] +fn user_defined_links_do_not_break_the_listing() { + let f = File::open(fixture("tall.h5")).unwrap(); + let g2 = f.group("g2").unwrap(); + let mut ds = g2.datasets().unwrap(); + ds.sort(); + assert_eq!(ds, ["dset2.1", "dset2.2"]); + assert!(g2.groups().unwrap().is_empty()); + assert!(f.dataset("g2/udlink").is_err()); + + let f = File::open(fixture("tudlink.h5")).unwrap(); + assert!(f.root().datasets().unwrap().is_empty()); + assert!(f.root().groups().unwrap().is_empty()); +} diff --git a/crates/clawhdf5/tests/fixtures/tall.h5 b/crates/clawhdf5/tests/fixtures/tall.h5 new file mode 100644 index 0000000000000000000000000000000000000000..918aeeeda416eafefdf960b0b6df7dbee30d8890 GIT binary patch literal 9968 zcmeHN%}-N75Z_m@6s%f6%y7C`rp+@eIMu&La!cU0)x5v3*L)!1(9UnI&sroe+w)t|1 zOWxTjH_Q*Z2h!s=GO#f#hAN&uM7)+y4RL)c$vLoxG-{u_e>jB6?f7jM{HB{&S5a^o zgx~cz%ZK>g{W~}aJ!;{1Mew6GP7PhQ2&UX3c(3@qckA|;V#P|F0P1sg4TgVKMSg#8{4amXML@8V71lji`+uKl%%DA^Q6TdBu2er zb=PXeu>|k@%%H@lAp9zbOiFf%>-m?{f% zPx8+uXBP9qP(15;sq#TN$1$gT9^3GJFG2l7K6^m#VOGQ;>v9himmMo1TX^ySPJE`jJKD()NOXKUX}lom*WN z<4ne_<~4H5pWdh9l!4&IyS~rlY_6{N`Jj)ztM4;XzRA}cdd>gpEntfF`=A`36>h_# zM7Vvz#C4Ys8SueFJK&EK$5A2xOrpNQ?G|Lv5By_9-~nBLkeUx&&;uFt$FC9KpC)pN z`iYP*AB+V(km(^hLv)U4fanqt62^osj0HWAfsTtrgG5(|kYE?)jQLPuy`g(>tv=z-7!p$GPt2lRf)MjV=Y{rg;m846U0RFvfJ0Tc*zfBOZo zmr{l^dHVOa#;kbw|GUyg7Kll9Dp4EK5s;&NHR3w}Ms2t!er&WQzf{8MlTM^KeHNx? z^0qKPSt=U;IZ}6F*eD^fH@O@>YkMFVuZ+L;zO&_XUsud@Jr|XZcv^9bO%zfEw}ZXh zf-PJnRpWNK!u*dV|GCWWPk@ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/tudlink.h5 b/crates/clawhdf5/tests/fixtures/tudlink.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5dc0c7071d8c1497f21b4bd468047b6564423590 GIT binary patch literal 904 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@pf(~Yg2#gPtPk=FS(2NYM42(cI38=gP)O-b~ zc^OdgkRVrA5EEuTjD|`xM2J9G958`VdPIeQhpS@%$iop(U&GSD2`1EF0Hrk$04s(t zDI%HRDHoO^VKl^8E(Q*8`i5D;22J0L9J|>|Q*tu%vJJsfY*2F$;zleCY5DmueE?eY BKrH|O literal 0 HcmV?d00001 diff --git a/docs/known-issues.md b/docs/known-issues.md index 774c70e..a437db4 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -76,6 +76,8 @@ the VDS item, which is marked. - **Old-style shared messages (version 1)** read the wrong address. - **Groups and links:** - Groups with a user-defined link type (e.g. 187) cannot be listed. + **Fixed 2026-09-25:** user-defined links are skipped; the rest of the + group lists. - Dense groups with more than about 22 000 links cannot be listed. **Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had the wrong row count, and v2 B-tree internal nodes at depth 3+ were read From 190918a478bfc8bc412f9c58e5a70b087e560d0b Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:57:32 -0500 Subject: [PATCH 08/24] feat(format): decode hyperslab selection versions 1 and 2 in VDS mappings libhdf5 serializes a VDS hyperslab as version 1 (irregular, 4-byte block corners) for the default format bounds, and as version 2 (regular, 8-byte) for unlimited selections in the 1.10 format. Only version 3 was accepted, so every h5py VDS written with default libver failed with "only version-3 hyperslab selections are supported" (5 libhdf5 test files in the sweep). Decode all three versions following H5S__hyper_deserialize, including irregular hyperslabs (a union of blocks, enumerated in row-major order as libhdf5 iterates them) and the all-ones "unlimited" count/block marker. SerializedSelection exposes the raw form for unlimited-mapping support. Test: vds_interop::vds_version1_irregular_hyperslab_selections compares default-libver h5py VDS reads (contiguous, strided and 2-D block mappings) with libhdf5's values. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 + crates/clawhdf5-format/src/selection.rs | 498 +++++++++++++++++++----- crates/clawhdf5/tests/vds_interop.rs | 156 ++++++++ docs/known-issues.md | 3 +- 4 files changed, 562 insertions(+), 104 deletions(-) create mode 100644 crates/clawhdf5/tests/vds_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b87415..a349fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,15 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` virtual datasets (VDS), checked against HDF5 2.0 through + h5py (`crates/clawhdf5/tests/vds_interop.rs`): + - Hyperslab selection versions 1 and 2 were refused ("only version-3 + hyperslab selections are supported"). Version 1 is what libhdf5 writes for + every VDS created with the default format bounds (h5py's default), so + those could not be read at all; version 2 is its encoding of an unlimited + selection. Both are decoded now, as are irregular hyperslabs (a union of + blocks, read in row-major order as libhdf5 iterates them). + `SerializedSelection` exposes the raw form, including unlimited counts. - `clawhdf5-format` reader — **values returned wrong with no error:** - Fixed Array and Extensible Array chunk indexes were laid out by the dataset's current shape instead of its max shape (23 libhdf5 test files, diff --git a/crates/clawhdf5-format/src/selection.rs b/crates/clawhdf5-format/src/selection.rs index e1f98aa..de31e46 100644 --- a/crates/clawhdf5-format/src/selection.rs +++ b/crates/clawhdf5-format/src/selection.rs @@ -229,44 +229,47 @@ impl Selection { /// self-describing in length, so the count lets a caller walk a packed list /// of selections — as the Virtual Dataset global-heap block does). /// - /// Only the forms needed for VDS assembly are decoded: `ALL`, `NONE`, and - /// **regular** hyperslabs serialized at **version 3** (the encoding HDF5 - /// 1.10+/2.0 emit). Point selections, irregular hyperslabs, and older - /// hyperslab versions return an error rather than mis-decoding. + /// Decodes `ALL`, `NONE`, and hyperslabs at every version libhdf5 writes + /// (1: irregular, 4-byte coordinates — the default-format encoding; 2: + /// regular, 8-byte; 3: either, variable width). A regular hyperslab maps + /// to [`Selection::Hyperslab`]; an *irregular* one (a union of blocks) + /// maps to a single-block hyperslab when it has one block, and otherwise to + /// [`Selection::Points`] listing the union in row-major order (the order + /// libhdf5 iterates it in). Unlimited counts/blocks decode as `u64::MAX` + /// (see [`SerializedSelection::decode`] for the raw form). Point + /// selections are refused: libhdf5 does not allow them in virtual datasets + /// either. pub fn decode_serialized(data: &[u8]) -> Result<(Selection, usize), FormatError> { - if data.len() < 8 { - return Err(FormatError::UnexpectedEof { - expected: 8, - available: data.len(), - }); - } - let sel_type = u32::from_le_bytes([data[0], data[1], data[2], data[3]]); - let version = u32::from_le_bytes([data[4], data[5], data[6], data[7]]); - - match sel_type { - // ALL / NONE: type(4) + version(4) + reserved(4) + length(4) = 16 bytes. - 3 | 0 => { - if data.len() < 16 { - return Err(FormatError::UnexpectedEof { - expected: 16, - available: data.len(), - }); - } - let sel = if sel_type == 3 { - Selection::All + let (raw, len) = SerializedSelection::decode(data)?; + let sel = match raw { + SerializedSelection::All => Selection::All, + SerializedSelection::None => Selection::None, + SerializedSelection::Regular { + start, + stride, + count, + block, + } => Selection::Hyperslab { + start, + stride, + count, + block, + }, + SerializedSelection::Blocks { rank, starts, ends } => { + if starts.len() == rank { + let block = starts.iter().zip(&ends).map(|(&s, &e)| e - s + 1).collect(); + Selection::Hyperslab { + start: starts, + stride: vec![1; rank], + count: vec![1; rank], + block, + } } else { - Selection::None - }; - Ok((sel, 16)) + Selection::Points(blocks_union_coords(rank, &starts, &ends)?) + } } - 2 => decode_hyperslab_serialized(data, version), - 1 => Err(FormatError::ChunkedReadError( - "VDS point selections are not supported".into(), - )), - _ => Err(FormatError::ChunkedReadError( - "unknown dataspace selection type".into(), - )), - } + }; + Ok((sel, len)) } /// Enumerate the selected element indices of a **1-D** dataspace of the @@ -314,6 +317,11 @@ impl Selection { "VDS selection rank does not match dataspace rank".into(), )); } + if count.iter().chain(block.iter()).any(|&v| v == UNLIMITED) { + return Err(FormatError::ChunkedReadError( + "unlimited selection must be clipped before it is enumerated".into(), + )); + } // Selected coordinates along each dimension, in order. let mut per_dim: Vec> = Vec::with_capacity(rank); for d in 0..rank { @@ -400,84 +408,279 @@ impl Selection { } } -/// Decode an `H5S_SEL_HYPER` selection in its serialized form. Only version-3 -/// **regular** hyperslabs are supported. -fn decode_hyperslab_serialized( - data: &[u8], - version: u32, -) -> Result<(Selection, usize), FormatError> { - if version != 3 { - return Err(FormatError::ChunkedReadError( - "only version-3 hyperslab selections are supported".into(), - )); +/// Hyperslab count/block value meaning "unlimited" (`H5S_UNLIMITED`). +pub const UNLIMITED: u64 = u64::MAX; + +/// Largest number of elements an irregular selection is expanded to when it +/// is converted to a point list by [`Selection::decode_serialized`]. +const MAX_EXPANDED_POINTS: u64 = 1 << 26; + +/// A selection exactly as `H5S_select_serialize` stores it, before it is +/// applied to any dataspace. +/// +/// Unlike [`Selection`] this keeps an irregular hyperslab as its list of +/// blocks, and a regular hyperslab's count/block may be [`UNLIMITED`] (the +/// unlimited selections used by unlimited and "printf" virtual dataset +/// mappings). +#[derive(Debug, Clone, PartialEq)] +pub enum SerializedSelection { + /// `H5S_SEL_ALL`. + All, + /// `H5S_SEL_NONE`. + None, + /// A regular hyperslab. `count[d]` or `block[d]` may be [`UNLIMITED`]. + Regular { + start: Vec, + stride: Vec, + count: Vec, + block: Vec, + }, + /// An irregular hyperslab: the union of `starts.len() / rank` blocks, each + /// given by its first (`starts`) and last (`ends`, inclusive) coordinate, + /// flattened block-major. + Blocks { + rank: usize, + starts: Vec, + ends: Vec, + }, +} + +fn sel_err(msg: &str) -> FormatError { + FormatError::ChunkedReadError(msg.into()) +} + +/// Bounds-checked little-endian reader over a serialized selection. +struct SelReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl SelReader<'_> { + fn take(&mut self, n: usize) -> Result<&[u8], FormatError> { + let end = self.pos.checked_add(n).filter(|&e| e <= self.data.len()); + let end = end.ok_or(FormatError::UnexpectedEof { + expected: self.pos.saturating_add(n), + available: self.data.len(), + })?; + let s = &self.data[self.pos..end]; + self.pos = end; + Ok(s) } - // type(4) ver(4) flags(1) enc_size(1) rank(4) [start,stride,count,block]*rank - if data.len() < 14 { - return Err(FormatError::UnexpectedEof { - expected: 14, - available: data.len(), - }); + + fn uint(&mut self, size: usize) -> Result { + let bytes = self.take(size)?; + Ok(bytes + .iter() + .enumerate() + .fold(0u64, |v, (i, &b)| v | (b as u64) << (i * 8))) } - let flags = data[8]; - let enc_size = data[9] as usize; - // Bit 0 set => regular hyperslab. Irregular hyperslabs list explicit blocks. - if flags & 0x01 == 0 { - return Err(FormatError::ChunkedReadError( - "irregular VDS hyperslab selections are not supported".into(), - )); + + fn remaining(&self) -> usize { + self.data.len() - self.pos } - if enc_size != 2 && enc_size != 4 && enc_size != 8 { - return Err(FormatError::ChunkedReadError( - "unsupported hyperslab coordinate encoding size".into(), - )); - } - let rank = u32::from_le_bytes([data[10], data[11], data[12], data[13]]) as usize; - // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything larger so a - // corrupt rank can't drive a huge allocation or read loop. - if rank > 32 { - return Err(FormatError::ChunkedReadError( - "hyperslab selection rank exceeds maximum (32)".into(), - )); - } - let mut pos = 14; - let read_coord = |data: &[u8], pos: usize| -> Result { - if pos + enc_size > data.len() { - return Err(FormatError::UnexpectedEof { - expected: pos + enc_size, - available: data.len(), - }); +} + +impl SerializedSelection { + /// Decode a serialized selection, returning it and the number of bytes it + /// occupies. Mirrors libhdf5's `H5S_select_deserialize`: `ALL`/`NONE` and + /// hyperslab versions 1-3 are decoded; point selections (which libhdf5 + /// refuses in virtual datasets) and malformed input are errors. + pub fn decode(data: &[u8]) -> Result<(SerializedSelection, usize), FormatError> { + let mut r = SelReader { data, pos: 0 }; + let sel_type = r.uint(4)?; + let version = r.uint(4)?; + match sel_type { + // ALL / NONE: type(4) + version(4) + reserved(4) + length(4). + 0 | 3 => { + r.take(8)?; + let sel = if sel_type == 3 { + SerializedSelection::All + } else { + SerializedSelection::None + }; + Ok((sel, r.pos)) + } + 2 => { + let sel = decode_hyperslab(&mut r, version)?; + Ok((sel, r.pos)) + } + 1 => Err(sel_err( + "VDS point selections are not supported (libhdf5 rejects them too)", + )), + _ => Err(sel_err("unknown dataspace selection type")), } - let mut v = 0u64; - for (i, &b) in data[pos..pos + enc_size].iter().enumerate() { - v |= (b as u64) << (i * 8); + } + + /// The single dimension in which this selection is unlimited, if any. + pub fn unlimited_dim(&self) -> Option { + match self { + SerializedSelection::Regular { count, block, .. } => count + .iter() + .zip(block) + .position(|(&c, &b)| c == UNLIMITED || b == UNLIMITED), + _ => None, } - Ok(v) + } + + /// The rank the selection was serialized with (`None` for ALL/NONE, which + /// carry no rank). + pub fn rank(&self) -> Option { + match self { + SerializedSelection::Regular { start, .. } => Some(start.len()), + SerializedSelection::Blocks { rank, .. } => Some(*rank), + _ => None, + } + } +} + +/// `H5S__hyper_deserialize`: after the type and version words. +fn decode_hyperslab(r: &mut SelReader, version: u64) -> Result { + const REGULAR: u8 = 0x01; + let (flags, enc_size) = match version { + // v1: reserved(4) + length(4), always irregular, 4-byte coordinates. + 1 => { + r.take(8)?; + (0u8, 4usize) + } + // v2: flags(1) + length(4), 8-byte coordinates. + 2 => { + let flags = r.take(1)?[0]; + r.take(4)?; + (flags, 8) + } + // v3: flags(1) + encoding size(1). + 3 => { + let flags = r.take(1)?[0]; + let enc = r.take(1)?[0] as usize; + (flags, enc) + } + _ => return Err(sel_err("unsupported hyperslab selection version")), }; - let (mut start, mut stride, mut count, mut block) = ( - Vec::with_capacity(rank), - Vec::with_capacity(rank), - Vec::with_capacity(rank), - Vec::with_capacity(rank), - ); - for _ in 0..rank { - start.push(read_coord(data, pos)?); - pos += enc_size; - stride.push(read_coord(data, pos)?); - pos += enc_size; - count.push(read_coord(data, pos)?); - pos += enc_size; - block.push(read_coord(data, pos)?); - pos += enc_size; + if flags & !REGULAR != 0 { + return Err(sel_err("unknown hyperslab selection flags")); } - Ok(( - Selection::Hyperslab { + if !matches!(enc_size, 2 | 4 | 8) { + return Err(sel_err("unsupported hyperslab coordinate encoding size")); + } + let rank = r.uint(4)? as usize; + // HDF5 caps dataspace rank at 32 (H5S_MAX_RANK). Reject anything else so a + // corrupt rank can't drive a huge allocation or read loop. + if rank == 0 || rank > 32 { + return Err(sel_err("hyperslab selection rank must be 1..=32")); + } + // The all-ones value of the encoding width means "unlimited". + let unlim_raw = if enc_size == 8 { + u64::MAX + } else { + (1u64 << (enc_size * 8)) - 1 + }; + + if flags & REGULAR != 0 { + let (mut start, mut stride, mut count, mut block) = ( + Vec::with_capacity(rank), + Vec::with_capacity(rank), + Vec::with_capacity(rank), + Vec::with_capacity(rank), + ); + for _ in 0..rank { + start.push(r.uint(enc_size)?); + stride.push(r.uint(enc_size)?); + let c = r.uint(enc_size)?; + count.push(if c == unlim_raw { UNLIMITED } else { c }); + let b = r.uint(enc_size)?; + block.push(if b == unlim_raw { UNLIMITED } else { b }); + } + let unlimited = count + .iter() + .zip(&block) + .filter(|&(&c, &b)| c == UNLIMITED || b == UNLIMITED) + .count(); + if unlimited > 1 { + return Err(sel_err( + "hyperslab selection is unlimited in more than one dimension", + )); + } + for d in 0..rank { + // Overlapping blocks are not a valid regular hyperslab. + if count[d] > 1 && block[d] != UNLIMITED && block[d] > stride[d] { + return Err(sel_err("regular hyperslab blocks overlap")); + } + } + return Ok(SerializedSelection::Regular { start, stride, count, block, - }, - pos, - )) + }); + } + + // Irregular: number of blocks, then each block's start and end corners. + let nblocks = r.uint(enc_size)?; + let per_block = (rank * 2 * enc_size) as u64; + // Untrusted count: it must fit in what is left of the buffer. + if nblocks + .checked_mul(per_block) + .is_none_or(|need| need > r.remaining() as u64) + { + return Err(FormatError::UnexpectedEof { + expected: r + .pos + .saturating_add(nblocks.saturating_mul(per_block) as usize), + available: r.data.len(), + }); + } + let n = nblocks as usize * rank; + let (mut starts, mut ends) = (Vec::with_capacity(n), Vec::with_capacity(n)); + for _ in 0..nblocks { + for _ in 0..rank { + starts.push(r.uint(enc_size)?); + } + for _ in 0..rank { + ends.push(r.uint(enc_size)?); + } + } + if starts.iter().zip(&ends).any(|(s, e)| e < s) { + return Err(sel_err("hyperslab block ends before it starts")); + } + Ok(SerializedSelection::Blocks { rank, starts, ends }) +} + +/// The coordinates of the union of the given blocks, in row-major order. +fn blocks_union_coords( + rank: usize, + starts: &[u64], + ends: &[u64], +) -> Result>, FormatError> { + let mut total = 0u64; + for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { + let vol = s + .iter() + .zip(e) + .try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1)); + total = vol + .and_then(|v| total.checked_add(v)) + .filter(|&t| t <= MAX_EXPANDED_POINTS) + .ok_or_else(|| sel_err("irregular hyperslab selection is too large to expand"))?; + } + let mut out = Vec::with_capacity(total as usize); + for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { + let mut cur = s.to_vec(); + 'block: loop { + out.push(cur.clone()); + for d in (0..rank).rev() { + if cur[d] < e[d] { + cur[d] += 1; + continue 'block; + } + cur[d] = s[d]; + } + break; + } + } + // Lexicographic order of coordinates is row-major order. + out.sort_unstable(); + out.dedup(); + Ok(out) } // --------------------------------------------------------------------------- @@ -642,11 +845,100 @@ mod tests { } #[test] - fn decode_irregular_hyperslab_rejected() { + fn decode_truncated_irregular_hyperslab_is_error() { + // Irregular, rank 1, but the block count is missing. let bytes = [0x02u8, 0, 0, 0, 0x03, 0, 0, 0, 0x00, 0x02, 0x01, 0, 0, 0]; assert!(Selection::decode_serialized(&bytes).is_err()); } + /// Version 1 as libhdf5 writes it for the default (earliest) format bounds: + /// type, version, reserved(4), length(4), rank(4), nblocks(4), then each + /// block's start and inclusive end corner as 4-byte values. + fn v1_blocks(rank: u32, blocks: &[(&[u32], &[u32])]) -> Vec { + let mut b = Vec::new(); + for w in [2u32, 1, 0, 0, rank, blocks.len() as u32] { + b.extend_from_slice(&w.to_le_bytes()); + } + for (s, e) in blocks { + for v in s.iter().chain(e.iter()) { + b.extend_from_slice(&v.to_le_bytes()); + } + } + b + } + + #[test] + fn decode_v1_irregular_single_block() { + // Exactly what h5py/HDF5 2.0 writes for `[0:4]` with default libver. + let bytes = v1_blocks(1, &[(&[0], &[3])]); + let (sel, used) = Selection::decode_serialized(&bytes).unwrap(); + assert_eq!(used, bytes.len()); + assert_eq!(sel.iter_linear_1d(8).unwrap(), vec![0, 1, 2, 3]); + } + + #[test] + fn decode_v1_irregular_union_is_row_major() { + // Blocks given out of order and overlapping still enumerate once each, + // in row-major order (libhdf5 iterates the union, not the list). + let bytes = v1_blocks(2, &[(&[1, 0], &[1, 1]), (&[0, 2], &[1, 2])]); + let (sel, used) = Selection::decode_serialized(&bytes).unwrap(); + assert_eq!(used, bytes.len()); + // (0,2) (1,0) (1,1) (1,2) in a 2x3 space. + assert_eq!(sel.iter_linear(&[2, 3]).unwrap(), vec![2, 3, 4, 5]); + } + + #[test] + fn decode_v2_regular_with_unlimited_count() { + // v2: flags(1) + length(4), then 8-byte start/stride/count/block. + let mut b = Vec::new(); + b.extend_from_slice(&2u32.to_le_bytes()); + b.extend_from_slice(&2u32.to_le_bytes()); + b.push(0x01); + b.extend_from_slice(&36u32.to_le_bytes()); + b.extend_from_slice(&1u32.to_le_bytes()); + for v in [0u64, 10, u64::MAX, 10] { + b.extend_from_slice(&v.to_le_bytes()); + } + let (raw, used) = SerializedSelection::decode(&b).unwrap(); + assert_eq!(used, b.len()); + assert_eq!(raw.unlimited_dim(), Some(0)); + assert_eq!( + raw, + SerializedSelection::Regular { + start: vec![0], + stride: vec![10], + count: vec![UNLIMITED], + block: vec![10], + } + ); + // An unclipped unlimited selection cannot be enumerated. + let (sel, _) = Selection::decode_serialized(&b).unwrap(); + assert!(sel.iter_linear_1d(100).is_err()); + } + + #[test] + fn decode_v3_two_byte_all_ones_is_unlimited() { + let bytes = [ + 0x02, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0x02, 0x01, 0, 0, 0, // + 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0xFF, 0xFF, + ]; + let (raw, _) = SerializedSelection::decode(&bytes).unwrap(); + assert_eq!(raw.unlimited_dim(), Some(0)); + } + + #[test] + fn decode_irregular_block_count_beyond_buffer_is_error() { + let mut b = v1_blocks(1, &[(&[0], &[3])]); + b[20..24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(Selection::decode_serialized(&b).is_err()); + } + + #[test] + fn decode_point_selection_is_refused() { + let bytes = [1u8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + assert!(Selection::decode_serialized(&bytes).is_err()); + } + #[test] fn iter_linear_2d_block_row_major() { // A 2x2 block at the top-left of a 4x4 space => linear 0,1,4,5. diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs new file mode 100644 index 0000000..43eaee4 --- /dev/null +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -0,0 +1,156 @@ +//! Virtual Dataset (VDS) reads checked against libhdf5 (through h5py). +//! +//! Each test has h5py build virtual datasets and their source files in a temp +//! directory, record what libhdf5 reads back (shape and values) next to them, +//! and then compares that with what clawhdf5 reads from the same files. +//! +//! Skipped when python3 or h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::path::Path; +use std::process::Command; + +use clawhdf5::File; + +/// The Python interpreter to drive interop checks with (`CLAWHDF5_PYTHON` +/// lets these run against a virtualenv holding h5py). +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency +/// is a test failure instead of a silent skip. +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Prelude for every generator script: `expect(file, dset, tag)` records what +/// libhdf5 reads for `file:dset` as `.expect` (shape line, values line). +const PRELUDE: &str = r#" +import h5py, numpy as np +def expect(fn, dset, tag): + with h5py.File(fn, "r") as f: + d = f[dset] + a = d[...] + with open(tag + ".expect", "w") as out: + out.write(" ".join(str(n) for n in d.shape) + "\n") + out.write(" ".join(repr(float(v)) for v in a.ravel()) + "\n") +"#; + +/// Run `body` (after [`PRELUDE`]) with `dir` as the working directory, so +/// relative source file names land next to the virtual file. +fn generate(dir: &Path, body: &str) { + let script = format!("{PRELUDE}\n{body}"); + let out = Command::new(python()) + .args(["-c", &script]) + .current_dir(dir) + .output() + .expect("failed to run python"); + assert!( + out.status.success(), + "generator failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); +} + +/// What libhdf5 read for `tag`: (shape, values as f64). +fn expected(dir: &Path, tag: &str) -> (Vec, Vec) { + let text = std::fs::read_to_string(dir.join(format!("{tag}.expect"))).unwrap(); + let mut lines = text.lines(); + let parse_line = |l: Option<&str>| -> Vec { + l.unwrap_or("") + .split_whitespace() + .map(str::to_string) + .collect() + }; + let shape = parse_line(lines.next()) + .iter() + .map(|s| s.parse().unwrap()) + .collect(); + let values = parse_line(lines.next()) + .iter() + .map(|s| s.parse().unwrap()) + .collect(); + (shape, values) +} + +/// Assert clawhdf5 reads `file:dset` exactly as libhdf5 did for `tag`. +fn assert_matches_libhdf5(dir: &Path, file: &str, dset: &str, tag: &str) { + let (shape, values) = expected(dir, tag); + let f = File::open(dir.join(file)).unwrap(); + let ds = f.dataset(dset).unwrap(); + assert_eq!( + ds.shape().unwrap(), + shape, + "{tag}: shape differs from libhdf5" + ); + let got = ds + .read_f64() + .unwrap_or_else(|e| panic!("{tag}: read failed: {e}")); + assert_eq!(got.len(), values.len(), "{tag}: element count differs"); + for (i, (g, e)) in got.iter().zip(&values).enumerate() { + assert!( + g == e || (g.is_nan() && e.is_nan()), + "{tag}: element {i} is {g}, libhdf5 reads {e}\n ours: {got:?}\n libhdf5: {values:?}" + ); + } +} + +// --------------------------------------------------------------------------- +// Selection encodings +// --------------------------------------------------------------------------- + +/// Files written with the default (earliest) format bounds serialize every +/// VDS hyperslab as a version-1 *irregular* selection (4-byte block corners), +/// and a strided selection as many blocks. These were refused outright. +#[test] +fn vds_version1_irregular_hyperslab_selections() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + generate( + dir.path(), + r#" +with h5py.File("src.h5", "w") as s: + s.create_dataset("a", data=np.arange(12.0)) + s.create_dataset("m", data=np.arange(20.0).reshape(4, 5)) +with h5py.File("v1.h5", "w") as f: # default libver: hyperslab version 1 + f.create_dataset("local", data=np.arange(10.0) * -1) + lay = h5py.VirtualLayout(shape=(12,), dtype="f8") + lay[0:4] = h5py.VirtualSource(".", "local", shape=(10,))[2:6] + lay[4:10] = h5py.VirtualSource("src.h5", "a", shape=(12,))[::2] + lay[10:12] = h5py.VirtualSource("src.h5", "a", shape=(12,))[10:12] + f.create_virtual_dataset("strided", lay) + lay = h5py.VirtualLayout(shape=(4, 6), dtype="f8") + lay[:, 0:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 3:5] + lay[:, 2:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 0:2] + lay[:, 3:6:2] = h5py.VirtualSource("src.h5", "m", shape=(4, 5))[:, 2:4] + f.create_virtual_dataset("grid", lay) +expect("v1.h5", "strided", "strided") +expect("v1.h5", "grid", "grid") +"#, + ); + assert_matches_libhdf5(dir.path(), "v1.h5", "strided", "strided"); + assert_matches_libhdf5(dir.path(), "v1.h5", "grid", "grid"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 6dfa369..c15e824 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -71,7 +71,8 @@ the VDS item, which is marked. - **Virtual datasets:** - **Wrong data:** unmapped regions read as 0 instead of the fill value. - `%b` printf-style source names are not expanded. - - Hyperslab selection versions 1 and 2 are refused. + - ~~Hyperslab selection versions 1 and 2 are refused.~~ Fixed 2026-09-25: + versions 1-3 and irregular hyperslabs are decoded. - **Files with a user block:** the base address is not applied. - **Old-style shared messages (version 1)** read the wrong address. - **Groups and links:** From 2c6c6c176e0e504ed7a91a11a98ee704ad4bb89f Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:59:15 -0500 Subject: [PATCH 09/24] fix(format): decode the version-1 VDS mapping list HDF5 2.0 writes With a 2.0 low version bound, libhdf5 stores the VDS mapping list as heap block version 1: every entry starts with a flags byte (0x04 same file, no file name; 0x01/0x02 file/dataset name shared with an earlier entry, whose index is stored in place of the name). The parser treated only a leading 0x04 byte as special, so a 0x00 flags byte read as an empty (same-file) name and shared names were read as garbage. Decode it as H5D__virtual_load_layout does, refusing unknown flags, forward references and block versions above 1. Test: vds_interop::vds_mapping_block_version1_shared_names (h5py libver=("v200","v200") with repeated long names; failed before with "unknown dataspace selection type") plus the exact heap block as a unit test. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 + crates/clawhdf5-format/src/data_layout.rs | 132 ++++++++++++++++++++-- crates/clawhdf5/tests/vds_interop.rs | 36 ++++++ docs/known-issues.md | 2 + 4 files changed, 162 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a349fba..fd8624f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -282,6 +282,10 @@ selection. Both are decoded now, as are irregular hyperslabs (a union of blocks, read in row-major order as libhdf5 iterates them). `SerializedSelection` exposes the raw form, including unlimited counts. + - The version-1 mapping list HDF5 2.0 writes (low version bound 2.0) was + misparsed: each entry's flags byte was read as the start of the source + file name, and names shared with an earlier entry (stored as that entry's + index) were not followed. Now decoded as `H5D__virtual_load_layout` does. - `clawhdf5-format` reader — **values returned wrong with no error:** - Fixed Array and Extensible Array chunk indexes were laid out by the dataset's current shape instead of its max shape (23 libhdf5 test files, diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 8429c5d..b58843c 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -72,21 +72,33 @@ pub enum DataLayout { }, } +/// Version-1 VDS mapping flag: the source file name is stored by an earlier +/// entry, whose index follows in place of the name. +const VDS_SOURCE_FILE_SHARED: u8 = 0x01; +/// Version-1 VDS mapping flag: likewise for the source dataset name. +const VDS_SOURCE_DSET_SHARED: u8 = 0x02; +/// Version-1 VDS mapping flag: the source is in the virtual file itself +/// (`"."`); no file name is stored. +const VDS_SOURCE_SAME_FILE: u8 = 0x04; +const VDS_ALL_FLAGS: u8 = VDS_SOURCE_FILE_SHARED | VDS_SOURCE_DSET_SHARED | VDS_SOURCE_SAME_FILE; + /// Parse VDS mappings from global-heap object data. /// /// The global-heap block holding a VDS mapping list is laid out as -/// (reverse-engineered and validated against HDF5 2.0): +/// (`H5D__virtual_store_layout` / `H5D__virtual_load_layout` in libhdf5): /// /// ```text /// version(1) · nused(length_size, LE) · entry[nused] · checksum(4) /// ``` /// /// Each entry is: -/// - source file name — a null-terminated string in **block version 0**; in -/// **block version 1** a same-file reference is encoded as a single `0x04` -/// marker byte (the source file is the virtual file itself) in place of the -/// name; -/// - source dataset name (null-terminated string); +/// - **block version 1 only:** a flags byte. `0x04`: the source is in the +/// virtual file itself and no file name is stored; `0x01`/`0x02`: the +/// source file/dataset name is that of an earlier entry, whose index +/// (`length_size` bytes) is stored instead of the name. libhdf5 2.0 writes +/// version 1 when the file's low version bound is 2.0 and it saves space; +/// - source file name (null-terminated string, unless flagged above); +/// - source dataset name (null-terminated string, unless flagged above); /// - source selection (serialized `H5S` dataspace selection — self-describing /// in length); /// - virtual selection (serialized `H5S` dataspace selection). @@ -112,7 +124,7 @@ pub fn parse_vds_mappings( // `nused` is untrusted; don't pre-allocate from it. Each entry consumes at // least a few bytes, so the loop is naturally bounded by the heap data and // a bogus `nused` simply errors out on the first short read. - let mut mappings = Vec::new(); + let mut mappings: Vec = Vec::new(); // Reads one self-describing selection at `pos`, returning its raw bytes and // advancing past it — bounds-checked so a corrupt selection can't overrun. let read_selection = |heap_data: &[u8], pos: &mut usize| -> Result, FormatError> { @@ -132,17 +144,57 @@ pub fn parse_vds_mappings( Ok(bytes) }; - for _ in 0..nused { - // Source file name (with the version-1 same-file marker handled). - let source_file = if version >= 1 && heap_data.get(pos) == Some(&0x04) { + if version > 1 { + return Err(FormatError::ChunkedReadError( + "unsupported VDS mapping block version".into(), + )); + } + for i in 0..nused { + // Version 1 prefixes each entry with a flags byte; a name may then be + // omitted (same file) or replaced by the index of an earlier entry + // holding the same name (`H5D__virtual_load_layout`). + let flags = if version >= 1 { + let f = *heap_data.get(pos).ok_or(FormatError::UnexpectedEof { + expected: pos + 1, + available: heap_data.len(), + })?; pos += 1; + if f & !VDS_ALL_FLAGS != 0 { + return Err(FormatError::ChunkedReadError( + "unknown VDS mapping flags".into(), + )); + } + f + } else { + 0 + }; + // Index of an earlier entry, for a shared name. + let earlier = |pos: &mut usize| -> Result { + let idx = read_length(heap_data, *pos, length_size)?; + *pos += ls; + if idx >= i { + return Err(FormatError::ChunkedReadError( + "VDS mapping shares a name with a later entry".into(), + )); + } + Ok(idx as usize) + }; + + let source_file = if flags & VDS_SOURCE_SAME_FILE != 0 { String::from(".") + } else if flags & VDS_SOURCE_FILE_SHARED != 0 { + let idx = earlier(&mut pos)?; + mappings[idx].source_file.clone() } else { read_null_terminated_string(heap_data, &mut pos)? }; - // Source dataset name. - let source_dataset = read_null_terminated_string(heap_data, &mut pos)?; + let source_dataset = if flags & VDS_SOURCE_DSET_SHARED != 0 { + let idx = earlier(&mut pos)?; + mappings[idx].source_dataset.clone() + } else { + read_null_terminated_string(heap_data, &mut pos)? + }; // Source selection, then virtual selection (both self-describing length). let source_selection = read_selection(heap_data, &mut pos)?; @@ -849,6 +901,62 @@ mod tests { assert_eq!(v1.iter_linear_1d(8).unwrap(), vec![4, 5, 6, 7]); } + #[test] + fn parse_vds_mappings_v1_shared_names() { + // Written by HDF5 2.0 (h5py, libver=("v200", "v200")) for three + // mappings from `a_rather_long_source_file.h5:a_rather_long_dataset_name` + // and one from the same file: the entries carry flags 0x00, 0x03, 0x03 + // and 0x06, so names after the first are stored as entry indices. + let blob: &[u8] = &[ + 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x5f, 0x72, 0x61, + 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x68, 0x35, 0x00, 0x61, 0x5f, 0x72, + 0x61, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x64, 0x61, 0x74, + 0x61, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x01, 0x00, 0x04, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, + 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x01, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x02, 0x00, + 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x00, 0x01, 0x02, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x04, 0x00, 0x8e, 0xa7, 0xea, 0x7a, + ]; + let mappings = parse_vds_mappings(blob, 8).unwrap(); + let names: Vec<(&str, &str)> = mappings + .iter() + .map(|m| (m.source_file.as_str(), m.source_dataset.as_str())) + .collect(); + let (file, dset) = ("a_rather_long_source_file.h5", "a_rather_long_dataset_name"); + assert_eq!( + names, + vec![(file, dset), (file, dset), (file, dset), (".", dset)] + ); + } + + #[test] + fn parse_vds_mappings_v1_forward_reference_is_error() { + // Entry 0 claiming to share entry 0's file name must not index past + // the entries decoded so far. + let mut blob = vec![0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x01]; + blob.extend_from_slice(&[0u8; 8]); + blob.extend_from_slice(b"d\0"); + assert!(parse_vds_mappings(&blob, 8).is_err()); + // Unknown flag bits are refused. + let blob = [0x01u8, 1, 0, 0, 0, 0, 0, 0, 0, 0x08, b'd', 0]; + assert!(parse_vds_mappings(&blob, 8).is_err()); + } + #[test] fn parse_vds_mappings_external_v0() { // Block version 0 with an explicit (external) source file name. diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs index 43eaee4..f3ce7e3 100644 --- a/crates/clawhdf5/tests/vds_interop.rs +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -154,3 +154,39 @@ expect("v1.h5", "grid", "grid") assert_matches_libhdf5(dir.path(), "v1.h5", "strided", "strided"); assert_matches_libhdf5(dir.path(), "v1.h5", "grid", "grid"); } + +// --------------------------------------------------------------------------- +// Mapping list encoding +// --------------------------------------------------------------------------- + +/// With a 2.0 low version bound libhdf5 writes the mapping list as block +/// version 1: a flags byte per entry, and repeated names stored as the index +/// of the entry that first spelled them out. The flags byte was mistaken for +/// an empty (same-file) name. +#[test] +fn vds_mapping_block_version1_shared_names() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + generate( + dir.path(), + r#" +name = "a_rather_long_dataset_name" +with h5py.File("a_rather_long_source_file.h5", "w") as s: + s.create_dataset(name, data=np.arange(12.0) + 100) +with h5py.File("shared.h5", "w", libver=("v200", "v200")) as f: + f.create_dataset(name, data=np.arange(12.0) * -1) + lay = h5py.VirtualLayout(shape=(4, 4), dtype="f8") + for i in range(3): + src = h5py.VirtualSource("a_rather_long_source_file.h5", name, shape=(12,)) + lay[i] = src[4 * i:4 * i + 4] + lay[3] = h5py.VirtualSource(".", name, shape=(12,))[0:4] + f.create_virtual_dataset("v", lay) +# the heap block must really be version 1 for this test to mean anything +raw = open("shared.h5", "rb").read() +gcol = raw.index(b"GCOL") +assert raw[gcol + 32] == 1, "expected a version-1 VDS mapping block" +expect("shared.h5", "v", "shared") +"#, + ); + assert_matches_libhdf5(dir.path(), "shared.h5", "v", "shared"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index c15e824..eb7d80f 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -73,6 +73,8 @@ the VDS item, which is marked. - `%b` printf-style source names are not expanded. - ~~Hyperslab selection versions 1 and 2 are refused.~~ Fixed 2026-09-25: versions 1-3 and irregular hyperslabs are decoded. + - ~~The version-1 mapping list written with a 2.0 low bound (flags byte, + shared names) was misparsed.~~ Found and fixed 2026-09-25. - **Files with a user block:** the base address is not applied. - **Old-style shared messages (version 1)** read the wrong address. - **Groups and links:** From aadfd18d4cc7e4b9ddec4fefb2b02cac8937bfdb Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:00:46 -0500 Subject: [PATCH 10/24] fix(format): list soft links as their targets, like h5py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group::datasets()/groups() (and the Mmap/Lazy handles) listed only hard links, so a soft link to a dataset or group was missing, and dataset(name) / group(name) on a group handle could not open one. In old-style (symbol table) groups a soft link's entry has no object header address, and the listing failed outright trying to parse one. The three facade handles each had their own copy of the child-listing code; they now share group_v2::resolve_group_children, which returns hard links plus soft links resolved to their targets (relative targets from the group holding the link, via the new resolve_path_from). A dangling or cyclic soft link, an external link and a user-defined link are left out — h5py lists their names but cannot open them. Any other error met while resolving is returned, not hidden. Path resolution now walks a relative soft link's target from the group holding it instead of rebuilding the path from the root (same result, one less re-walk), and ignores "." components. Regression test: soft_links_are_listed_as_their_targets (h5py writes absolute, relative, group, dangling, cyclic and external links with libver latest and earliest; listings compared with h5py for File, MmapFile and LazyFile). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 ++ crates/clawhdf5-format/src/group_v1.rs | 65 ++++++++- crates/clawhdf5-format/src/group_v2.rs | 136 +++++++++++++++--- crates/clawhdf5/src/lazy.rs | 43 +----- crates/clawhdf5/src/mmap_file.rs | 43 +----- crates/clawhdf5/src/reader.rs | 44 +----- .../clawhdf5/tests/dense_storage_interop.rs | 66 +++++++++ docs/known-issues.md | 3 +- 8 files changed, 277 insertions(+), 133 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95da497..b3ba430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -327,6 +327,16 @@ cannot be followed without the application that registered the class, so they are now left out of `datasets()`/`groups()` and path lookup, as h5py leaves out links it cannot open; reserved link types are still an error. +- `clawhdf5` — soft links are listed, as h5py lists them: `datasets()` and + `groups()` on `Group`/`MmapGroup`/`LazyGroup` include each soft link under + its own name as the kind of object it resolves to, and `dataset(name)` / + `group(name)` open through it. Relative targets resolve from the group + holding the link. Dangling or cyclic soft links, external links and + user-defined links are left out (h5py lists their names but cannot open + them). Previously soft links were missing from the listings, and in + old-style (symbol table) groups a soft link made the listing fail. New + `group_v2::resolve_group_children` / `resolve_path_from` and + `group_v1::v1_soft_links` in `clawhdf5-format`. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index 989f826..edb84f6 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -73,6 +73,53 @@ pub fn find_v1_soft_link( offset_size: u8, length_size: u8, ) -> Result, FormatError> { + let mut found = None; + for_each_v1_soft_link( + file_data, + sym_table_msg, + offset_size, + length_size, + |link_name| link_name == name, + |_, target| { + found = Some(target); + false + }, + )?; + Ok(found) +} + +/// Every soft link in a v1 group, as `(name, target path)`. +pub fn v1_soft_links( + file_data: &[u8], + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, +) -> Result, FormatError> { + let mut links = Vec::new(); + for_each_v1_soft_link( + file_data, + sym_table_msg, + offset_size, + length_size, + |_| true, + |name, target| { + links.push((String::from(name), target)); + true + }, + )?; + Ok(links) +} + +/// Visit the soft links of a v1 group whose name passes `wanted`, with their +/// target paths, until `visit` returns false. +fn for_each_v1_soft_link( + file_data: &[u8], + sym_table_msg: &SymbolTableMessage, + offset_size: u8, + length_size: u8, + wanted: impl Fn(&str) -> bool, + mut visit: impl FnMut(&str, String) -> bool, +) -> Result<(), FormatError> { let heap = LocalHeap::parse( file_data, sym_table_msg.local_heap_address as usize, @@ -91,7 +138,8 @@ pub fn find_v1_soft_link( if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; } - if heap.read_string(file_data, entry.link_name_offset)? != name { + let name = heap.read_string(file_data, entry.link_name_offset)?; + if !wanted(&name) { continue; } let value_offset = u32::from_le_bytes([ @@ -100,12 +148,19 @@ pub fn find_v1_soft_link( entry.scratch_pad[2], entry.scratch_pad[3], ]); - return heap - .read_string(file_data, u64::from(value_offset)) - .map(Some); + let target = heap.read_string(file_data, u64::from(value_offset))?; + if !visit(&name, target) { + return Ok(()); + } } } - Ok(None) + Ok(()) +} + +/// Whether a v1 symbol-table entry is a soft link (no object header of its +/// own; its target path is in the local heap). +pub fn is_v1_soft_link(entry: &GroupEntry) -> bool { + entry.cache_type == CACHE_TYPE_SOFT_LINK } /// Extract the SymbolTableMessage from an object header's messages. diff --git a/crates/clawhdf5-format/src/group_v2.rs b/crates/clawhdf5-format/src/group_v2.rs index bcfa1be..57119c9 100644 --- a/crates/clawhdf5-format/src/group_v2.rs +++ b/crates/clawhdf5-format/src/group_v2.rs @@ -255,32 +255,135 @@ pub fn resolve_path_any( superblock: &Superblock, path: &str, ) -> Result { - resolve_path_following_links(file_data, superblock, path, 0) + resolve_path_following_links( + file_data, + superblock, + superblock.root_group_address, + path, + 0, + ) +} + +/// Resolve `path` relative to the group at `group_address` (an absolute path +/// starts at the root group instead), following soft links. This is how a +/// relative soft link's target is resolved: from the group holding the link. +pub fn resolve_path_from( + file_data: &[u8], + superblock: &Superblock, + group_address: u64, + path: &str, +) -> Result { + let start = if path.starts_with('/') { + superblock.root_group_address + } else { + group_address + }; + resolve_path_following_links(file_data, superblock, start, path, 0) +} + +/// The children of the group at `group_address` that can be opened, as h5py +/// lists them: hard links, and soft links resolved to the object they point +/// at (under the soft link's own name). Links that cannot be followed are +/// left out rather than failing the listing — a dangling or cyclic soft link +/// (h5py lists its name but cannot open it), an external link (another +/// file), and a user-defined link. An object header that is not a group has +/// no children. +/// +/// Any other error, such as a corrupt structure met while resolving a soft +/// link, is returned. +pub fn resolve_group_children( + file_data: &[u8], + superblock: &Superblock, + group_address: u64, +) -> Result, FormatError> { + let os = superblock.offset_size; + let ls = superblock.length_size; + let header = ObjectHeader::parse(file_data, group_address as usize, os, ls)?; + + let mut entries = Vec::new(); + let mut soft = Vec::new(); + if is_v1_group(&header) { + let sym_msg = header + .messages + .iter() + .find(|m| m.msg_type == MessageType::SymbolTable) + .ok_or_else(|| FormatError::PathNotFound(String::from("no symbol table message")))?; + let stm = SymbolTableMessage::parse(&sym_msg.data, os)?; + let all = group_v1::resolve_v1_group_entries(file_data, &stm, os, ls)?; + if all.iter().any(group_v1::is_v1_soft_link) { + soft = group_v1::v1_soft_links(file_data, &stm, os, ls)?; + } + entries.extend(all.into_iter().filter(|e| !group_v1::is_v1_soft_link(e))); + } else if is_v2_group(&header) { + let mut visit = |link: LinkMessage| match link.link_target { + LinkTarget::Hard { + object_header_address, + } => entries.push(GroupEntry { + name: link.name, + object_header_address, + cache_type: 0, + }), + LinkTarget::Soft { target_path } => soft.push((link.name, target_path)), + LinkTarget::External { .. } => {} + }; + let link_info = find_link_info(&header, os)?; + if let Some(fh_addr) = link_info.fractal_heap_address { + for_each_dense_link(file_data, &link_info, fh_addr, os, ls, visit)?; + } else { + for msg in &header.messages { + if msg.msg_type == MessageType::Link + && let Some(link) = parse_link(&msg.data, os)? + { + visit(link); + } + } + } + } + + for (name, target) in soft { + match resolve_path_from(file_data, superblock, group_address, &target) { + Ok(object_header_address) => entries.push(GroupEntry { + name, + object_header_address, + cache_type: 0, + }), + // Dangling, cyclic, or ending in another file: not openable here. + Err( + FormatError::PathNotFound(_) + | FormatError::NestingDepthExceeded + | FormatError::ExternalLinkUnsupported { .. }, + ) => {} + Err(e) => return Err(e), + } + } + Ok(entries) } /// Soft links followed while resolving one path. Guards against link cycles /// (`a -> b -> a`), which are legal to create. const MAX_SOFT_LINK_DEPTH: u8 = 16; +/// Walk `path` from the group at `start`, following soft links. fn resolve_path_following_links( file_data: &[u8], superblock: &Superblock, + start: u64, path: &str, depth: u8, ) -> Result { - let components: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect(); + let components: Vec<&str> = path + .split('/') + .filter(|s| !s.is_empty() && *s != ".") + .collect(); if components.is_empty() { - return Ok(superblock.root_group_address); + return Ok(start); } let os = superblock.offset_size; let ls = superblock.length_size; - let root_header = - ObjectHeader::parse(file_data, superblock.root_group_address as usize, os, ls)?; - - let mut current_addr = superblock.root_group_address; - let mut current_header = root_header; + let mut current_addr = start; + let mut current_header = ObjectHeader::parse(file_data, start as usize, os, ls)?; for (i, component) in components.iter().enumerate() { let entries = resolve_group_entries(file_data, ¤t_header, os, ls)?; @@ -304,20 +407,17 @@ fn resolve_path_following_links( } // A relative target is relative to the group holding // the link; then the rest of the original path. - let mut full = String::new(); - if !target_path.starts_with('/') { - for parent in &components[..i] { - full.push('/'); - full.push_str(parent); - } - } - full.push('/'); - full.push_str(&target_path); + let from = if target_path.starts_with('/') { + superblock.root_group_address + } else { + current_addr + }; + let mut full = target_path; for rest in &components[i + 1..] { full.push('/'); full.push_str(rest); } - resolve_path_following_links(file_data, superblock, &full, depth + 1) + resolve_path_following_links(file_data, superblock, from, &full, depth + 1) } Some(LinkTarget::External { filename, diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 4a7421e..314234c 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -19,13 +19,12 @@ use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v1::GroupEntry; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::symbol_table::SymbolTableMessage; use clawhdf5_io::HDF5Read; @@ -275,12 +274,14 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { }) } + /// This group's links that can be opened: hard links, and soft links + /// resolved to their targets (see + /// [`group_v2::resolve_group_children`]); dangling, external and + /// user-defined links are left out. fn children(&self) -> Result, Error> { - let hdr = self.file.get_or_parse_header(self.address)?; let data = self.file.reader.as_bytes(); - let os = self.file.offset_size(); - let ls = self.file.length_size(); - resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) + group_v2::resolve_group_children(data, &self.file.superblock, self.address) + .map_err(Error::Format) } } @@ -530,33 +531,3 @@ fn is_group(header: &ObjectHeader) -> bool { || m.msg_type == MessageType::SymbolTable }) } - -fn resolve_group_entries( - file_data: &[u8], - object_header: &ObjectHeader, - offset_size: u8, - length_size: u8, -) -> Result, FormatError> { - let is_v1 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::SymbolTable); - let is_v2 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link); - - if is_v1 { - let sym_msg = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) - } else if is_v2 { - group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size) - } else { - Ok(Vec::new()) - } -} diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 9119cdf..bf098cf 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -14,13 +14,12 @@ use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::Datatype; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v1::GroupEntry; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::symbol_table::SymbolTableMessage; use clawhdf5_io::MmapReader; @@ -197,12 +196,14 @@ impl<'f> MmapGroup<'f> { }) } + /// This group's links that can be opened: hard links, and soft links + /// resolved to their targets (see + /// [`group_v2::resolve_group_children`]); dangling, external and + /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.reader.as_bytes(); - let hdr = self.file.parse_header(self.address)?; - let os = self.file.offset_size(); - let ls = self.file.length_size(); - resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) + group_v2::resolve_group_children(data, &self.file.superblock, self.address) + .map_err(Error::Format) } } @@ -470,33 +471,3 @@ fn is_group(header: &ObjectHeader) -> bool { || m.msg_type == MessageType::SymbolTable }) } - -fn resolve_group_entries( - file_data: &[u8], - object_header: &ObjectHeader, - offset_size: u8, - length_size: u8, -) -> Result, FormatError> { - let is_v1 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::SymbolTable); - let is_v2 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link); - - if is_v1 { - let sym_msg = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) - } else if is_v2 { - group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size) - } else { - Ok(Vec::new()) - } -} diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index bb6a8fd..9c9a805 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -15,13 +15,12 @@ use clawhdf5_format::dataspace::Dataspace; use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder}; use clawhdf5_format::error::FormatError; use clawhdf5_format::filter_pipeline::FilterPipeline; -use clawhdf5_format::group_v1::{self, GroupEntry}; +use clawhdf5_format::group_v1::GroupEntry; use clawhdf5_format::group_v2; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; -use clawhdf5_format::symbol_table::SymbolTableMessage; use crate::error::Error; use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; @@ -337,12 +336,14 @@ impl<'f> Group<'f> { }) } + /// This group's links that can be opened: hard links, and soft links + /// resolved to their targets (see + /// [`group_v2::resolve_group_children`]); dangling, external and + /// user-defined links are left out. fn children(&self) -> Result, Error> { let data = self.file.data.as_bytes(); - let hdr = self.file.parse_header(self.address)?; - let os = self.file.offset_size(); - let ls = self.file.length_size(); - resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) + group_v2::resolve_group_children(data, &self.file.superblock, self.address) + .map_err(Error::Format) } } @@ -959,37 +960,6 @@ fn is_group(header: &ObjectHeader) -> bool { }) } -fn resolve_group_entries( - file_data: &[u8], - object_header: &ObjectHeader, - offset_size: u8, - length_size: u8, -) -> Result, FormatError> { - let is_v1 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::SymbolTable); - let is_v2 = object_header - .messages - .iter() - .any(|m| m.msg_type == MessageType::LinkInfo || m.msg_type == MessageType::Link); - - if is_v1 { - let sym_msg = object_header - .messages - .iter() - .find(|m| m.msg_type == MessageType::SymbolTable) - .ok_or_else(|| FormatError::PathNotFound("no symbol table message".into()))?; - let stm = SymbolTableMessage::parse(&sym_msg.data, offset_size)?; - group_v1::resolve_v1_group_entries(file_data, &stm, offset_size, length_size) - } else if is_v2 { - group_v2::resolve_v2_group_entries(file_data, object_header, offset_size, length_size) - } else { - // Empty group or unrecognized — return empty - Ok(Vec::new()) - } -} - #[cfg(test)] mod sibling_file_name_tests { use super::sibling_file_name; diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index 92af6ee..cbb3b63 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -298,3 +298,69 @@ fn user_defined_links_do_not_break_the_listing() { assert!(f.root().datasets().unwrap().is_empty()); assert!(f.root().groups().unwrap().is_empty()); } + +/// Soft links (absolute, relative, to a dataset and to a group), a dangling +/// one, a cycle and an external link, in an old-style (symbol table) or +/// new-style (link message) group. +fn soft_link_file(libver: &str) -> (tempfile::TempDir, String) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("soft.h5").display().to_string(); + run_python(&format!( + "import h5py, numpy as np\n\ + with h5py.File(r'{path}', 'w', libver='{libver}') as f:\n\ + \x20 f.create_dataset('a/b/deep', data=np.arange(3.0))\n\ + \x20 f.create_dataset('plain', data=[1.0])\n\ + \x20 f['soft_ds'] = h5py.SoftLink('/a/b/deep')\n\ + \x20 f['soft_grp'] = h5py.SoftLink('/a')\n\ + \x20 f['a/rel'] = h5py.SoftLink('b/deep')\n\ + \x20 f['a/rel_grp'] = h5py.SoftLink('b')\n\ + \x20 f['dangling'] = h5py.SoftLink('/nope')\n\ + \x20 f['loop1'] = h5py.SoftLink('/loop2')\n\ + \x20 f['loop2'] = h5py.SoftLink('/loop1')\n\ + \x20 f['ext'] = h5py.ExternalLink('elsewhere.h5', '/x')\n" + )); + (dir, path) +} + +fn check_soft_links(libver: &str) { + let (_dir, path) = soft_link_file(libver); + assert_same_listing(&path, "/"); + assert_same_listing(&path, "a"); + + let f = File::open(&path).unwrap(); + let root = f.root(); + let deep = vec![0.0, 1.0, 2.0]; + assert_eq!(root.dataset("soft_ds").unwrap().read_f64().unwrap(), deep); + let a = root.group("soft_grp").unwrap(); + assert_eq!(a.dataset("rel").unwrap().read_f64().unwrap(), deep); + assert_eq!(a.group("rel_grp").unwrap().datasets().unwrap(), ["deep"]); + // A dangling link is not listed and cannot be opened. + assert!(root.dataset("dangling").is_err()); + assert!(root.dataset("loop1").is_err()); + + // The memory-mapped and lazy handles list the same way. + let (ds, gs) = h5py_listing(&path, "/"); + let m = clawhdf5::MmapFile::open(&path).unwrap(); + let mut mds = m.root().datasets().unwrap(); + let mut mgs = m.root().groups().unwrap(); + mds.sort(); + mgs.sort(); + assert_eq!((mds, mgs), (ds.clone(), gs.clone())); + let l = clawhdf5::LazyFile::from_bytes(std::fs::read(&path).unwrap()).unwrap(); + let mut lds = l.root().datasets().unwrap(); + let mut lgs = l.root().groups().unwrap(); + lds.sort(); + lgs.sort(); + assert_eq!((lds, lgs), (ds, gs)); +} + +/// Soft links were left out of `datasets()`/`groups()` (and could not be +/// opened by name from a group handle); in old-style groups, where a soft +/// link has no object header address, they made the listing fail. h5py +/// lists a soft link under its own name as whatever it points at. +#[test] +fn soft_links_are_listed_as_their_targets() { + skip_if_no_python!(); + check_soft_links("latest"); + check_soft_links("earliest"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index a437db4..3dba53d 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -82,7 +82,8 @@ the VDS item, which is marked. **Fixed 2026-09-25:** two bugs — fractal-heap child indirect blocks had the wrong row count, and v2 B-tree internal nodes at depth 3+ were read with the wrong pointer widths. - - Soft links are left out of `datasets()`. + - Soft links are left out of `datasets()`. **Fixed 2026-09-25:** soft + links are listed as their targets; dangling ones are left out. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF files (`issue671.nc`). **Fixed 2026-09-25:** huge and tiny heap objects, From d54a0f473716572a0b48b69e052795267d99f243 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:03:09 -0500 Subject: [PATCH 11/24] feat(format): read the other attributes when one cannot be read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit attrs() read every attribute of an object through extract_attributes_full, so one attribute it could not read (a corrupt or unsupported attribute message, or a heap object it could not locate) failed all of them — the same shape as the huge-object bug, where one 8 KiB attribute hid every attribute on a NetCDF file's root group. - clawhdf5-format: new attribute::extract_attributes_tolerant returns the attributes it could read plus one error per attribute it could not. Errors in the attribute index itself (Attribute Info message, dense heap header, B-tree) still fail, since then it is unknown which attributes exist. extract_attributes_full is unchanged (strict); both share one implementation. - clawhdf5: attrs() on Group/Dataset, MmapGroup/MmapDataset and LazyGroup/LazyDataset leaves an unreadable attribute out (documented), and the new attrs_with_errors() returns the map with the per-attribute errors. A value is either returned complete or not at all. Regression test: one_unreadable_attribute_does_not_hide_the_others (h5py writes 11 dense attributes; one message's version byte is corrupted; before: attrs() failed with InvalidAttributeVersion(127), after: the 10 others come back with their values and one error is reported). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 ++ crates/clawhdf5-format/src/attribute.rs | 130 ++++++++++++------ crates/clawhdf5/src/lazy.rs | 54 +++++--- crates/clawhdf5/src/mmap_file.rs | 56 +++++--- crates/clawhdf5/src/reader.rs | 56 +++++--- crates/clawhdf5/src/types.rs | 27 ++++ .../clawhdf5/tests/dense_storage_interop.rs | 52 +++++++ docs/known-issues.md | 4 +- 8 files changed, 290 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3ba430..ce688d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -337,6 +337,14 @@ old-style (symbol table) groups a soft link made the listing fail. New `group_v2::resolve_group_children` / `resolve_path_from` and `group_v1::v1_soft_links` in `clawhdf5-format`. +- `clawhdf5` — one unreadable attribute no longer fails `attrs()` for every + attribute on its object: it is left out of the map, and the new + `attrs_with_errors()` (on every group and dataset handle) returns the map + plus one error per attribute left out. Returned values are always complete. + An error in the attribute index itself (attribute info message, dense heap + header or B-tree) still fails the call. `clawhdf5-format` gains + `attribute::extract_attributes_tolerant`; `extract_attributes_full` stays + strict. - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index bb96bd5..62ee2bf 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -394,42 +394,80 @@ pub fn find_attribute<'a>( /// /// Use this instead of `extract_attributes` when reading files that may use dense storage /// (e.g., objects with many attributes, typically >8). +/// +/// Fails if any attribute cannot be read; see [`extract_attributes_tolerant`] +/// to read the others. pub fn extract_attributes_full( file_data: &[u8], header: &ObjectHeader, offset_size: u8, length_size: u8, +) -> Result, FormatError> { + extract_attributes_with(file_data, header, offset_size, length_size, &mut Err) +} + +/// Like [`extract_attributes_full`], but an attribute that cannot be read +/// (a corrupt or unsupported attribute message, or a heap object that cannot +/// be located) is left out and its error returned alongside the attributes +/// that could be read, instead of failing them all. +/// +/// Errors in the structures that index the attributes (the Attribute Info +/// message, the dense-storage heap header or B-tree) still fail the call: +/// then it is unknown which attributes exist at all. +pub fn extract_attributes_tolerant( + file_data: &[u8], + header: &ObjectHeader, + offset_size: u8, + length_size: u8, +) -> Result<(Vec, Vec), FormatError> { + let mut errors = Vec::new(); + let attrs = extract_attributes_with(file_data, header, offset_size, length_size, &mut |e| { + errors.push(e); + Ok(()) + })?; + Ok((attrs, errors)) +} + +/// Read every attribute; each one that fails goes to `on_error`, which +/// either stops the read (returns the error) or skips that attribute. +fn extract_attributes_with( + file_data: &[u8], + header: &ObjectHeader, + offset_size: u8, + length_size: u8, + on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, ) -> Result, FormatError> { let mut attrs = Vec::new(); // Collect compact attributes (inline in OH) for msg in &header.messages { if msg.msg_type == MessageType::Attribute { - if shared_message::is_shared(msg.flags) { + let attr = if shared_message::is_shared(msg.flags) { // Shared attribute: resolve the reference to get actual attribute data - let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?; - let resolved_data = shared_message::resolve_shared_message( - file_data, - &shared_ref, - MessageType::Attribute, - offset_size, - length_size, - )?; - let attr = AttributeMessage::parse_in_file( - &resolved_data, - file_data, - offset_size, - length_size, - )?; - attrs.push(attr); + shared_message::parse_shared_ref(&msg.data, offset_size) + .and_then(|shared_ref| { + shared_message::resolve_shared_message( + file_data, + &shared_ref, + MessageType::Attribute, + offset_size, + length_size, + ) + }) + .and_then(|resolved| { + AttributeMessage::parse_in_file( + &resolved, + file_data, + offset_size, + length_size, + ) + }) } else { - let attr = AttributeMessage::parse_in_file( - &msg.data, - file_data, - offset_size, - length_size, - )?; - attrs.push(attr); + AttributeMessage::parse_in_file(&msg.data, file_data, offset_size, length_size) + }; + match attr { + Ok(attr) => attrs.push(attr), + Err(e) => on_error(e)?, } } } @@ -439,9 +477,15 @@ pub fn extract_attributes_full( if let Some(info) = attr_info && let Some(fh_addr) = info.fractal_heap_address { - let dense_attrs = - extract_dense_attributes(file_data, &info, fh_addr, offset_size, length_size)?; - attrs.extend(dense_attrs); + extract_dense_attributes( + file_data, + &info, + fh_addr, + offset_size, + length_size, + &mut attrs, + on_error, + )?; } Ok(attrs) @@ -468,7 +512,9 @@ fn extract_dense_attributes( fh_addr: u64, offset_size: u8, length_size: u8, -) -> Result, FormatError> { + attrs: &mut Vec, + on_error: &mut dyn FnMut(FormatError) -> Result<(), FormatError>, +) -> Result<(), FormatError> { // Parse fractal heap let fh = FractalHeapHeader::parse(file_data, fh_addr as usize, offset_size, length_size)?; @@ -482,28 +528,32 @@ fn extract_dense_attributes( let btree_hdr = BTreeV2Header::parse(file_data, btree_addr as usize, offset_size, length_size)?; let records = collect_btree_v2_records(file_data, &btree_hdr, offset_size, length_size)?; - let mut attrs = Vec::new(); for record in &records { // Per HDF5 spec, both type 8 and type 9 records start with heap_id: // Type 8: heap_id(8) + msg_flags(1) + creation_order(4) + hash(4) // Type 9: heap_id(8) + msg_flags(1) + creation_order(4) - let id_offset = 0; - - if record.data.len() < id_offset + fh.heap_id_length as usize { + let id_len = fh.heap_id_length as usize; + let Some(id_bytes) = record.data.get(..id_len) else { + on_error(FormatError::UnexpectedEof { + expected: id_len, + available: record.data.len(), + })?; continue; - } - let id_bytes = &record.data[id_offset..id_offset + fh.heap_id_length as usize]; - - // Read attribute message from fractal heap - let attr_data = fh.read_managed_object(file_data, id_bytes, offset_size)?; + }; // The data in the heap is a complete attribute message - let attr = - AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size)?; - attrs.push(attr); + let attr = fh + .read_managed_object(file_data, id_bytes, offset_size) + .and_then(|attr_data| { + AttributeMessage::parse_in_file(&attr_data, file_data, offset_size, length_size) + }); + match attr { + Ok(attr) => attrs.push(attr), + Err(e) => on_error(e)?, + } } - Ok(attrs) + Ok(()) } #[cfg(test)] diff --git a/crates/clawhdf5/src/lazy.rs b/crates/clawhdf5/src/lazy.rs index 314234c..6ceea55 100644 --- a/crates/clawhdf5/src/lazy.rs +++ b/crates/clawhdf5/src/lazy.rs @@ -12,7 +12,6 @@ use std::cell::RefCell; use std::collections::HashMap; -use clawhdf5_format::attribute::extract_attributes_full; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_read; use clawhdf5_format::dataspace::Dataspace; @@ -29,7 +28,7 @@ use clawhdf5_format::superblock::Superblock; use clawhdf5_io::HDF5Read; use crate::error::Error; -use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; +use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; /// A lazy HDF5 file handle that parses metadata on demand. /// @@ -231,17 +230,26 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { } /// Read all attributes of this group. + /// + /// An attribute that cannot be read — a corrupt or unsupported attribute + /// message, or a dense-storage heap object that cannot be located — is + /// left out of the map instead of failing every attribute on the object; + /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed. + /// Values that are returned are complete (never partially decoded). An + /// error in the index of the attributes itself (the attribute info + /// message, the dense heap header or B-tree) still fails the call. pub fn attrs(&self) -> Result, Error> { + self.attrs_with_errors().map(|(attrs, _)| attrs) + } + + /// Like [`attrs`](Self::attrs), also returning one error for each + /// attribute that could not be read and was left out. + pub fn attrs_with_errors( + &self, + ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.get_or_parse_header(self.address)?; let data = self.file.reader.as_bytes(); - let attr_msgs = - extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?; - Ok(attrs_to_map( - &attr_msgs, - data, - self.file.offset_size(), - self.file.length_size(), - )) + read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) } /// Get a dataset within this group by name. @@ -401,20 +409,30 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { } /// Read all attributes of this dataset. + /// + /// An attribute that cannot be read — a corrupt or unsupported attribute + /// message, or a dense-storage heap object that cannot be located — is + /// left out of the map instead of failing every attribute on the object; + /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed. + /// Values that are returned are complete (never partially decoded). An + /// error in the index of the attributes itself (the attribute info + /// message, the dense heap header or B-tree) still fails the call. pub fn attrs(&self) -> Result, Error> { + self.attrs_with_errors().map(|(attrs, _)| attrs) + } + + /// Like [`attrs`](Self::attrs), also returning one error for each + /// attribute that could not be read and was left out. + pub fn attrs_with_errors( + &self, + ) -> Result<(HashMap, Vec), Error> { let data = self.file.reader.as_bytes(); - let attr_msgs = extract_attributes_full( + read_attrs( data, &self.header, self.file.offset_size(), self.file.length_size(), - )?; - Ok(attrs_to_map( - &attr_msgs, - data, - self.file.offset_size(), - self.file.length_size(), - )) + ) } /// A header message's payload, resolved through the shared-message diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index bf098cf..b57d8b0 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -7,7 +7,6 @@ use std::collections::HashMap; -use clawhdf5_format::attribute::extract_attributes_full; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_read; use clawhdf5_format::dataspace::Dataspace; @@ -24,7 +23,7 @@ use clawhdf5_format::superblock::Superblock; use clawhdf5_io::MmapReader; use crate::error::Error; -use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; +use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; /// An HDF5 file opened via memory mapping. /// @@ -153,17 +152,26 @@ impl<'f> MmapGroup<'f> { } /// Read all attributes of this group. + /// + /// An attribute that cannot be read — a corrupt or unsupported attribute + /// message, or a dense-storage heap object that cannot be located — is + /// left out of the map instead of failing every attribute on the object; + /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed. + /// Values that are returned are complete (never partially decoded). An + /// error in the index of the attributes itself (the attribute info + /// message, the dense heap header or B-tree) still fails the call. pub fn attrs(&self) -> Result, Error> { - let data = self.file.reader.as_bytes(); + self.attrs_with_errors().map(|(attrs, _)| attrs) + } + + /// Like [`attrs`](Self::attrs), also returning one error for each + /// attribute that could not be read and was left out. + pub fn attrs_with_errors( + &self, + ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let attr_msgs = - extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?; - Ok(attrs_to_map( - &attr_msgs, - data, - self.file.offset_size(), - self.file.length_size(), - )) + let data = self.file.reader.as_bytes(); + read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) } /// Get a dataset within this group by name. @@ -342,20 +350,30 @@ impl<'f> MmapDataset<'f> { } /// Read all attributes of this dataset. + /// + /// An attribute that cannot be read — a corrupt or unsupported attribute + /// message, or a dense-storage heap object that cannot be located — is + /// left out of the map instead of failing every attribute on the object; + /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed. + /// Values that are returned are complete (never partially decoded). An + /// error in the index of the attributes itself (the attribute info + /// message, the dense heap header or B-tree) still fails the call. pub fn attrs(&self) -> Result, Error> { + self.attrs_with_errors().map(|(attrs, _)| attrs) + } + + /// Like [`attrs`](Self::attrs), also returning one error for each + /// attribute that could not be read and was left out. + pub fn attrs_with_errors( + &self, + ) -> Result<(HashMap, Vec), Error> { let data = self.file.reader.as_bytes(); - let attr_msgs = extract_attributes_full( + read_attrs( data, &self.header, self.file.offset_size(), self.file.length_size(), - )?; - Ok(attrs_to_map( - &attr_msgs, - data, - self.file.offset_size(), - self.file.length_size(), - )) + ) } /// A header message's payload, resolved through the shared-message diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 9c9a805..b8e7829 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -7,7 +7,6 @@ use std::collections::HashMap; -use clawhdf5_format::attribute::extract_attributes_full; use clawhdf5_format::chunk_cache::ChunkCache; use clawhdf5_format::data_layout::DataLayout; use clawhdf5_format::data_read; @@ -23,7 +22,7 @@ use clawhdf5_format::signature; use clawhdf5_format::superblock::Superblock; use crate::error::Error; -use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; +use crate::types::{AttrValue, DType, classify_datatype, read_attrs}; // --------------------------------------------------------------------------- // FileData — internal storage for either owned bytes or an mmap @@ -293,17 +292,26 @@ impl<'f> Group<'f> { } /// Read all attributes of this group. + /// + /// An attribute that cannot be read — a corrupt or unsupported attribute + /// message, or a dense-storage heap object that cannot be located — is + /// left out of the map instead of failing every attribute on the object; + /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed. + /// Values that are returned are complete (never partially decoded). An + /// error in the index of the attributes itself (the attribute info + /// message, the dense heap header or B-tree) still fails the call. pub fn attrs(&self) -> Result, Error> { - let data = self.file.data.as_bytes(); + self.attrs_with_errors().map(|(attrs, _)| attrs) + } + + /// Like [`attrs`](Self::attrs), also returning one error for each + /// attribute that could not be read and was left out. + pub fn attrs_with_errors( + &self, + ) -> Result<(HashMap, Vec), Error> { let hdr = self.file.parse_header(self.address)?; - let attr_msgs = - extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?; - Ok(attrs_to_map( - &attr_msgs, - data, - self.file.offset_size(), - self.file.length_size(), - )) + let data = self.file.data.as_bytes(); + read_attrs(data, &hdr, self.file.offset_size(), self.file.length_size()) } /// Get a dataset within this group by name. @@ -732,20 +740,30 @@ impl<'f> Dataset<'f> { } /// Read all attributes of this dataset. + /// + /// An attribute that cannot be read — a corrupt or unsupported attribute + /// message, or a dense-storage heap object that cannot be located — is + /// left out of the map instead of failing every attribute on the object; + /// [`attrs_with_errors`](Self::attrs_with_errors) reports which failed. + /// Values that are returned are complete (never partially decoded). An + /// error in the index of the attributes itself (the attribute info + /// message, the dense heap header or B-tree) still fails the call. pub fn attrs(&self) -> Result, Error> { + self.attrs_with_errors().map(|(attrs, _)| attrs) + } + + /// Like [`attrs`](Self::attrs), also returning one error for each + /// attribute that could not be read and was left out. + pub fn attrs_with_errors( + &self, + ) -> Result<(HashMap, Vec), Error> { let data = self.file.data.as_bytes(); - let attr_msgs = extract_attributes_full( + read_attrs( data, &self.header, self.file.offset_size(), self.file.length_size(), - )?; - Ok(attrs_to_map( - &attr_msgs, - data, - self.file.offset_size(), - self.file.length_size(), - )) + ) } /// Verify this dataset's content against its stored provenance hash diff --git a/crates/clawhdf5/src/types.rs b/crates/clawhdf5/src/types.rs index 91cb884..7023b14 100644 --- a/crates/clawhdf5/src/types.rs +++ b/crates/clawhdf5/src/types.rs @@ -155,6 +155,33 @@ pub(crate) fn classify_datatype(dt: &clawhdf5_format::datatype::Datatype) -> DTy /// Read attribute messages into a `HashMap`. /// /// Best-effort: attributes that can't be decoded are silently skipped. +/// The attributes of the object with header `header` that could be read, +/// and one error for each that could not (see +/// [`extract_attributes_tolerant`](clawhdf5_format::attribute::extract_attributes_tolerant)). +pub(crate) fn read_attrs( + file_data: &[u8], + header: &clawhdf5_format::object_header::ObjectHeader, + offset_size: u8, + length_size: u8, +) -> Result< + ( + HashMap, + Vec, + ), + crate::Error, +> { + let (msgs, errors) = clawhdf5_format::attribute::extract_attributes_tolerant( + file_data, + header, + offset_size, + length_size, + )?; + Ok(( + attrs_to_map(&msgs, file_data, offset_size, length_size), + errors, + )) +} + pub(crate) fn attrs_to_map( attrs: &[clawhdf5_format::attribute::AttributeMessage], file_data: &[u8], diff --git a/crates/clawhdf5/tests/dense_storage_interop.rs b/crates/clawhdf5/tests/dense_storage_interop.rs index cbb3b63..b5aa8d1 100644 --- a/crates/clawhdf5/tests/dense_storage_interop.rs +++ b/crates/clawhdf5/tests/dense_storage_interop.rs @@ -364,3 +364,55 @@ fn soft_links_are_listed_as_their_targets() { check_soft_links("latest"); check_soft_links("earliest"); } + +/// One unreadable attribute used to fail `attrs()` for every attribute on +/// the object. Now it is left out (and reported by `attrs_with_errors`), +/// and the others are returned with their full values. +#[test] +fn one_unreadable_attribute_does_not_hide_the_others() { + skip_if_no_python!(); + let (dir, path) = h5py_file( + "d = f.create_dataset('d', data=[1.0])\n\ + for i in range(10):\n\ + \x20 d.attrs['a%d' % i] = float(i)\n\ + d.attrs['zz_broken_attribute'] = 42.0\n", + ); + // Dense attributes live in a fractal heap whose blocks carry no checksum + // by default: give the one named `zz_broken_attribute` a nonexistent + // attribute message version (the byte 9 before its name in a v3 message). + let mut bytes = std::fs::read(&path).unwrap(); + let needle = b"zz_broken_attribute"; + let at = bytes + .windows(needle.len()) + .position(|w| w == needle) + .expect("attribute name in the file"); + assert_eq!(bytes[at - 9], 3, "expected a version-3 attribute message"); + bytes[at - 9] = 0x7f; + let broken = dir.path().join("broken.h5"); + std::fs::write(&broken, &bytes).unwrap(); + + for file in [ + File::open(&broken).unwrap(), + File::from_bytes(bytes.clone()).unwrap(), + ] { + let ds = file.dataset("d").unwrap(); + let (attrs, errors) = ds.attrs_with_errors().unwrap(); + assert_eq!(errors.len(), 1, "{errors:?}"); + assert_eq!(attrs.len(), 10); + for i in 0..10 { + assert!( + matches!(attrs[&format!("a{i}")], AttrValue::F64(v) if v == f64::from(i)), + "a{i}" + ); + } + assert!(!attrs.contains_key("zz_broken_attribute")); + assert_eq!(ds.attrs().unwrap().len(), 10); + } + let m = clawhdf5::MmapFile::open(&broken).unwrap(); + assert_eq!( + m.dataset("d").unwrap().attrs_with_errors().unwrap().1.len(), + 1 + ); + let l = clawhdf5::LazyFile::from_bytes(bytes).unwrap(); + assert_eq!(l.dataset("d").unwrap().attrs().unwrap().len(), 10); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 3dba53d..a05688f 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -87,7 +87,9 @@ the VDS item, which is marked. - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF files (`issue671.nc`). **Fixed 2026-09-25:** huge and tiny heap objects, - and filtered heaps, are read. + and filtered heaps, are read; and an attribute that still cannot be read + is left out of `attrs()` (reported by `attrs_with_errors()`) instead of + failing the others. - **Other readers:** - VL-string datasets are not readable through `File`. - Metadata cache images are not supported. From e94a52a88b214f9fdaa93c2817b16fe61ebcbfe0 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:03:47 -0500 Subject: [PATCH 12/24] fix(format): read unmapped VDS elements as the virtual dataset's fill value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Elements of a virtual dataset that no mapping supplies (unmapped regions, a missing source file, a missing source dataset) read as 0 instead of the fill value libhdf5 returns — silent wrong data for any VDS created with a non-zero fillvalue (read-matrix cases 0471/0472: -1 and 7 read as 0). A missing source dataset was an error; libhdf5 reads it as fill. Move VDS assembly into a new vds module following H5Dvirtual.c: vds::read_virtual_dataset takes the dataset's fill value and a VdsFileResolver that can refuse a name, and reports how many elements were unmapped. Sources are read with their own fill value, and a source whose datatype differs from the virtual dataset's is an error (libhdf5 converts). File passes the dataset's fill value, resolves source names against the virtual file's directory, and refuses names that leave it with an error instead of reading them as fill. read_selection on a VDS goes through the same fill-aware path. The raw-read API (read_raw_data_full*) has no fill value, so it now errors for a VDS with unmapped elements instead of guessing zeros. Tests: vds_interop::vds_unmapped_regions_read_as_fill_value (external, same-file, missing file/dataset, sparse source with its own fill, int fill; earliest and latest format) and vds_source_outside_directory_is_an_error_not_fill, both against h5py; integration_test::v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 + crates/clawhdf5-format/src/data_read.rs | 181 ++----- crates/clawhdf5-format/src/lib.rs | 1 + crates/clawhdf5-format/src/vds.rs | 497 ++++++++++++++++++ .../clawhdf5-format/tests/integration_test.rs | 75 ++- crates/clawhdf5/src/reader.rs | 76 ++- crates/clawhdf5/tests/vds_interop.rs | 99 ++++ docs/known-issues.md | 8 +- 8 files changed, 785 insertions(+), 167 deletions(-) create mode 100644 crates/clawhdf5-format/src/vds.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index fd8624f..de54b91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -275,6 +275,21 @@ ### Correctness - `clawhdf5-format` virtual datasets (VDS), checked against HDF5 2.0 through h5py (`crates/clawhdf5/tests/vds_interop.rs`): + - **Wrong data:** elements no mapping supplies — unmapped regions, and + mappings whose source file or dataset is missing — read as 0 instead of + the virtual dataset's fill value (e.g. h5py `fillvalue=-1`). Assembly moved + to the new `vds` module: `vds::read_virtual_dataset` takes the fill value + and a resolver that can refuse a name (`VdsFileResolver`), and `File` + passes the dataset's fill value. A missing source *dataset* read as an + error; it is fill now, as in libhdf5. Source datasets are read with their + own fill value for unallocated chunks, and a source whose datatype differs + from the virtual dataset's is an error (libhdf5 converts; we do not). + `File` now refuses a source name that leaves the virtual file's directory + (`../x.h5`, absolute paths), or any external source of a `File::from_bytes` + file, with an error — these used to read as fill. + **Behaviour change:** the raw-read API (`read_raw_data_full*`), which has + no fill value, now returns an error for a virtual dataset with unmapped + elements instead of zeros. - Hyperslab selection versions 1 and 2 were refused ("only version-3 hyperslab selections are supported"). Version 1 is what libhdf5 writes for every VDS created with the default format bounds (h5py's default), so diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 82e6544..76c2b0e 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -191,14 +191,9 @@ fn read_raw_data_full_impl( offset_size, length_size, ), - DataLayout::Virtual { - global_heap_address, - global_heap_index, - .. - } => read_virtual_data( + DataLayout::Virtual { .. } => read_virtual_data( file_data, - *global_heap_address, - *global_heap_index, + layout, dataspace, datatype, offset_size, @@ -465,158 +460,54 @@ pub fn read_raw_data_selection( } } -/// Assemble a **Virtual Dataset (VDS)** from its source mappings. +/// Assemble a **Virtual Dataset (VDS)** through the raw-read API, which has no +/// access to the dataset's fill value message. /// -/// Supports virtual datasets of any rank. Same-file sources are read directly; -/// **external-file** sources are read through the caller-supplied `resolver`, -/// which maps a stored source file name to that file's bytes. Each mapping's -/// selected source elements are scattered into the virtual buffer at the -/// positions given by the virtual selection (both enumerated in row-major -/// order, as HDF5 pairs them). Unmapped regions are left at the zero fill value. -/// -/// A mapping whose external source file the resolver cannot supply (`None`) is -/// skipped, leaving its region at fill — matching HDF5's tolerance of missing -/// sources. An external source with no resolver at all is a hard error. +/// Delegates to [`crate::vds::read_virtual_dataset`]. Because the fill value +/// is unknown here, a virtual dataset with any element no mapping supplies +/// (an unmapped region, or a missing source file or dataset) is an error +/// rather than a guess at the fill value; so is one whose extent libhdf5 +/// would report differently from the stored dataspace (unlimited mappings). +/// Use [`crate::vds::read_virtual_dataset`] to read those. #[allow(clippy::too_many_arguments)] fn read_virtual_data( file_data: &[u8], - global_heap_address: Option, - global_heap_index: u32, + layout: &DataLayout, dataspace: &Dataspace, datatype: &Datatype, offset_size: u8, length_size: u8, resolver: Option<&VdsSourceResolver>, ) -> Result, FormatError> { - use crate::data_layout::parse_vds_mappings; - use crate::global_heap::GlobalHeapCollection; - use crate::selection::Selection; - - let elem_size = datatype.type_size() as usize; - let mut out = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len( - dataspace.checked_num_elements()?, - elem_size, - )?)?; - - let virtual_dims = &dataspace.dimensions; - - let addr = global_heap_address.ok_or_else(|| { - FormatError::ChunkedReadError("virtual dataset has no mapping global heap".into()) - })?; - let coll = GlobalHeapCollection::parse(file_data, addr as usize, length_size)?; - let obj = - coll.get_object(global_heap_index as u16) - .ok_or(FormatError::GlobalHeapObjectNotFound { - collection_address: addr, - index: global_heap_index as u16, - })?; - let mappings = parse_vds_mappings(&obj.data, length_size)?; - - for m in &mappings { - let same_file = m.source_file.is_empty() || m.source_file == "."; - - // Resolve the bytes of the file holding this source dataset. - let external; - let src_file_data: &[u8] = if same_file { - file_data - } else { - let r = resolver.ok_or_else(|| { - FormatError::ChunkedReadError( - "external-file virtual dataset sources require a file resolver".into(), - ) - })?; - match r(&m.source_file) { - Some(bytes) => { - external = bytes; - &external - } - // Source file unavailable: leave this region at fill value. - None => continue, - } - }; - - let (vsel, _) = Selection::decode_serialized(&m.virtual_selection)?; - let (ssel, _) = Selection::decode_serialized(&m.source_selection)?; - - let (src_raw, src_dims) = - read_named_dataset_raw(src_file_data, &m.source_dataset, offset_size, length_size)?; - - let vidx = vsel.iter_linear(virtual_dims)?; - let sidx = ssel.iter_linear(&src_dims)?; - if vidx.len() != sidx.len() { - return Err(FormatError::ChunkedReadError( - "virtual/source selection element counts differ".into(), - )); - } - - for (&v, &s) in vidx.iter().zip(sidx.iter()) { - let (vo, so) = (v as usize * elem_size, s as usize * elem_size); - if vo + elem_size > out.len() || so + elem_size > src_raw.len() { - return Err(FormatError::ChunkedReadError( - "virtual dataset selection out of bounds".into(), - )); - } - out[vo..vo + elem_size].copy_from_slice(&src_raw[so..so + elem_size]); - } - } - - Ok(out) -} - -/// Read a named dataset's raw (decoded) bytes and its dimensions, navigating -/// from the superblock. Used to pull VDS source datasets out of the same file. -fn read_named_dataset_raw( - file_data: &[u8], - path: &str, - _offset_size: u8, - _length_size: u8, -) -> Result<(Vec, Vec), FormatError> { - use crate::filter_pipeline::FilterPipeline; - use crate::group_v2::resolve_path_any; - use crate::message_type::MessageType; - use crate::object_header::ObjectHeader; - use crate::signature::find_signature; - use crate::superblock::Superblock; - - let sig = find_signature(file_data)?; - let sb = Superblock::parse(file_data, sig)?; - let addr = resolve_path_any(file_data, &sb, path)?; - let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?; - - let find = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t); - let ds_msg = find(MessageType::Dataspace) - .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no dataspace".into()))?; - let dataspace = Dataspace::parse(&ds_msg.data, sb.length_size)?; - let dt_msg = find(MessageType::Datatype) - .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no datatype".into()))?; - let (datatype, _) = Datatype::parse(&dt_msg.data)?; - let dl_msg = find(MessageType::DataLayout) - .ok_or_else(|| FormatError::ChunkedReadError("VDS source has no data layout".into()))?; - let layout = DataLayout::parse(&dl_msg.data, sb.offset_size, sb.length_size)?; - // A virtual dataset whose source is itself another virtual dataset could - // form a cycle (A -> B -> A) and recurse into a stack overflow. Nested - // virtual sources are exotic and unsupported, so stop here cleanly. - if matches!(layout, DataLayout::Virtual { .. }) { + let wrapped = + resolver.map(|r| move |name: &str| -> Result>, FormatError> { Ok(r(name)) }); + let wrapped_ref = wrapped.as_ref().map(|w| w as &crate::vds::VdsFileResolver); + let v = crate::vds::read_virtual_dataset( + file_data, + layout, + dataspace, + datatype, + None, + offset_size, + length_size, + wrapped_ref, + )?; + if v.dims != dataspace.dimensions { return Err(FormatError::ChunkedReadError( - "virtual dataset source is itself virtual (unsupported)".into(), + "virtual dataset extent differs from its stored dataspace; \ + read it with vds::read_virtual_dataset" + .into(), )); } - let pipeline = find(MessageType::FilterPipeline) - .map(|m| FilterPipeline::parse(&m.data)) - .transpose()?; - - let raw = read_raw_data_full( - file_data, - &layout, - &dataspace, - &datatype, - pipeline.as_ref(), - sb.offset_size, - sb.length_size, - )?; - Ok((raw, dataspace.dimensions.clone())) + if v.unmapped > 0 { + return Err(FormatError::ChunkedReadError( + "virtual dataset has elements no source supplies, which read as its \ + fill value; read it with vds::read_virtual_dataset and the fill value" + .into(), + )); + } + Ok(v.data) } - /// Extract selected elements from a full dataset buffer. pub fn extract_selection_from_buffer( full_data: &[u8], diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 4a6c2a1..879708d 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -100,6 +100,7 @@ pub mod signature; pub mod superblock; pub mod symbol_table; pub mod type_builders; +pub mod vds; pub mod vl_data; #[cfg(feature = "provenance")] diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs new file mode 100644 index 0000000..3ac0232 --- /dev/null +++ b/crates/clawhdf5-format/src/vds.rs @@ -0,0 +1,497 @@ +//! Virtual Dataset (VDS) assembly, following libhdf5's `H5Dvirtual.c`. +//! +//! A virtual dataset stores no data of its own: a list of mappings (kept in +//! the global heap) pairs a selection of the virtual dataspace with a +//! selection of a *source* dataset, in the same file (`"."`) or another one. +//! Reading it means reading each source and scattering the selected source +//! elements into the virtual buffer, pairing the two selections element by +//! element in row-major order. Elements no mapping supplies — unmapped +//! regions, and mappings whose source file or dataset does not exist — read +//! as the virtual dataset's **fill value**, as in libhdf5. +//! +//! Source files other than the virtual file itself are obtained through a +//! caller-supplied [`VdsFileResolver`], since this crate has no filesystem. + +#[cfg(not(feature = "std"))] +use alloc::{format, string::String, vec, vec::Vec}; + +use crate::data_layout::{DataLayout, VdsMapping, parse_vds_mappings}; +use crate::dataspace::Dataspace; +use crate::datatype::Datatype; +use crate::error::FormatError; +use crate::selection::{SerializedSelection, UNLIMITED}; + +/// Resolves the name of an external VDS source file, as stored in the +/// mapping, to that file's bytes. +/// +/// `Ok(None)` means the file does not exist; its mappings then read as the +/// fill value, as libhdf5 does for a missing source. `Err` refuses the name +/// (e.g. a path the caller will not follow) and fails the read, so that a +/// refused source is never passed off as fill. +pub type VdsFileResolver<'a> = dyn Fn(&str) -> Result>, FormatError> + 'a; + +/// A fully assembled virtual dataset. +#[derive(Debug, Clone, PartialEq)] +pub struct VirtualData { + /// The virtual dataset's extent. + pub dims: Vec, + /// Raw element bytes, row-major, in the virtual dataset's datatype. + pub data: Vec, + /// Number of elements no mapping supplied; they hold the fill value. + pub unmapped: u64, +} + +fn vds_err(msg: impl Into) -> FormatError { + FormatError::ChunkedReadError(msg.into()) +} + +/// One mapping with its selections decoded. +struct Mapping { + file: String, + dataset: String, + vsel: SerializedSelection, + ssel: SerializedSelection, +} + +/// Load and decode the mapping list of a virtual layout. +fn load_mappings( + file_data: &[u8], + layout: &DataLayout, + length_size: u8, +) -> Result, FormatError> { + let DataLayout::Virtual { + global_heap_address, + global_heap_index, + .. + } = layout + else { + return Err(vds_err("not a virtual dataset layout")); + }; + let Some(addr) = *global_heap_address else { + return Ok(Vec::new()); + }; + let coll = + crate::global_heap::GlobalHeapCollection::parse(file_data, addr as usize, length_size)?; + let index = u16::try_from(*global_heap_index) + .map_err(|_| vds_err("VDS mapping heap index out of range"))?; + let obj = coll + .get_object(index) + .ok_or(FormatError::GlobalHeapObjectNotFound { + collection_address: addr, + index, + })?; + parse_vds_mappings(&obj.data, length_size)? + .into_iter() + .map(|m: VdsMapping| { + let (vsel, _) = SerializedSelection::decode(&m.virtual_selection)?; + let (ssel, _) = SerializedSelection::decode(&m.source_selection)?; + Ok(Mapping { + file: m.source_file, + dataset: m.source_dataset, + vsel, + ssel, + }) + }) + .collect() +} + +/// The virtual dataset's extent as libhdf5 reports it (`H5Dget_space`). +/// +/// For a virtual dataset whose mappings are all of fixed size this is the +/// stored dataspace. +pub fn virtual_dataset_extent( + file_data: &[u8], + layout: &DataLayout, + dataspace: &Dataspace, + _offset_size: u8, + length_size: u8, + _resolver: Option<&VdsFileResolver>, +) -> Result, FormatError> { + let mappings = load_mappings(file_data, layout, length_size)?; + check_fixed(&mappings)?; + Ok(dataspace.dimensions.clone()) +} + +fn check_fixed(mappings: &[Mapping]) -> Result<(), FormatError> { + if mappings + .iter() + .any(|m| m.vsel.unlimited_dim().is_some() || m.ssel.unlimited_dim().is_some()) + { + return Err(vds_err( + "unlimited virtual dataset mappings are not supported", + )); + } + Ok(()) +} + +/// Read a whole virtual dataset. +/// +/// `fill` is the virtual dataset's fill value (from its fill value message; +/// `None` for the default of zeros); every element no mapping supplies holds +/// it. External source files are read through `resolver`; without one, a +/// mapping to another file is an error. +#[allow(clippy::too_many_arguments)] +pub fn read_virtual_dataset( + file_data: &[u8], + layout: &DataLayout, + dataspace: &Dataspace, + datatype: &Datatype, + fill: Option<&[u8]>, + offset_size: u8, + length_size: u8, + resolver: Option<&VdsFileResolver>, +) -> Result { + let mappings = load_mappings(file_data, layout, length_size)?; + check_fixed(&mappings)?; + let dims = virtual_dataset_extent( + file_data, + layout, + dataspace, + offset_size, + length_size, + resolver, + )?; + + let elem_size = datatype.type_size() as usize; + let total = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or_else(|| FormatError::Overflow("virtual dataset extent".into()))?; + let mut data = crate::chunked_read::alloc_output(crate::chunked_read::checked_byte_len( + total, elem_size, + )?)?; + if let Some(fill) = fill.filter(|f| f.len() == elem_size && f.iter().any(|&b| b != 0)) { + for element in data.chunks_exact_mut(elem_size) { + element.copy_from_slice(fill); + } + } + let mut mapped = vec![false; usize::try_from(total).map_err(|_| vds_err("VDS too large"))?]; + + let mut sources = Sources::new(file_data, resolver); + for m in &mappings { + let Some(src) = sources.dataset(&m.file, &m.dataset, datatype)? else { + continue; // missing source file or dataset: fill + }; + let vidx = selection_indices(&m.vsel, &dims, None)?; + let sidx = selection_indices(&m.ssel, &src.dims, None)?; + scatter(&mut data, &mut mapped, &src.raw, &vidx, &sidx, elem_size)?; + } + + let unmapped = mapped.iter().filter(|&&m| !m).count() as u64; + Ok(VirtualData { + dims, + data, + unmapped, + }) +} + +/// Copy source element `sidx[i]` to virtual element `vidx[i]` for every `i`. +fn scatter( + out: &mut [u8], + mapped: &mut [bool], + src: &[u8], + vidx: &[u64], + sidx: &[u64], + elem_size: usize, +) -> Result<(), FormatError> { + if vidx.len() != sidx.len() { + return Err(vds_err("virtual/source selection element counts differ")); + } + for (&v, &s) in vidx.iter().zip(sidx) { + let (vo, so) = (v as usize * elem_size, s as usize * elem_size); + if vo + elem_size > out.len() || so + elem_size > src.len() { + return Err(vds_err("virtual dataset selection out of bounds")); + } + out[vo..vo + elem_size].copy_from_slice(&src[so..so + elem_size]); + mapped[v as usize] = true; + } + Ok(()) +} + +/// Row-major linear indices of the elements `sel` selects in a dataspace of +/// shape `dims`, in the order libhdf5 iterates them (row-major). +/// +/// `clip` = `(dim, limit)` drops every coordinate `>= limit` in `dim` — the +/// clipping libhdf5 applies to an unlimited selection +/// (`H5S_hyper_clip_unlim`); an unlimited selection must be clipped. +fn selection_indices( + sel: &SerializedSelection, + dims: &[u64], + clip: Option<(usize, u64)>, +) -> Result, FormatError> { + let overflow = || FormatError::Overflow("VDS selection index overflow".into()); + let rank = dims.len(); + let total = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d)) + .ok_or_else(overflow)?; + let mut row_stride = vec![1u64; rank]; + for d in (0..rank.saturating_sub(1)).rev() { + row_stride[d] = row_stride[d + 1] + .checked_mul(dims[d + 1]) + .ok_or_else(overflow)?; + } + if sel.rank().is_some_and(|r| r != rank) { + return Err(vds_err("VDS selection rank does not match dataspace rank")); + } + let limit = |d: usize| match clip { + Some((cd, l)) if cd == d => l, + _ => u64::MAX, + }; + + match sel { + SerializedSelection::All => Ok((0..total).collect()), + SerializedSelection::None => Ok(Vec::new()), + SerializedSelection::Regular { + start, + stride, + count, + block, + } => { + // Selected coordinates along each dimension, in order. + let mut per_dim: Vec> = Vec::with_capacity(rank); + for d in 0..rank { + let lim = limit(d); + if (count[d] == UNLIMITED || block[d] == UNLIMITED) && lim == u64::MAX { + return Err(vds_err("unlimited VDS selection was not clipped")); + } + let mut coords = Vec::new(); + let mut ci = 0u64; + 'blocks: while ci < count[d] { + let base = ci + .checked_mul(stride[d]) + .and_then(|o| start[d].checked_add(o)) + .ok_or_else(overflow)?; + if base >= lim { + break; + } + let mut bi = 0u64; + while bi < block[d] { + let coord = base.checked_add(bi).ok_or_else(overflow)?; + if coord >= lim { + break 'blocks; + } + // Past the extent is malformed; bail before the list + // can grow without bound. + if coord >= dims[d] { + return Err(vds_err("VDS selection exceeds the dataspace extent")); + } + coords.push(coord); + bi += 1; + } + ci += 1; + } + per_dim.push(coords); + } + if per_dim.iter().any(|c| c.is_empty()) { + return Ok(Vec::new()); + } + let n = per_dim + .iter() + .try_fold(1usize, |acc, c| acc.checked_mul(c.len())) + .ok_or_else(overflow)?; + let mut out = Vec::with_capacity(n); + let mut idx = vec![0usize; rank]; + loop { + let lin: u64 = (0..rank).map(|d| per_dim[d][idx[d]] * row_stride[d]).sum(); + out.push(lin); + // Mixed-radix increment, last dimension fastest. + let mut d = rank; + loop { + if d == 0 { + return Ok(out); + } + d -= 1; + idx[d] += 1; + if idx[d] < per_dim[d].len() { + break; + } + idx[d] = 0; + } + } + } + SerializedSelection::Blocks { + rank: _, + starts, + ends, + } => { + // libhdf5 serializes the union as disjoint blocks, so their volumes + // never add up to more than the dataspace. + let mut volume = 0u64; + for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { + for d in 0..rank { + if e[d] >= dims[d] { + return Err(vds_err("VDS selection exceeds the dataspace extent")); + } + } + let v = s + .iter() + .zip(e) + .try_fold(1u64, |acc, (&s, &e)| acc.checked_mul(e - s + 1)) + .ok_or_else(overflow)?; + volume = volume.checked_add(v).ok_or_else(overflow)?; + if volume > total { + return Err(vds_err("VDS selection blocks overlap")); + } + } + let mut out = Vec::with_capacity(volume as usize); + for (s, e) in starts.chunks_exact(rank).zip(ends.chunks_exact(rank)) { + let mut cur = s.to_vec(); + 'block: loop { + if (0..rank).all(|d| cur[d] < limit(d)) { + out.push((0..rank).map(|d| cur[d] * row_stride[d]).sum()); + } + for d in (0..rank).rev() { + if cur[d] < e[d] { + cur[d] += 1; + continue 'block; + } + cur[d] = s[d]; + } + break; + } + } + out.sort_unstable(); + out.dedup(); + Ok(out) + } + } +} + +/// A source dataset's decoded contents. +struct SourceData { + dims: Vec, + raw: Vec, +} + +/// Source files and datasets, fetched on demand. The most recently used +/// external file is kept, since consecutive mappings usually share one. +struct Sources<'a, 'r> { + file_data: &'a [u8], + resolver: Option<&'r VdsFileResolver<'r>>, + cached_file: Option<(String, Option>)>, +} + +impl<'a, 'r> Sources<'a, 'r> { + fn new(file_data: &'a [u8], resolver: Option<&'r VdsFileResolver<'r>>) -> Self { + Sources { + file_data, + resolver, + cached_file: None, + } + } + + /// The bytes of source file `name`, or `None` if it does not exist. + fn file(&mut self, name: &str) -> Result, FormatError> { + if name == "." { + return Ok(Some(self.file_data)); + } + if self.cached_file.as_ref().is_none_or(|(n, _)| n != name) { + let resolver = self.resolver.ok_or_else(|| { + vds_err("external-file virtual dataset sources require a file resolver") + })?; + self.cached_file = Some((String::from(name), resolver(name)?)); + } + Ok(self.cached_file.as_ref().and_then(|(_, b)| b.as_deref())) + } + + /// Read source dataset `path` from file `file`, or `None` when either + /// does not exist. Its datatype must be the virtual dataset's: libhdf5 + /// converts between types here, which is not supported. + fn dataset( + &mut self, + file: &str, + path: &str, + datatype: &Datatype, + ) -> Result, FormatError> { + let Some(bytes) = self.file(file)? else { + return Ok(None); + }; + read_source(bytes, path, datatype) + } +} + +/// Read source dataset `path` of the file in `file_data` in full (its own +/// fill value applied to unallocated chunks), or `None` if it does not exist. +fn read_source( + file_data: &[u8], + path: &str, + datatype: &Datatype, +) -> Result, FormatError> { + use crate::filter_pipeline::FilterPipeline; + use crate::message_type::MessageType; + use crate::object_header::ObjectHeader; + use crate::shared_message::message_data_with_sohm; + + let sig = crate::signature::find_signature(file_data)?; + let sb = crate::superblock::Superblock::parse(file_data, sig)?; + let (os, ls) = (sb.offset_size, sb.length_size); + let addr = match crate::group_v2::resolve_path_any(file_data, &sb, path) { + Ok(a) => a, + Err(FormatError::PathNotFound(_)) => return Ok(None), + Err(e) => return Err(e), + }; + let hdr = ObjectHeader::parse(file_data, addr as usize, os, ls)?; + let msg = |t: MessageType| { + hdr.messages + .iter() + .find(|m| m.msg_type == t) + .ok_or_else(|| vds_err(format!("VDS source {path} has no {t:?} message"))) + }; + let dataspace = Dataspace::parse( + &message_data_with_sohm(file_data, msg(MessageType::Dataspace)?, os, ls)?, + ls, + )?; + let (src_type, _) = Datatype::parse(&message_data_with_sohm( + file_data, + msg(MessageType::Datatype)?, + os, + ls, + )?)?; + if &src_type != datatype { + return Err(vds_err(format!( + "VDS source {path} has a different datatype from the virtual dataset \ + (type conversion is not supported)" + ))); + } + let layout = DataLayout::parse(&msg(MessageType::DataLayout)?.data, os, ls)?; + // A source that is itself virtual could form a cycle (A -> B -> A) and + // recurse without bound. Nested virtual sources are not supported. + if matches!(layout, DataLayout::Virtual { .. }) { + return Err(vds_err( + "virtual dataset source is itself virtual (unsupported)", + )); + } + let pipeline = hdr + .messages + .iter() + .find(|m| m.msg_type == MessageType::FilterPipeline) + .map(|m| { + message_data_with_sohm(file_data, m, os, ls).and_then(|d| FilterPipeline::parse(&d)) + }) + .transpose()?; + let raw = crate::fill_value::read_full_with_fill( + &hdr.messages, + file_data, + &layout, + &dataspace, + src_type.type_size() as usize, + os, + ls, + || { + crate::data_read::read_raw_data_full( + file_data, + &layout, + &dataspace, + &src_type, + pipeline.as_ref(), + os, + ls, + ) + }, + )?; + Ok(Some(SourceData { + dims: dataspace.dimensions, + raw, + })) +} diff --git a/crates/clawhdf5-format/tests/integration_test.rs b/crates/clawhdf5-format/tests/integration_test.rs index 7aab9f0..a43ecda 100644 --- a/crates/clawhdf5-format/tests/integration_test.rs +++ b/crates/clawhdf5-format/tests/integration_test.rs @@ -83,6 +83,45 @@ fn read_chunked_dataset(file_data: &[u8], dataset_path: &str) -> (Vec, Datat (raw, datatype, dataspace) } +/// Helper: read a virtual dataset with `vds::read_virtual_dataset`, giving it +/// the dataset's own fill value (same-file sources only). +fn read_virtual_fixture(file_data: &[u8], path: &str) -> (Vec, Datatype) { + let sig = find_signature(file_data).unwrap(); + let sb = Superblock::parse(file_data, sig).unwrap(); + let addr = resolve_path_any(file_data, &sb, path).unwrap(); + let hdr = + ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap(); + let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap(); + let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap(); + let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap(); + let layout = DataLayout::parse( + &msg(MessageType::DataLayout).data, + sb.offset_size, + sb.length_size, + ) + .unwrap(); + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + file_data, + &hdr.messages, + sb.offset_size, + sb.length_size, + ) + .unwrap(); + let v = clawhdf5_format::vds::read_virtual_dataset( + file_data, + &layout, + &ds, + &dt, + fill.as_deref(), + sb.offset_size, + sb.length_size, + None, + ) + .unwrap(); + assert_eq!(v.dims, ds.dimensions); + (v.data, dt) +} + /// Helper: read any dataset (contiguous or chunked) as f64. fn read_dataset_f64_any(bytes: &[u8], path: &str) -> Vec { let sig = find_signature(bytes).unwrap(); @@ -672,7 +711,7 @@ fn v4_virtual_dataset_same_file_read() { // virt[4:8] <- (unmapped) => fill 0 // virt[8:12] <- src_b[0:4] (ALL) => 20,21,22,23 let file_data = include_bytes!("fixtures/vds_same_file.h5"); - let (raw, datatype, _) = read_chunked_dataset(file_data, "virt"); + let (raw, datatype) = read_virtual_fixture(file_data, "virt"); let values = read_as_i32(&raw, &datatype).unwrap(); assert_eq!( values, @@ -681,6 +720,38 @@ fn v4_virtual_dataset_same_file_read() { ); } +#[test] +fn v4_virtual_dataset_raw_api_refuses_to_guess_the_fill_value() { + // The raw read API has no fill value message, so a virtual dataset with an + // unmapped region is an error there instead of zeros that may be wrong. + let file_data = include_bytes!("fixtures/vds_same_file.h5"); + let sig = find_signature(file_data).unwrap(); + let sb = Superblock::parse(file_data, sig).unwrap(); + let addr = resolve_path_any(file_data, &sb, "virt").unwrap(); + let hdr = + ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size).unwrap(); + let msg = |t: MessageType| hdr.messages.iter().find(|m| m.msg_type == t).unwrap(); + let ds = Dataspace::parse(&msg(MessageType::Dataspace).data, sb.length_size).unwrap(); + let (dt, _) = Datatype::parse(&msg(MessageType::Datatype).data).unwrap(); + let layout = DataLayout::parse( + &msg(MessageType::DataLayout).data, + sb.offset_size, + sb.length_size, + ) + .unwrap(); + let err = read_raw_data_full( + file_data, + &layout, + &ds, + &dt, + None, + sb.offset_size, + sb.length_size, + ) + .unwrap_err(); + assert!(err.to_string().contains("fill value"), "{err}"); +} + #[test] fn v4_virtual_dataset_2d_same_file_read() { // A 4x4 virtual dataset assembled from two 2x2 same-file sources placed as @@ -689,7 +760,7 @@ fn v4_virtual_dataset_2d_same_file_read() { // virt[2:4,2:4] <- src_b = [[5,6],[7,8]] // everything else -> fill 0 let file_data = include_bytes!("fixtures/vds_2d_same_file.h5"); - let (raw, datatype, _) = read_chunked_dataset(file_data, "virt"); + let (raw, datatype) = read_virtual_fixture(file_data, "virt"); let values = read_as_i32(&raw, &datatype).unwrap(); assert_eq!( values, diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index bb6a8fd..2e7bd3d 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -485,6 +485,7 @@ impl<'f> Dataset<'f> { self.file.length_size(), )?; let fill_matters = !clawhdf5_format::fill_value::has_storage(&dl) + || matches!(dl, DataLayout::Virtual { .. }) || (matches!(dl, DataLayout::Chunked { .. }) && !clawhdf5_format::fill_value::is_default(fill.as_deref())); if fill_matters { @@ -837,24 +838,9 @@ impl<'f> Dataset<'f> { let pipeline = self.filter_pipeline()?; // Virtual datasets are assembled from source datasets; the per-file - // chunk cache does not apply. Route them through the resolver path so - // external sibling files resolve relative to this file's directory. + // chunk cache does not apply. if matches!(dl, DataLayout::Virtual { .. }) { - let base_dir = self.file.base_dir.clone(); - let resolver = move |name: &str| -> Option> { - let dir = base_dir.as_ref()?; - std::fs::read(dir.join(sibling_file_name(name)?)).ok() - }; - return Ok(data_read::read_raw_data_full_with_resolver( - self.file.data.as_bytes(), - &dl, - &ds, - &dt, - pipeline.as_ref(), - self.file.offset_size(), - self.file.length_size(), - Some(&resolver), - )?); + return self.read_virtual(&dl, &ds, &dt); } // Unallocated storage reads as the dataset's fill value. @@ -880,6 +866,62 @@ impl<'f> Dataset<'f> { }, ) } + + /// Resolver for external Virtual Dataset source files: names are + /// resolved against the directory of the file that holds the virtual + /// dataset, as libhdf5 does. A missing file is `Ok(None)` (its mappings + /// read as the fill value); a name that would leave that directory is + /// refused with an error rather than read as fill. + fn vds_resolver(&self) -> impl Fn(&str) -> Result>, FormatError> + use<> { + let base_dir = self.file.base_dir.clone(); + move |name: &str| { + let Some(dir) = base_dir.as_ref() else { + return Err(FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} cannot be resolved for an in-memory file" + ))); + }; + let rel = sibling_file_name(name).ok_or_else(|| { + FormatError::ChunkedReadError(format!( + "virtual dataset source file {name:?} is outside the virtual file's \ + directory and is not followed" + )) + })?; + match std::fs::read(dir.join(rel)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(FormatError::ChunkedReadError(format!( + "cannot read virtual dataset source file {name:?}: {e}" + ))), + } + } + } + + /// Read a whole virtual dataset; unmapped elements hold its fill value. + fn read_virtual( + &self, + dl: &DataLayout, + ds: &Dataspace, + dt: &Datatype, + ) -> Result, Error> { + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + self.file.data.as_bytes(), + &self.header.messages, + self.file.offset_size(), + self.file.length_size(), + )?; + let resolver = self.vds_resolver(); + let v = clawhdf5_format::vds::read_virtual_dataset( + self.file.data.as_bytes(), + dl, + ds, + dt, + fill.as_deref(), + self.file.offset_size(), + self.file.length_size(), + Some(&resolver), + )?; + Ok(v.data) + } } // --------------------------------------------------------------------------- diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs index f3ce7e3..032d77c 100644 --- a/crates/clawhdf5/tests/vds_interop.rs +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -190,3 +190,102 @@ expect("shared.h5", "v", "shared") ); assert_matches_libhdf5(dir.path(), "shared.h5", "v", "shared"); } + +// --------------------------------------------------------------------------- +// Fill value +// --------------------------------------------------------------------------- + +/// Elements no mapping supplies read as the virtual dataset's fill value, not +/// as 0: unmapped regions, a missing source file, a missing source dataset. +/// A source's own unallocated chunks read as *its* fill value. +#[test] +fn vds_unmapped_regions_read_as_fill_value() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + generate( + dir.path(), + r#" +for i in range(3): + with h5py.File(f"src_{i}.h5", "w") as s: + s.create_dataset("data", data=np.arange(10.0) + i * 100) +with h5py.File("sparse_src.h5", "w") as s: + d = s.create_dataset("data", shape=(10,), chunks=(5,), dtype="f8", fillvalue=42.0) + d[0:5] = np.arange(5.0) + 1000 # the second chunk is never written +for libver in ["earliest", "latest"]: + with h5py.File(f"fill_{libver}.h5", "w", libver=libver) as f: + f.create_dataset("local", data=np.arange(10.0) * -1) + lay = h5py.VirtualLayout(shape=(6, 10), dtype="f8") + for i in range(3): + lay[i] = h5py.VirtualSource(f"src_{i}.h5", "data", shape=(10,)) + lay[3] = h5py.VirtualSource("no_such_file.h5", "data", shape=(10,)) + lay[4] = h5py.VirtualSource("src_0.h5", "no_such_dataset", shape=(10,)) + # row 5 is not mapped at all + f.create_virtual_dataset("files", lay, fillvalue=-1.0) + lay = h5py.VirtualLayout(shape=(20,), dtype="f8") + lay[0:10] = h5py.VirtualSource(".", "local", shape=(10,)) + f.create_virtual_dataset("same_file", lay, fillvalue=7.0) + lay = h5py.VirtualLayout(shape=(12,), dtype="f8") + lay[1:11] = h5py.VirtualSource("sparse_src.h5", "data", shape=(10,)) + f.create_virtual_dataset("sparse_source", lay, fillvalue=-3.5) + lay = h5py.VirtualLayout(shape=(3, 4), dtype="i4") + lay[1, :] = h5py.VirtualSource(".", "ints", shape=(4,)) + f.create_dataset("ints", data=np.arange(4, dtype="i4") + 1) + f.create_virtual_dataset("int_fill", lay, fillvalue=-99) + for name in ["files", "same_file", "sparse_source", "int_fill"]: + expect(f"fill_{libver}.h5", name, f"{name}_{libver}") +"#, + ); + for libver in ["earliest", "latest"] { + let file = format!("fill_{libver}.h5"); + for name in ["files", "same_file", "sparse_source", "int_fill"] { + assert_matches_libhdf5(dir.path(), &file, name, &format!("{name}_{libver}")); + } + } + + // A selection read goes through the same fill-aware assembly. + let f = File::open(dir.path().join("fill_latest.h5")).unwrap(); + let sel = clawhdf5::Selection::slice(std::slice::from_ref(&(8..14))); + let got = f + .dataset("same_file") + .unwrap() + .read_f64_selection(&sel) + .unwrap(); + assert_eq!(got, vec![-8.0, -9.0, 7.0, 7.0, 7.0, 7.0]); +} + +/// A source name that would leave the virtual file's directory is refused +/// with an error; it used to be skipped and read silently as fill. +#[test] +fn vds_source_outside_directory_is_an_error_not_fill() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("sub")).unwrap(); + generate( + dir.path(), + r#" +with h5py.File("src.h5", "w") as s: + s.create_dataset("data", data=np.arange(4.0)) +with h5py.File("sub/up.h5", "w", libver="latest") as f: + lay = h5py.VirtualLayout(shape=(4,), dtype="f8") + lay[:] = h5py.VirtualSource("../src.h5", "data", shape=(4,)) + f.create_virtual_dataset("v", lay, fillvalue=-1.0) +with h5py.File("nested.h5", "w", libver="latest") as f: + lay = h5py.VirtualLayout(shape=(4,), dtype="f8") + lay[:] = h5py.VirtualSource("sub/inner.h5", "data", shape=(4,)) + f.create_virtual_dataset("v", lay, fillvalue=-1.0) +with h5py.File("sub/inner.h5", "w") as s: + s.create_dataset("data", data=np.arange(4.0) + 10) +expect("nested.h5", "v", "nested") +"#, + ); + // libhdf5 resolves "../src.h5" (and would read [0, 1, 2, 3]); we refuse + // to leave the directory, and say so. + let f = File::open(dir.path().join("sub/up.h5")).unwrap(); + let err = f.dataset("v").unwrap().read_f64().unwrap_err(); + assert!( + err.to_string().contains("not followed"), + "unexpected error: {err}" + ); + // A relative name below the virtual file's directory resolves there. + assert_matches_libhdf5(dir.path(), "nested.h5", "v", "nested"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index eb7d80f..b1de913 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -63,13 +63,15 @@ segfault or abort. ## Gaps found by the 2026-09-25 HDF5 audit (open) -**Status:** open. These fail with an error; none returns wrong data, except -the VDS item, which is marked. +**Status:** open. These fail with an error; none returns wrong data (the VDS +fill-value item that did is fixed). - **Layout message versions 1 and 2** (HDF5 1.6-era files): 84 of the 686 sweep files, `InvalidLayoutVersion`. This is the largest single gap. - **Virtual datasets:** - - **Wrong data:** unmapped regions read as 0 instead of the fill value. + - ~~**Wrong data:** unmapped regions read as 0 instead of the fill value.~~ + Fixed 2026-09-25: unmapped elements and missing sources read as the + virtual dataset's fill value. - `%b` printf-style source names are not expanded. - ~~Hyperslab selection versions 1 and 2 are refused.~~ Fixed 2026-09-25: versions 1-3 and irregular hyperslabs are decoded. From efc2dc53c969ef57b56a29de5469161016b1d8d8 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:59:54 -0500 Subject: [PATCH 13/24] fix(format): read array members of version-1 compound datatypes HDF5 1.6 encoded a compound member that is a fixed-size array through legacy per-member fields (dimensionality, permutation, four dimension sizes) that the v1 decoder skipped, so a [4] i32 member read as one i32 with the wrong size. Build the array type from those fields as libhdf5 does (ignoring the permutation) and refuse more than four dimensions. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 ++ crates/clawhdf5-format/src/datatype.rs | 91 +++++++++++++++++++++++++- docs/known-issues.md | 2 + 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b87415..ac26b3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,12 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` reader: array members of version-1 compound datatypes + (HDF5 1.6-era files, e.g. libhdf5's `tcompound.h5`) were read as a single + element: a `[4] i32` member came back as one `i32`, with the wrong size. + The legacy per-member dimension fields are now decoded into an array type, + as libhdf5 does; more than four dimensions, or a zero-sized one, is an + error. - `clawhdf5-format` reader — **values returned wrong with no error:** - Fixed Array and Extensible Array chunk indexes were laid out by the dataset's current shape instead of its max shape (23 libhdf5 test files, diff --git a/crates/clawhdf5-format/src/datatype.rs b/crates/clawhdf5-format/src/datatype.rs index 436a6cf..ba85afa 100644 --- a/crates/clawhdf5-format/src/datatype.rs +++ b/crates/clawhdf5-format/src/datatype.rs @@ -423,13 +423,44 @@ impl Datatype { ensure_len(data, pos, 4)?; let byte_offset = LittleEndian::read_u32(&data[pos..pos + 4]) as u64; pos += 4; + // v1 members can be fixed-size arrays of the member + // type (libhdf5 builds an array type from these + // fields; the permutation is ignored, as libhdf5 + // does). Skipping them read a `[4] i32` member as + // one `i32`. + let mut array_dims = Vec::new(); if version == 1 { ensure_len(data, pos, 28)?; + let ndims = data[pos] as usize; + // libhdf5 refuses more than four dimensions and, + // when building the array type, a zero-sized one. + let zero_dim = (0..ndims.min(4)).any(|j| { + let at = pos + 12 + 4 * j; + LittleEndian::read_u32(&data[at..at + 4]) == 0 + }); + if ndims > 4 || zero_dim { + return Err(FormatError::InvalidDatatypeVersion { + class: class_id, + version, + }); + } + array_dims = (0..ndims) + .map(|j| { + let at = pos + 12 + 4 * j; + LittleEndian::read_u32(&data[at..at + 4]) + }) + .collect(); pos += 28; } - let (member_dt, consumed) = + let (mut member_dt, consumed) = Self::parse_with_depth(&data[pos..], depth + 1)?; pos += consumed; + if !array_dims.is_empty() { + member_dt = Datatype::Array { + base_type: Box::new(member_dt), + dimensions: array_dims, + }; + } members.push(CompoundMember { name, byte_offset, @@ -1336,6 +1367,64 @@ mod tests { assert_xyid_compound(dt); } + #[test] + fn test_compound_v1_member_array_fields() { + // HDF5 1.6 wrote array members of a v1 compound through the legacy + // per-member fields (as in libhdf5's tools/test/testfiles/ + // tcompound.h5 `type2`: `int_array` [4] i32, `float_array` [5][6] + // f32). They used to be skipped, reading each member as a scalar. + let i32le: [u8; 12] = [ + 0x10, 0x08, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + ]; + let mut b = vec![0x16, 0x02, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00]; + for (name, offset, dims) in [ + (&b"int_array"[..], 0u32, &[4u32][..]), + (&b"xy"[..], 16, &[5u32, 6][..]), + ] { + let mut padded = name.to_vec(); + padded.resize((name.len() + 1 + 7) & !7, 0); + b.extend_from_slice(&padded); + b.extend_from_slice(&offset.to_le_bytes()); + b.push(dims.len() as u8); + b.extend_from_slice(&[0u8; 3 + 4 + 4]); // reserved, permutation, reserved + for j in 0..4 { + b.extend_from_slice(&dims.get(j).copied().unwrap_or(0).to_le_bytes()); + } + b.extend_from_slice(&i32le); + } + let (dt, consumed) = Datatype::parse(&b).unwrap(); + assert_eq!(consumed, b.len()); + let Datatype::Compound { members, .. } = dt else { + panic!("expected Compound, got {dt:?}"); + }; + let got: Vec<(&str, u64, u32, Option>)> = members + .iter() + .map(|m| { + let dims = match &m.datatype { + Datatype::Array { dimensions, .. } => Some(dimensions.clone()), + _ => None, + }; + (m.name.as_str(), m.byte_offset, m.datatype.type_size(), dims) + }) + .collect(); + assert_eq!( + got, + vec![ + ("int_array", 0, 16, Some(vec![4])), + ("xy", 16, 120, Some(vec![5, 6])), + ] + ); + + // More than four dimensions cannot be encoded, and libhdf5 refuses a + // zero-sized dimension (a fuzzed tcompound.h5, cve-2024-32616.h5). + let mut bad = b.clone(); + bad[8 + 16 + 4] = 5; + assert!(Datatype::parse(&bad).is_err()); + let mut bad = b.clone(); + bad[8 + 16 + 4] = 2; // [4, 0] + assert!(Datatype::parse(&bad).is_err()); + } + #[test] fn test_compound_v1_truncated_is_error_not_panic() { let bytes = compound_v1_bytes(); diff --git a/docs/known-issues.md b/docs/known-issues.md index 6dfa369..660dab0 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -74,6 +74,8 @@ the VDS item, which is marked. - Hyperslab selection versions 1 and 2 are refused. - **Files with a user block:** the base address is not applied. - **Old-style shared messages (version 1)** read the wrong address. +- **Array members of version-1 compound datatypes** (found while fixing the + item above) were read as one element. **Fixed 2026-09-25.** - **Groups and links:** - Groups with a user-defined link type (e.g. 187) cannot be listed. - Dense groups with more than about 22 000 links cannot be listed. From 055579485041e96862386ae7a268d4f481b05482 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:01:22 -0500 Subject: [PATCH 14/24] fix(format): read version-1 shared message addresses after the heap offset A version-1 shared message (HDF5 1.6) embeds the target as a symbol-table entry: after six reserved bytes comes a length-sized local-heap offset, then the object header address. We read the heap offset as the address, so datasets using a committed datatype in 1.6-era files (tcompound.h5, tcompound2.h5) failed with InvalidObjectHeaderVersion. parse_shared_ref now takes length_size and skips the offset, as libhdf5 does. Resolving a reference also no longer falls back to the first message of any type in the target header: a missing target message is SharedMessageTargetMissing instead of garbage. Fixture: tcompound.h5 from libhdf5's tools/test/testfiles (8 KiB). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++ crates/clawhdf5-format/src/attribute.rs | 5 +- crates/clawhdf5-format/src/error.rs | 8 ++ crates/clawhdf5-format/src/shared_message.rs | 67 +++++----- .../tests/fixtures/tcompound.h5 | Bin 0 -> 8192 bytes crates/clawhdf5/tests/shared_message_v1.rs | 121 ++++++++++++++++++ docs/known-issues.md | 3 + 7 files changed, 178 insertions(+), 35 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/tcompound.h5 create mode 100644 crates/clawhdf5/tests/shared_message_v1.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ac26b3b..6b054b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,15 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` reader: version-1 shared messages (HDF5 1.6-era files, + e.g. a dataset using a committed datatype in libhdf5's `tcompound.h5`) + read the heap-offset field of the embedded symbol-table entry as the + target address and failed with `InvalidObjectHeaderVersion`. The address + is now read after it, as libhdf5 does. **Breaking (format crate):** + `shared_message::parse_shared_ref` takes `length_size`. A reference whose + target header has no message of the referenced type is now + `FormatError::SharedMessageTargetMissing` instead of returning the first + other message found there (which decoded as garbage). - `clawhdf5-format` reader: array members of version-1 compound datatypes (HDF5 1.6-era files, e.g. libhdf5's `tcompound.h5`) were read as a single element: a `[4] i32` member came back as one `i32`, with the wrong size. diff --git a/crates/clawhdf5-format/src/attribute.rs b/crates/clawhdf5-format/src/attribute.rs index bb96bd5..06132bb 100644 --- a/crates/clawhdf5-format/src/attribute.rs +++ b/crates/clawhdf5-format/src/attribute.rs @@ -97,7 +97,7 @@ impl AttributeMessage { return Ok(Cow::Borrowed(bytes)); } let (file_data, offset_size) = file.ok_or(FormatError::UnresolvedSharedMessage)?; - let shared_ref = shared_message::parse_shared_ref(bytes, offset_size)?; + let shared_ref = shared_message::parse_shared_ref(bytes, offset_size, length_size)?; shared_message::resolve_shared_message( file_data, &shared_ref, @@ -407,7 +407,8 @@ pub fn extract_attributes_full( if msg.msg_type == MessageType::Attribute { if shared_message::is_shared(msg.flags) { // Shared attribute: resolve the reference to get actual attribute data - let shared_ref = shared_message::parse_shared_ref(&msg.data, offset_size)?; + let shared_ref = + shared_message::parse_shared_ref(&msg.data, offset_size, length_size)?; let resolved_data = shared_message::resolve_shared_message( file_data, &shared_ref, diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index 6d10c3d..7ae57f4 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -117,6 +117,9 @@ pub enum FormatError { /// A message is marked shared but was parsed without access to the file, /// so the reference to the real message could not be followed. UnresolvedSharedMessage, + /// A shared-message reference points at an object header that holds no + /// (unshared) message of the referenced type (raw message type id). + SharedMessageTargetMissing(u16), /// A selection does not fit the dataset it was applied to (wrong rank, or /// it reaches past a dimension's extent). SelectionOutOfBounds(String), @@ -339,6 +342,11 @@ impl fmt::Display for FormatError { FormatError::SelectionOutOfBounds(msg) => { write!(f, "selection out of bounds: {msg}") } + FormatError::SharedMessageTargetMissing(t) => write!( + f, + "shared message reference points at an object header with no message of type \ + {t:#06x}" + ), FormatError::UnresolvedSharedMessage => write!( f, "message is shared but no file data was available to resolve it" diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 33334fa..a77edc1 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -154,13 +154,24 @@ pub fn is_shared(msg_flags: u8) -> bool { /// /// When the shared flag is set on a message, the data contains a reference /// instead of the actual message content. -pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result { +/// +/// `length_size` is needed for version 1 references, which embed a +/// symbol-table-entry-shaped pointer whose first field is a length-sized +/// heap offset. +pub fn parse_shared_ref( + data: &[u8], + offset_size: u8, + length_size: u8, +) -> Result { ensure_len(data, 0, 2)?; let version = data[0]; let ref_type = data[1]; // Layouts (HDF5 spec IV.A.2 "Shared Message", and libhdf5's decoder): - // v1: version, type, reserved(6), address — always "committed" + // v1: version, type, reserved(6), then the HDF5 1.6 "symbol table + // entry" encoding of the target: a length-sized local-heap name + // offset (unused, skipped by libhdf5) followed by the object + // header address — always "committed" // v2: version, type, address — always "committed" // v3: version, type, then a fractal-heap ID if type == SOHM, otherwise // an address @@ -177,7 +188,7 @@ pub fn parse_shared_ref(data: &[u8], offset_size: u8) -> Result address_at(2 + 6), + 1 => address_at(2 + 6 + length_size as usize), 2 => address_at(2), 3 if ref_type == SHARE_TYPE_SOHM => { ensure_len(data, 2, FHEAP_ID_LEN)?; @@ -434,7 +445,7 @@ pub fn message_data_with_sohm<'a>( if !is_shared(msg.flags) { return Ok(Cow::Borrowed(&msg.data)); } - let shared_ref = parse_shared_ref(&msg.data, offset_size)?; + let shared_ref = parse_shared_ref(&msg.data, offset_size, length_size)?; let table = if shared_ref.heap_id.is_some() { load_sohm_table(file_data, offset_size, length_size)? } else { @@ -514,7 +525,7 @@ pub fn message_data<'a>( if !is_shared(msg.flags) { return Ok(Cow::Borrowed(&msg.data)); } - let shared_ref = parse_shared_ref(&msg.data, offset_size)?; + let shared_ref = parse_shared_ref(&msg.data, offset_size, length_size)?; resolve_shared_message( file_data, &shared_ref, @@ -571,24 +582,13 @@ pub fn resolve_shared_message_with_sohm( return Ok(msg.data.clone()); } } - // The message at that OH address is the message itself - // In many cases with type 1, the entire OH at that address IS the shared message - // Try returning the first message of any type that isn't Nil - for msg in &target_header.messages { - if msg.msg_type == target_msg_type { - return Ok(msg.data.clone()); - } - } - // Fall back to first non-nil message - for msg in &target_header.messages { - if msg.msg_type != MessageType::Nil { - return Ok(msg.data.clone()); - } - } - Err(FormatError::UnexpectedEof { - expected: 1, - available: 0, - }) + // The referenced header has no (unshared) message of the wanted + // type: the reference is wrong or the file is damaged. Handing + // back some other message's bytes — or another reference's — + // would decode as garbage, so refuse. + Err(FormatError::SharedMessageTargetMissing( + target_msg_type.to_u16(), + )) } (None, Some(heap_id)) => { let table = sohm_table.ok_or(FormatError::InvalidSharedMessageVersion(2))?; @@ -627,7 +627,7 @@ mod tests { data.push(SHARE_TYPE_COMMITTED); // message lives in another object header data.extend_from_slice(&0x1234u64.to_le_bytes()); // address - let shared = parse_shared_ref(&data, 8).unwrap(); + let shared = parse_shared_ref(&data, 8, 8).unwrap(); assert_eq!(shared.version, 3); assert_eq!(shared.ref_type, SHARE_TYPE_COMMITTED); assert_eq!(shared.object_header_address, Some(0x1234)); @@ -641,7 +641,7 @@ mod tests { data.push(SHARE_TYPE_HERE); // stored here but sharable: an address data.extend_from_slice(&0xABCDu64.to_le_bytes()); - let shared = parse_shared_ref(&data, 8).unwrap(); + let shared = parse_shared_ref(&data, 8, 8).unwrap(); assert_eq!(shared.version, 3); assert_eq!(shared.ref_type, 3); assert_eq!(shared.object_header_address, Some(0xABCD)); @@ -653,9 +653,10 @@ mod tests { data.push(1); // version data.push(0); // type data.extend_from_slice(&[0u8; 6]); // reserved + data.extend_from_slice(&0x10u64.to_le_bytes()); // heap name offset data.extend_from_slice(&0x5678u64.to_le_bytes()); - let shared = parse_shared_ref(&data, 8).unwrap(); + let shared = parse_shared_ref(&data, 8, 8).unwrap(); assert_eq!(shared.version, 1); assert_eq!(shared.object_header_address, Some(0x5678)); } @@ -668,7 +669,7 @@ mod tests { data.push(SHARE_TYPE_COMMITTED); data.extend_from_slice(&0x9000u32.to_le_bytes()); - let shared = parse_shared_ref(&data, 4).unwrap(); + let shared = parse_shared_ref(&data, 4, 4).unwrap(); assert_eq!(shared.version, 2); assert_eq!(shared.object_header_address, Some(0x9000)); } @@ -679,7 +680,7 @@ mod tests { // as written by h5py 3.16 / HDF5 2.0 (libver='latest'): header flags // 0x03 (shared), payload `02 02 <8-byte object header address>`. let data = [0x02, 0x02, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; - let shared = parse_shared_ref(&data, 8).unwrap(); + let shared = parse_shared_ref(&data, 8, 8).unwrap(); assert_eq!(shared.object_header_address, Some(0xb3)); assert!(shared.heap_id.is_none()); } @@ -691,7 +692,7 @@ mod tests { data.push(SHARE_TYPE_SOHM); // message lives in the SOHM fractal heap data.extend_from_slice(&[0xAA, 0xBB, 0xCC, 0xDD, 0x11, 0x22, 0x33, 0x44]); - let shared = parse_shared_ref(&data, 8).unwrap(); + let shared = parse_shared_ref(&data, 8, 8).unwrap(); assert_eq!(shared.version, 3); assert_eq!(shared.ref_type, SHARE_TYPE_SOHM); assert_eq!(shared.object_header_address, None); @@ -708,21 +709,21 @@ mod tests { data.push(SHARE_TYPE_SOHM); data.extend_from_slice(&[0xAA, 0xBB]); // only 2 bytes, need 8 - let err = parse_shared_ref(&data, 8).unwrap_err(); + let err = parse_shared_ref(&data, 8, 8).unwrap_err(); assert!(matches!(err, FormatError::UnexpectedEof { .. })); } #[test] fn invalid_version() { let data = vec![99, 0]; - let err = parse_shared_ref(&data, 8).unwrap_err(); + let err = parse_shared_ref(&data, 8, 8).unwrap_err(); assert_eq!(err, FormatError::InvalidSharedMessageVersion(99)); } #[test] fn truncated_data() { let data = vec![3u8]; // too short - let err = parse_shared_ref(&data, 8).unwrap_err(); + let err = parse_shared_ref(&data, 8, 8).unwrap_err(); assert!(matches!(err, FormatError::UnexpectedEof { .. })); } @@ -733,7 +734,7 @@ mod tests { data.push(SHARE_TYPE_COMMITTED); data.extend_from_slice(&0x1000u32.to_le_bytes()); - let shared = parse_shared_ref(&data, 4).unwrap(); + let shared = parse_shared_ref(&data, 4, 4).unwrap(); assert_eq!(shared.object_header_address, Some(0x1000)); } diff --git a/crates/clawhdf5-format/tests/fixtures/tcompound.h5 b/crates/clawhdf5-format/tests/fixtures/tcompound.h5 new file mode 100644 index 0000000000000000000000000000000000000000..d1ec6504cafee27eeda2e6ea97f95bd9bfc8c97d GIT binary patch literal 8192 zcmeHMJ8u&~5T5g0f(Z{f1xa`x9i>SEazlJ45tIT!#G}Rr!b5^23PFMjmmpE31giXu zl(a}eN`pk{G9@J)d~-8%an84P3`B{*Bkj)Y?CkB_?9A-m-rJcgSC0&x7$SyZkpe1_ zpERWUsX$?-tuku`B^+pGI-cdOn)a6!ubooDfo|WNo+k3h<~MBOGl5W{G5YwwvVcbg zcn6tV(lGp%+wav1HN}QJ8ch17BKY`PLXN=MOAxBxov%NeGif(29VEmELrC{@jJl$8 z(D1pl>6pKf|~<^&kLfp<3gC+`%!7v6!RJ~tmrstwb!AtHYMY=3+yq+gJ-ho zBGtpzc~;kXCDkuXsZK;T|C&9U`aIXzZuxf=all~fD6M||zgWPPf3xvx_Tb!*#I6Rg zPvzvCVe!1v`1Lfuc{@K6@@3NqlO`Yn%>)U#+R?iKc`@7BLy@o!PkzYfK(eevs=!!jX{=do{e{;iXD%GgbUmEkw>`79BEh*0r4 zB3ecHghD8&w(*+zyqIMhD61%+P?|){<8w$JFAmYk9}TJla1f!|&H1u=#Ub=7W6~BK zKo8{U@H_A4ny=M1aVR>(5oYG2t{ho8+d(t zK>&|)4xqqi8i3C^1rTtqwEGvNwFJygvuBmzK;7 String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn compound(fields: &[(&str, DType)]) -> DType { + DType::Compound( + fields + .iter() + .map(|(n, t)| (n.to_string(), t.clone())) + .collect(), + ) +} + +fn expected() -> Vec<(&'static str, DType)> { + let int_float = |i: &str, f: &str| compound(&[(i, DType::I32), (f, DType::F32)]); + vec![ + ("group1/dset2", int_float("int_name", "float_name")), + ( + "group1/dset3", + compound(&[ + ("int_array", DType::Array(Box::new(DType::I32), vec![4])), + ( + "float_array", + DType::Array(Box::new(DType::F32), vec![5, 6]), + ), + ]), + ), + ("group1/dset4", int_float("int", "float")), + ("group2/dset5", int_float("int", "float")), + ] +} + +#[test] +fn v1_shared_datatype_resolves_to_the_committed_type() { + // Reading the heap offset as the address used to land on the superblock + // and fail with InvalidObjectHeaderVersion. + let file = File::from_bytes(FIXTURE.to_vec()).unwrap(); + for (path, dtype) in expected() { + assert_eq!( + file.dataset(path).unwrap().dtype().unwrap(), + dtype, + "{path}" + ); + } +} + +#[test] +fn v1_shared_datatype_field_names_match_h5py() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("tcompound.h5"); + std::fs::write(&path, FIXTURE).unwrap(); + let script = format!( + r#" +import h5py +with h5py.File("{path}", "r") as f: + for p in ("group1/dset2", "group1/dset3", "group1/dset4", "group2/dset5"): + print(p, *f[p].dtype.names) +"#, + path = path.display() + ); + let out = Command::new(python()) + .args(["-c", &script]) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout); + let theirs: Vec<&str> = stdout.lines().collect(); + let ours: Vec = expected() + .into_iter() + .map(|(p, t)| match t { + DType::Compound(fields) => { + let names: Vec = fields.into_iter().map(|(n, _)| n).collect(); + format!("{p} {}", names.join(" ")) + } + other => panic!("{other:?}"), + }) + .collect(); + assert_eq!(theirs, ours); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 660dab0..0783e91 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -74,6 +74,9 @@ the VDS item, which is marked. - Hyperslab selection versions 1 and 2 are refused. - **Files with a user block:** the base address is not applied. - **Old-style shared messages (version 1)** read the wrong address. + **Fixed 2026-09-25:** the address follows a length-sized heap offset + (`tcompound.h5`, `tcompound2.h5`; their datasets now stop at the layout + v1 gap above). - **Array members of version-1 compound datatypes** (found while fixing the item above) were read as one element. **Fixed 2026-09-25.** - **Groups and links:** From a6e90f3ee30b8b7a39000814b4a54688280ca058 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:02:47 -0500 Subject: [PATCH 15/24] fix(format): apply the base address of files with a user block A file may start with a user block (h5py userblock_size, h5jam), putting the superblock at 512, 1024, ...; every address in the file is then relative to the superblock. The signature search found it, but every reader passed the whole file to the parsers, so addresses landed userblock bytes early and the root group failed with InvalidObjectHeaderVersion (twithub.h5, twithub513.h5, h5clear_fsm_persist_user_*.h5). Readers now view the file from the superblock on, taking the signature's position as the base address as libhdf5 does: File (mmap, buffered, from_bytes), MmapFile, LazyFile, AsyncHDF5File, the VOL and MPI VOL readers, the HNSW loader and external VDS source files. File, MmapFile and LazyFile gain user_block_size(). The new signature::split_user_block returns the two parts, and Superblock::parse refuses a non-zero offset (UserBlockNotStripped) so a format-level caller cannot silently apply superblock-relative addresses to the whole file. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 + crates/clawhdf5-ann/src/hnsw.rs | 7 +- crates/clawhdf5-format/src/data_read.rs | 8 +- crates/clawhdf5-format/src/error.rs | 10 + crates/clawhdf5-format/src/lib.rs | 11 +- crates/clawhdf5-format/src/shared_message.rs | 4 +- crates/clawhdf5-format/src/signature.rs | 36 +++ crates/clawhdf5-format/src/superblock.rs | 23 +- crates/clawhdf5-io/src/async_read.rs | 22 +- crates/clawhdf5-io/src/mpi_vol.rs | 9 +- crates/clawhdf5-io/src/vol.rs | 7 +- crates/clawhdf5/src/lazy.rs | 42 ++- crates/clawhdf5/src/mmap_file.rs | 50 ++- crates/clawhdf5/src/reader.rs | 60 +++- crates/clawhdf5/tests/userblock_interop.rs | 314 +++++++++++++++++++ docs/known-issues.md | 3 + 16 files changed, 540 insertions(+), 79 deletions(-) create mode 100644 crates/clawhdf5/tests/userblock_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b054b1..5a5283b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,19 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- **Files with a user block** (`h5py.File(..., userblock_size=N)`, `h5jam`; + the superblock at 512, 1024, …) could not be read: every address in the + file is relative to the superblock, but it was applied from byte 0 + (`InvalidObjectHeaderVersion` on the root group). `File` (mmap, buffered, + `from_bytes`), `MmapFile`, `LazyFile`, `AsyncHDF5File`, the VOL readers, + the HNSW loader and external VDS sources now view the file from the + superblock on, using the signature's position as the base address as + libhdf5 does; `user_block_size()` reports the user block (h5py's + `userblock_size`), and `as_bytes()` returns the bytes from the superblock + on. **Breaking (format crate):** `Superblock::parse` refuses a non-zero + signature offset with `FormatError::UserBlockNotStripped`, since the + addresses it returns would be applied to the wrong bytes; pass the slice + from `signature::split_user_block` (new) and parse at offset 0. - `clawhdf5-format` reader: version-1 shared messages (HDF5 1.6-era files, e.g. a dataset using a committed datatype in libhdf5's `tcompound.h5`) read the heap-offset field of the embedded symbol-table entry as the diff --git a/crates/clawhdf5-ann/src/hnsw.rs b/crates/clawhdf5-ann/src/hnsw.rs index a526bee..a725b3f 100644 --- a/crates/clawhdf5-ann/src/hnsw.rs +++ b/crates/clawhdf5-ann/src/hnsw.rs @@ -13,7 +13,7 @@ use clawhdf5_format::filter_pipeline::FilterPipeline; use clawhdf5_format::group_v2::resolve_path_any; use clawhdf5_format::message_type::MessageType; use clawhdf5_format::object_header::ObjectHeader; -use clawhdf5_format::signature::find_signature; +use clawhdf5_format::signature::split_user_block; use clawhdf5_format::superblock::Superblock; use clawhdf5_io::FileWriter as IoFileWriter; @@ -861,8 +861,9 @@ impl HnswIndex { /// The HDF5 data must contain the `/ann/vectors`, `/ann/graph_layer_*`, /// and `/ann/config` datasets as produced by [`to_hdf5_bytes`]. pub fn load_from_hdf5(data: &[u8]) -> Result { - let sig_offset = find_signature(data)?; - let sb = Superblock::parse(data, sig_offset)?; + // Addresses are relative to the superblock: skip any user block. + let (_, data) = split_user_block(data)?; + let sb = Superblock::parse(data, 0)?; // Read config dataset and its attributes let config_attrs = read_dataset_attrs(data, &sb, "ann/config")?; diff --git a/crates/clawhdf5-format/src/data_read.rs b/crates/clawhdf5-format/src/data_read.rs index 82e6544..8d3b16e 100644 --- a/crates/clawhdf5-format/src/data_read.rs +++ b/crates/clawhdf5-format/src/data_read.rs @@ -575,11 +575,13 @@ fn read_named_dataset_raw( use crate::group_v2::resolve_path_any; use crate::message_type::MessageType; use crate::object_header::ObjectHeader; - use crate::signature::find_signature; + use crate::signature::split_user_block; use crate::superblock::Superblock; - let sig = find_signature(file_data)?; - let sb = Superblock::parse(file_data, sig)?; + // An external source file is handed over whole, user block included; + // its addresses are relative to its superblock. + let (_, file_data) = split_user_block(file_data)?; + let sb = Superblock::parse(file_data, 0)?; let addr = resolve_path_any(file_data, &sb, path)?; let hdr = ObjectHeader::parse(file_data, addr as usize, sb.offset_size, sb.length_size)?; diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index 7ae57f4..a54ca72 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -120,6 +120,11 @@ pub enum FormatError { /// A shared-message reference points at an object header that holds no /// (unshared) message of the referenced type (raw message type id). SharedMessageTargetMissing(u16), + /// A superblock was parsed at a non-zero offset of the buffer (the file + /// has a user block of this many bytes). HDF5 addresses are relative to + /// the superblock, so the buffer must start there: see + /// `signature::split_user_block`. + UserBlockNotStripped(u64), /// A selection does not fit the dataset it was applied to (wrong rank, or /// it reaches past a dimension's extent). SelectionOutOfBounds(String), @@ -342,6 +347,11 @@ impl fmt::Display for FormatError { FormatError::SelectionOutOfBounds(msg) => { write!(f, "selection out of bounds: {msg}") } + FormatError::UserBlockNotStripped(n) => write!( + f, + "file has a {n}-byte user block: parse the bytes from the superblock on \ + (signature::split_user_block)" + ), FormatError::SharedMessageTargetMissing(t) => write!( f, "shared message reference points at an object header with no message of type \ diff --git a/crates/clawhdf5-format/src/lib.rs b/crates/clawhdf5-format/src/lib.rs index 4a6c2a1..ffed10f 100644 --- a/crates/clawhdf5-format/src/lib.rs +++ b/crates/clawhdf5-format/src/lib.rs @@ -26,12 +26,13 @@ //! use clawhdf5_format::{signature, superblock, object_header, group_v2, //! datatype, dataspace, data_layout, data_read, message_type::MessageType}; //! -//! let file_data = std::fs::read("output.h5").unwrap(); -//! let sig = signature::find_signature(&file_data).unwrap(); -//! let sb = superblock::Superblock::parse(&file_data, sig).unwrap(); -//! let addr = group_v2::resolve_path_any(&file_data, &sb, "data").unwrap(); +//! let bytes = std::fs::read("output.h5").unwrap(); +//! // Addresses are relative to the superblock: skip any user block. +//! let (_user_block, file_data) = signature::split_user_block(&bytes).unwrap(); +//! let sb = superblock::Superblock::parse(file_data, 0).unwrap(); +//! let addr = group_v2::resolve_path_any(file_data, &sb, "data").unwrap(); //! let hdr = object_header::ObjectHeader::parse( -//! &file_data, addr as usize, sb.offset_size, sb.length_size).unwrap(); +//! file_data, addr as usize, sb.offset_size, sb.length_size).unwrap(); //! ``` //! //! # Features diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index a77edc1..f760021 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -408,8 +408,8 @@ pub fn load_sohm_table( offset_size: u8, length_size: u8, ) -> Result, FormatError> { - let sig = crate::signature::find_signature(file_data)?; - let sb = crate::superblock::Superblock::parse(file_data, sig)?; + // `file_data` starts at the superblock (see `signature::split_user_block`). + let sb = crate::superblock::Superblock::parse(file_data, 0)?; let Some(ext_addr) = sb .superblock_extension_address .filter(|&a| !is_undefined(a, offset_size)) diff --git a/crates/clawhdf5-format/src/signature.rs b/crates/clawhdf5-format/src/signature.rs index 27d5e75..600b650 100644 --- a/crates/clawhdf5-format/src/signature.rs +++ b/crates/clawhdf5-format/src/signature.rs @@ -11,6 +11,16 @@ pub const HDF5_SIGNATURE: [u8; 8] = [0x89, b'H', b'D', b'F', b'\r', b'\n', 0x1A, /// (powers of two starting at 512, plus offset 0). /// /// Returns the byte offset where the signature was found. +/// +/// A non-zero offset means the file starts with a *user block*, and every +/// address inside the file is relative to the superblock's position, not to +/// byte 0 (libhdf5 uses the signature's position as the base address even +/// when the stored base-address field disagrees). The parsers in this crate +/// take addresses as indices into `file_data`, so they must be handed the +/// bytes from the signature on — use [`split_user_block`]. [`Superblock::parse`] +/// refuses a non-zero offset for this reason. +/// +/// [`Superblock::parse`]: crate::superblock::Superblock::parse pub fn find_signature(data: &[u8]) -> Result { // Check offset 0 if data.len() >= 8 && data[..8] == HDF5_SIGNATURE { @@ -29,6 +39,17 @@ pub fn find_signature(data: &[u8]) -> Result { Err(FormatError::SignatureNotFound) } +/// Split a file into its user block and its HDF5 bytes. +/// +/// Returns `(user_block, hdf5)`: `user_block` is everything before the +/// superblock signature (empty for most files) and `hdf5` is the rest, in +/// which every HDF5 address is a plain index. Pass `hdf5` as `file_data` to +/// every parser in this crate, and parse the superblock at offset 0 of it. +pub fn split_user_block(data: &[u8]) -> Result<(&[u8], &[u8]), FormatError> { + let offset = find_signature(data)?; + Ok(data.split_at(offset)) +} + #[cfg(test)] mod tests { use super::*; @@ -88,6 +109,21 @@ mod tests { assert_eq!(find_signature(&data), Err(FormatError::SignatureNotFound)); } + #[test] + fn split_user_block_rebases_at_the_signature() { + let mut data = vec![7u8; 1024]; + data[512..520].copy_from_slice(&HDF5_SIGNATURE); + let (ub, hdf5) = split_user_block(&data).unwrap(); + assert_eq!(ub.len(), 512); + assert_eq!(hdf5.len(), 512); + assert_eq!(&hdf5[..8], &HDF5_SIGNATURE); + + data[..8].copy_from_slice(&HDF5_SIGNATURE); + let (ub, hdf5) = split_user_block(&data).unwrap(); + assert!(ub.is_empty()); + assert_eq!(hdf5.len(), 1024); + } + #[test] fn signature_prefers_earliest() { // Signature at both 0 and 512, should return 0 diff --git a/crates/clawhdf5-format/src/superblock.rs b/crates/clawhdf5-format/src/superblock.rs index e971495..d2ec4d6 100644 --- a/crates/clawhdf5-format/src/superblock.rs +++ b/crates/clawhdf5-format/src/superblock.rs @@ -174,8 +174,18 @@ impl Superblock { /// Parse a superblock from `data` starting at `signature_offset`. /// - /// The signature must be present at the given offset. + /// The signature must be present at the given offset, and that offset + /// must be 0: every address in an HDF5 file is relative to the + /// superblock, so when a file has a user block (signature at 512, 1024, + /// …) the caller must pass the bytes from the signature on — see + /// [`crate::signature::split_user_block`] — and use that slice as + /// `file_data` everywhere. A non-zero offset is refused with + /// [`FormatError::UserBlockNotStripped`] because the addresses in the + /// returned superblock would otherwise be applied to the wrong bytes. pub fn parse(data: &[u8], signature_offset: usize) -> Result { + if signature_offset != 0 { + return Err(FormatError::UserBlockNotStripped(signature_offset as u64)); + } let d = data .get(signature_offset..) .ok_or(FormatError::UnexpectedEof { @@ -676,7 +686,16 @@ mod tests { let mut data = vec![0u8; 1024]; let v0 = build_v0_bytes(8); data[512..512 + v0.len()].copy_from_slice(&v0); - let sb = Superblock::parse(&data, 512).unwrap(); + // Addresses are relative to the superblock, so parsing in place + // (where they would be applied to the whole buffer) is refused... + assert_eq!( + Superblock::parse(&data, 512), + Err(FormatError::UserBlockNotStripped(512)) + ); + // ...and the caller parses the bytes from the signature on. + let (ub, hdf5) = crate::signature::split_user_block(&data).unwrap(); + assert_eq!(ub.len(), 512); + let sb = Superblock::parse(hdf5, 0).unwrap(); assert_eq!(sb.version, 0); assert_eq!(sb.root_group_address, 96); } diff --git a/crates/clawhdf5-io/src/async_read.rs b/crates/clawhdf5-io/src/async_read.rs index 94afff0..d9e2ffe 100644 --- a/crates/clawhdf5-io/src/async_read.rs +++ b/crates/clawhdf5-io/src/async_read.rs @@ -268,28 +268,26 @@ impl AsyncHDF5File { /// /// Reads the entire file into memory, then parses the superblock. pub async fn open(reader: &R) -> Result { - let data = reader.read_all().await?; - let sig_offset = find_signature(&data)?; - let superblock = Superblock::parse(&data, sig_offset)?; - Ok(Self { data, superblock }) + Self::from_bytes(reader.read_all().await?) } /// Open an HDF5 file asynchronously from a file path. pub async fn open_path>(path: P) -> Result { - let data = tokio::fs::read(path).await?; - let sig_offset = find_signature(&data)?; - let superblock = Superblock::parse(&data, sig_offset)?; - Ok(Self { data, superblock }) + Self::from_bytes(tokio::fs::read(path).await?) } /// Open an HDF5 file from bytes already in memory. - pub fn from_bytes(data: Vec) -> Result { - let sig_offset = find_signature(&data)?; - let superblock = Superblock::parse(&data, sig_offset)?; + pub fn from_bytes(mut data: Vec) -> Result { + // HDF5 addresses are relative to the superblock: drop any user block + // so they index `data` directly. + let user_block = find_signature(&data)?; + data.drain(..user_block); + let superblock = Superblock::parse(&data, 0)?; Ok(Self { data, superblock }) } - /// Access the raw file bytes. + /// Access the file bytes from the superblock on (any user block is + /// dropped on open). pub fn as_bytes(&self) -> &[u8] { &self.data } diff --git a/crates/clawhdf5-io/src/mpi_vol.rs b/crates/clawhdf5-io/src/mpi_vol.rs index a1ddf72..7dac39e 100644 --- a/crates/clawhdf5-io/src/mpi_vol.rs +++ b/crates/clawhdf5-io/src/mpi_vol.rs @@ -180,7 +180,7 @@ fn mpi_collective_read(vol: &MpiVol, location: &str, path: &str) -> Result Result { reader: R, + /// Offset of the superblock in the file (the user-block size); every + /// HDF5 address is relative to it. + base: usize, superblock: Superblock, root_header: ObjectHeader, /// Cache of parsed object headers, keyed by address. @@ -73,9 +76,9 @@ impl LazyFile { /// /// Parses only the superblock and root group object header. pub fn open(reader: R) -> Result { - let data = reader.as_bytes(); - let sig_offset = signature::find_signature(data)?; - let superblock = Superblock::parse(data, sig_offset)?; + let (user_block, data) = signature::split_user_block(reader.as_bytes())?; + let base = user_block.len(); + let superblock = Superblock::parse(data, 0)?; let root_header = ObjectHeader::parse( data, superblock.root_group_address as usize, @@ -84,15 +87,26 @@ impl LazyFile { )?; Ok(Self { reader, + base, superblock, root_header, header_cache: RefCell::new(HashMap::new()), }) } - /// Returns the raw file bytes. + /// Returns the file's bytes from the superblock on (after any user + /// block), which is the space every HDF5 address in the file indexes. pub fn as_bytes(&self) -> &[u8] { - self.reader.as_bytes() + self.hdf5_bytes() + } + + /// Size of the user block before the superblock (0 for most files). + pub fn user_block_size(&self) -> u64 { + self.base as u64 + } + + fn hdf5_bytes(&self) -> &[u8] { + &self.reader.as_bytes()[self.base..] } /// Returns a reference to the parsed superblock. @@ -110,7 +124,7 @@ impl LazyFile { /// Resolve a path and return a `LazyDataset` handle. pub fn dataset(&self, path: &str) -> Result, Error> { - let data = self.reader.as_bytes(); + let data = self.hdf5_bytes(); let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; let hdr = self.get_or_parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { @@ -124,7 +138,7 @@ impl LazyFile { /// Resolve a path and return a `LazyGroup` handle. pub fn group(&self, path: &str) -> Result, Error> { - let data = self.reader.as_bytes(); + let data = self.hdf5_bytes(); let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; Ok(LazyGroup { file: self, @@ -163,7 +177,7 @@ impl LazyFile { } // Parse and cache - let data = self.reader.as_bytes(); + let data = self.hdf5_bytes(); let hdr = ObjectHeader::parse( data, address as usize, @@ -187,7 +201,7 @@ impl LazyFile { impl std::fmt::Debug for LazyFile { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LazyFile") - .field("size", &self.reader.as_bytes().len()) + .field("size", &self.hdf5_bytes().len()) .field("superblock_version", &self.superblock.version) .field("cached_headers", &self.header_cache.borrow().len()) .finish() @@ -234,7 +248,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { /// Read all attributes of this group. pub fn attrs(&self) -> Result, Error> { let hdr = self.file.get_or_parse_header(self.address)?; - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let attr_msgs = extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?; Ok(attrs_to_map( @@ -277,7 +291,7 @@ impl<'f, R: HDF5Read> LazyGroup<'f, R> { fn children(&self) -> Result, Error> { let hdr = self.file.get_or_parse_header(self.address)?; - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let os = self.file.offset_size(); let ls = self.file.length_size(); resolve_group_entries(data, &hdr, os, ls).map_err(Error::Format) @@ -360,7 +374,7 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { let dl = self.data_layout()?; let ds = self.dataspace()?; let dt = self.datatype()?; - let slice = data_read::read_raw_data_zerocopy(self.file.reader.as_bytes(), &dl, &ds, &dt)?; + let slice = data_read::read_raw_data_zerocopy(self.file.hdf5_bytes(), &dl, &ds, &dt)?; Ok(slice) } @@ -401,7 +415,7 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { /// Read all attributes of this dataset. pub fn attrs(&self) -> Result, Error> { - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let attr_msgs = extract_attributes_full( data, &self.header, @@ -479,7 +493,7 @@ impl<'f, R: HDF5Read> LazyDataset<'f, R> { let ds = self.dataspace()?; let dl = self.data_layout()?; let pipeline = self.filter_pipeline()?; - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); // Unallocated storage reads as the dataset's fill value. clawhdf5_format::fill_value::read_full_with_fill( &self.header.messages, diff --git a/crates/clawhdf5/src/mmap_file.rs b/crates/clawhdf5/src/mmap_file.rs index 9119cdf..b7251ac 100644 --- a/crates/clawhdf5/src/mmap_file.rs +++ b/crates/clawhdf5/src/mmap_file.rs @@ -34,6 +34,9 @@ use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; /// `&[u8]` slice via [`MmapDataset::read_raw_slice`]. pub struct MmapFile { reader: MmapReader, + /// Offset of the superblock in the mapped file (the user-block size); + /// every HDF5 address is relative to it. + base: usize, superblock: Superblock, } @@ -41,10 +44,25 @@ impl MmapFile { /// Open an HDF5 file using memory-mapped I/O. pub fn open>(path: P) -> Result { let reader = MmapReader::open(path).map_err(Error::Io)?; - let data = reader.as_bytes(); - let sig_offset = signature::find_signature(data)?; - let superblock = Superblock::parse(data, sig_offset)?; - Ok(Self { reader, superblock }) + let (user_block, data) = signature::split_user_block(reader.as_bytes())?; + let base = user_block.len(); + let superblock = Superblock::parse(data, 0)?; + Ok(Self { + reader, + base, + superblock, + }) + } + + /// The file's bytes from the superblock on — the space HDF5 addresses + /// index into. + fn hdf5_bytes(&self) -> &[u8] { + &self.reader.as_bytes()[self.base..] + } + + /// Size of the user block before the superblock (0 for most files). + pub fn user_block_size(&self) -> u64 { + self.base as u64 } /// Returns a handle to the root group. @@ -57,7 +75,7 @@ impl MmapFile { /// Resolve a path and return a `MmapDataset` handle. pub fn dataset(&self, path: &str) -> Result, Error> { - let data = self.reader.as_bytes(); + let data = self.hdf5_bytes(); let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; let hdr = self.parse_header(addr)?; if !has_message(&hdr, MessageType::DataLayout) { @@ -71,7 +89,7 @@ impl MmapFile { /// Resolve a path and return a `MmapGroup` handle. pub fn group(&self, path: &str) -> Result, Error> { - let data = self.reader.as_bytes(); + let data = self.hdf5_bytes(); let addr = group_v2::resolve_path_any(data, &self.superblock, path)?; Ok(MmapGroup { file: self, @@ -79,9 +97,11 @@ impl MmapFile { }) } - /// Returns the raw file bytes (zero-copy from mmap). + /// Returns the file's bytes from the superblock on (after any user + /// block), zero-copy from the mmap. Every HDF5 address in the file + /// indexes this slice. pub fn as_bytes(&self) -> &[u8] { - self.reader.as_bytes() + self.hdf5_bytes() } /// Returns a reference to the parsed superblock. @@ -91,7 +111,7 @@ impl MmapFile { fn parse_header(&self, address: u64) -> Result { ObjectHeader::parse( - self.reader.as_bytes(), + self.hdf5_bytes(), address as usize, self.superblock.offset_size, self.superblock.length_size, @@ -155,7 +175,7 @@ impl<'f> MmapGroup<'f> { /// Read all attributes of this group. pub fn attrs(&self) -> Result, Error> { - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let hdr = self.file.parse_header(self.address)?; let attr_msgs = extract_attributes_full(data, &hdr, self.file.offset_size(), self.file.length_size())?; @@ -198,7 +218,7 @@ impl<'f> MmapGroup<'f> { } fn children(&self) -> Result, Error> { - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let hdr = self.file.parse_header(self.address)?; let os = self.file.offset_size(); let ls = self.file.length_size(); @@ -326,7 +346,7 @@ impl<'f> MmapDataset<'f> { actual: sz, })); } - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let a = addr as usize; if a + sz > data.len() { return Err(Error::Format(FormatError::UnexpectedEof { @@ -342,7 +362,7 @@ impl<'f> MmapDataset<'f> { /// Read all attributes of this dataset. pub fn attrs(&self) -> Result, Error> { - let data = self.file.reader.as_bytes(); + let data = self.file.hdf5_bytes(); let attr_msgs = extract_attributes_full( data, &self.header, @@ -423,7 +443,7 @@ impl<'f> MmapDataset<'f> { // Unallocated storage reads as the dataset's fill value. clawhdf5_format::fill_value::read_full_with_fill( &self.header.messages, - self.file.reader.as_bytes(), + self.file.hdf5_bytes(), &dl, &ds, dt.type_size() as usize, @@ -431,7 +451,7 @@ impl<'f> MmapDataset<'f> { self.file.length_size(), || { Ok(data_read::read_raw_data_full( - self.file.reader.as_bytes(), + self.file.hdf5_bytes(), &dl, &ds, &dt, diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index bb6a8fd..b3d7338 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -31,20 +31,43 @@ use crate::types::{AttrValue, DType, attrs_to_map, classify_datatype}; // --------------------------------------------------------------------------- /// Internal storage: either an owned `Vec` or a memory-mapped region. -enum FileData { +enum Backing { Owned(Vec), #[cfg(feature = "mmap")] Mmap(clawhdf5_io::MmapReader), } -impl FileData { - fn as_bytes(&self) -> &[u8] { +impl Backing { + fn whole_file(&self) -> &[u8] { match self { - FileData::Owned(v) => v, + Backing::Owned(v) => v, #[cfg(feature = "mmap")] - FileData::Mmap(r) => r.as_bytes(), + Backing::Mmap(r) => r.as_bytes(), } } +} + +/// The file's bytes, viewed from the superblock on. A file may start with a +/// user block (the superblock at 512, 1024, …); every HDF5 address is +/// relative to the superblock, so all parsing goes through [`Self::as_bytes`]. +struct FileData { + backing: Backing, + /// Offset of the superblock in the file (the user-block size). + base: usize, +} + +impl FileData { + /// Locate the superblock and parse it. + fn new(backing: Backing) -> Result<(Self, Superblock), Error> { + let (user_block, hdf5) = signature::split_user_block(backing.whole_file())?; + let base = user_block.len(); + let superblock = Superblock::parse(hdf5, 0)?; + Ok((Self { backing, base }, superblock)) + } + + fn as_bytes(&self) -> &[u8] { + &self.backing.whole_file()[self.base..] + } fn len(&self) -> usize { self.as_bytes().len() @@ -81,11 +104,9 @@ impl File { #[cfg(feature = "mmap")] { let reader = clawhdf5_io::MmapReader::open(path).map_err(Error::Io)?; - let data_ref = reader.as_bytes(); - let sig_offset = signature::find_signature(data_ref)?; - let superblock = Superblock::parse(data_ref, sig_offset)?; + let (data, superblock) = FileData::new(Backing::Mmap(reader))?; Ok(Self { - data: FileData::Mmap(reader), + data, superblock, chunk_cache: ChunkCache::new(), base_dir, @@ -116,10 +137,9 @@ impl File { /// In-memory files have no directory, so external Virtual Dataset sources /// cannot be resolved automatically (same-file VDS still works). pub fn from_bytes(data: Vec) -> Result { - let sig_offset = signature::find_signature(&data)?; - let superblock = Superblock::parse(&data, sig_offset)?; + let (data, superblock) = FileData::new(Backing::Owned(data))?; Ok(Self { - data: FileData::Owned(data), + data, superblock, chunk_cache: ChunkCache::new(), base_dir: None, @@ -209,11 +229,19 @@ impl File { Ok(results.into_iter().map(|(_, data)| data).collect()) } - /// Returns the raw file bytes. + /// Returns the file's bytes from the superblock on (after any user + /// block). Every HDF5 address in the file indexes this slice, so it is + /// what the `clawhdf5_format` parsers expect as `file_data`. pub fn as_bytes(&self) -> &[u8] { self.data.as_bytes() } + /// Size of the user block before the superblock (0 for most files). + /// Matches h5py's `File.userblock_size`. + pub fn user_block_size(&self) -> u64 { + self.data.base as u64 + } + /// Returns a reference to the parsed superblock. pub fn superblock(&self) -> &Superblock { &self.superblock @@ -221,10 +249,10 @@ impl File { /// Returns `true` when the file is backed by memory-mapped I/O. pub fn is_mmap(&self) -> bool { - match &self.data { - FileData::Owned(_) => false, + match &self.data.backing { + Backing::Owned(_) => false, #[cfg(feature = "mmap")] - FileData::Mmap(_) => true, + Backing::Mmap(_) => true, } } diff --git a/crates/clawhdf5/tests/userblock_interop.rs b/crates/clawhdf5/tests/userblock_interop.rs new file mode 100644 index 0000000..c9130b9 --- /dev/null +++ b/crates/clawhdf5/tests/userblock_interop.rs @@ -0,0 +1,314 @@ +//! Files that start with a user block (`h5py.File(..., userblock_size=N)`, +//! `h5jam`): the superblock sits at 512, 1024, ... and every address in the +//! file is relative to it. Each reader (buffered, mmap, `MmapFile`, +//! `LazyFile`) must apply that base, and read the same values h5py does. +//! +//! h5py writes the files; skipped when python3 with h5py is unavailable, +//! unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::collections::HashMap; +use std::path::Path; +use std::process::Command; + +use clawhdf5::{AttrValue, File, LazyFile, MmapFile}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +/// Run `script` and return its stdout as `key -> values` (one +/// `key v1 v2 ...` line per key). +fn run_python(script: &str) -> HashMap> { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| { + let mut words = line.split_whitespace().map(str::to_string); + Some((words.next()?, words.collect())) + }) + .collect() +} + +fn parse(values: &[String]) -> Vec +where + T::Err: std::fmt::Debug, +{ + values.iter().map(|v| v.parse().unwrap()).collect() +} + +/// Write a file with a user block of `userblock` bytes holding contiguous, +/// chunked (deflate), compact and committed-type datasets, nested groups, +/// and attributes (compact and, under `latest`, dense). Prints what h5py +/// reads back. +fn write_file(path: &Path, userblock: u32, libver: &str) -> HashMap> { + let script = format!( + r#" +import h5py, numpy as np +path = "{path}" +with h5py.File(path, "w", userblock_size={userblock}, libver={libver}) as f: + f.attrs["title"] = "user block" + f.attrs["answer"] = np.int64(42) + f.create_dataset("contig", data=np.arange(12, dtype=", key: &str) -> String { + match map.get(key) { + Some(AttrValue::I64(v)) => format!("i64 {v}"), + Some(AttrValue::F64(v)) => format!("f64 {v}"), + Some(AttrValue::String(v)) => format!("str {v}"), + other => format!("{other:?}"), + } +} + +fn i64s(v: &[String]) -> Vec { + parse(v) +} + +/// Everything read through the `File` API must match h5py. +fn check_file(file: &File, expected: &HashMap>, label: &str) { + let ub: u64 = expected["userblock"][0].parse().unwrap(); + assert_eq!(file.user_block_size(), ub, "{label}: user block size"); + assert_eq!( + file.dataset("contig").unwrap().read_f64().unwrap(), + parse::(&expected["contig"]), + "{label}: contiguous" + ); + assert_eq!( + file.dataset("chunked") + .unwrap() + .read_i32() + .unwrap() + .iter() + .map(|&v| v as i64) + .collect::>(), + i64s(&expected["chunked"]), + "{label}: chunked" + ); + assert_eq!( + file.dataset("compact").unwrap().read_i64().unwrap(), + i64s(&expected["compact"]), + "{label}: compact" + ); + assert_eq!( + file.dataset("committed").unwrap().read_f32().unwrap(), + parse::(&expected["committed"]), + "{label}: committed datatype" + ); + assert_eq!( + file.dataset("a/b/deep").unwrap().read_i64().unwrap(), + i64s(&expected["deep"]), + "{label}: nested group" + ); + let many: Vec = (0..20) + .map(|i| { + file.dataset(&format!("many/d{i:02}")) + .unwrap() + .read_i32() + .unwrap()[0] as i64 + }) + .collect(); + assert_eq!(many, i64s(&expected["many"]), "{label}: many links"); + + let root = file.root().attrs().unwrap(); + assert_eq!(attr(&root, "title"), "str user block", "{label}"); + assert_eq!(attr(&root, "answer"), "i64 42", "{label}"); + let a = file.group("a").unwrap().attrs().unwrap(); + let k: Vec = (0..12) + .map(|i| match &a[&format!("k{i:02}")] { + AttrValue::I64(v) => *v, + _ => panic!("{label}: k{i:02} is not an i64"), + }) + .collect(); + assert_eq!(k, i64s(&expected["k"]), "{label}: attributes"); + assert_eq!( + attr(&file.group("a/b").unwrap().attrs().unwrap(), "scale"), + "f64 2.5", + "{label}" + ); + assert_eq!( + attr(&file.dataset("contig").unwrap().attrs().unwrap(), "units"), + "str m", + "{label}" + ); +} + +fn check_all_readers(path: &Path, expected: &HashMap>, label: &str) { + check_file( + &File::open(path).unwrap(), + expected, + &format!("{label} File::open"), + ); + check_file( + &File::open_buffered(path).unwrap(), + expected, + &format!("{label} File::open_buffered"), + ); + check_file( + &File::from_bytes(std::fs::read(path).unwrap()).unwrap(), + expected, + &format!("{label} File::from_bytes"), + ); + + let ub: u64 = expected["userblock"][0].parse().unwrap(); + + let mm = MmapFile::open(path).unwrap(); + assert_eq!(mm.user_block_size(), ub, "{label} MmapFile"); + assert_eq!( + mm.dataset("contig").unwrap().read_f64().unwrap(), + parse::(&expected["contig"]), + "{label} MmapFile contiguous" + ); + assert_eq!( + mm.dataset("compact").unwrap().read_i64().unwrap(), + i64s(&expected["compact"]), + "{label} MmapFile compact" + ); + assert_eq!( + mm.dataset("committed").unwrap().read_f32().unwrap(), + parse::(&expected["committed"]), + "{label} MmapFile committed" + ); + assert_eq!( + mm.dataset("a/b/deep").unwrap().read_i64().unwrap(), + i64s(&expected["deep"]), + "{label} MmapFile nested" + ); + assert_eq!( + attr(&mm.root().attrs().unwrap(), "answer"), + "i64 42", + "{label} MmapFile attrs" + ); + + let lazy = LazyFile::open_mmap(path).unwrap(); + assert_eq!(lazy.user_block_size(), ub, "{label} LazyFile"); + assert_eq!( + lazy.dataset("contig").unwrap().read_f64().unwrap(), + parse::(&expected["contig"]), + "{label} LazyFile contiguous" + ); + assert_eq!( + lazy.dataset("chunked") + .unwrap() + .read_i32() + .unwrap() + .iter() + .map(|&v| v as i64) + .collect::>(), + i64s(&expected["chunked"]), + "{label} LazyFile chunked" + ); + assert_eq!( + lazy.dataset("committed").unwrap().read_f32().unwrap(), + parse::(&expected["committed"]), + "{label} LazyFile committed" + ); + assert_eq!( + lazy.dataset("a/b/deep").unwrap().read_i64().unwrap(), + i64s(&expected["deep"]), + "{label} LazyFile nested" + ); + assert_eq!( + attr(&lazy.root().attrs().unwrap(), "answer"), + "i64 42", + "{label} LazyFile attrs" + ); +} + +#[test] +fn user_block_files_read_like_h5py() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + for userblock in [512u32, 4096] { + for libver in ["default", "latest"] { + let label = format!("userblock={userblock} libver={libver}"); + let path = dir.path().join(format!("ub_{userblock}_{libver}.h5")); + let expected = write_file(&path, userblock, libver); + assert_eq!(expected["userblock"], [userblock.to_string()], "{label}"); + check_all_readers(&path, &expected, &label); + } + } +} + +#[test] +fn file_without_user_block_reports_zero() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no_ub.h5"); + let expected = write_file(&path, 0, "default"); + assert_eq!(expected["userblock"], ["0"]); + check_all_readers(&path, &expected, "userblock=0"); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 0783e91..6f84c6a 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -73,6 +73,9 @@ the VDS item, which is marked. - `%b` printf-style source names are not expanded. - Hyperslab selection versions 1 and 2 are refused. - **Files with a user block:** the base address is not applied. + **Fixed 2026-09-25:** every reader views the file from the superblock on + (`twithub.h5`, `twithub513.h5`, `h5clear_fsm_persist_user_*.h5`; the + `twithub` files still stop at the user-defined link type below). - **Old-style shared messages (version 1)** read the wrong address. **Fixed 2026-09-25:** the address follows a length-sized heap offset (`tcompound.h5`, `tcompound2.h5`; their datasets now stop at the layout From 90e050944f1ab7d6335e031f665af866b632ee53 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:07:27 -0500 Subject: [PATCH 16/24] fix(format): refuse a local heap whose free list leaves the heap libhdf5 walks a local heap's free list when it loads the heap's data and refuses the heap ("bad heap free list") when a free block starts or ends outside the data segment, or links to offset 0. We never looked at the free list, so a damaged old-style group listed names read from the broken heap: once the user block of cve-2021-36977.h5 was applied, its root listed eight garbage names where libhdf5 fails. LocalHeap::validate_free_list (new) mirrors H5HL__fl_deserialize, with a cycle bound, and accepts H5HL_FREE_NULL (1) or an all-ones head as the end of the list. Like libhdf5 it runs when the first name is needed, not on parse, so an empty group with a damaged heap still lists as empty (cve-2018-13871.h5, cve-2024-29166.h5, gh-4431-poc-03.h5 keep matching h5py). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 ++ crates/clawhdf5-format/src/error.rs | 6 + crates/clawhdf5-format/src/group_v1.rs | 12 ++ crates/clawhdf5-format/src/local_heap.rs | 99 ++++++++++++++- crates/clawhdf5/tests/local_heap_interop.rs | 131 ++++++++++++++++++++ docs/known-issues.md | 5 + 6 files changed, 258 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5/tests/local_heap_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a5283b..a4a5fa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,6 +273,13 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` reader: an old-style group whose local heap has a free + list pointing outside the heap was listed with names read from the broken + heap (garbage names on `cve-2021-36977.h5` once its user block was + applied). libhdf5 refuses such a heap ("bad heap free list"); so do we now, + with `FormatError::InvalidLocalHeapFreeList`. As in libhdf5 the free list + is checked when the first name is read (`LocalHeap::validate_free_list`, + new), so an empty group with a damaged heap still lists as empty. - **Files with a user block** (`h5py.File(..., userblock_size=N)`, `h5jam`; the superblock at 512, 1024, …) could not be read: every address in the file is relative to the superblock, but it was applied from byte 0 diff --git a/crates/clawhdf5-format/src/error.rs b/crates/clawhdf5-format/src/error.rs index a54ca72..bb81939 100644 --- a/crates/clawhdf5-format/src/error.rs +++ b/crates/clawhdf5-format/src/error.rs @@ -80,6 +80,9 @@ pub enum FormatError { InvalidLocalHeapSignature, /// Invalid local heap version. InvalidLocalHeapVersion(u8), + /// A local heap's free list points outside its data segment (libhdf5: + /// "bad heap free list"). + InvalidLocalHeapFreeList, /// Invalid B-tree v1 signature. InvalidBTreeSignature, /// Invalid B-tree node type. @@ -278,6 +281,9 @@ impl fmt::Display for FormatError { FormatError::InvalidLocalHeapSignature => { write!(f, "invalid local heap signature") } + FormatError::InvalidLocalHeapFreeList => { + write!(f, "bad local heap free list") + } FormatError::InvalidLocalHeapVersion(v) => { write!(f, "invalid local heap version: {v}") } diff --git a/crates/clawhdf5-format/src/group_v1.rs b/crates/clawhdf5-format/src/group_v1.rs index 989f826..e55ad16 100644 --- a/crates/clawhdf5-format/src/group_v1.rs +++ b/crates/clawhdf5-format/src/group_v1.rs @@ -45,9 +45,16 @@ pub fn resolve_v1_group_entries( )?; let mut entries = Vec::new(); + let mut heap_checked = false; for snod_addr in snod_addrs { let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; for entry in &snod.entries { + // Like libhdf5, look at the heap's free list only once a name is + // needed: an empty group with a damaged heap still lists. + if !heap_checked { + heap.validate_free_list(file_data, length_size)?; + heap_checked = true; + } let name = heap.read_string(file_data, entry.link_name_offset)?; entries.push(GroupEntry { name, @@ -85,12 +92,17 @@ pub fn find_v1_soft_link( offset_size, length_size, )?; + let mut heap_checked = false; for snod_addr in snod_addrs { let snod = SymbolTableNode::parse(file_data, snod_addr as usize, offset_size)?; for entry in &snod.entries { if entry.cache_type != CACHE_TYPE_SOFT_LINK { continue; } + if !heap_checked { + heap.validate_free_list(file_data, length_size)?; + heap_checked = true; + } if heap.read_string(file_data, entry.link_name_offset)? != name { continue; } diff --git a/crates/clawhdf5-format/src/local_heap.rs b/crates/clawhdf5-format/src/local_heap.rs index 33e3433..39e9b26 100644 --- a/crates/clawhdf5-format/src/local_heap.rs +++ b/crates/clawhdf5-format/src/local_heap.rs @@ -87,6 +87,57 @@ impl LocalHeap { }) } + /// Walk the free list the way libhdf5 does when it loads a heap's data + /// (`H5HL__fl_deserialize`), rejecting a heap whose free list points + /// outside the data segment. libhdf5 refuses such a heap ("bad heap free + /// list"), and names read from it would be garbage. + /// + /// libhdf5 only loads a heap when it needs a name from it (an empty + /// group's broken heap goes unnoticed), so call this before the first + /// [`Self::read_string`], not on parse. + /// + /// The end of the list is `H5HL_FREE_NULL` (1); an all-ones value (the + /// undefined address) is accepted as "no free list" too. + pub fn validate_free_list(&self, file_data: &[u8], length_size: u8) -> Result<(), FormatError> { + const FREE_NULL: u64 = 1; + let ls = length_size as usize; + let undefined = if ls >= 8 { + u64::MAX + } else { + (1u64 << (8 * ls)) - 1 + }; + let size = self.data_segment_size; + let seg = self.data_segment_address; + let mut next = self.free_list_head_offset; + // Each free block holds two lengths, so a list longer than this + // revisits a block: a cycle. + let max_blocks = size / (2 * ls as u64) + 1; + let mut walked = 0u64; + while next != FREE_NULL && next != undefined { + if next >= size || walked >= max_blocks { + return Err(FormatError::InvalidLocalHeapFreeList); + } + walked += 1; + let at = seg + .checked_add(next) + .and_then(|a| usize::try_from(a).ok()) + .ok_or(FormatError::InvalidLocalHeapFreeList)?; + let block_offset = next; + next = read_offset(file_data, at, length_size)?; + if next == 0 { + return Err(FormatError::InvalidLocalHeapFreeList); + } + let block_size = read_offset(file_data, at + ls, length_size)?; + if block_offset + .checked_add(block_size) + .is_none_or(|end| end > size) + { + return Err(FormatError::InvalidLocalHeapFreeList); + } + } + Ok(()) + } + /// Read a null-terminated string from the heap's data segment at the given byte offset. pub fn read_string(&self, file_data: &[u8], string_offset: u64) -> Result { let seg_addr = self.data_segment_address as usize; @@ -162,8 +213,8 @@ mod tests { // data_segment_size write_val(&mut file, pos, data_seg_size as u64, length_size); pos += length_size as usize; - // free_list_head_offset - write_val(&mut file, pos, 0xFFFFFFFF, length_size); + // free_list_head_offset: H5HL_FREE_NULL (no free space) + write_val(&mut file, pos, 1, length_size); pos += length_size as usize; // data_segment_address write_val(&mut file, pos, data_seg_offset as u64, offset_size); @@ -243,6 +294,50 @@ mod tests { assert_eq!(s, "test"); } + /// Heap with data segment `[a, b, c, 0-padding]` whose free list starts + /// at `head` and has one block `(next, size)` at offset 8. + fn heap_with_free_block(head: u64, next: u64, size: u64) -> Vec { + let mut file = build_heap_file(0, 100, &["abcdefg"], 8, 8); + file.resize(200, 0); + write_val(&mut file, 8, 32, 8); // data segment size + write_val(&mut file, 16, head, 8); + write_val(&mut file, 108, next, 8); + write_val(&mut file, 116, size, 8); + file + } + + #[test] + fn free_list_inside_the_segment_is_accepted() { + let file = heap_with_free_block(8, 1, 24); + let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); + heap.validate_free_list(&file, 8).unwrap(); + assert_eq!(heap.read_string(&file, 0).unwrap(), "abcdefg"); + // An all-ones head is "no free list" too. + let file = heap_with_free_block(u64::MAX, 0, 0); + let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); + assert!(heap.validate_free_list(&file, 8).is_ok()); + } + + #[test] + fn bad_free_list_is_rejected_like_libhdf5() { + for (head, next, size, why) in [ + (40, 1, 8, "head past the segment"), + (8, 1, 25, "block runs past the segment"), + (8, 0, 8, "next offset of zero"), + (8, 8, 8, "cycle"), + (8, 999, 8, "next past the segment"), + ] { + let file = heap_with_free_block(head, next, size); + // The header itself parses; the free list is checked on use. + let heap = LocalHeap::parse(&file, 0, 8, 8).unwrap(); + assert_eq!( + heap.validate_free_list(&file, 8).unwrap_err(), + FormatError::InvalidLocalHeapFreeList, + "{why}" + ); + } + } + #[test] fn invalid_version() { let mut file = build_heap_file(0, 100, &["x"], 8, 8); diff --git a/crates/clawhdf5/tests/local_heap_interop.rs b/crates/clawhdf5/tests/local_heap_interop.rs new file mode 100644 index 0000000..02aa6e0 --- /dev/null +++ b/crates/clawhdf5/tests/local_heap_interop.rs @@ -0,0 +1,131 @@ +//! Old-style (symbol-table) groups keep link names in a local heap. libhdf5 +//! validates the heap's free list when it loads the heap and refuses the +//! group ("bad heap free list") when the list points outside the heap; we +//! must refuse too instead of listing names read from a broken heap. Like +//! libhdf5, the check happens when a name is needed, so an empty group with +//! a broken heap still lists. +//! +//! h5py writes the files; skipped when python3 with h5py is unavailable, +//! unless `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::File; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +macro_rules! skip_if_no_python { + () => { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + }; +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + assert!( + output.status.success(), + "python failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +#[test] +fn local_heap_free_list_checked_like_libhdf5() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let good = dir.path().join("good.h5"); + // Writes `good.h5` (a deleted link leaves a real free block in the root + // group's heap) and two copies whose root heap free list is broken; for + // each prints what h5py lists, or `ERROR`. + let script = format!( + r#" +import h5py, struct +good = "{good}" +with h5py.File(good, "w", libver="earliest") as f: + for name in ("alpha", "beta", "gamma"): + f.create_group(name) + del f["beta"] +data = bytearray(open(good, "rb").read()) +heap = data.find(b"HEAP") # the root group's heap is written first +size, head, seg = struct.unpack_from(" = out.lines().collect(); + assert_eq!( + lines, + [ + "good alpha gamma", + "bad_head ERROR", + "bad_block ERROR", + "bad_empty" + ], + "h5py's view changed" + ); + + let file = File::open(&good).unwrap(); + let mut groups = file.root().groups().unwrap(); + groups.sort(); + assert_eq!(groups, ["alpha", "gamma"]); + + for name in ["bad_head", "bad_block"] { + let file = File::open(dir.path().join(format!("{name}.h5"))).unwrap(); + let listed = file.root().groups(); + assert!( + listed.is_err(), + "{name}: listed {listed:?} from a heap libhdf5 rejects" + ); + } + + let file = File::open(dir.path().join("bad_empty.h5")).unwrap(); + assert_eq!(file.root().groups().unwrap(), Vec::::new()); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 6f84c6a..4d57750 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -86,6 +86,11 @@ the VDS item, which is marked. - Groups with a user-defined link type (e.g. 187) cannot be listed. - Dense groups with more than about 22 000 links cannot be listed. - Soft links are left out of `datasets()`. + - **Wrong data (found while fixing user blocks):** an old-style group whose + local-heap free list points outside the heap listed garbage names where + libhdf5 refuses the heap. **Fixed 2026-09-25** + (`InvalidLocalHeapFreeList`, checked when a name is first read, as + libhdf5 does). - **Dense attributes:** a large attribute stored as a fractal-heap "huge" object makes every attribute on the object fail. This affects real NetCDF files (`issue671.nc`). From 17fa783dced4c982cb51ee102c7d9a99b658bb37 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:08:45 -0500 Subject: [PATCH 17/24] fix(format): resolve SOHM-shared messages on every path A message shared through the file's SOHM heap (H5Pset_shared_mesg_index) is referenced by heap ID, which needs the SOHM table from the superblock extension. Only message_data_with_sohm (used for fill values) loaded it; resolve_shared_message passed no table, so a SOHM-shared datatype, dataspace, filter pipeline or attribute failed with "invalid shared message version: 2" and the dataset or attribute could not be read. resolve_shared_message now loads the table when the reference carries a heap ID. Found while making attrs() tolerant: SOHM attributes turned from an error into missing keys in the audit read matrix. With this fix all 36 SOHM cases there match h5py (datasets, fill values and attributes, every shareable message type, libver earliest and latest). Regression test: sohm_shared_messages_resolve (h5py writes files sharing each message type on its own and all of them; values and attributes checked). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 + crates/clawhdf5-format/src/shared_message.rs | 10 +- crates/clawhdf5/tests/sohm_interop.rs | 141 +++++++++++++++++++ 3 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5/tests/sohm_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ce688d1..dc2dfa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -345,6 +345,13 @@ header or B-tree) still fails the call. `clawhdf5-format` gains `attribute::extract_attributes_tolerant`; `extract_attributes_full` stays strict. +- `clawhdf5-format` reader — files with shared object header messages + (SOHM, `H5Pset_shared_mesg_index`): a datatype, dataspace, filter pipeline + or attribute stored in the file's SOHM heap failed with "invalid shared + message version: 2" — only shared fill values loaded the SOHM table — so + such files' datasets and attributes could not be read. + `shared_message::resolve_shared_message` now loads the table when a + reference needs it (36 cases of the audit's read matrix). - `clawhdf5-format` writer — **files libhdf5 rejects or reads wrong:** - Extensible Array (one unlimited dimension): chunks from index 244 on were written but never indexed and read as 0, by libhdf5 and by us. diff --git a/crates/clawhdf5-format/src/shared_message.rs b/crates/clawhdf5-format/src/shared_message.rs index 33334fa..fc7842d 100644 --- a/crates/clawhdf5-format/src/shared_message.rs +++ b/crates/clawhdf5-format/src/shared_message.rs @@ -529,7 +529,8 @@ pub fn message_data<'a>( /// /// For type 1/3 (shared in another object header), reads the target object header /// and finds the message of the specified type. -/// For type 2 (SOHM), uses the fractal heap from the SOHM table. +/// For type 2 (SOHM), uses the fractal heap from the file's SOHM table, +/// loaded from the superblock extension on demand. pub fn resolve_shared_message( file_data: &[u8], shared_ref: &SharedMessageRef, @@ -537,13 +538,18 @@ pub fn resolve_shared_message( offset_size: u8, length_size: u8, ) -> Result, FormatError> { + let table = if shared_ref.heap_id.is_some() { + load_sohm_table(file_data, offset_size, length_size)? + } else { + None + }; resolve_shared_message_with_sohm( file_data, shared_ref, target_msg_type, offset_size, length_size, - None, + table.as_ref(), ) } diff --git a/crates/clawhdf5/tests/sohm_interop.rs b/crates/clawhdf5/tests/sohm_interop.rs new file mode 100644 index 0000000..a259313 --- /dev/null +++ b/crates/clawhdf5/tests/sohm_interop.rs @@ -0,0 +1,141 @@ +//! Files with shared object header messages (SOHM: datatypes, dataspaces, +//! filter pipelines and attributes stored once in a file-wide heap and +//! referenced by heap ID), written by libhdf5 through h5py. +//! +//! Skipped when python3 with h5py is unavailable, unless +//! `CLAWHDF5_REQUIRE_INTEROP=1`. + +use std::process::Command; + +use clawhdf5::{AttrValue, File}; + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn run_python(script: &str) -> String { + let output = Command::new(python()) + .args(["-c", script]) + .output() + .expect("failed to run python"); + if !output.status.success() { + panic!( + "Python script failed:\nSTDOUT: {}\nSTDERR: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// h5py has no binding for the SOHM property-list calls, so they go through +/// the libhdf5 that h5py bundles. `None` when that library is not found. +fn sohm_file(path: &str, libver: &str, mesg_types: u32) -> Option<()> { + let out = run_python(&format!( + "import ctypes, glob, os, h5py, numpy as np\n\ + libs = glob.glob(os.path.join(os.path.dirname(h5py.__file__), '..', 'h5py.libs', 'libhdf5-*.so*'))\n\ + if not libs:\n\ + \x20 print('nolib'); raise SystemExit\n\ + lib = ctypes.CDLL(libs[0])\n\ + lib.H5Pset_shared_mesg_nindexes.argtypes = [ctypes.c_int64, ctypes.c_uint]\n\ + lib.H5Pset_shared_mesg_index.argtypes = [ctypes.c_int64, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint]\n\ + fcpl = h5py.h5p.create(h5py.h5p.FILE_CREATE)\n\ + assert lib.H5Pset_shared_mesg_nindexes(fcpl.id, 1) >= 0\n\ + assert lib.H5Pset_shared_mesg_index(fcpl.id, 0, {mesg_types}, 1) >= 0\n\ + fapl = h5py.h5p.create(h5py.h5p.FILE_ACCESS)\n\ + low = h5py.h5f.LIBVER_EARLIEST if '{libver}' == 'earliest' else h5py.h5f.LIBVER_LATEST\n\ + fapl.set_libver_bounds(low, h5py.h5f.LIBVER_LATEST)\n\ + fid = h5py.h5f.create(r'{path}'.encode(), h5py.h5f.ACC_TRUNC, fcpl=fcpl, fapl=fapl)\n\ + with h5py.File(fid) as f:\n\ + \x20 for i in range(4):\n\ + \x20 ds = f.create_dataset('d%d' % i, shape=(50,), dtype=' = (0..20).map(|v| f64::from(v) + 2.0).collect(); + expected.resize(50, -9.0); + assert_eq!( + d2.read_f64().unwrap_or_else(|e| panic!("{case}: {e}")), + expected, + "{case}" + ); + let (attrs, errors) = d2.attrs_with_errors().unwrap(); + assert!(errors.is_empty(), "{case}: {errors:?}"); + let shared: Vec = (0..10).map(f64::from).collect(); + assert!( + matches!(&attrs["shared_attr"], AttrValue::F64Array(v) if *v == shared), + "{case}: {:?}", + attrs.get("shared_attr") + ); + assert!( + matches!(&attrs["units"], AttrValue::String(s) if s == "m/s"), + "{case}: {:?}", + attrs.get("units") + ); + assert_eq!( + f.dataset("contig").unwrap().read_i32().unwrap(), + [0, 2, 4, 6, 8, 10, 12], + "{case}" + ); + } + } +} From b4a44a2e66adb86b87c45d8e1337a7d7f37fe546 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:10:09 -0500 Subject: [PATCH 18/24] feat(format): read unlimited and printf-style VDS mappings like libhdf5 Unlimited VDS mappings were refused, and printf-style source names ("f-%b.h5") were not expanded, so those regions read as fill (read-matrix case 0470: 29 of 30 values wrong). All 7 virtual datasets in the libhdf5 test set use such mappings. Implement H5Dvirtual.c's semantics in the vds module: - %b is the block number, %% a literal %, other specifiers are an error; block j of the virtual selection comes from the source named with j, probing from 0 to the first missing source (printf gap 0); - unlimited source/virtual selections are clipped to what the source's current extent fills (H5S_hyper_get_clip_extent_match, partial last block included); - the extent is recomputed as H5Dget_space does (view "last available": the largest clip, never below what limited mappings need), exposed as vds::virtual_dataset_extent and used by Dataset::shape(); - a source in the other byte order is byte-swapped; other conversions stay an error. Tests: vds_interop::vds_printf_source_names, vds_unlimited_mappings_follow_source_extents (h5py low-level API, earliest and latest format) and vds_libhdf5_test_files (vds-eiger, 4_vds and vds-percival-unlim-maxmin from HDF5's tools/test/testfiles/vds, committed as fixtures) all compare shape and values with h5py; unit tests for the clip arithmetic, name parsing and mapping rules. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 + crates/clawhdf5-format/src/vds.rs | 762 ++++++++++++++++-- crates/clawhdf5/src/reader.rs | 17 +- crates/clawhdf5/tests/fixtures/vds/4_0.h5 | Bin 0 -> 4581 bytes crates/clawhdf5/tests/fixtures/vds/4_1.h5 | Bin 0 -> 4581 bytes crates/clawhdf5/tests/fixtures/vds/4_2.h5 | Bin 0 -> 4581 bytes crates/clawhdf5/tests/fixtures/vds/4_vds.h5 | Bin 0 -> 5496 bytes crates/clawhdf5/tests/fixtures/vds/README.md | 14 + crates/clawhdf5/tests/fixtures/vds/a.h5 | Bin 0 -> 7736 bytes crates/clawhdf5/tests/fixtures/vds/b.h5 | Bin 0 -> 7736 bytes crates/clawhdf5/tests/fixtures/vds/c.h5 | Bin 0 -> 7736 bytes crates/clawhdf5/tests/fixtures/vds/d.h5 | Bin 0 -> 7736 bytes crates/clawhdf5/tests/fixtures/vds/f-0.h5 | Bin 0 -> 4144 bytes crates/clawhdf5/tests/fixtures/vds/f-3.h5 | Bin 0 -> 4144 bytes .../clawhdf5/tests/fixtures/vds/vds-eiger.h5 | Bin 0 -> 5496 bytes .../fixtures/vds/vds-percival-unlim-maxmin.h5 | Bin 0 -> 5496 bytes crates/clawhdf5/tests/vds_interop.rs | 159 ++++ docs/known-issues.md | 8 +- 18 files changed, 889 insertions(+), 84 deletions(-) create mode 100644 crates/clawhdf5/tests/fixtures/vds/4_0.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/4_1.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/4_2.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/4_vds.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/README.md create mode 100644 crates/clawhdf5/tests/fixtures/vds/a.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/b.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/c.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/d.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/f-0.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/f-3.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/vds-eiger.h5 create mode 100644 crates/clawhdf5/tests/fixtures/vds/vds-percival-unlim-maxmin.h5 diff --git a/CHANGELOG.md b/CHANGELOG.md index de54b91..aabbce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -290,6 +290,19 @@ **Behaviour change:** the raw-read API (`read_raw_data_full*`), which has no fill value, now returns an error for a virtual dataset with unmapped elements instead of zeros. + - Unlimited and printf-style mappings are supported (all 7 VDS files in the + libhdf5 test set are such mappings, e.g. Eiger/Percival detector layouts). + `%b` in a source file or dataset name is the block number and `%%` a + literal `%` (other `%` sequences are an error, as in libhdf5); block `j` + is read from the source named with `j`, probing from 0 up to the first + missing source. Unlimited source/virtual selections cover as much as the + source's current extent fills, including a partial last block. As + libhdf5 does on `H5Dget_space`, the extent is recomputed from the sources + present (default "last available" view, printf gap 0) — + `vds::virtual_dataset_extent`, used by `Dataset::shape()` — so e.g. + `vds-eiger.h5` is `[5, 10, 10]`, not its stored `[20, 10, 10]`. A source + stored in the other byte order is byte-swapped (libhdf5 converts); + other type conversions remain an error. - Hyperslab selection versions 1 and 2 were refused ("only version-3 hyperslab selections are supported"). Version 1 is what libhdf5 writes for every VDS created with the default format bounds (h5py's default), so diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index 3ac0232..099c14a 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -45,12 +45,149 @@ fn vds_err(msg: impl Into) -> FormatError { FormatError::ChunkedReadError(msg.into()) } +/// Upper bound on the printf-style source datasets probed for one mapping. +const MAX_PRINTF_BLOCKS: u64 = 1 << 20; + +/// A source file or dataset name, parsed for printf-style `%b` block-number +/// substitutions (`H5D_virtual_parse_source_name`): `%b` is the block +/// number, `%%` a literal `%`, and any other `%` sequence is invalid. +#[derive(Debug, Clone, PartialEq)] +struct SourceName { + /// Literal text around the substitutions: `segments.len() == subs + 1`. + segments: Vec, +} + +impl SourceName { + fn parse(name: &str) -> Result { + let mut segments = vec![String::new()]; + let mut chars = name.chars(); + while let Some(c) = chars.next() { + if c != '%' { + segments.last_mut().expect("never empty").push(c); + continue; + } + match chars.next() { + Some('b') => segments.push(String::new()), + Some('%') => segments.last_mut().expect("never empty").push('%'), + _ => { + return Err(vds_err(format!( + "invalid format specifier in VDS source name {name:?}" + ))); + } + } + } + Ok(SourceName { segments }) + } + + /// Number of `%b` substitutions. + fn subs(&self) -> usize { + self.segments.len() - 1 + } + + /// The name with every `%b` replaced by `block`. + fn build(&self, block: u64) -> String { + let mut out = String::new(); + for (i, seg) in self.segments.iter().enumerate() { + if i > 0 { + out.push_str(&format!("{block}")); + } + out.push_str(seg); + } + out + } +} + +/// How a mapping's selections relate, as libhdf5 classifies them. +#[derive(Debug, Clone, Copy, PartialEq)] +enum Kind { + /// Both selections have a fixed size. + Fixed, + /// Both are unlimited (in `vdim` / `sdim`): the mapping grows with the + /// source dataset's extent. + Unlimited { vdim: usize, sdim: usize }, + /// The virtual selection repeats a block without limit in `vdim`; block + /// `j` comes from the source named by substituting `j` for `%b`. + Printf { vdim: usize }, +} + /// One mapping with its selections decoded. struct Mapping { - file: String, - dataset: String, + file: SourceName, + dataset: SourceName, vsel: SerializedSelection, ssel: SerializedSelection, + kind: Kind, +} + +impl Mapping { + fn new(m: VdsMapping) -> Result { + let (vsel, _) = SerializedSelection::decode(&m.virtual_selection)?; + let (ssel, _) = SerializedSelection::decode(&m.source_selection)?; + let file = SourceName::parse(&m.source_file)?; + let dataset = SourceName::parse(&m.source_dataset)?; + let subs = file.subs() + dataset.subs(); + // The checks of H5D_virtual_check_mapping_pre/_post. + let kind = match (vsel.unlimited_dim(), ssel.unlimited_dim()) { + (Some(vdim), None) => { + if subs == 0 { + return Err(vds_err( + "unlimited virtual selection with a limited source selection \ + and no %b in the source names", + )); + } + match &vsel { + SerializedSelection::Regular { count, block, .. } + if count[vdim] == UNLIMITED && block[vdim] != UNLIMITED => {} + _ => { + return Err(vds_err( + "printf VDS mapping needs a virtual selection with an unlimited count", + )); + } + } + Kind::Printf { vdim } + } + (Some(vdim), Some(sdim)) => { + if non_unlimited_elements(&vsel, vdim) != non_unlimited_elements(&ssel, sdim) { + return Err(vds_err( + "unlimited VDS mapping: virtual and source selections differ \ + outside the unlimited dimension", + )); + } + Kind::Unlimited { vdim, sdim } + } + (None, Some(_)) => { + return Err(vds_err( + "VDS mapping with an unlimited source selection and a limited \ + virtual selection is not supported", + )); + } + (None, None) => Kind::Fixed, + }; + if subs > 0 && !matches!(kind, Kind::Printf { .. }) { + return Err(vds_err( + "%b in a VDS source name without an unlimited virtual selection", + )); + } + Ok(Mapping { + file, + dataset, + vsel, + ssel, + kind, + }) + } +} + +/// Elements a regular selection selects outside dimension `skip`. +fn non_unlimited_elements(sel: &SerializedSelection, skip: usize) -> Option { + let SerializedSelection::Regular { count, block, .. } = sel else { + return None; + }; + (0..count.len()) + .filter(|&d| d != skip) + .try_fold(1u64, |acc, d| { + acc.checked_mul(count[d].checked_mul(block[d])?) + }) } /// Load and decode the mapping list of a virtual layout. @@ -82,49 +219,269 @@ fn load_mappings( })?; parse_vds_mappings(&obj.data, length_size)? .into_iter() - .map(|m: VdsMapping| { - let (vsel, _) = SerializedSelection::decode(&m.virtual_selection)?; - let (ssel, _) = SerializedSelection::decode(&m.source_selection)?; - Ok(Mapping { - file: m.source_file, - dataset: m.source_dataset, - vsel, - ssel, - }) - }) + .map(Mapping::new) .collect() } +/// `H5S__hyper_get_clip_diminfo`: the count and block a regular selection has +/// in its unlimited dimension once clipped to `clip`. +fn clip_diminfo(start: u64, stride: u64, count: u64, block: u64, clip: u64) -> (u64, u64) { + if start >= clip { + if block == UNLIMITED { + (count, 0) + } else { + (0, block) + } + } else if block == UNLIMITED || block == stride { + (1, clip - start) + } else { + ((clip - start).div_ceil(stride.max(1)), block) + } +} + +/// The unlimited-dimension parameters (start, stride, count, block) of a +/// regular selection. +fn unlim_diminfo(sel: &SerializedSelection, dim: usize) -> Result<[u64; 4], FormatError> { + match sel { + SerializedSelection::Regular { + start, + stride, + count, + block, + } => Ok([start[dim], stride[dim], count[dim], block[dim]]), + _ => Err(vds_err( + "unlimited VDS selection is not a regular hyperslab", + )), + } +} + +/// `H5S_hyper_get_clip_extent_match` with `incl_trail = false` (the +/// "last available" view): the extent to clip `clip_sel` (unlimited in +/// `clip_dim`) to so that it holds as many slices as `match_sel` (unlimited +/// in `match_dim`) holds when clipped to `match_clip`. +fn clip_extent_match( + clip_sel: &SerializedSelection, + clip_dim: usize, + match_sel: &SerializedSelection, + match_dim: usize, + match_clip: u64, +) -> Result { + let overflow = || FormatError::Overflow("VDS clip extent overflow".into()); + let [mstart, mstride, mcount, mblock] = unlim_diminfo(match_sel, match_dim)?; + let (count, block) = clip_diminfo(mstart, mstride, mcount, mblock, match_clip); + let slices = if block == 0 || count == 0 { + 0 + } else if count == 1 { + block + } else { + let mut n = block.checked_mul(count).ok_or_else(overflow)?; + let span = mstride + .checked_mul(count - 1) + .and_then(|s| s.checked_add(block)) + .ok_or_else(overflow)?; + let room = match_clip - mstart; + if span > room { + n -= span - room; + } + n + }; + + // H5S__hyper_get_clip_extent_real + let [start, stride, _, block] = unlim_diminfo(clip_sel, clip_dim)?; + if slices == 0 { + return Ok(0); + } + let extent = if block == UNLIMITED || block == stride { + start.checked_add(slices) + } else { + let full = slices / block; + let rem = slices - full * block; + if rem > 0 { + full.checked_mul(stride) + .and_then(|o| start.checked_add(o)) + .and_then(|e| e.checked_add(rem)) + } else { + (full - 1) + .checked_mul(stride) + .and_then(|o| start.checked_add(o)) + .and_then(|e| e.checked_add(block)) + } + }; + extent.ok_or_else(overflow) +} + +/// How each mapping is read, and the resulting extent. +struct Plan { + dims: Vec, + steps: Vec, +} + +#[derive(Clone, Copy)] +enum Step { + Fixed, + /// Read with the virtual selection clipped to `vclip` and the source + /// selection to the source's extent; `None` if the source is missing. + Unlimited(Option), + /// Read blocks `0..blocks`. + Printf(u64), +} + +/// Work out the extent libhdf5 gives the virtual dataset +/// (`H5D__virtual_set_extent_unlim`, default view `H5D_VDS_LAST_AVAILABLE` +/// with a printf gap of 0) and how much of each unlimited mapping is read. +fn plan(mappings: &[Mapping], stored: &[u64], sources: &mut Sources) -> Result { + let overflow = || FormatError::Overflow("VDS extent overflow".into()); + let rank = stored.len(); + let mut new_dims: Vec> = vec![None; rank]; + // Minimum extent needed by the limited parts of every virtual selection + // (H5D_virtual_update_min_dims). + let mut min_dims = vec![0u64; rank]; + let mut steps = Vec::with_capacity(mappings.len()); + + for m in mappings { + let skip = match m.kind { + Kind::Unlimited { vdim, .. } | Kind::Printf { vdim } => Some(vdim), + Kind::Fixed => None, + }; + if let Some(ends) = selection_bounds_end(&m.vsel)? { + if ends.len() != rank { + return Err(vds_err("VDS selection rank does not match dataspace rank")); + } + for d in (0..rank).filter(|&d| Some(d) != skip) { + min_dims[d] = min_dims[d].max(ends[d].checked_add(1).ok_or_else(overflow)?); + } + } + + let (vdim, clip, step) = match m.kind { + Kind::Fixed => { + steps.push(Step::Fixed); + continue; + } + Kind::Unlimited { vdim, sdim } => { + match sources.dims(&m.file.build(0), &m.dataset.build(0))? { + Some(src_dims) => { + let extent = *src_dims.get(sdim).ok_or_else(|| { + vds_err("VDS source rank does not match its selection") + })?; + let clip = clip_extent_match(&m.vsel, vdim, &m.ssel, sdim, extent)?; + (vdim, clip, Step::Unlimited(Some(clip))) + } + None => (vdim, 0, Step::Unlimited(None)), + } + } + Kind::Printf { vdim } => { + // With a gap of 0 the search stops at the first missing + // source dataset. + let mut found = 0u64; + while sources + .dims(&m.file.build(found), &m.dataset.build(found))? + .is_some() + { + found += 1; + if found > MAX_PRINTF_BLOCKS { + return Err(vds_err("too many printf-style VDS source datasets")); + } + } + let clip = if found == 0 { + 0 + } else { + // End of block `found - 1` in the unlimited dimension. + let [start, stride, _, block] = unlim_diminfo(&m.vsel, vdim)?; + (found - 1) + .checked_mul(stride) + .and_then(|o| start.checked_add(o)) + .and_then(|e| e.checked_add(block)) + .ok_or_else(overflow)? + }; + (vdim, clip, Step::Printf(found)) + } + }; + if vdim >= rank { + return Err(vds_err("VDS selection rank does not match dataspace rank")); + } + new_dims[vdim] = Some(new_dims[vdim].map_or(clip, |n| n.max(clip))); + steps.push(step); + } + + let dims = (0..rank) + .map(|d| match new_dims[d] { + None => stored[d], + Some(n) => n.max(min_dims[d]), + }) + .collect(); + Ok(Plan { dims, steps }) +} + +/// The last selected coordinate in each dimension (`H5S_SELECT_BOUNDS`), +/// ignoring any unlimited dimension; `None` for ALL/NONE and empty selections. +fn selection_bounds_end(sel: &SerializedSelection) -> Result>, FormatError> { + let overflow = || FormatError::Overflow("VDS selection bounds overflow".into()); + match sel { + SerializedSelection::All | SerializedSelection::None => Ok(None), + SerializedSelection::Regular { + start, + stride, + count, + block, + } => { + if count.contains(&0) || block.contains(&0) { + return Ok(None); + } + let mut ends = Vec::with_capacity(start.len()); + for d in 0..start.len() { + if count[d] == UNLIMITED || block[d] == UNLIMITED { + ends.push(0); + continue; + } + let end = (count[d] - 1) + .checked_mul(stride[d]) + .and_then(|o| start[d].checked_add(o)) + .and_then(|e| e.checked_add(block[d] - 1)) + .ok_or_else(overflow)?; + ends.push(end); + } + Ok(Some(ends)) + } + SerializedSelection::Blocks { rank, ends, .. } => { + if ends.is_empty() { + return Ok(None); + } + let mut max = vec![0u64; *rank]; + for e in ends.chunks_exact(*rank) { + for d in 0..*rank { + max[d] = max[d].max(e[d]); + } + } + Ok(Some(max)) + } + } +} + /// The virtual dataset's extent as libhdf5 reports it (`H5Dget_space`). /// /// For a virtual dataset whose mappings are all of fixed size this is the -/// stored dataspace. +/// stored dataspace. With unlimited or printf-style mappings libhdf5 +/// recomputes the unlimited dimension from the sources present (the default +/// "last available" view: the largest extent any mapping can fill), which +/// needs the source files, read through `resolver`. pub fn virtual_dataset_extent( file_data: &[u8], layout: &DataLayout, dataspace: &Dataspace, _offset_size: u8, length_size: u8, - _resolver: Option<&VdsFileResolver>, + resolver: Option<&VdsFileResolver>, ) -> Result, FormatError> { let mappings = load_mappings(file_data, layout, length_size)?; - check_fixed(&mappings)?; - Ok(dataspace.dimensions.clone()) -} - -fn check_fixed(mappings: &[Mapping]) -> Result<(), FormatError> { - if mappings - .iter() - .any(|m| m.vsel.unlimited_dim().is_some() || m.ssel.unlimited_dim().is_some()) - { - return Err(vds_err( - "unlimited virtual dataset mappings are not supported", - )); + if mappings.iter().all(|m| m.kind == Kind::Fixed) { + return Ok(dataspace.dimensions.clone()); } - Ok(()) + let mut sources = Sources::new(file_data, resolver); + Ok(plan(&mappings, &dataspace.dimensions, &mut sources)?.dims) } -/// Read a whole virtual dataset. +/// Read a whole virtual dataset, at the extent +/// [`virtual_dataset_extent`] reports. /// /// `fill` is the virtual dataset's fill value (from its fill value message; /// `None` for the default of zeros); every element no mapping supplies holds @@ -137,20 +494,13 @@ pub fn read_virtual_dataset( dataspace: &Dataspace, datatype: &Datatype, fill: Option<&[u8]>, - offset_size: u8, + _offset_size: u8, length_size: u8, resolver: Option<&VdsFileResolver>, ) -> Result { let mappings = load_mappings(file_data, layout, length_size)?; - check_fixed(&mappings)?; - let dims = virtual_dataset_extent( - file_data, - layout, - dataspace, - offset_size, - length_size, - resolver, - )?; + let mut sources = Sources::new(file_data, resolver); + let Plan { dims, steps } = plan(&mappings, &dataspace.dimensions, &mut sources)?; let elem_size = datatype.type_size() as usize; let total = dims @@ -167,14 +517,46 @@ pub fn read_virtual_dataset( } let mut mapped = vec![false; usize::try_from(total).map_err(|_| vds_err("VDS too large"))?]; - let mut sources = Sources::new(file_data, resolver); - for m in &mappings { - let Some(src) = sources.dataset(&m.file, &m.dataset, datatype)? else { - continue; // missing source file or dataset: fill - }; - let vidx = selection_indices(&m.vsel, &dims, None)?; - let sidx = selection_indices(&m.ssel, &src.dims, None)?; - scatter(&mut data, &mut mapped, &src.raw, &vidx, &sidx, elem_size)?; + for (m, step) in mappings.iter().zip(&steps) { + match (*step, m.kind) { + (Step::Fixed, _) => { + let (file, dset) = (m.file.build(0), m.dataset.build(0)); + let Some(src) = sources.dataset(&file, &dset, datatype)? else { + continue; // missing source file or dataset: fill + }; + let vidx = selection_indices(&m.vsel, &dims, None)?; + let sidx = selection_indices(&m.ssel, &src.dims, None)?; + scatter(&mut data, &mut mapped, &src.raw, &vidx, &sidx, elem_size)?; + } + (Step::Unlimited(Some(vclip)), Kind::Unlimited { vdim, sdim }) => { + let (file, dset) = (m.file.build(0), m.dataset.build(0)); + let Some(src) = sources.dataset(&file, &dset, datatype)? else { + continue; + }; + let vidx = selection_indices(&m.vsel, &dims, Some((vdim, vclip)))?; + let extent = *src + .dims + .get(sdim) + .ok_or_else(|| vds_err("VDS source rank does not match its selection"))?; + let sidx = selection_indices(&m.ssel, &src.dims, Some((sdim, extent)))?; + scatter(&mut data, &mut mapped, &src.raw, &vidx, &sidx, elem_size)?; + } + (Step::Unlimited(None), _) => {} + (Step::Printf(blocks), Kind::Printf { vdim }) => { + for j in 0..blocks { + let Some(src) = + sources.dataset(&m.file.build(j), &m.dataset.build(j), datatype)? + else { + continue; + }; + let vblock = unlim_block(&m.vsel, vdim, j)?; + let vidx = selection_indices(&vblock, &dims, None)?; + let sidx = selection_indices(&m.ssel, &src.dims, None)?; + scatter(&mut data, &mut mapped, &src.raw, &vidx, &sidx, elem_size)?; + } + } + _ => return Err(vds_err("internal error: VDS plan does not match mapping")), + } } let unmapped = mapped.iter().filter(|&&m| !m).count() as u64; @@ -185,6 +567,37 @@ pub fn read_virtual_dataset( }) } +/// `H5S_hyper_get_unlim_block`: block `j` of a selection whose count is +/// unlimited in `dim`. +fn unlim_block( + sel: &SerializedSelection, + dim: usize, + j: u64, +) -> Result { + let SerializedSelection::Regular { + start, + stride, + count, + block, + } = sel + else { + return Err(vds_err("printf VDS selection is not a regular hyperslab")); + }; + let mut start = start.clone(); + let mut count = count.clone(); + start[dim] = j + .checked_mul(stride[dim]) + .and_then(|o| start[dim].checked_add(o)) + .ok_or_else(|| FormatError::Overflow("VDS block start overflow".into()))?; + count[dim] = 1; + Ok(SerializedSelection::Regular { + start, + stride: stride.clone(), + count, + block: block.clone(), + }) +} + /// Copy source element `sidx[i]` to virtual element `vidx[i]` for every `i`. fn scatter( out: &mut [u8], @@ -395,6 +808,15 @@ impl<'a, 'r> Sources<'a, 'r> { Ok(self.cached_file.as_ref().and_then(|(_, b)| b.as_deref())) } + /// The extent of source dataset `path` in file `file`, or `None` when + /// either does not exist. + fn dims(&mut self, file: &str, path: &str) -> Result>, FormatError> { + let Some(bytes) = self.file(file)? else { + return Ok(None); + }; + Ok(open_source(bytes, path)?.map(|s| s.dataspace.dimensions)) + } + /// Read source dataset `path` from file `file`, or `None` when either /// does not exist. Its datatype must be the virtual dataset's: libhdf5 /// converts between types here, which is not supported. @@ -407,20 +829,37 @@ impl<'a, 'r> Sources<'a, 'r> { let Some(bytes) = self.file(file)? else { return Ok(None); }; - read_source(bytes, path, datatype) + let Some(src) = open_source(bytes, path)? else { + return Ok(None); + }; + read_source(bytes, src, path, datatype).map(Some) } } -/// Read source dataset `path` of the file in `file_data` in full (its own -/// fill value applied to unallocated chunks), or `None` if it does not exist. -fn read_source( - file_data: &[u8], +/// An opened source dataset's object header. +struct OpenSource { + offset_size: u8, + length_size: u8, + header: crate::object_header::ObjectHeader, + dataspace: Dataspace, +} + +fn source_message<'h>( + src: &'h OpenSource, path: &str, - datatype: &Datatype, -) -> Result, FormatError> { - use crate::filter_pipeline::FilterPipeline; + t: crate::message_type::MessageType, +) -> Result<&'h crate::object_header::HeaderMessage, FormatError> { + src.header + .messages + .iter() + .find(|m| m.msg_type == t) + .ok_or_else(|| vds_err(format!("VDS source {path} has no {t:?} message"))) +} + +/// Open source dataset `path` of the file in `file_data`, or `None` if there +/// is no such object (libhdf5 reads a missing source as fill). +fn open_source(file_data: &[u8], path: &str) -> Result, FormatError> { use crate::message_type::MessageType; - use crate::object_header::ObjectHeader; use crate::shared_message::message_data_with_sohm; let sig = crate::signature::find_signature(file_data)?; @@ -431,30 +870,55 @@ fn read_source( Err(FormatError::PathNotFound(_)) => return Ok(None), Err(e) => return Err(e), }; - let hdr = ObjectHeader::parse(file_data, addr as usize, os, ls)?; - let msg = |t: MessageType| { - hdr.messages - .iter() - .find(|m| m.msg_type == t) - .ok_or_else(|| vds_err(format!("VDS source {path} has no {t:?} message"))) + let header = crate::object_header::ObjectHeader::parse(file_data, addr as usize, os, ls)?; + let mut src = OpenSource { + offset_size: os, + length_size: ls, + header, + dataspace: Dataspace { + space_type: crate::dataspace::DataspaceType::Null, + rank: 0, + dimensions: Vec::new(), + max_dimensions: None, + }, }; - let dataspace = Dataspace::parse( - &message_data_with_sohm(file_data, msg(MessageType::Dataspace)?, os, ls)?, - ls, - )?; - let (src_type, _) = Datatype::parse(&message_data_with_sohm( - file_data, - msg(MessageType::Datatype)?, - os, - ls, - )?)?; - if &src_type != datatype { + let ds_msg = source_message(&src, path, MessageType::Dataspace)?; + src.dataspace = Dataspace::parse(&message_data_with_sohm(file_data, ds_msg, os, ls)?, ls)?; + Ok(Some(src)) +} + +/// Read an opened source dataset in full (its own fill value applied to +/// unallocated chunks). +fn read_source( + file_data: &[u8], + src: OpenSource, + path: &str, + datatype: &Datatype, +) -> Result { + use crate::filter_pipeline::FilterPipeline; + use crate::message_type::MessageType; + use crate::shared_message::message_data_with_sohm; + + let (os, ls) = (src.offset_size, src.length_size); + let dt_msg = source_message(&src, path, MessageType::Datatype)?; + let (src_type, _) = Datatype::parse(&message_data_with_sohm(file_data, dt_msg, os, ls)?)?; + // libhdf5 converts each source to the virtual dataset's type. Only the + // conversion that is a pure byte swap is done here. + let swap = if &src_type == datatype { + false + } else if differs_only_in_byte_order(&src_type, datatype) { + true + } else { return Err(vds_err(format!( "VDS source {path} has a different datatype from the virtual dataset \ - (type conversion is not supported)" + (only a byte-order conversion is supported)" ))); - } - let layout = DataLayout::parse(&msg(MessageType::DataLayout)?.data, os, ls)?; + }; + let layout = DataLayout::parse( + &source_message(&src, path, MessageType::DataLayout)?.data, + os, + ls, + )?; // A source that is itself virtual could form a cycle (A -> B -> A) and // recurse without bound. Nested virtual sources are not supported. if matches!(layout, DataLayout::Virtual { .. }) { @@ -462,7 +926,8 @@ fn read_source( "virtual dataset source is itself virtual (unsupported)", )); } - let pipeline = hdr + let pipeline = src + .header .messages .iter() .find(|m| m.msg_type == MessageType::FilterPipeline) @@ -471,10 +936,10 @@ fn read_source( }) .transpose()?; let raw = crate::fill_value::read_full_with_fill( - &hdr.messages, + &src.header.messages, file_data, &layout, - &dataspace, + &src.dataspace, src_type.type_size() as usize, os, ls, @@ -482,7 +947,7 @@ fn read_source( crate::data_read::read_raw_data_full( file_data, &layout, - &dataspace, + &src.dataspace, &src_type, pipeline.as_ref(), os, @@ -490,8 +955,141 @@ fn read_source( ) }, )?; - Ok(Some(SourceData { - dims: dataspace.dimensions, + let mut raw = raw; + if swap { + let size = datatype.type_size() as usize; + for element in raw.chunks_exact_mut(size) { + element.reverse(); + } + } + Ok(SourceData { + dims: src.dataspace.dimensions, raw, - })) + }) +} + +/// Whether two numeric types are identical apart from their byte order (both +/// little- or big-endian), so converting one to the other is a byte swap. +fn differs_only_in_byte_order(a: &Datatype, b: &Datatype) -> bool { + use crate::datatype::DatatypeByteOrder::{BigEndian, LittleEndian}; + let mut a = a.clone(); + match &mut a { + Datatype::FixedPoint { byte_order, .. } + | Datatype::FloatingPoint { byte_order, .. } + | Datatype::BitField { byte_order, .. } => { + *byte_order = match byte_order { + LittleEndian => BigEndian, + BigEndian => LittleEndian, + _ => return false, + } + } + _ => return false, + } + &a == b +} + +#[cfg(test)] +mod tests { + use super::*; + + fn regular(start: u64, stride: u64, count: u64, block: u64) -> SerializedSelection { + SerializedSelection::Regular { + start: vec![start], + stride: vec![stride], + count: vec![count], + block: vec![block], + } + } + + #[test] + fn printf_names_follow_libhdf5() { + let n = SourceName::parse("f-%b.h5").unwrap(); + assert_eq!(n.subs(), 1); + assert_eq!(n.build(12), "f-12.h5"); + let n = SourceName::parse("100%%_%b_%b").unwrap(); + assert_eq!(n.build(3), "100%_3_3"); + let n = SourceName::parse("plain%%name").unwrap(); + assert_eq!((n.subs(), n.build(7)), (0, "plain%name".to_string())); + // Anything else after '%' (or a trailing '%') is invalid. + assert!(SourceName::parse("a%d").is_err()); + assert!(SourceName::parse("a%").is_err()); + } + + #[test] + fn clip_extent_matches_libhdf5_arithmetic() { + // 7 source slices (contiguous unlimited source) into blocks of 3 + // every 4: two full blocks and one slice of a third -> extent 9. + let src = regular(0, 1, UNLIMITED, 1); + let v = regular(0, 4, UNLIMITED, 3); + assert_eq!(clip_extent_match(&v, 0, &src, 0, 7).unwrap(), 9); + // Exactly two blocks: the extent ends at the end of the last block. + assert_eq!(clip_extent_match(&v, 0, &src, 0, 6).unwrap(), 7); + // An empty source gives an empty mapping. + assert_eq!(clip_extent_match(&v, 0, &src, 0, 0).unwrap(), 0); + // Unlimited block: the extent is start + slices. + let vb = regular(2, 1, 1, UNLIMITED); + assert_eq!(clip_extent_match(&vb, 0, &src, 0, 5).unwrap(), 7); + // A strided source clipped mid-block counts only the selected slices. + let src2 = regular(1, 4, UNLIMITED, 2); // 1,2, 5,6, 9,10 ... + let dense = regular(0, 1, UNLIMITED, 1); + assert_eq!(clip_extent_match(&dense, 0, &src2, 0, 6).unwrap(), 3); + } + + #[test] + fn clipped_selection_drops_the_partial_tail() { + let v = regular(0, 4, UNLIMITED, 3); + assert_eq!( + selection_indices(&v, &[20], Some((0, 9))).unwrap(), + vec![0, 1, 2, 4, 5, 6, 8] + ); + // Unclipped unlimited selections cannot be enumerated. + assert!(selection_indices(&v, &[20], None).is_err()); + } + + #[test] + fn unlimited_mapping_rules() { + let sel = |s: &SerializedSelection| -> Vec { + // Serialize as version 2 (8-byte regular). + let SerializedSelection::Regular { + start, + stride, + count, + block, + } = s + else { + unreachable!() + }; + let mut b = Vec::new(); + b.extend_from_slice(&2u32.to_le_bytes()); + b.extend_from_slice(&2u32.to_le_bytes()); + b.push(1); + b.extend_from_slice(&0u32.to_le_bytes()); + b.extend_from_slice(&(start.len() as u32).to_le_bytes()); + for d in 0..start.len() { + for v in [start[d], stride[d], count[d], block[d]] { + b.extend_from_slice(&v.to_le_bytes()); + } + } + b + }; + let mapping = |file: &str, v: &SerializedSelection, s: &SerializedSelection| VdsMapping { + source_file: file.into(), + source_dataset: "d".into(), + source_selection: sel(s), + virtual_selection: sel(v), + }; + let unlim = regular(0, 10, UNLIMITED, 10); + let fixed = regular(0, 1, 1, 10); + // Unlimited virtual + limited source needs %b in a name ... + assert!(Mapping::new(mapping("f.h5", &unlim, &fixed)).is_err()); + let m = Mapping::new(mapping("f%b.h5", &unlim, &fixed)).unwrap(); + assert_eq!(m.kind, Kind::Printf { vdim: 0 }); + // ... and %b is refused anywhere else. + assert!(Mapping::new(mapping("f%b.h5", &fixed, &fixed)).is_err()); + let src_unlim = regular(0, 1, UNLIMITED, 1); + let m = Mapping::new(mapping("f.h5", &unlim, &src_unlim)).unwrap(); + assert_eq!(m.kind, Kind::Unlimited { vdim: 0, sdim: 0 }); + // An unlimited source into a limited virtual selection is refused. + assert!(Mapping::new(mapping("f.h5", &fixed, &src_unlim)).is_err()); + } } diff --git a/crates/clawhdf5/src/reader.rs b/crates/clawhdf5/src/reader.rs index 2e7bd3d..20e2c33 100644 --- a/crates/clawhdf5/src/reader.rs +++ b/crates/clawhdf5/src/reader.rs @@ -809,7 +809,22 @@ impl<'f> Dataset<'f> { fn dataspace(&self) -> Result { let data = self.required_payload(MessageType::Dataspace)?; - Ok(Dataspace::parse(&data, self.file.length_size())?) + let mut ds = Dataspace::parse(&data, self.file.length_size())?; + // libhdf5 reports a virtual dataset with unlimited or printf-style + // mappings at the extent its sources currently fill, not the stored + // one (`H5Dget_space`). + if let Ok(dl @ DataLayout::Virtual { .. }) = self.data_layout() { + let resolver = self.vds_resolver(); + ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent( + self.file.data.as_bytes(), + &dl, + &ds, + self.file.offset_size(), + self.file.length_size(), + Some(&resolver), + )?; + } + Ok(ds) } fn data_layout(&self) -> Result { diff --git a/crates/clawhdf5/tests/fixtures/vds/4_0.h5 b/crates/clawhdf5/tests/fixtures/vds/4_0.h5 new file mode 100644 index 0000000000000000000000000000000000000000..5e71d20ca6a499ef7a2d9dc5ed9b4f7023ef4e27 GIT binary patch literal 4581 zcmeHKy-EW?5T3og;1Llog7{P2BUoB0W@X7X3^N7TkonUY^TZ-a%<&msZT|0@L=>GC=qc!y6d_4LGbCSr6B zkB|2A-Bu^>I!NkrNeI961|nC#exM3n^m(n%Y&jp7C-#rGpYhnGim{MAH*p&(*cX&B zuz?|*>Yo-ue!pPjkT{;G(* z+uxn);L@l!-mOf16pe3dPqZFX~&VVi5cax(w4XoYreJp$~hXKa$^s&G%w9POX zqi5#Jg8a->JY+@hQ5{tQRX`O`1yli5Kow90Q~_1sFBIs#yzOP#>m~k$m_C6%hHVV5 SFr2x78w{Hm_A#8hhlei-8D*^i literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/4_1.h5 b/crates/clawhdf5/tests/fixtures/vds/4_1.h5 new file mode 100644 index 0000000000000000000000000000000000000000..edad46e9fa9c7f16d74a51abaeda99560661c468 GIT binary patch literal 4581 zcmeHKy-EW?5T3oe;1Urp+KAQFB4}x8ASbE(7wo(vIV(Xh5z^Y}V~7Yof}M8p2`sEE z^flC(-5E^oFkoTj%#od)Z@-<(ewkUwRlBt@Q(vqDnY9o>?4~k&vb{GvBC)JzJTdGq z+1~|zVt^ii1;!hkzs%d;O;bRogW(F8{0nN%U#~rxd4_fQ4Y`(gCJ`M4FL-1Dt*#1So}CK+I)^n9K|{51KrI`3C}Uiwl5_5&$M- zh*|}(23BCc0GY_d%)|&1{|n3jAPE+z{V*|Z1_iJ>MySUi?qg&~NlnX1EJ2<*^kNVI literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/4_vds.h5 b/crates/clawhdf5/tests/fixtures/vds/4_vds.h5 new file mode 100644 index 0000000000000000000000000000000000000000..64c2288f7e6138ce0ba6f643fa22e33cb6e1547f GIT binary patch literal 5496 zcmeHHJxc>Y5S_if;1Up{AOx#xY}G=+QZc6~#E4(mMKl3B!Ba_Vr+>oQe_(j@@yutA@$a~R3poYp z2Nv}H+KfZmP5(!;`P4hTD=ymu;;~o}%n%nEMYj3BwFACDXTd3(B z6aZN7FJG=)Z_sLWbANj^OyXwIqoIf~{|LxqD8@aE^P6LtVHq(1LkzhhM;%MylVnl- t`_Rey$JvX>cih>Y8;2Gc1IBX8(fH7bU7z4Fo;1}6}Rr>${ literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/README.md b/crates/clawhdf5/tests/fixtures/vds/README.md new file mode 100644 index 0000000..25a1371 --- /dev/null +++ b/crates/clawhdf5/tests/fixtures/vds/README.md @@ -0,0 +1,14 @@ +# VDS test files from libhdf5 + +Copied unchanged from the HDF Group's HDF5 repository, +`tools/test/testfiles/vds/` (the h5dump/h5ls VDS test data). HDF5 is +distributed under a BSD-style license (see `COPYING` in the HDF5 source). + +| File | What it exercises | +|---|---| +| `vds-eiger.h5` + `f-0.h5`, `f-3.h5` | printf-style source name `f-%b.h5`; `f-3.h5` lies past the first missing source and must be ignored (extent 5, not 20) | +| `4_vds.h5` + `4_0.h5`..`4_2.h5` | printf-style `4_%b.h5` with version-2 (1.10 format) hyperslab selections | +| `vds-percival-unlim-maxmin.h5` + `a.h5`..`d.h5` | four interleaved unlimited mappings whose sources have different lengths | + +Used by `crates/clawhdf5/tests/vds_interop.rs::vds_libhdf5_test_files`, +which compares our reads with h5py's. diff --git a/crates/clawhdf5/tests/fixtures/vds/a.h5 b/crates/clawhdf5/tests/fixtures/vds/a.h5 new file mode 100644 index 0000000000000000000000000000000000000000..fa1953587aa29feb90477ff684cb3e34311c7cfd GIT binary patch literal 7736 zcmeI0!Ab)$5QZnowuFMvdQp0iK7z*{Ep%m1Dk{>WkKrTeOL+Fudv9KSByYNtncq@H zuu|AV|16tHCX)&I__OTpQ!_t57>oy^C1f7||QWO0c962vdl z-;Xv25D#6#!oL~K9tn^D36KB@kN^pg011#lmjv{Ck2e`m1JnRDKn-lG0sKg$UH??= Sl0}#A5?q@ENPq;kA@B)9LQp6G literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/b.h5 b/crates/clawhdf5/tests/fixtures/vds/b.h5 new file mode 100644 index 0000000000000000000000000000000000000000..08449ca3909de3755669ea173f0e637a533cb794 GIT binary patch literal 7736 zcmeH~%}PTt5QQhnZ3zXT6+v+!eFT?Xs`ScTsZm^T{COzKYp1&(p-3uR1kB+B$qj3+%Y2in#L_vMu1&N>DcY6Tp2NfPh7J&FSAil|a z-kje+Jh+5~e?FQkDnJFO02QDDRDcRl0V+TRmZ^aL?PaG1WPl8i0W$Dg1~$s~2Rk(& S179<+%-?oAn+p7Q1zrIkUtMwl literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/c.h5 b/crates/clawhdf5/tests/fixtures/vds/c.h5 new file mode 100644 index 0000000000000000000000000000000000000000..ba9af3051ac5f5d045dad907559abe237f0a68da GIT binary patch literal 7736 zcmeI0!Ab)$5QZnowuXYxdQm(`AECz{6}z%0l`7)V$M6x_m+0AxcW+*OByYMinO~_z zP$?AYpCy@OGMSK{FH5`gW^%UMJM4*;oTZY!b=#94|z#p=Fe5f-j#wEArj z*=yHn(M8XXiilA&HyZbFnAX0%^$N(8217zU04Cv>4R%$>7 Q$iRPKV3WVKx%a99??BLUi~s-t literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/d.h5 b/crates/clawhdf5/tests/fixtures/vds/d.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8eceb4af89bff0b062fa2c3040f0bcd661dacd3e GIT binary patch literal 7736 zcmeH}u}T9$5Qb-WFJTD?iH%|*DJ?BiD%{CcDk@@`$H*h-OIX|3+u8a^cDge=zY;|- z60pd>VRv?CXXey=uc$V z{S&YMHSX%7>!d`YKCGw7VN+)r4%&@Upzn^)^}siNYEb&(oI#A%C*ZNi#vg5vq48S> zC3I$$LaI*ykM#nHz=B__VN>B>Gsh*dB0aTw1Igx(DO%wy)t_Rs7+8dbYZ>i+8$`BC zo;FQ-e!Rc_RLIc} z2I9dbEd0ySTu}ikKn17(6`%rCfC^9nD)65Q=>K$fYCs0a02v?yt1@s78{ZWRwgtt)2y;I*C(1p`&Y|l};)u;&`8$|QQ017iQG18q}n@WMyBHB!4btr88#Gqr8BjlgMkVtCk^HDJ8ez_h19Da5#yVbhJ liRL*hOy@_+0n!2KKz}=ce+YS{2f~0bAPfit!a$E1cmwT^HAesd literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/vds-eiger.h5 b/crates/clawhdf5/tests/fixtures/vds/vds-eiger.h5 new file mode 100644 index 0000000000000000000000000000000000000000..23d1fd32d6b3dae1a25aca7b84be19a130ded793 GIT binary patch literal 5496 zcmeH{y-EW?5XWclF0w?#C<#r1B_ug=u~&N*Y}oy zF85LJOB-lT^s5sqbWb{jTsyB=KfORiu+T4J*gWZ<-)toztEAh~p+QmdS(ZuTX!oSZ z#|>RV*IQ0?JUF_Te;E>uB%akYIPRBxqj3-0`K(s{Tk9Mh^m}rm_PN#5C3?m-k+|t> zTy2eZsY^^6S*~K`^FOeTrjNk{^Z6}BoXTXN2`sev)AP;uod?v^i}2h!0Vm)DoPZN> V0#3jQH~}Z%1e|~qa01^Em;ePNd literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/fixtures/vds/vds-percival-unlim-maxmin.h5 b/crates/clawhdf5/tests/fixtures/vds/vds-percival-unlim-maxmin.h5 new file mode 100644 index 0000000000000000000000000000000000000000..b7f8827c74ca881b765574d74a236d322ec5eecc GIT binary patch literal 5496 zcmeHIJxc>Y5S{yo96=!pf>@ z(wp6R7D7mM;V`oyJF~O1v-@7&Zl3zl@m77mPTYckd}WVrXmydVPzS0eCq`Ws9h@t`f+#2fG-7gOYE2EOolX$M&a=G zZjwGEv+#aCOS3Rb@+_T80(9EJf+g0W4WnP%u|f~c3dggRa9ivJB7%i}F^2UD{qxF3 zA!IA*rm0t>z{FkGHH8DYC%%{0OoAL>r^w^RNuO`eW1^`Uuc{lJ4I;kL_7p@-Z&f@0 zuYFE>gH!WEi-3$@Kzs+E^4?L>o*qz_Smyk7IF6jc6jx2+__io?o~j z4hDsz@Unu-=C2v(Q3ZekP+34H4767Ppm;jV0(!$hqY41UeJKm*$7>t^FHm`F6P{XS YKp9X5lmTTx8Bhk40cAiLSa$|~0Ko{0Bme*a literal 0 HcmV?d00001 diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs index 032d77c..2346a5e 100644 --- a/crates/clawhdf5/tests/vds_interop.rs +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -289,3 +289,162 @@ expect("nested.h5", "v", "nested") // A relative name below the virtual file's directory resolves there. assert_matches_libhdf5(dir.path(), "nested.h5", "v", "nested"); } + +// --------------------------------------------------------------------------- +// Unlimited and printf-style mappings +// --------------------------------------------------------------------------- + +/// Helpers for building unlimited VDS mappings through h5py's low-level API. +const UNLIMITED_HELPERS: &str = r#" +U = h5py.h5s.UNLIMITED +def space(dims, maxdims, start=None, count=None, stride=None, block=None): + s = h5py.h5s.create_simple(dims, maxdims) + if start is not None: + s.select_hyperslab(start, count, stride, block) + return s +def make_vds(fn, name, dims, maxdims, maps, fill, libver="latest", mode="w"): + # maps: [(vsel_kwargs, source_file, source_dataset, source_space)] + with h5py.File(fn, mode, libver=libver) as f: + dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE) + for vsel, sfile, sdset, sspace in maps: + dcpl.set_virtual(space(dims, maxdims, **vsel), sfile.encode(), sdset.encode(), sspace) + dcpl.set_fill_value(np.array(fill, dtype="f8")) + h5py.h5d.create(f.id, name.encode(), h5py.h5t.IEEE_F64LE, + h5py.h5s.create_simple(dims, maxdims), dcpl=dcpl) +"#; + +/// printf-style names: block `j` of the virtual selection comes from the +/// source named with `j` in place of `%b` (`%%` is a literal `%`), probing +/// j = 0, 1, ... until the first missing source. libhdf5 also recomputes the +/// extent from what it finds, so the stored dataspace is not the shape. +#[test] +fn vds_printf_source_names() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let body = format!( + "{UNLIMITED_HELPERS}{}", + r#" +for i in [0, 1, 2, 4]: # 3 is missing: 4 is past the first gap and unused + with h5py.File(f"vds_src_{i}.h5", "w") as s: + s.create_dataset("data", data=np.arange(10.0) + i * 100) + with h5py.File(f"p%c_{i}.h5", "w") as s: + s.create_dataset("data", data=np.arange(10.0) - i * 100) +for libver in ["earliest", "latest"]: + fn = f"printf_{libver}.h5" + make_vds(fn, "files", (10,), (U,), + [(dict(start=(0,), count=(U,), stride=(10,), block=(10,)), "vds_src_%b.h5", "data", + space((10,), (10,), (0,), (1,), (1,), (10,)))], -1.0, libver) + # interleaved blocks with gaps between them, and an escaped percent sign + make_vds(fn, "escaped", (4,), (U,), + [(dict(start=(1,), count=(U,), stride=(6,), block=(4,)), "p%%c_%b.h5", "data", + space((10,), (10,), (2,), (1,), (1,), (4,)))], -5.0, libver, "a") + # printf in the dataset name, same file, 2-D frames + with h5py.File(fn, "a") as f: + for i in range(3): + f.create_dataset(f"frame_{i}", data=np.arange(6.0).reshape(2, 3) + 10 * i) + with h5py.File(fn, "a", libver=libver) as f: + dcpl = h5py.h5p.create(h5py.h5p.DATASET_CREATE) + dcpl.set_virtual(space((1, 2, 3), (U, 2, 3), (0, 0, 0), (U, 1, 1), (1, 1, 1), (1, 2, 3)), + b".", b"frame_%b", space((2, 3), (2, 3))) + h5py.h5d.create(f.id, b"frames", h5py.h5t.IEEE_F64LE, + h5py.h5s.create_simple((1, 2, 3), (U, 2, 3)), dcpl=dcpl) + for name in ["files", "escaped", "frames"]: + expect(fn, name, f"{name}_{libver}") +"# + ); + generate(dir.path(), &body); + for libver in ["earliest", "latest"] { + for name in ["files", "escaped", "frames"] { + let file = format!("printf_{libver}.h5"); + assert_matches_libhdf5(dir.path(), &file, name, &format!("{name}_{libver}")); + } + } + // The shape libhdf5 reports: three 10-element blocks. + let f = File::open(dir.path().join("printf_latest.h5")).unwrap(); + assert_eq!(f.dataset("files").unwrap().shape().unwrap(), vec![30]); +} + +/// Unlimited source and virtual selections: each mapping covers as much as +/// its source's current extent fills (a partial last block included), the +/// extent is the largest of them but never smaller than the limited +/// mappings need, and a missing source contributes nothing. +#[test] +fn vds_unlimited_mappings_follow_source_extents() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let body = format!( + "{UNLIMITED_HELPERS}{}", + r#" +with h5py.File("grow.h5", "w") as s: + s.create_dataset("a", data=np.arange(7.0) + 1, maxshape=(None,)) + s.create_dataset("b", data=np.arange(5.0) + 100, maxshape=(None,)) + s.create_dataset("rows", data=np.arange(12.0).reshape(4, 3) + 50, maxshape=(None, 3)) +unlim_src = lambda: space((1,), (U,), (0,), (U,), (1,), (1,)) +for libver in ["earliest", "latest"]: + fn = f"unlim_{libver}.h5" + make_vds(fn, "interleaved", (1,), (U,), [ + # blocks of 3 every 4: 7 source elements end mid-block + (dict(start=(0,), count=(U,), stride=(4,), block=(3,)), "grow.h5", "a", unlim_src()), + (dict(start=(3,), count=(U,), stride=(4,), block=(1,)), "grow.h5", "b", unlim_src()), + (dict(start=(0,), count=(U,), stride=(1,), block=(1,)), "missing.h5", "a", unlim_src()), + ], -2.0, libver) + make_vds(fn, "rows", (6, 3), (U, 3), [ + # an unlimited *block*, plus a limited mapping reaching row 5 + (dict(start=(0, 0), count=(1, 1), stride=(1, 1), block=(U, 3)), "grow.h5", "rows", + space((1, 3), (U, 3), (0, 0), (1, 1), (1, 1), (U, 3))), + (dict(start=(5, 0), count=(1, 1), stride=(1, 1), block=(1, 3)), "grow.h5", "rows", + space((4, 3), (U, 3), (1, 0), (1, 1), (1, 1), (1, 3))), + ], -4.0, libver, "a") + for name in ["interleaved", "rows"]: + expect(fn, name, f"{name}_{libver}") +"# + ); + generate(dir.path(), &body); + for libver in ["earliest", "latest"] { + for name in ["interleaved", "rows"] { + let file = format!("unlim_{libver}.h5"); + assert_matches_libhdf5(dir.path(), &file, name, &format!("{name}_{libver}")); + } + } + // "a" (7 elements, blocks of 3 every 4) ends at 9; "b" (5 elements from + // 3, every 4) at 20. "rows" fills 4 rows but a limited mapping needs 6. + let f = File::open(dir.path().join("unlim_latest.h5")).unwrap(); + assert_eq!(f.dataset("interleaved").unwrap().shape().unwrap(), vec![20]); + assert_eq!(f.dataset("rows").unwrap().shape().unwrap(), vec![6, 3]); +} + +/// libhdf5's own VDS test files (HDF5 `tools/test/testfiles/vds`): +/// printf-style Eiger frames (with a source past the first gap that must be +/// ignored), a printf mapping in the 1.10 format, and Percival's four +/// interleaved unlimited sources of different lengths. +#[test] +fn vds_libhdf5_test_files() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + let fixtures = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/vds"); + for entry in std::fs::read_dir(&fixtures).unwrap() { + let path = entry.unwrap().path(); + if path.extension().is_some_and(|e| e == "h5") { + std::fs::copy(&path, dir.path().join(path.file_name().unwrap())).unwrap(); + } + } + let cases = [ + ("vds-eiger.h5", "/VDS-Eiger"), + ("4_vds.h5", "/vds_dset"), + ("vds-percival-unlim-maxmin.h5", "/VDS-Percival-unlim-maxmin"), + ]; + let mut body = String::new(); + for (i, (file, dset)) in cases.iter().enumerate() { + body.push_str(&format!("expect({file:?}, {dset:?}, \"case{i}\")\n")); + } + generate(dir.path(), &body); + for (i, (file, dset)) in cases.iter().enumerate() { + assert_matches_libhdf5(dir.path(), file, dset, &format!("case{i}")); + } + // Stored as 20 frames; only f-0.h5 is found before the first gap. + let f = File::open(dir.path().join("vds-eiger.h5")).unwrap(); + assert_eq!( + f.dataset("VDS-Eiger").unwrap().shape().unwrap(), + vec![5, 10, 10] + ); +} diff --git a/docs/known-issues.md b/docs/known-issues.md index b1de913..dc5a005 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -72,7 +72,13 @@ fill-value item that did is fixed). - ~~**Wrong data:** unmapped regions read as 0 instead of the fill value.~~ Fixed 2026-09-25: unmapped elements and missing sources read as the virtual dataset's fill value. - - `%b` printf-style source names are not expanded. + - ~~`%b` printf-style source names are not expanded.~~ Fixed 2026-09-25: + printf-style and unlimited mappings are read, and the extent is + recomputed from the sources as libhdf5 does. Still open: the + "first missing" view and a printf gap other than 0 (libhdf5 access + properties we always read at their defaults), source-to-virtual type + conversion other than a byte swap, nested virtual sources, and source + files outside the virtual file's directory (refused with an error). - ~~Hyperslab selection versions 1 and 2 are refused.~~ Fixed 2026-09-25: versions 1-3 and irregular hyperslabs are decoded. - ~~The version-1 mapping list written with a 2.0 low bound (flags byte, From 256e7b89e43eb2a8294f8404f1e950274a43f0b8 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:42:57 -0500 Subject: [PATCH 19/24] fix(format): refuse variable-length and reference VDS data from another file Their elements are global-heap IDs and object addresses in the source file. The VDS reader copied them raw, so anything decoding them against the virtual dataset's file got another object's data with no error. Same-file sources are unaffected. Found by the adversarial review. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-format/src/vds.rs | 25 ++++++++++++++++++ crates/clawhdf5/tests/vds_interop.rs | 38 ++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/crates/clawhdf5-format/src/vds.rs b/crates/clawhdf5-format/src/vds.rs index 23a3a52..f5d5237 100644 --- a/crates/clawhdf5-format/src/vds.rs +++ b/crates/clawhdf5-format/src/vds.rs @@ -831,6 +831,16 @@ impl<'a, 'r> Sources<'a, 'r> { path: &str, datatype: &Datatype, ) -> Result, FormatError> { + // Variable-length and reference elements are addresses into the file + // that holds them (global-heap IDs, object addresses). Copied out of + // another file they would be decoded against the virtual dataset's + // file and name some other object, so refuse rather than return them. + if file != "." && holds_file_addresses(datatype) { + return Err(vds_err(format!( + "VDS source {path} in {file}: variable-length and reference data \ + from another file is not supported" + ))); + } let Some(bytes) = self.file(file)? else { return Ok(None); }; @@ -841,6 +851,21 @@ impl<'a, 'r> Sources<'a, 'r> { } } +/// Whether elements of `dt` contain addresses into their own file: +/// variable-length data (global-heap IDs) or references. +fn holds_file_addresses(dt: &Datatype) -> bool { + match dt { + Datatype::VariableLength { .. } | Datatype::Reference { .. } => true, + Datatype::Compound { members, .. } => { + members.iter().any(|m| holds_file_addresses(&m.datatype)) + } + Datatype::Array { base_type, .. } | Datatype::Enumeration { base_type, .. } => { + holds_file_addresses(base_type) + } + _ => false, + } +} + /// An opened source dataset's object header. struct OpenSource { offset_size: u8, diff --git a/crates/clawhdf5/tests/vds_interop.rs b/crates/clawhdf5/tests/vds_interop.rs index 2346a5e..17069b0 100644 --- a/crates/clawhdf5/tests/vds_interop.rs +++ b/crates/clawhdf5/tests/vds_interop.rs @@ -290,6 +290,44 @@ expect("nested.h5", "v", "nested") assert_matches_libhdf5(dir.path(), "nested.h5", "v", "nested"); } +/// Variable-length and reference elements are addresses into their own file. +/// Copied raw from an external source they would be decoded against the +/// virtual dataset's file and name another object, so they are refused. +#[test] +fn vds_external_variable_length_source_is_an_error_not_foreign_addresses() { + skip_if_no_python!(); + let dir = tempfile::tempdir().unwrap(); + generate( + dir.path(), + r#" +st = h5py.string_dtype() +with h5py.File("src.h5", "w") as s: + s.create_dataset("names", data=np.array(["alpha", "beta", "gamma"], dtype=object), dtype=st) + s.create_dataset("refs", data=[s.ref, s.ref], dtype=h5py.ref_dtype) +with h5py.File("v.h5", "w", libver="latest") as f: + f.create_dataset("pad", data=np.arange(64.0)) + lay = h5py.VirtualLayout(shape=(3,), dtype=st) + lay[:] = h5py.VirtualSource("src.h5", "names", shape=(3,)) + f.create_virtual_dataset("names", lay) + lay = h5py.VirtualLayout(shape=(2,), dtype=h5py.ref_dtype) + lay[:] = h5py.VirtualSource("src.h5", "refs", shape=(2,)) + f.create_virtual_dataset("refs", lay) +"#, + ); + let f = File::open(dir.path().join("v.h5")).unwrap(); + for name in ["names", "refs"] { + let err = f + .dataset(name) + .unwrap() + .read_selection(&clawhdf5_format::selection::Selection::All) + .expect_err("raw addresses from another file must not be returned"); + assert!( + err.to_string().contains("from another file"), + "{name}: unexpected error: {err}" + ); + } +} + // --------------------------------------------------------------------------- // Unlimited and printf-style mappings // --------------------------------------------------------------------------- From d6e426e6d5e08736ade9f52ce8ec4ddebdc4db61 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:43:53 -0500 Subject: [PATCH 20/24] fix(agent): fail to open a store whose /meta has an unreadable attribute Group::attrs now leaves out an attribute it cannot decode. The agent read its settings through it, so a store whose float16 (or compression, quantized_index, WAL mark, signature...) attribute could not be decoded opened with the default in its place, and no error. /meta is now read with attrs_with_errors and any unreadable attribute is a Schema error, as it was before attrs became tolerant. Found by the adversarial review. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5-agent/src/schema.rs | 58 ++++++++++++-------- crates/clawhdf5-agent/tests/float16_store.rs | 40 ++++++++++++++ 2 files changed, 75 insertions(+), 23 deletions(-) diff --git a/crates/clawhdf5-agent/src/schema.rs b/crates/clawhdf5-agent/src/schema.rs index 432b2d8..9f33a0a 100644 --- a/crates/clawhdf5-agent/src/schema.rs +++ b/crates/clawhdf5-agent/src/schema.rs @@ -477,10 +477,34 @@ fn write_string_dataset( } } +/// `/meta`'s attributes, failing if any of them cannot be read. +/// +/// `Group::attrs` leaves out an attribute it cannot decode. For the store's +/// settings that would silently fall back to defaults (e.g. `float16`, the +/// WAL mark), so an unreadable attribute is an error here, as it was before +/// `attrs` became tolerant. +fn meta_attrs( + file: &clawhdf5::File, +) -> Result, MemoryError> { + let meta = file + .group("meta") + .map_err(|e| MemoryError::Schema(format!("missing /meta group: {e}")))?; + let (attrs, errors) = meta + .attrs_with_errors() + .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?; + if let Some(e) = errors.first() { + return Err(MemoryError::Schema(format!( + "cannot read /meta attrs: {} unreadable, first: {e}", + errors.len() + ))); + } + Ok(attrs) +} + /// Validate an HDF5 file has the correct schema and load all data. /// Read the checkpoint's [`WalMark`] from `/meta`, if it has one. pub fn read_wal_mark(file: &clawhdf5::File) -> Option { - let attrs = file.group("meta").ok()?.attrs().ok()?; + let attrs = meta_attrs(file).ok()?; let len = match attrs.get(WAL_APPLIED_LEN_ATTR)? { AttrValue::I64(v) => u64::try_from(*v).ok()?, _ => return None, @@ -498,10 +522,7 @@ pub fn read_signature( file: &clawhdf5::File, ) -> Result, MemoryError> { use crate::signing::{Manifest, StoredSignature, from_hex}; - let attrs = file - .group("meta") - .and_then(|g| g.attrs()) - .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?; + let attrs = meta_attrs(file)?; let version = match attrs.get(SIG_VERSION_ATTR) { None => return Ok(None), Some(AttrValue::I64(v)) => *v, @@ -552,18 +573,14 @@ pub fn read_signature( /// Read the checkpoint bookkeeping from `/meta`. pub fn read_checkpoint_meta(file: &clawhdf5::File) -> CheckpointMeta { - let ann_generation = file - .group("meta") - .ok() - .and_then(|g| g.attrs().ok()) - .and_then(|attrs| match attrs.get(ANN_GENERATION_ATTR) { - Some(AttrValue::I64(v)) => Some(*v as u64), - _ => None, - }); - let signed = file - .group("meta") - .and_then(|g| g.attrs()) - .is_ok_and(|attrs| attrs.contains_key(SIG_VERSION_ATTR)); + let ann_generation = + meta_attrs(file) + .ok() + .and_then(|attrs| match attrs.get(ANN_GENERATION_ATTR) { + Some(AttrValue::I64(v)) => Some(*v as u64), + _ => None, + }); + let signed = meta_attrs(file).is_ok_and(|attrs| attrs.contains_key(SIG_VERSION_ATTR)); CheckpointMeta { wal_applied: read_wal_mark(file), ann_generation, @@ -575,12 +592,7 @@ pub fn validate_and_load( file: &clawhdf5::File, ) -> Result<(MemoryConfig, MemoryCache, SessionCache, KnowledgeCache), MemoryError> { // Read /meta group attributes - let meta = file - .group("meta") - .map_err(|e| MemoryError::Schema(format!("missing /meta group: {e}")))?; - let attrs = meta - .attrs() - .map_err(|e| MemoryError::Schema(format!("cannot read /meta attrs: {e}")))?; + let attrs = meta_attrs(file)?; let schema_version = match attrs.get("schema_version") { Some(AttrValue::String(s)) => s.clone(), diff --git a/crates/clawhdf5-agent/tests/float16_store.rs b/crates/clawhdf5-agent/tests/float16_store.rs index 0c20d62..9bc3168 100644 --- a/crates/clawhdf5-agent/tests/float16_store.rs +++ b/crates/clawhdf5-agent/tests/float16_store.rs @@ -258,3 +258,43 @@ fn an_existing_f32_store_stays_f32() { assert_eq!(&values[..before.1.len()], before.1.as_slice()); assert_eq!(&values[before.1.len()..], odd.as_slice()); } + +/// `Group::attrs` leaves out an attribute it cannot decode. A store whose +/// `float16` setting is unreadable must not open as `float16 = false` (or with +/// any other default in place of a setting it has): it is an error. +#[test] +fn unreadable_meta_attribute_fails_open_instead_of_defaulting() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("store.h5"); + { + let mut m = HDF5Memory::create(config(&dir, "store.h5", true)).unwrap(); + m.save(entry(1)).unwrap(); + m.flush_wal().unwrap(); + } + assert!(HDF5Memory::open_read_only(&path).is_ok()); + + // Give the `float16` attribute message an unknown version (the name is + // at +8 in a version-1 message and +9 in a version-3 one). + let mut bytes = std::fs::read(&path).unwrap(); + let name = b"float16\0"; + let mut hit = false; + let positions: Vec = (9..bytes.len() - name.len()) + .filter(|&p| &bytes[p..p + name.len()] == name) + .collect(); + for pos in positions { + for (back, version) in [(8, 1u8), (9, 3u8)] { + if bytes[pos - back] == version { + bytes[pos - back] = 0x7f; + hit = true; + } + } + } + assert!(hit, "float16 attribute message not found"); + std::fs::write(&path, &bytes).unwrap(); + + match HDF5Memory::open_read_only(&path) { + Err(MemoryError::Schema(msg)) => assert!(msg.contains("/meta"), "{msg}"), + Err(e) => panic!("unexpected error: {e}"), + Ok(_) => panic!("store opened with an unreadable float16 setting"), + } +} From 883980f2bd963819d6e92779cebf9ab15a3efd22 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:44:05 -0500 Subject: [PATCH 21/24] test: compare h5py's v1 compound field names with what clawhdf5 reads The test compared h5py against its own expected table, so it passed with the fix reverted. Found by the adversarial review. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/clawhdf5/tests/shared_message_v1.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/clawhdf5/tests/shared_message_v1.rs b/crates/clawhdf5/tests/shared_message_v1.rs index 0873d03..80da855 100644 --- a/crates/clawhdf5/tests/shared_message_v1.rs +++ b/crates/clawhdf5/tests/shared_message_v1.rs @@ -107,8 +107,12 @@ with h5py.File("{path}", "r") as f: ); let stdout = String::from_utf8_lossy(&out.stdout); let theirs: Vec<&str> = stdout.lines().collect(); + // Read the types through clawhdf5, not from `expected()`, so this checks + // our reader against libhdf5 rather than the table against h5py. + let file = File::from_bytes(FIXTURE.to_vec()).unwrap(); let ours: Vec = expected() .into_iter() + .map(|(p, _)| (p, file.dataset(p).unwrap().dtype().unwrap())) .map(|(p, t)| match t { DType::Compound(fields) => { let names: Vec = fields.into_iter().map(|(n, _)| n).collect(); From 10d1029ead524e2fe64c2cd7f61b28067d9e449c Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:46:08 -0500 Subject: [PATCH 22/24] conformance: probe files with a user block and VDS the library's way Superblock::parse now refuses a user-block offset, and the raw read path no longer guesses a VDS fill value. The probe looks at the file from the superblock on and reads virtual datasets with vds::read_virtual_dataset, the dataset's fill value and its source-derived extent. Co-Authored-By: Claude Opus 5.5 (1M context) --- conformance/probe/src/main.rs | 82 +++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/conformance/probe/src/main.rs b/conformance/probe/src/main.rs index 3591dc5..1298cf0 100644 --- a/conformance/probe/src/main.rs +++ b/conformance/probe/src/main.rs @@ -286,6 +286,29 @@ impl<'a> Ctx<'a> { Ok(()) } + /// VDS source files resolve next to the virtual file; like the library, + /// refuse absolute paths and `..`. + fn vds_resolver( + &self, + ) -> impl Fn(&str) -> Result>, clawhdf5_format::error::FormatError> + use<> { + let base = self.base_dir.clone(); + move |name: &str| { + use clawhdf5_format::error::FormatError; + let p = std::path::Path::new(name); + if p.is_absolute() + || p.components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err(FormatError::ChunkedReadError(format!("refused {name}"))); + } + match std::fs::read(base.join(p)) { + Ok(b) => Ok(Some(b)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(FormatError::ChunkedReadError(err.to_string())), + } + } + } + fn read_dataset(&self, h: &ObjectHeader, rec: &mut Map) -> Result<(), String> { let dtb = self .payload(h, MessageType::Datatype)? @@ -295,7 +318,27 @@ impl<'a> Ctx<'a> { let dsb = self .payload(h, MessageType::Dataspace)? .ok_or("MissingMessage(Dataspace)")?; - let ds = Dataspace::parse(&dsb, self.ls).map_err(e)?; + let mut ds = Dataspace::parse(&dsb, self.ls).map_err(e)?; + // A virtual dataset's extent can come from its sources (unlimited / + // printf mappings), as h5py reports it, rather than the stored one. + if let Some(lm) = h + .messages + .iter() + .find(|m| m.msg_type == MessageType::DataLayout) + && let Ok(dl @ DataLayout::Virtual { .. }) = + DataLayout::parse(&lm.data, self.os, self.ls) + { + let resolver = self.vds_resolver(); + ds.dimensions = clawhdf5_format::vds::virtual_dataset_extent( + self.data, + &dl, + &ds, + self.os, + self.ls, + Some(&resolver), + ) + .map_err(e)?; + } let (shape, n) = Self::shape(&ds); rec.insert("shape".into(), shape); if n.saturating_mul(dt.type_size() as u64) > MAX_BYTES { @@ -331,28 +374,26 @@ impl<'a> Ctx<'a> { ); } let raw = if matches!(dl, DataLayout::Virtual { .. }) { - let base = self.base_dir.clone(); - let resolver = move |name: &str| -> Option> { - let p = std::path::Path::new(name); - if p.is_absolute() - || p.components() - .any(|c| matches!(c, std::path::Component::ParentDir)) - { - return None; - } - std::fs::read(base.join(p)).ok() - }; - data_read::read_raw_data_full_with_resolver( + let resolver = self.vds_resolver(); + let fill = clawhdf5_format::fill_value::dataset_fill_value_in( + self.data, + &h.messages, + self.os, + self.ls, + ) + .map_err(e)?; + clawhdf5_format::vds::read_virtual_dataset( self.data, &dl, &ds, &dt, - pipeline.as_ref(), + fill.as_deref(), self.os, self.ls, Some(&resolver), ) .map_err(e)? + .data } else { let cache = clawhdf5_format::chunk_cache::ChunkCache::new(); clawhdf5_format::fill_value::read_full_with_fill::( @@ -660,10 +701,13 @@ fn main() { return; } }; - let sb = guarded(|| { - let off = signature::find_signature(&data).map_err(e)?; - Superblock::parse(&data, off).map_err(e) - }); + // Every address is relative to the superblock: look at the file from + // there on (past any user block), as libhdf5 does. + let hdf5: &[u8] = match signature::find_signature(&data) { + Ok(off) => &data[off..], + Err(_) => &data, + }; + let sb = guarded(|| Superblock::parse(hdf5, 0).map_err(e)); let sb = match sb { Ok(sb) => sb, Err(msg) => { @@ -674,7 +718,7 @@ fn main() { }; top.insert("superblock_version".into(), json!(sb.version)); let ctx = Ctx { - data: &data, + data: hdf5, os: sb.offset_size, ls: sb.length_size, base_dir: std::path::Path::new(&path) From a7de15534cc866789ad25e3724f7945eb53b29d3 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:46:37 -0500 Subject: [PATCH 23/24] docs: conformance report after the read-gap fixes (569 of 697 ok) Regenerated on tank at 10d1029: ok 467 -> 569, our-error 123 -> 14, mismatch 15 -> 22 (six user-defined-link files moved from our-error to a listing difference), no panics, hangs, crashes or OOM. Baseline raised. Co-Authored-By: Claude Opus 5.5 (1M context) --- CONFORMANCE.md | 68 ++++++++++---------- conformance/baseline.json | 132 +++++++++++++++++++++++++++++++++----- 2 files changed, 149 insertions(+), 51 deletions(-) diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 1e52273..aca1b5d 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -13,15 +13,15 @@ fatal. This file is generated by `conformance/run.sh`; do not edit it by hand. | | | |---|---| -| date | 2026-09-26 03:05 UTC | -| clawhdf5 commit | `42b81d9f1c3d9bef6050ad8a1326ac8c97f641d3` | +| date | 2026-09-26 03:46 UTC | +| clawhdf5 commit | `10d1029ead524e2fe64c2cd7f61b28067d9e449c` | | machine | `tank`: AMD Ryzen 7 7800X3D 8-Core Processor, 16 CPUs, 61 GiB, Linux 7.0.0-34-generic x86_64 | -| command | `conformance/run.sh --update-baseline` | +| command | `conformance/run.sh --no-fetch --update-baseline` | | rustc | rustc 1.98.1 (48a229cea 2026-09-01) | | reference | h5py 3.16.0, HDF5 2.0.0, numpy 2.5.3, hdf5plugin 7.1.0, Python 3.14.4 | | h5dump | Version 1.14.6 (CVE corpus only) | | limits | 20 s timeout (SIGKILL), 4096 MiB address space, per process; 16 files in parallel | -| runtime | 25 s probing + comparing (0 s fetch/build before it) | +| runtime | 22 s probing + comparing (0 s fetch/build before it) | ## Results @@ -35,17 +35,17 @@ A file's class is the first that applies: | corpus | files | ok | our-error | mismatch | h5py-cannot-read | panic | hang | crash | oom | |---|---|---|---|---|---|---|---|---|---| -| NCAS-CMS_pyfive | 33 | 31 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | -| cve_hdf5 | 147 | 87 | 17 | 11 | 32 | 0 | 0 | 0 | 0 | +| NCAS-CMS_pyfive | 33 | 32 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | +| cve_hdf5 | 147 | 100 | 6 | 9 | 32 | 0 | 0 | 0 | 0 | | h5py_data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| hdf5 | 466 | 300 | 103 | 3 | 60 | 0 | 0 | 0 | 0 | +| hdf5 | 466 | 386 | 8 | 12 | 60 | 0 | 0 | 0 | 0 | | netcdf-c | 20 | 20 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| netcdf4-python | 18 | 16 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | +| netcdf4-python | 18 | 18 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | usnistgov_h5wasm | 5 | 5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | | xarray-data | 4 | 4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| **all** | **697** | **467** | **123** | **15** | **92** | **0** | **0** | **0** | **0** | +| **all** | **697** | **569** | **14** | **22** | **92** | **0** | **0** | **0** | **0** | -2 of the 15 mismatches are a known h5py bug, not ours (see *Known not-our-bug*). +2 of the 22 mismatches are a known h5py bug, not ours (see *Known not-our-bug*). Corpora (fetched by `conformance/fetch-corpus.sh` into the gitignored `conformance/.cache/`): @@ -70,29 +70,26 @@ Grouped by normalised error message. *files* counts files whose class this cause | files | objects | error | examples | |---:|---:|---|---| -| 84 | 205 | `InvalidLayoutVersion(N)` | `cve_hdf5/cvefiles/cve-2016-4330.h5`, `cve_hdf5/cvefiles/cve-2016-4333.h5`, `cve_hdf5/cvefiles/cve-2018-11206-old.h5` (+81 more) | -| 10 | 10 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5`, `hdf5/tools/test/testfiles/vds/1_vds.h5` (+7 more) | -| 9 | 17 | `InvalidObjectHeaderVersion(N)` | `cve_hdf5/cvefiles/cve-2021-36977.h5`, `cve_hdf5/cvefiles/unknown-1.h5`, `hdf5/tools/test/testfiles/h5clear_fsm_persist_user_equal.h5` (+6 more) | | 6 | 6 | `UnsupportedFilter(N)` | `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc.h5`, `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_blosc2.h5`, `hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_bshuf.h5` (+3 more) | -| 6 | 6 | `InvalidLinkType(N)` | `hdf5/tools/test/testfiles/bigendian/tall.h5`, `hdf5/tools/test/testfiles/h5diff_types.h5`, `hdf5/tools/test/testfiles/tall.h5` (+3 more) | -| 5 | 5 | `UnexpectedEof { expected: N, available: N }` | `NCAS-CMS_pyfive/tests/data/cmip_bad_eg.nc`, `cve_hdf5/cvefiles/cve-2019-9151.h5`, `hdf5/tools/test/testfiles/h5stat_newgrat.h5` (+2 more) | | 3 | 3 | `DataSizeMismatch { expected: N, actual: N }` | `cve_hdf5/cvefiles/cve-2020-18494.h5`, `cve_hdf5/cvefiles/cve-2024-32623.h5`, `cve_hdf5/cvefiles/cve-2025-2309.h5` | +| 2 | 2 | `ChunkedReadError("…")` | `cve_hdf5/cvefiles/cve-2025-2308.h5`, `hdf5/test/testfiles/bad_nbit_parms_walk.h5` | +| 1 | 1 | `UnexpectedEof { expected: N, available: N }` | `cve_hdf5/cvefiles/cve-2019-9151.h5` | | 1 | 1 | `MissingMessage(Dataspace)` | `cve_hdf5/cvefiles/cve-2024-33874.h5` | -| 1 | 2 | `InvalidSharedMessageVersion(N)` | `hdf5/tools/test/testfiles/h5stat_tsohm.h5` | +| 1 | 1 | `InvalidObjectHeaderVersion(N)` | `hdf5/tools/test/testfiles/h5clear_mdc_image.h5` | ## Mismatch root causes | files | objects | cause | examples | |---:|---:|---|---| -| 9 | 27 | `missing-object` | `cve_hdf5/cvefiles/cve-2019-8397.h5`, `cve_hdf5/cvefiles/cve-2019-8398.h5`, `cve_hdf5/cvefiles/cve-2021-46243.h5` (+6 more) | -| 5 | 11 | `extra-object` | `cve_hdf5/cvefiles/cve-2021-46244.h5`, `cve_hdf5/cvefiles/cve-2024-32613.h5`, `cve_hdf5/cvefiles/cve-2024-32616.h5` (+2 more) | +| 13 | 14 | `missing-object` | `cve_hdf5/cvefiles/cve-2019-8397.h5`, `cve_hdf5/cvefiles/cve-2019-8398.h5`, `cve_hdf5/cvefiles/cve-2021-46243.h5` (+10 more) | | 3 | 7 | `extra-attr` | `cve_hdf5/cvefiles/cve-2018-17438`, `cve_hdf5/cvefiles/cve-2018-17439`, `cve_hdf5/cvefiles/cve-2024-33874.h5` | -| 2 | 4 | `missing-attr` | `hdf5/tools/test/testfiles/twithub.h5`, `hdf5/tools/test/testfiles/twithub513.h5` | +| 3 | 6 | `extra-object` | `cve_hdf5/cvefiles/cve-2021-46244.h5`, `hdf5/tools/test/testfiles/h5clear_fsm_persist_less.h5`, `hdf5/tools/test/testfiles/h5stat_err_refcount.h5` | | 1 | 1 | `attr-values: ours=vlen(>u8) h5py=object layout=- filters=-` | `NCAS-CMS_pyfive/tests/data/attr_datatypes.hdf5` | | 1 | 1 | `values: ours=i2 h5py=>i2 layout=chunked filters=[6]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` | | 1 | 1 | `values: ours=>f4 h5py=>f4 layout=chunked filters=[2]` | `cve_hdf5/cvefiles/cve-2025-44905.h5` | | 1 | 1 | `values: ours=f4,i:>f4}8) h5py=object layout=contiguous filters=-` | `hdf5/tools/test/testfiles/tcomplex_be.h5` | ## CVE corpus: clawhdf5 vs h5dump vs h5py @@ -113,12 +110,12 @@ columns are. | file | h5dump | h5py | clawhdf5 | class | |---|---|---|---|---| -| cvefiles/cve-2016-4330.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2016-4330.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2016-4331.h5 | error exit | read 25 obj, 1 errors | read 25 obj, 1 errors | ok | | cvefiles/cve-2016-4332-mtime-new.h5 | error exit | read 25 obj, 1 errors | read 25 obj | ok | -| cvefiles/cve-2016-4332-mtime.h5 | error exit | read 4 obj, 3 errors | read 4 obj, 1 errors | ok | +| cvefiles/cve-2016-4332-mtime.h5 | error exit | read 4 obj, 3 errors | read 4 obj | ok | | cvefiles/cve-2016-4332-stab.h5 | error exit | open error | read 65 obj | h5py-cannot-read | -| cvefiles/cve-2016-4333.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2016-4333.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2017-17505.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2017-17506.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2017-17507.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | @@ -129,23 +126,23 @@ columns are. | cvefiles/cve-2018-11204.h5 | error exit | read 2 obj, 1 errors | read 2 obj | ok | | cvefiles/cve-2018-11205.h5 | error exit | read 2 obj, 1 errors | read 2 obj | ok | | cvefiles/cve-2018-11206-new.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | -| cvefiles/cve-2018-11206-old.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2018-11206-old.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-11207.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2018-13866.h5 | error exit | open error | open error | h5py-cannot-read | | cvefiles/cve-2018-13867.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | -| cvefiles/cve-2018-13868.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2018-13868.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-13869.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2018-13870.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2018-13871.h5 | error exit | read 2 obj | read 2 obj | ok | | cvefiles/cve-2018-13872.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2018-13873.h5 | error exit | read 1 obj, 1 errors | read 1 obj | ok | | cvefiles/cve-2018-13874.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2018-13875.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2018-13875.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-13876.h5 | error exit | open error | read 2 obj, 1 errors | h5py-cannot-read | -| cvefiles/cve-2018-14031.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2018-14031.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-14033.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-14034.h5 | error exit | read 1 obj, 2 errors | read 1 obj | ok | -| cvefiles/cve-2018-14035.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2018-14035.h5 | error exit | read 3 obj, 1 errors | read 3 obj, 1 errors | ok | | cvefiles/cve-2018-14460.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok | | cvefiles/cve-2018-15671.h5 | ok | read 1 obj | read 1 obj | ok | | cvefiles/cve-2018-15672.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | @@ -172,7 +169,7 @@ columns are. | cvefiles/cve-2020-10812.h5 | error exit | open error | read 2 obj | h5py-cannot-read | | cvefiles/cve-2020-18232.h5 | error exit | read 3 obj, 2 errors | read 3 obj, 2 errors | ok | | cvefiles/cve-2020-18494.h5 | ok | read 2 obj | read 2 obj, 1 errors | our-error | -| cvefiles/cve-2021-36977.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | our-error | +| cvefiles/cve-2021-36977.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2021-37501.h5 | error exit | read 18 obj, 1 errors | read 18 obj, 1 errors | ok | | cvefiles/cve-2021-45829.h5 | error exit | read 1 obj, 2 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2021-45830.h5 | error exit | open error | read 1 obj, 1 errors | h5py-cannot-read | @@ -187,7 +184,7 @@ columns are. | cvefiles/cve-2024-29161.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 2 errors | ok | | cvefiles/cve-2024-29162.h5 | error exit | read 17 obj, 4 errors | read 17 obj, 3 errors | ok | | cvefiles/cve-2024-29163.h5 | error exit | read 7 obj, 1 errors | read 7 obj, 1 errors | ok | -| cvefiles/cve-2024-29164.h5 | ok | read 3 obj | read 3 obj, 2 errors | our-error | +| cvefiles/cve-2024-29164.h5 | ok | read 3 obj | read 3 obj | ok | | cvefiles/cve-2024-29165.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2024-29166.h5 | error exit | read 17 obj, 2 errors | read 17 obj | ok | | cvefiles/cve-2024-32605.h5 | ok | read 6 obj, 1 errors | read 6 obj, 1 errors | ok | @@ -195,14 +192,14 @@ columns are. | cvefiles/cve-2024-32607-1.h5 | ok | read 10 obj | read 10 obj | ok | | cvefiles/cve-2024-32607-2.h5 | error exit | read 9 obj, 1 errors | read 9 obj, 1 errors | ok | | cvefiles/cve-2024-32608.h5 | error exit | read 6 obj, 1 errors | read 6 obj | ok | -| cvefiles/cve-2024-32609.h5 | error exit | SIGSEGV | read 4 obj, 1 errors | h5py-cannot-read | +| cvefiles/cve-2024-32609.h5 | error exit | SIGSEGV | read 3 obj, 1 errors | h5py-cannot-read | | cvefiles/cve-2024-32610.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | | cvefiles/cve-2024-32611.h5 | ok | read 6 obj | read 6 obj | ok | -| cvefiles/cve-2024-32612.h5 | ok | read 3 obj | read 3 obj, 2 errors | our-error | -| cvefiles/cve-2024-32613.h5 | error exit | read 7 obj, 1 errors | read 11 obj | mismatch | +| cvefiles/cve-2024-32612.h5 | ok | read 3 obj | read 3 obj | ok | +| cvefiles/cve-2024-32613.h5 | error exit | read 7 obj, 1 errors | read 7 obj, 1 errors | ok | | cvefiles/cve-2024-32614.h5 | error exit | read 25 obj, 2 errors | read 25 obj, 1 errors | ok | | cvefiles/cve-2024-32615.h5 | error exit | read 4 obj, 1 errors | read 4 obj, 1 errors | ok | -| cvefiles/cve-2024-32616.h5 | error exit | read 10 obj, 7 errors | read 11 obj, 6 errors | mismatch | +| cvefiles/cve-2024-32616.h5 | error exit | read 10 obj, 7 errors | read 10 obj, 5 errors | ok | | cvefiles/cve-2024-32617.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2024-32618.h5 | error exit | read 4 obj, 2 errors | read 3 obj | mismatch | | cvefiles/cve-2024-32619.h5 | error exit | read 3 obj, 2 errors | read 3 obj | ok | @@ -251,7 +248,7 @@ columns are. | cvefiles/cve-2026-26200.h5 | error exit | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | cvefiles/cve-2026-34734.h5 | error exit | read 2 obj, 1 errors | read 2 obj | ok | | cvefiles/cve-2026-92627.h5 | error exit | read 2 obj, 1 errors | read 2 obj, 1 errors | ok | -| cvefiles/unknown-1.h5 | error exit | read 11 obj, 1 errors | read 11 obj, 5 errors | our-error | +| cvefiles/unknown-1.h5 | error exit | read 11 obj, 1 errors | read 11 obj, 1 errors | ok | | fuzzerfiles/gh-4431-poc-03.h5 | error exit | read 1 obj | read 1 obj | ok | | fuzzerfiles/gh-4432-poc-05.h5 | SIGSEGV | read 1 obj, 1 errors | read 1 obj, 1 errors | ok | | fuzzerfiles/gh-4433-poc-08.h5 | error exit | read 1 obj, 1 errors | read 1 obj | ok | @@ -282,10 +279,11 @@ columns are. ## Objects h5py fails on but clawhdf5 reads +- 19 x `KeyError: '…'` - 19 x `OSError: Can't synchronously read data (no appropriate function for conversion path)` -- 17 x `KeyError: '…'` - 1 x `TypeError: unhandled dtype kind M (dtype('…'))` - 1 x `OSError: Can't synchronously read data (bad coordinate offset)` +- 1 x `TypeError: No NumPy equivalent for TypeTimeID exists` - 1 x `KeyError: "…"` - 1 x `ValueError: Insufficient precision in available types to represent (N, N, N, N, N)` diff --git a/conformance/baseline.json b/conformance/baseline.json index 6395d18..d17769a 100644 --- a/conformance/baseline.json +++ b/conformance/baseline.json @@ -1,43 +1,41 @@ { "comment": "conformance/run.sh fails if the ok count drops below `ok` or a file in `ok_files` stops being ok. Regenerate with `conformance/run.sh --update-baseline` after an intended change.", - "commit": "42b81d9f1c3d9bef6050ad8a1326ac8c97f641d3", - "date": "2026-09-26 03:05 UTC", + "commit": "10d1029ead524e2fe64c2cd7f61b28067d9e449c", + "date": "2026-09-26 03:46 UTC", "reference": "h5py 3.16.0 / HDF5 2.0.0", "files": 697, - "ok": 467, + "ok": 569, "counts": { "h5py-cannot-read": 92, - "mismatch": 15, - "ok": 467, - "our-error": 123 + "mismatch": 22, + "ok": 569, + "our-error": 14 }, "per_corpus": { "NCAS-CMS_pyfive": { "mismatch": 1, - "ok": 31, - "our-error": 1 + "ok": 32 }, "cve_hdf5": { "h5py-cannot-read": 32, - "mismatch": 11, - "ok": 87, - "our-error": 17 + "mismatch": 9, + "ok": 100, + "our-error": 6 }, "h5py_data": { "ok": 4 }, "hdf5": { "h5py-cannot-read": 60, - "mismatch": 3, - "ok": 300, - "our-error": 103 + "mismatch": 12, + "ok": 386, + "our-error": 8 }, "netcdf-c": { "ok": 20 }, "netcdf4-python": { - "ok": 16, - "our-error": 2 + "ok": 18 }, "usnistgov_h5wasm": { "ok": 5 @@ -50,6 +48,7 @@ "NCAS-CMS_pyfive/tests/compact.hdf5", "NCAS-CMS_pyfive/tests/data/btreev2.hdf5", "NCAS-CMS_pyfive/tests/data/chunked.hdf5", + "NCAS-CMS_pyfive/tests/data/cmip_bad_eg.nc", "NCAS-CMS_pyfive/tests/data/compressed.hdf5", "NCAS-CMS_pyfive/tests/data/compressed_v1.hdf5", "NCAS-CMS_pyfive/tests/data/dataset_datatypes.hdf5", @@ -78,9 +77,11 @@ "NCAS-CMS_pyfive/tests/data/resizable.hdf5", "NCAS-CMS_pyfive/tests/opaque_datetime.hdf5", "NCAS-CMS_pyfive/tests/opaque_fixed.hdf5", + "cve_hdf5/cvefiles/cve-2016-4330.h5", "cve_hdf5/cvefiles/cve-2016-4331.h5", "cve_hdf5/cvefiles/cve-2016-4332-mtime-new.h5", "cve_hdf5/cvefiles/cve-2016-4332-mtime.h5", + "cve_hdf5/cvefiles/cve-2016-4333.h5", "cve_hdf5/cvefiles/cve-2017-17505.h5", "cve_hdf5/cvefiles/cve-2017-17506.h5", "cve_hdf5/cvefiles/cve-2017-17507.h5", @@ -91,15 +92,20 @@ "cve_hdf5/cvefiles/cve-2018-11204.h5", "cve_hdf5/cvefiles/cve-2018-11205.h5", "cve_hdf5/cvefiles/cve-2018-11206-new.h5", + "cve_hdf5/cvefiles/cve-2018-11206-old.h5", "cve_hdf5/cvefiles/cve-2018-11207.h5", "cve_hdf5/cvefiles/cve-2018-13867.h5", + "cve_hdf5/cvefiles/cve-2018-13868.h5", "cve_hdf5/cvefiles/cve-2018-13869.h5", "cve_hdf5/cvefiles/cve-2018-13870.h5", "cve_hdf5/cvefiles/cve-2018-13871.h5", "cve_hdf5/cvefiles/cve-2018-13872.h5", "cve_hdf5/cvefiles/cve-2018-13873.h5", + "cve_hdf5/cvefiles/cve-2018-13875.h5", + "cve_hdf5/cvefiles/cve-2018-14031.h5", "cve_hdf5/cvefiles/cve-2018-14033.h5", "cve_hdf5/cvefiles/cve-2018-14034.h5", + "cve_hdf5/cvefiles/cve-2018-14035.h5", "cve_hdf5/cvefiles/cve-2018-14460.h5", "cve_hdf5/cvefiles/cve-2018-15671.h5", "cve_hdf5/cvefiles/cve-2018-15672.h5", @@ -115,6 +121,7 @@ "cve_hdf5/cvefiles/cve-2019-9152.h5", "cve_hdf5/cvefiles/cve-2020-10811.h5", "cve_hdf5/cvefiles/cve-2020-18232.h5", + "cve_hdf5/cvefiles/cve-2021-36977.h5", "cve_hdf5/cvefiles/cve-2021-37501.h5", "cve_hdf5/cvefiles/cve-2021-45829.h5", "cve_hdf5/cvefiles/cve-2021-45833.h5", @@ -125,6 +132,7 @@ "cve_hdf5/cvefiles/cve-2024-29161.h5", "cve_hdf5/cvefiles/cve-2024-29162.h5", "cve_hdf5/cvefiles/cve-2024-29163.h5", + "cve_hdf5/cvefiles/cve-2024-29164.h5", "cve_hdf5/cvefiles/cve-2024-29165.h5", "cve_hdf5/cvefiles/cve-2024-29166.h5", "cve_hdf5/cvefiles/cve-2024-32605.h5", @@ -134,8 +142,11 @@ "cve_hdf5/cvefiles/cve-2024-32608.h5", "cve_hdf5/cvefiles/cve-2024-32610.h5", "cve_hdf5/cvefiles/cve-2024-32611.h5", + "cve_hdf5/cvefiles/cve-2024-32612.h5", + "cve_hdf5/cvefiles/cve-2024-32613.h5", "cve_hdf5/cvefiles/cve-2024-32614.h5", "cve_hdf5/cvefiles/cve-2024-32615.h5", + "cve_hdf5/cvefiles/cve-2024-32616.h5", "cve_hdf5/cvefiles/cve-2024-32617.h5", "cve_hdf5/cvefiles/cve-2024-32619.h5", "cve_hdf5/cvefiles/cve-2024-32620.h5", @@ -159,6 +170,7 @@ "cve_hdf5/cvefiles/cve-2026-26200.h5", "cve_hdf5/cvefiles/cve-2026-34734.h5", "cve_hdf5/cvefiles/cve-2026-92627.h5", + "cve_hdf5/cvefiles/unknown-1.h5", "cve_hdf5/fuzzerfiles/gh-4431-poc-03.h5", "cve_hdf5/fuzzerfiles/gh-4432-poc-05.h5", "cve_hdf5/fuzzerfiles/gh-4433-poc-08.h5", @@ -174,6 +186,20 @@ "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_jpeg.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_lz4.h5", "hdf5/HDF5Examples/C/H5FLT/tfiles/h5ex_d_zstd.h5", + "hdf5/HDF5Examples/C/H5G/16/h5ex_g_iterate.h5", + "hdf5/HDF5Examples/C/H5G/16/h5ex_g_traverse.h5", + "hdf5/HDF5Examples/C/H5G/h5ex_g_iterate.h5", + "hdf5/HDF5Examples/C/H5G/h5ex_g_traverse.h5", + "hdf5/HDF5Examples/C/H5G/h5ex_g_visit.h5", + "hdf5/HDF5Examples/FORTRAN/H5G/h5ex_g_iterate.h5", + "hdf5/HDF5Examples/FORTRAN/H5G/h5ex_g_traverse.h5", + "hdf5/HDF5Examples/FORTRAN/H5G/h5ex_g_visit.h5", + "hdf5/HDF5Examples/JAVA/H5G/h5ex_g_iterate.h5", + "hdf5/HDF5Examples/JAVA/H5G/h5ex_g_visit.h5", + "hdf5/HDF5Examples/JAVA/compat/H5G/110/h5ex_g_iterate.h5", + "hdf5/HDF5Examples/JAVA/compat/H5G/110/h5ex_g_visit.h5", + "hdf5/HDF5Examples/JAVA/compat/H5G/h5ex_g_iterate.h5", + "hdf5/HDF5Examples/JAVA/compat/H5G/h5ex_g_visit.h5", "hdf5/c++/test/th5s.h5", "hdf5/hl/test/testfiles/test_ds_be.h5", "hdf5/hl/test/testfiles/test_ds_be_new_ref-32bit.h5", @@ -181,6 +207,9 @@ "hdf5/hl/test/testfiles/test_ds_le.h5", "hdf5/hl/test/testfiles/test_ds_le_new_ref.h5", "hdf5/hl/test/testfiles/test_ld.h5", + "hdf5/hl/test/testfiles/test_table_be.h5", + "hdf5/hl/test/testfiles/test_table_cray.h5", + "hdf5/hl/test/testfiles/test_table_le.h5", "hdf5/test/testfiles/aggr.h5", "hdf5/test/testfiles/bad_chunk_ndims.h5", "hdf5/test/testfiles/bad_compound.h5", @@ -188,12 +217,16 @@ "hdf5/test/testfiles/be_data.h5", "hdf5/test/testfiles/be_extlink1.h5", "hdf5/test/testfiles/be_extlink2.h5", + "hdf5/test/testfiles/btree_idx_1_6.h5", "hdf5/test/testfiles/btree_idx_1_8.h5", "hdf5/test/testfiles/charsets.h5", "hdf5/test/testfiles/corrupt_stab_msg.h5", + "hdf5/test/testfiles/deflate.h5", "hdf5/test/testfiles/file_image_core_test.h5", + "hdf5/test/testfiles/filespace_1_6.h5", "hdf5/test/testfiles/filespace_1_8.h5", "hdf5/test/testfiles/fill18.h5", + "hdf5/test/testfiles/fill_old.h5", "hdf5/test/testfiles/filter_error.h5", "hdf5/test/testfiles/fsm_aggr_nopersist.h5", "hdf5/test/testfiles/fsm_aggr_persist.h5", @@ -214,17 +247,27 @@ "hdf5/test/testfiles/paged_nopersist.h5", "hdf5/test/testfiles/paged_persist.h5", "hdf5/test/testfiles/specmetaread.h5", + "hdf5/test/testfiles/tarrold.h5", "hdf5/test/testfiles/tbad_msg_count.h5", "hdf5/test/testfiles/tbogus.h5", + "hdf5/test/testfiles/test_filters_be.h5", + "hdf5/test/testfiles/test_filters_le.h5", "hdf5/test/testfiles/th5s.h5", "hdf5/test/testfiles/tlayouto.h5", "hdf5/test/testfiles/tmisc38a.h5", "hdf5/test/testfiles/tmisc38b.h5", + "hdf5/test/testfiles/tmtimen.h5", + "hdf5/test/testfiles/tmtimeo.h5", "hdf5/test/testfiles/tnullspace.h5", "hdf5/test/testfiles/tsizeslheap.h5", "hdf5/tools/test/testfiles/bigendian/tdset2.h5", + "hdf5/tools/test/testfiles/binfp64.h5", + "hdf5/tools/test/testfiles/binin16.h5", "hdf5/tools/test/testfiles/binin32.h5", + "hdf5/tools/test/testfiles/binin8.h5", "hdf5/tools/test/testfiles/binin8w.h5", + "hdf5/tools/test/testfiles/binuin16.h5", + "hdf5/tools/test/testfiles/binuin32.h5", "hdf5/tools/test/testfiles/bounds_latest_latest.h5", "hdf5/tools/test/testfiles/charsets.h5", "hdf5/tools/test/testfiles/compounds_array_vlen1.h5", @@ -234,6 +277,8 @@ "hdf5/tools/test/testfiles/filter_fail.h5", "hdf5/tools/test/testfiles/h5clear_fsm_persist_equal.h5", "hdf5/tools/test/testfiles/h5clear_fsm_persist_noclose.h5", + "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_equal.h5", + "hdf5/tools/test/testfiles/h5clear_fsm_persist_user_less.h5", "hdf5/tools/test/testfiles/h5clear_sec2_v0.h5", "hdf5/tools/test/testfiles/h5clear_sec2_v2.h5", "hdf5/tools/test/testfiles/h5copy_extlinks_src.h5", @@ -343,7 +388,9 @@ "hdf5/tools/test/testfiles/h5stat_err_old_layout.h5", "hdf5/tools/test/testfiles/h5stat_filters.h5", "hdf5/tools/test/testfiles/h5stat_idx.h5", + "hdf5/tools/test/testfiles/h5stat_newgrat.h5", "hdf5/tools/test/testfiles/h5stat_threshold.h5", + "hdf5/tools/test/testfiles/h5stat_tsohm.h5", "hdf5/tools/test/testfiles/mod_h5clear_mdc_image.h5", "hdf5/tools/test/testfiles/non_comparables1.h5", "hdf5/tools/test/testfiles/non_comparables2.h5", @@ -359,8 +406,13 @@ "hdf5/tools/test/testfiles/t128bit_float.h5", "hdf5/tools/test/testfiles/tCVE-2021-37501_attr_decode.h5", "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_new.h5", + "hdf5/tools/test/testfiles/tCVE_2018_11206_fill_old.h5", "hdf5/tools/test/testfiles/taindices.h5", + "hdf5/tools/test/testfiles/tarray1.h5", "hdf5/tools/test/testfiles/tarray1_big.h5", + "hdf5/tools/test/testfiles/tarray2.h5", + "hdf5/tools/test/testfiles/tarray4.h5", + "hdf5/tools/test/testfiles/tarray5.h5", "hdf5/tools/test/testfiles/tarray8.h5", "hdf5/tools/test/testfiles/tattr.h5", "hdf5/tools/test/testfiles/tattr2.h5", @@ -372,15 +424,20 @@ "hdf5/tools/test/testfiles/tbigdims.h5", "hdf5/tools/test/testfiles/tbinary.h5", "hdf5/tools/test/testfiles/tbitnopaque.h5", + "hdf5/tools/test/testfiles/tchar.h5", "hdf5/tools/test/testfiles/tcmpdattrintsize.h5", "hdf5/tools/test/testfiles/tcmpdintarray.h5", "hdf5/tools/test/testfiles/tcmpdints.h5", "hdf5/tools/test/testfiles/tcmpdintsize.h5", "hdf5/tools/test/testfiles/tcomplex.h5", + "hdf5/tools/test/testfiles/tcompound.h5", + "hdf5/tools/test/testfiles/tcompound_complex.h5", "hdf5/tools/test/testfiles/tcompound_complex2.h5", + "hdf5/tools/test/testfiles/tdatareg.h5", "hdf5/tools/test/testfiles/tdset.h5", "hdf5/tools/test/testfiles/tdset2.h5", "hdf5/tools/test/testfiles/tdset_idx.h5", + "hdf5/tools/test/testfiles/tempty.h5", "hdf5/tools/test/testfiles/textlink.h5", "hdf5/tools/test/testfiles/textlinkfar.h5", "hdf5/tools/test/testfiles/textlinksrc.h5", @@ -412,6 +469,7 @@ "hdf5/tools/test/testfiles/tloop.h5", "hdf5/tools/test/testfiles/tnamed_dtype_attr.h5", "hdf5/tools/test/testfiles/tnestedcmpddt.h5", + "hdf5/tools/test/testfiles/tnestedcomp.h5", "hdf5/tools/test/testfiles/tno-subset.h5", "hdf5/tools/test/testfiles/tnullspace.h5", "hdf5/tools/test/testfiles/torderattr.h5", @@ -426,6 +484,7 @@ "hdf5/tools/test/testfiles/trefer_param.h5", "hdf5/tools/test/testfiles/trefer_reg.h5", "hdf5/tools/test/testfiles/trefer_reg_1d.h5", + "hdf5/tools/test/testfiles/tsaf.h5", "hdf5/tools/test/testfiles/tscalarattrintsize.h5", "hdf5/tools/test/testfiles/tscalarintattrsize.h5", "hdf5/tools/test/testfiles/tscalarintsize.h5", @@ -435,39 +494,78 @@ "hdf5/tools/test/testfiles/tst_onion_dset_1d.h5", "hdf5/tools/test/testfiles/tst_onion_dset_ext.h5", "hdf5/tools/test/testfiles/tst_onion_objs.h5", + "hdf5/tools/test/testfiles/tstr.h5", + "hdf5/tools/test/testfiles/tstr2.h5", "hdf5/tools/test/testfiles/tstr3.h5", "hdf5/tools/test/testfiles/tudfilter.h5", "hdf5/tools/test/testfiles/tudfilter2.h5", + "hdf5/tools/test/testfiles/tvldtypes1.h5", + "hdf5/tools/test/testfiles/tvldtypes2.h5", + "hdf5/tools/test/testfiles/tvldtypes3.h5", + "hdf5/tools/test/testfiles/tvldtypes4.h5", + "hdf5/tools/test/testfiles/tvldtypes5.h5", "hdf5/tools/test/testfiles/tvlenstr_array.h5", "hdf5/tools/test/testfiles/tvlstr.h5", "hdf5/tools/test/testfiles/tvms.h5", + "hdf5/tools/test/testfiles/txtfp32.h5", + "hdf5/tools/test/testfiles/txtfp64.h5", + "hdf5/tools/test/testfiles/txtin16.h5", + "hdf5/tools/test/testfiles/txtin32.h5", + "hdf5/tools/test/testfiles/txtin8.h5", "hdf5/tools/test/testfiles/txtstr.h5", + "hdf5/tools/test/testfiles/txtuin16.h5", + "hdf5/tools/test/testfiles/txtuin32.h5", "hdf5/tools/test/testfiles/vds/1_a.h5", "hdf5/tools/test/testfiles/vds/1_b.h5", "hdf5/tools/test/testfiles/vds/1_c.h5", "hdf5/tools/test/testfiles/vds/1_d.h5", "hdf5/tools/test/testfiles/vds/1_e.h5", "hdf5/tools/test/testfiles/vds/1_f.h5", + "hdf5/tools/test/testfiles/vds/1_vds.h5", "hdf5/tools/test/testfiles/vds/2_a.h5", "hdf5/tools/test/testfiles/vds/2_b.h5", "hdf5/tools/test/testfiles/vds/2_c.h5", "hdf5/tools/test/testfiles/vds/2_d.h5", "hdf5/tools/test/testfiles/vds/2_e.h5", + "hdf5/tools/test/testfiles/vds/2_vds.h5", + "hdf5/tools/test/testfiles/vds/3_1_vds.h5", + "hdf5/tools/test/testfiles/vds/3_2_vds.h5", "hdf5/tools/test/testfiles/vds/4_0.h5", "hdf5/tools/test/testfiles/vds/4_1.h5", "hdf5/tools/test/testfiles/vds/4_2.h5", + "hdf5/tools/test/testfiles/vds/4_vds.h5", "hdf5/tools/test/testfiles/vds/5_a.h5", "hdf5/tools/test/testfiles/vds/5_b.h5", "hdf5/tools/test/testfiles/vds/5_c.h5", + "hdf5/tools/test/testfiles/vds/5_vds.h5", "hdf5/tools/test/testfiles/vds/a.h5", "hdf5/tools/test/testfiles/vds/b.h5", "hdf5/tools/test/testfiles/vds/c.h5", "hdf5/tools/test/testfiles/vds/d.h5", "hdf5/tools/test/testfiles/vds/f-0.h5", "hdf5/tools/test/testfiles/vds/f-3.h5", + "hdf5/tools/test/testfiles/vds/vds-eiger.h5", + "hdf5/tools/test/testfiles/vds/vds-percival-unlim-maxmin.h5", + "hdf5/tools/test/testfiles/xml/tbitfields.h5", + "hdf5/tools/test/testfiles/xml/tcompound2.h5", + "hdf5/tools/test/testfiles/xml/tdset2.h5", + "hdf5/tools/test/testfiles/xml/tenum.h5", "hdf5/tools/test/testfiles/xml/test35.nc", "hdf5/tools/test/testfiles/xml/tloop2.h5", + "hdf5/tools/test/testfiles/xml/tname-amp.h5", + "hdf5/tools/test/testfiles/xml/tname-apos.h5", + "hdf5/tools/test/testfiles/xml/tname-gt.h5", + "hdf5/tools/test/testfiles/xml/tname-lt.h5", + "hdf5/tools/test/testfiles/xml/tname-quot.h5", + "hdf5/tools/test/testfiles/xml/tname-sp.h5", + "hdf5/tools/test/testfiles/xml/tnodata.h5", + "hdf5/tools/test/testfiles/xml/tobjref.h5", "hdf5/tools/test/testfiles/xml/topaque.h5", + "hdf5/tools/test/testfiles/xml/tref-escapes-at.h5", + "hdf5/tools/test/testfiles/xml/tref-escapes.h5", + "hdf5/tools/test/testfiles/xml/tref.h5", + "hdf5/tools/test/testfiles/xml/tstring-at.h5", + "hdf5/tools/test/testfiles/xml/tstring.h5", "hdf5/tools/test/testfiles/zerodim.h5", "netcdf-c/h5_test/ref_tst_h_compounds.h5", "netcdf-c/h5_test/ref_tst_h_compounds2.h5", @@ -504,6 +602,8 @@ "netcdf4-python/examples/data/rtofs_glo_3dz_f006_6hrly_reg3.nc", "netcdf4-python/test/20171025_2056.Cloud_Top_Height.nc", "netcdf4-python/test/issue1152.nc", + "netcdf4-python/test/issue671.nc", + "netcdf4-python/test/issue672.nc", "netcdf4-python/test/test_gold.nc", "usnistgov_h5wasm/test/array.h5", "usnistgov_h5wasm/test/compressed.h5", From bb78d70b994063c84f1a81df5403cedaf7f41a60 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 22:46:44 -0500 Subject: [PATCH 24/24] docs: changelog for the review follow-up fixes Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8568f7..1fc429d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -296,6 +296,14 @@ - CI keeps zlib-ng building and tested; the arm64 job no longer needs cmake. ### Correctness +- `clawhdf5-format` VDS: variable-length and reference data from a source in + another file is refused. Those elements are global-heap IDs and object + addresses in the source file; copied into the virtual dataset they would + be decoded against the wrong file and name another object. +- `clawhdf5-agent`: a store whose `/meta` has an attribute that cannot be + decoded fails to open (`MemoryError::Schema`). With `attrs()` now leaving + unreadable attributes out, it would otherwise have opened with defaults in + place of its settings (`float16`, `compression`, the WAL mark, ...). - `clawhdf5-format` reader: an old-style group whose local heap has a free list pointing outside the heap was listed with names read from the broken heap (garbage names on `cve-2021-36977.h5` once its user block was