From 85eb7f5ce2e0c237f07697af3b209f691782e9e2 Mon Sep 17 00:00:00 2001 From: osobh Date: Fri, 25 Sep 2026 21:52:19 -0500 Subject: [PATCH] feat(format): read Data Layout message versions 1 and 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HDF5 1.4/1.6-era files store the layout as version 1 or 2: version, dimensionality, class, 5 reserved bytes, an address (contiguous and chunked only), dimensionality 32-bit sizes (with the trailing element-size dimension) and, for compact storage, a 32-bit size and the raw data. They failed with InvalidLayoutVersion — 84 of the 686 files in the audit sweep, 205 datasets. Map them onto the existing variants: chunked uses the same version-1 B-tree chunk index as version 3 and is reported as version 3, so every chunked read path (filters, selections, caches) applies unchanged. Contiguous size is the product of the stored dimensions, which is what libhdf5 computes from the dataspace; a disagreement fails the reader's size check instead of returning wrong data. Fixtures are HDF5's own deflate.h5 (v1, chunked + deflate) and h5ex_g_iterate.h5 (v2, contiguous, one unallocated dataset); the new interop test compares every dataset byte for byte against h5py. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 + crates/clawhdf5-format/src/data_layout.rs | 190 +++++++++++++++++- .../tests/fixtures/legacy/README.md | 11 + .../tests/fixtures/legacy/deflate.h5 | Bin 0 -> 6240 bytes .../tests/fixtures/legacy/h5ex_g_iterate.h5 | Bin 0 -> 2928 bytes .../clawhdf5/tests/legacy_format_interop.rs | 121 +++++++++++ docs/known-issues.md | 2 + 7 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/README.md create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/deflate.h5 create mode 100644 crates/clawhdf5-format/tests/fixtures/legacy/h5ex_g_iterate.h5 create mode 100644 crates/clawhdf5/tests/legacy_format_interop.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b87415..cf1b36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -236,6 +236,12 @@ - A pipeline with Fletcher32 ahead of the compressor (h5py `set_fletcher32()` then `set_deflate()`) no longer fails with "deflate: output exceeds size limit". +- `clawhdf5-format`: **HDF5 1.4/1.6-era files are readable.** Data Layout + message versions 1 and 2 (compact, contiguous, and chunked through the + version-1 B-tree) failed with `InvalidLayoutVersion` — 84 of the 686 files in + the 2026-09-25 audit sweep, 205 datasets. They now read as libhdf5 does; + checked byte for byte against h5py on HDF5's own test files + (`tests/legacy_format_interop.rs`). ### Storage - `clawhdf5-format`: **half-precision datasets.** diff --git a/crates/clawhdf5-format/src/data_layout.rs b/crates/clawhdf5-format/src/data_layout.rs index 8429c5d..ece868c 100644 --- a/crates/clawhdf5-format/src/data_layout.rs +++ b/crates/clawhdf5-format/src/data_layout.rs @@ -1,7 +1,7 @@ //! HDF5 Data Layout message parsing (message type 0x0008). #[cfg(not(feature = "std"))] -use alloc::{string::String, vec::Vec}; +use alloc::{format, string::String, vec::Vec}; #[cfg(feature = "std")] use std::string::String; @@ -45,7 +45,9 @@ pub enum DataLayout { chunk_dimensions: Vec, /// B-tree address, or `None` if undefined. btree_address: Option, - /// Layout version (3 or 4). + /// Layout version (3 or 4). Version 1/2 messages (HDF5 1.4/1.6-era) + /// use the same version-1 B-tree chunk index as version 3 and are + /// reported as 3. version: u8, /// Chunk index type (v4 only). chunk_index_type: Option, @@ -261,6 +263,7 @@ impl DataLayout { let layout_class = data[1]; match version { + 1 | 2 => Self::parse_v1_v2(data, offset_size), 3 => Self::parse_v3(data, layout_class, offset_size, length_size), // v5 (emitted by HDF5 1.14+/2.0 with `libver=latest`) uses the same // message structure as v4 — only the version number was bumped. @@ -269,6 +272,87 @@ impl DataLayout { } } + /// Layout message versions 1 and 2 (HDF5 before 1.6.3): + /// + /// ```text + /// version(1) · dimensionality(1) · layout class(1) · reserved(5) + /// · address(offset_size) — contiguous and chunked only + /// · dimension sizes(4 × dimensionality) + /// · compact data size(4) · compact raw data — compact only + /// ``` + /// + /// The dimension sizes are the dataset's (contiguous/compact) or the + /// chunk's (chunked) extent plus a trailing element-size dimension, as in + /// version 3's chunked form. libhdf5 ignores them for contiguous storage + /// and sizes the data from the dataspace; the product of the stored + /// dimensions is that same size, and a disagreement (a dimension that was + /// truncated to 32 bits) is caught by the reader's size check rather than + /// returning wrong data. + fn parse_v1_v2(data: &[u8], offset_size: u8) -> Result { + ensure_len(data, 0, 8)?; + let dimensionality = data[1] as usize; + let layout_class = data[2]; + // H5O_LAYOUT_NDIMS: 32 dataspace dimensions + the element-size one. + if dimensionality > 33 { + return Err(FormatError::Overflow(format!( + "data layout dimensionality {dimensionality} exceeds 33" + ))); + } + let mut p = 8; + let os = offset_size as usize; + let address = match layout_class { + 1 | 2 => { + ensure_len(data, p, os)?; + let a = if is_undefined(data, p, offset_size) { + None + } else { + Some(read_offset(data, p, offset_size)?) + }; + p += os; + a + } + 0 => None, + _ => return Err(FormatError::InvalidLayoutClass(layout_class)), + }; + ensure_len(data, p, dimensionality * 4)?; + let dims: Vec = data[p..p + dimensionality * 4] + .as_chunks::<4>() + .0 + .iter() + .map(|c| u32::from_le_bytes(*c)) + .collect(); + p += dimensionality * 4; + match layout_class { + 0 => { + ensure_len(data, p, 4)?; + let size = + u32::from_le_bytes([data[p], data[p + 1], data[p + 2], data[p + 3]]) as usize; + ensure_len(data, p + 4, size)?; + Ok(DataLayout::Compact { + data: data[p + 4..p + 4 + size].to_vec(), + }) + } + 1 => { + let size = dims + .iter() + .try_fold(1u64, |acc, &d| acc.checked_mul(d as u64)) + .ok_or_else(|| { + FormatError::Overflow(format!("contiguous layout size {dims:?}")) + })?; + Ok(DataLayout::Contiguous { address, size }) + } + _ => Ok(DataLayout::Chunked { + chunk_dimensions: dims, + btree_address: address, + version: 3, + chunk_index_type: None, + single_chunk_filtered_size: None, + single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, + }), + } + } + fn parse_v3( data: &[u8], layout_class: u8, @@ -546,6 +630,108 @@ impl DataLayout { mod tests { use super::*; + /// Version 1/2 header: version, dimensionality, class, reserved(5). + fn v1v2_header(version: u8, ndims: u8, class: u8) -> Vec { + vec![version, ndims, class, 0, 0, 0, 0, 0] + } + + #[test] + fn v2_compact() { + let mut buf = v1v2_header(2, 2, 0); + // dims (3 elements of 2 bytes) — no address for compact + buf.extend_from_slice(&3u32.to_le_bytes()); + buf.extend_from_slice(&2u32.to_le_bytes()); + buf.extend_from_slice(&6u32.to_le_bytes()); // compact size (u32 in v1/v2) + buf.extend_from_slice(&[1, 0, 2, 0, 3, 0]); + assert_eq!( + DataLayout::parse(&buf, 8, 8).unwrap(), + DataLayout::Compact { + data: vec![1, 0, 2, 0, 3, 0] + } + ); + } + + #[test] + fn v1_contiguous_size_from_dimensions() { + let mut buf = v1v2_header(1, 3, 1); + buf.extend_from_slice(&0x800u32.to_le_bytes()); // 4-byte address + for d in [10u32, 20, 4] { + buf.extend_from_slice(&d.to_le_bytes()); + } + assert_eq!( + DataLayout::parse(&buf, 4, 4).unwrap(), + DataLayout::Contiguous { + address: Some(0x800), + size: 800, + } + ); + } + + #[test] + fn v1_contiguous_undefined_address() { + let mut buf = v1v2_header(1, 2, 1); + buf.extend_from_slice(&[0xFF; 8]); + buf.extend_from_slice(&5u32.to_le_bytes()); + buf.extend_from_slice(&8u32.to_le_bytes()); + assert_eq!( + DataLayout::parse(&buf, 8, 8).unwrap(), + DataLayout::Contiguous { + address: None, + size: 40, + } + ); + } + + #[test] + fn v1_chunked_maps_to_btree_v1_index() { + let mut buf = v1v2_header(1, 3, 2); + buf.extend_from_slice(&0x1234u64.to_le_bytes()); + for d in [50u32, 50, 4] { + buf.extend_from_slice(&d.to_le_bytes()); + } + assert_eq!( + DataLayout::parse(&buf, 8, 8).unwrap(), + DataLayout::Chunked { + chunk_dimensions: vec![50, 50, 4], + btree_address: Some(0x1234), + version: 3, + chunk_index_type: None, + single_chunk_filtered_size: None, + single_chunk_filter_mask: None, + dont_filter_partial_edge_chunks: false, + } + ); + } + + #[test] + fn v1v2_rejects_bad_class_dimensionality_and_truncation() { + assert_eq!( + DataLayout::parse(&v1v2_header(1, 1, 3), 8, 8).unwrap_err(), + FormatError::InvalidLayoutClass(3) + ); + assert!(matches!( + DataLayout::parse(&v1v2_header(2, 34, 1), 8, 8).unwrap_err(), + FormatError::Overflow(_) + )); + // Chunked, dims cut short. + let mut buf = v1v2_header(1, 2, 2); + buf.extend_from_slice(&0x10u64.to_le_bytes()); + buf.extend_from_slice(&7u32.to_le_bytes()); + assert!(matches!( + DataLayout::parse(&buf, 8, 8).unwrap_err(), + FormatError::UnexpectedEof { .. } + )); + // Compact, raw data shorter than its declared size. + let mut buf = v1v2_header(2, 1, 0); + buf.extend_from_slice(&4u32.to_le_bytes()); + buf.extend_from_slice(&100u32.to_le_bytes()); + buf.extend_from_slice(&[0; 4]); + assert!(matches!( + DataLayout::parse(&buf, 8, 8).unwrap_err(), + FormatError::UnexpectedEof { .. } + )); + } + #[test] fn v3_compact() { let mut buf = vec![3u8, 0]; // version=3, class=0 (compact) diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/README.md b/crates/clawhdf5-format/tests/fixtures/legacy/README.md new file mode 100644 index 0000000..b7ee62d --- /dev/null +++ b/crates/clawhdf5-format/tests/fixtures/legacy/README.md @@ -0,0 +1,11 @@ +# Legacy (HDF5 1.4/1.6-era) fixtures + +Unmodified copies of the HDF Group's own test files from +https://github.com/HDFGroup/hdf5 at a3cf1ea82cc7a66e50029a688121e1b105a7ce88 +(BSD-style license, see that repository's `LICENSE`). Current libraries cannot +write these structures, so they are kept as files. + +| File | Upstream path | Exercises | +|---|---|---| +| `deflate.h5` | `test/testfiles/deflate.h5` | Data Layout message v1, chunked + deflate (v1 B-tree index) | +| `h5ex_g_iterate.h5` | `HDF5Examples/C/H5G/h5ex_g_iterate.h5` | Data Layout message v2, contiguous; an unallocated dataset | diff --git a/crates/clawhdf5-format/tests/fixtures/legacy/deflate.h5 b/crates/clawhdf5-format/tests/fixtures/legacy/deflate.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2f62e2599eee6a832a07f6048b0daba59bd8cf08 GIT binary patch literal 6240 zcmeD5aB<`1lHy_j0S*oZ76t(ZW-tdr{D*=B2~<8z$pWZiMyNmol#u}Cd$>9VfSFKn zs4)x;PhoLXWC!F@5o#|Z*jz@2l+?7G#FA77PN+IQp!pzRWME)qXlQ6= zWNd6?3e43UF#XIB3pCgu8jL_{ff$<1fh-S*1eM5OKYtfSpvz(T=K^x!MkPB&f-#_S z3KWX4@(D&;6YzWjBsnmks{_S3GMJ4+9V{Kf)Lz4(ZW>Ghlok|(Fktqg+XqwbgF_v< z`gR=Z(A{?khdOlk{e`N7xdUbnOdTRWz*LOVqaiRF0;3@?8UmvsFd71*Auu#TpyJls z6DK(t3J5PjYa!3siJs1TLQv~YqT?ZOr)L~IU_l9D4tqJbMwQgRt2N{&HEfdo6A zc?6 File { + File::open(format!("{FIXTURES}/{name}")).unwrap() +} + +fn python() -> String { + std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +fn interop_required() -> bool { + std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1") +} + +fn python_available() -> bool { + Command::new(python()) + .args(["-c", "import h5py, numpy"]) + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +/// Layout v1, chunked (50x50 chunks of a 100x200 dataset), deflate: every +/// read path goes through the version-1 B-tree chunk index. +#[test] +fn layout_v1_chunked_deflate() { + let file = open("deflate.h5"); + let ds = file.dataset("Dataset1").unwrap(); + assert_eq!(ds.shape().unwrap(), [100, 200]); + let expected: Vec = (0..100).flat_map(|_| (0..200).map(|j| j % 5)).collect(); + assert_eq!(ds.read_i32().unwrap(), expected); + + // A hyperslab that straddles four chunks. + let slab = Selection::Hyperslab { + start: vec![48, 48], + stride: vec![1, 1], + count: vec![4, 4], + block: vec![1, 1], + }; + let raw = ds.read_selection(&slab).unwrap(); + let got: Vec = raw + .as_chunks::<4>() + .0 + .iter() + .map(|b| i32::from_le_bytes(*b)) + .collect(); + assert_eq!(got, [3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1]); +} + +/// Layout v2, contiguous: one dataset with storage, one never written (reads +/// as its fill value, 0). +#[test] +fn layout_v2_contiguous() { + let file = open("h5ex_g_iterate.h5"); + assert_eq!(file.dataset("G1/DS2").unwrap().read_i32().unwrap(), [1]); + assert_eq!(file.dataset("DS1").unwrap().read_i32().unwrap(), [0]); +} + +/// Every dataset in every fixture, byte for byte against h5py. +#[test] +fn legacy_fixtures_match_h5py() { + if !python_available() { + assert!( + !interop_required(), + "CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available" + ); + eprintln!("SKIP: python3 with h5py not available"); + return; + } + for (name, datasets) in [ + ("deflate.h5", &["Dataset1"][..]), + ("h5ex_g_iterate.h5", &["DS1", "G1/DS2"][..]), + ] { + let path = format!("{FIXTURES}/{name}"); + let script = format!( + r#" +import h5py, numpy as np +f = h5py.File({path:?}, "r") +for n in {datasets:?}: + print(n, np.ascontiguousarray(f[n][()]).tobytes().hex()) +"# + ); + let out = Command::new(python()) + .args(["-c", &script]) + .output() + .unwrap(); + assert!( + out.status.success(), + "h5py: {}", + String::from_utf8_lossy(&out.stderr) + ); + let file = File::open(&path).unwrap(); + for line in String::from_utf8(out.stdout).unwrap().lines() { + let (ds, hex) = line.split_once(' ').unwrap(); + let ours = file + .dataset(ds) + .unwrap() + .read_selection(&Selection::All) + .unwrap(); + let ours: String = ours.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(ours, hex, "{name}:{ds}"); + } + } +} diff --git a/docs/known-issues.md b/docs/known-issues.md index 6dfa369..135ff49 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -68,6 +68,8 @@ the VDS item, which is marked. - **Layout message versions 1 and 2** (HDF5 1.6-era files): 84 of the 686 sweep files, `InvalidLayoutVersion`. This is the largest single gap. + **Fixed 2026-09-25:** versions 1 and 2 are parsed (compact, contiguous, + chunked via the v1 B-tree). - **Virtual datasets:** - **Wrong data:** unmapped regions read as 0 instead of the fill value. - `%b` printf-style source names are not expanded.